From ff6a63147b13a1c89db23e10f07327936238aa26 Mon Sep 17 00:00:00 2001 From: User Date: Thu, 16 Apr 2026 19:16:11 +0800 Subject: [PATCH] =?UTF-8?q?fix(voice-gateway):=20S2S=20idle=20timeout=20+?= =?UTF-8?q?=20upstream=20send=20lock=20+=20iOS=20AudioContext=20suspended?= =?UTF-8?q?=20+=20port=203012=E2=86=923013?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - P0: S2S DialogAudioIdleTimeoutError now notifies client instead of force-closing, sets upstreamReady=false and cancels keepalive - P0: Reduce audioKeepaliveIntervalMs from 20s to 8s to prevent S2S idle timeout - P1: Add upstreamSendLock to prevent concurrent IllegalStateException: Send pending - P1: iOS AudioContext suspended handling - buffer audio chunks and try resume after user interaction - P1: disconnect() clears pendingAudioChunks and _resuming to prevent memory leak - Fix: Frontend hardcoded port 3012→3013 in videoApi.js and vite.config.js - Add complete Java backend source code to git tracking --- delivery/client/src/services/chatApi.js | 122 ++ .../client/src/services/nativeVoiceService.js | 426 +++++ delivery/client/src/services/videoApi.js | 80 + delivery/client/src/services/voiceApi.js | 61 + delivery/client/vite.config.js | 27 + .../bigwo/javaserver/BigwoApplication.java | 23 + .../com/bigwo/javaserver/api/ApiEnvelope.java | 27 + .../config/AssistantProfileProperties.java | 71 + .../bigwo/javaserver/config/CorsConfig.java | 24 + .../javaserver/config/CozeProperties.java | 49 + .../javaserver/config/DatabaseContext.java | 240 +++ .../InfrastructureAvailabilityState.java | 19 + .../config/KnowledgeBaseProperties.java | 127 ++ .../javaserver/config/MysqlProperties.java | 97 + .../javaserver/config/RedisClientManager.java | 86 + .../javaserver/config/RedisProperties.java | 62 + .../config/VideoGenerationProperties.java | 143 ++ .../config/VoiceGatewayProperties.java | 103 ++ .../AssistantProfileController.java | 77 + .../javaserver/controller/ChatController.java | 75 + .../controller/HealthController.java | 77 + .../controller/SessionController.java | 88 + .../controller/VideoController.java | 156 ++ .../controller/VoiceController.java | 61 + .../exception/BadRequestException.java | 8 + .../exception/GlobalExceptionHandler.java | 53 + .../javaserver/model/AssistantProfile.java | 56 + .../model/AssistantProfileResult.java | 11 + .../javaserver/model/ChatHistoryResponse.java | 4 + .../javaserver/model/ChatSendResponse.java | 4 + .../javaserver/model/ChatSessionState.java | 124 ++ .../javaserver/model/ChatStartResponse.java | 4 + .../javaserver/model/ChatStreamEvent.java | 23 + .../bigwo/javaserver/model/ChatSubtitle.java | 4 + .../javaserver/model/CozeChatResult.java | 4 + .../javaserver/model/KnowledgeChunk.java | 15 + .../javaserver/model/KnowledgeQueryInfo.java | 14 + .../model/KnowledgeSearchResult.java | 22 + .../bigwo/javaserver/model/LlmMessage.java | 4 + .../javaserver/model/RedisContextMessage.java | 4 + .../model/RenderedVideoPayload.java | 9 + .../javaserver/model/SessionFullMessage.java | 13 + .../model/SessionHistoryResult.java | 6 + .../javaserver/model/SessionListItem.java | 12 + .../javaserver/model/SessionSwitchResult.java | 6 + .../model/VideoAdminConfigResponse.java | 16 + .../model/VideoGenerateResponse.java | 4 + .../model/VideoHistoryResponse.java | 6 + .../model/VideoHistoryRowResponse.java | 20 + .../javaserver/model/VideoPromptDetail.java | 16 + .../javaserver/model/VideoTaskSnapshot.java | 24 + .../javaserver/repository/ChatRepository.java | 111 ++ .../repository/SessionRepository.java | 144 ++ .../repository/VideoTaskRepository.java | 224 +++ .../service/AssistantProfileService.java | 220 +++ .../service/ChatContentSafetyService.java | 133 ++ .../bigwo/javaserver/service/ChatService.java | 440 +++++ .../service/ContextKeywordTracker.java | 118 ++ .../javaserver/service/CozeChatClient.java | 230 +++ .../javaserver/service/FastAsrCorrector.java | 153 ++ .../KnowledgeBaseRetrieverService.java | 666 +++++++ .../service/KnowledgeKeywordCatalog.java | 170 ++ .../service/KnowledgeQueryResolver.java | 327 ++++ .../service/KnowledgeRouteDecider.java | 63 + .../service/PinyinProductMatcher.java | 181 ++ .../service/ProductLinkTrigger.java | 95 + .../javaserver/service/RedisContextStore.java | 159 ++ .../javaserver/service/SessionService.java | 70 + .../service/VideoGenerationService.java | 1353 ++++++++++++++ .../service/VoiceAssistantProfileSupport.java | 123 ++ .../service/VoiceGatewayService.java | 1567 +++++++++++++++++ .../javaserver/service/VoiceSessionState.java | 150 ++ .../bigwo/javaserver/util/VolcSignerV4.java | 76 + .../web/ApiRequestLoggingFilter.java | 31 + .../AssistantProfileRefreshRequest.java | 4 + .../web/request/ChatSendRequest.java | 4 + .../web/request/ChatStartRequest.java | 7 + .../web/request/SessionSwitchRequest.java | 4 + .../web/request/VideoAdminConfigRequest.java | 4 + .../AssistantProfileResponseData.java | 14 + .../web/response/HealthFeaturesResponse.java | 15 + .../web/response/HealthResponse.java | 10 + .../websocket/VoiceWebSocketConfig.java | 22 + .../websocket/VoiceWebSocketHandler.java | 47 + .../websocket/VolcRealtimeProtocol.java | 169 ++ .../src/main/resources/application.yml | 89 + .../src/main/resources/logback-spring.xml | 12 + .../javaserver/ApiContractSmokeTest.java | 64 + .../javaserver/ChatApiContractSmokeTest.java | 61 + .../javaserver/KnowledgeServicesTest.java | 77 + .../javaserver/VideoApiContractSmokeTest.java | 79 + .../javaserver/VoiceGatewaySmokeTest.java | 183 ++ .../client/src/services/nativeVoiceService.js | 104 +- 93 files changed, 10557 insertions(+), 23 deletions(-) create mode 100644 delivery/client/src/services/chatApi.js create mode 100644 delivery/client/src/services/nativeVoiceService.js create mode 100644 delivery/client/src/services/videoApi.js create mode 100644 delivery/client/src/services/voiceApi.js create mode 100644 delivery/client/vite.config.js create mode 100644 java-server/src/main/java/com/bigwo/javaserver/BigwoApplication.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/api/ApiEnvelope.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/AssistantProfileProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/CorsConfig.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/CozeProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/DatabaseContext.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/InfrastructureAvailabilityState.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/KnowledgeBaseProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/MysqlProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/RedisClientManager.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/RedisProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/VideoGenerationProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/config/VoiceGatewayProperties.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/AssistantProfileController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/ChatController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/HealthController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/SessionController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/VideoController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/controller/VoiceController.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/exception/BadRequestException.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/exception/GlobalExceptionHandler.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfile.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfileResult.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatHistoryResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatSendResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatSessionState.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatStartResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatStreamEvent.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/ChatSubtitle.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/CozeChatResult.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeChunk.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeQueryInfo.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeSearchResult.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/LlmMessage.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/RedisContextMessage.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/RenderedVideoPayload.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/SessionFullMessage.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/SessionHistoryResult.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/SessionListItem.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/SessionSwitchResult.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoAdminConfigResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoGenerateResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryRowResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoPromptDetail.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/model/VideoTaskSnapshot.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/repository/ChatRepository.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/repository/SessionRepository.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/repository/VideoTaskRepository.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/AssistantProfileService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/ChatContentSafetyService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/ChatService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/ContextKeywordTracker.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/CozeChatClient.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/FastAsrCorrector.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeBaseRetrieverService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeKeywordCatalog.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeQueryResolver.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeRouteDecider.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/PinyinProductMatcher.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/ProductLinkTrigger.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/RedisContextStore.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/SessionService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/VideoGenerationService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/VoiceAssistantProfileSupport.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/VoiceGatewayService.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/service/VoiceSessionState.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/util/VolcSignerV4.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/ApiRequestLoggingFilter.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/request/AssistantProfileRefreshRequest.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/request/ChatSendRequest.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/request/ChatStartRequest.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/request/SessionSwitchRequest.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/request/VideoAdminConfigRequest.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/response/AssistantProfileResponseData.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/response/HealthFeaturesResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/web/response/HealthResponse.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketConfig.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketHandler.java create mode 100644 java-server/src/main/java/com/bigwo/javaserver/websocket/VolcRealtimeProtocol.java create mode 100644 java-server/src/main/resources/application.yml create mode 100644 java-server/src/main/resources/logback-spring.xml create mode 100644 java-server/src/test/java/com/bigwo/javaserver/ApiContractSmokeTest.java create mode 100644 java-server/src/test/java/com/bigwo/javaserver/ChatApiContractSmokeTest.java create mode 100644 java-server/src/test/java/com/bigwo/javaserver/KnowledgeServicesTest.java create mode 100644 java-server/src/test/java/com/bigwo/javaserver/VideoApiContractSmokeTest.java create mode 100644 java-server/src/test/java/com/bigwo/javaserver/VoiceGatewaySmokeTest.java diff --git a/delivery/client/src/services/chatApi.js b/delivery/client/src/services/chatApi.js new file mode 100644 index 0000000..2cb4ef2 --- /dev/null +++ b/delivery/client/src/services/chatApi.js @@ -0,0 +1,122 @@ +import axios from 'axios'; + +function resolveApiBaseURL(configured, path) { + if (configured) { + return configured; + } + if (typeof window === 'undefined') { + return path; + } + const hostname = window.location.hostname; + const port = window.location.port; + const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:'; + if (window.location.protocol === 'file:') { + return `http://127.0.0.1:3013${path}`; + } + if (!port || port === '80' || port === '443' || port === '3013') { + return path; + } + return `${protocol}//${hostname || '127.0.0.1'}:3013${path}`; +} + +const chatApiBaseURL = resolveApiBaseURL(import.meta.env.VITE_CHAT_API_BASE_URL, '/api/chat'); + +const api = axios.create({ + baseURL: chatApiBaseURL, + timeout: 30000, +}); + +export async function startChatSession(sessionId, voiceSubtitles = [], systemPrompt = '') { + const { data } = await api.post('/start', { sessionId, voiceSubtitles, systemPrompt }); + return data; +} + +export async function sendMessage(sessionId, message) { + const { data } = await api.post('/send', { sessionId, message }); + return data; +} + +/** + * SSE 流式发送消息，逐块回调 + * @param {string} sessionId + * @param {string} message + * @param {object} callbacks - { onChunk, onToolCall, onDone, onError } + * @returns {function} abort - 调用可取消请求 + */ +export function sendMessageStream(sessionId, message, { onChunk, onToolCall, onStreamReset, onDone, onError }) { + const controller = new AbortController(); + + (async () => { + try { + const response = await fetch(`${chatApiBaseURL}/send-stream`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sessionId, message }), + signal: controller.signal, + }); + + if (!response.ok) { + const err = await response.json().catch(() => ({ error: response.statusText })); + onError?.(err.error || 'Request failed'); + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed.startsWith('data: ')) continue; + + try { + const data = JSON.parse(trimmed.slice(6)); + switch (data.type) { + case 'chunk': + onChunk?.(data.content); + break; + case 'tool_call': + onToolCall?.(data.tools); + break; + case 'stream_reset': + onStreamReset?.(data.reason); + break; + case 'done': + onDone?.(data.content); + break; + case 'error': + onError?.(data.error); + break; + } + } catch (e) { + // skip malformed SSE + } + } + } + } catch (err) { + if (err.name !== 'AbortError') { + onError?.(err.message || 'Stream failed'); + } + } + })(); + + return () => controller.abort(); +} + +export async function getChatHistory(sessionId) { + const { data } = await api.get(`/history/${sessionId}`); + return data.data; +} + +export async function deleteChatSession(sessionId) { + const { data } = await api.delete(`/${sessionId}`); + return data; +} diff --git a/delivery/client/src/services/nativeVoiceService.js b/delivery/client/src/services/nativeVoiceService.js new file mode 100644 index 0000000..046b894 --- /dev/null +++ b/delivery/client/src/services/nativeVoiceService.js @@ -0,0 +1,426 @@ +class NativeVoiceService { + constructor() { + this.ws = null; + this.mediaStream = null; + this.captureContext = null; + this.captureSource = null; + this.captureProcessor = null; + this.captureSilenceGain = null; + this.playbackContext = null; + this.playbackTime = 0; + this.activeSources = new Set(); + this.pendingSamples = []; + this.pendingAudioChunks = []; + this._resuming = false; + this.readyResolver = null; + this.readyRejector = null; + this.callbacks = { + onSubtitle: null, + onConnectionStateChange: null, + onError: null, + onAssistantPending: null, + onDiagnostic: null, + onIdleTimeout: null, + onProductLink: null, + }; + } + + resolveWebSocketUrl(sessionId, userId) { + const query = new URLSearchParams({ + sessionId, + userId: userId || '', + }); + const configuredBase = import.meta.env.VITE_VOICE_WS_BASE_URL || import.meta.env.VITE_VOICE_API_BASE_URL || ''; + if (configuredBase && !configuredBase.startsWith('/')) { + let base = configuredBase.replace(/\/$/, ''); + if (base.startsWith('https://')) { + base = `wss://${base.slice('https://'.length)}`; + } else if (base.startsWith('http://')) { + base = `ws://${base.slice('http://'.length)}`; + } + if (base.endsWith('/api/voice')) { + base = base.slice(0, -'/api/voice'.length); + } else if (base.endsWith('/api')) { + base = base.slice(0, -'/api'.length); + } + return `${base}/ws/realtime-dialog?${query.toString()}`; + } + const hostname = window.location.hostname; + const port = window.location.port; + const isLocalHost = hostname === 'localhost' || hostname === '127.0.0.1'; + if ((window.location.protocol === 'file:' || isLocalHost) && port !== '3013') { + return `ws://${hostname || '127.0.0.1'}:3013/ws/realtime-dialog?${query.toString()}`; + } + const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + return `${protocol}//${window.location.host}/ws/realtime-dialog?${query.toString()}`; + } + + emitConnectionState(state) { + this.callbacks.onConnectionStateChange?.(state); + } + + emitDiagnostic(type, payload) { + this.callbacks.onDiagnostic?.({ type, payload, timestamp: Date.now() }); + } + + resetPlaybackQueue() { + this.activeSources.forEach((source) => { + try { + source.stop(); + } catch (_) {} + try { + source.disconnect(); + } catch (_) {} + }); + this.activeSources.clear(); + if (this.playbackContext) { + this.playbackTime = this.playbackContext.currentTime + 0.02; + } else { + this.playbackTime = 0; + } + } + + async connect({ sessionId, userId, botName, systemRole, speakingStyle, modelVersion, speaker, greetingText }) { + await this.disconnect(); + const wsUrl = this.resolveWebSocketUrl(sessionId, userId); + this.emitConnectionState('connecting'); + this.playbackContext = new (window.AudioContext || window.webkitAudioContext)(); + if (this.playbackContext.state === 'suspended') { + await this.playbackContext.resume().catch(() => {}); + } + this.playbackTime = this.playbackContext.currentTime; + + // 并行: 同时预获取麦克风和建立WS连接，节省500ms+ + const micPromise = navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + noiseSuppression: true, + echoCancellation: true, + autoGainControl: true, + }, + video: false, + }).catch((err) => { + console.warn('[NativeVoice] Pre-fetch getUserMedia failed:', err.message); + return null; + }); + + const CONNECTION_TIMEOUT_MS = 12000; + + await new Promise((resolve, reject) => { + this.readyResolver = resolve; + this.readyRejector = reject; + const ws = new WebSocket(wsUrl); + ws.binaryType = 'arraybuffer'; + this.ws = ws; + + // 超时兜底：避免无限等待 + const timeoutId = setTimeout(() => { + if (this.readyResolver) { + console.warn(`[NativeVoice] Connection timeout (${CONNECTION_TIMEOUT_MS}ms), forcing ready`); + this.readyResolver(); + this.readyResolver = null; + this.readyRejector = null; + } + }, CONNECTION_TIMEOUT_MS); + + const clearTimeoutOnSettle = () => clearTimeout(timeoutId); + + ws.onopen = () => { + this.emitConnectionState('connected'); + ws.send(JSON.stringify({ + type: 'start', + sessionId, + userId, + botName, + systemRole, + speakingStyle, + modelVersion, + speaker, + greetingText, + })); + }; + + ws.onerror = () => { + clearTimeoutOnSettle(); + const error = new Error('WebSocket connection failed'); + this.callbacks.onError?.(error); + this.readyRejector?.(error); + this.readyResolver = null; + this.readyRejector = null; + reject(error); + }; + + ws.onclose = () => { + clearTimeoutOnSettle(); + this.emitConnectionState('disconnected'); + if (this.readyRejector) { + this.readyRejector(new Error('WebSocket closed before ready')); + this.readyResolver = null; + this.readyRejector = null; + } + }; + + ws.onmessage = (event) => { + if (typeof event.data === 'string') { + const peek = event.data; + if (peek.includes('"ready"')) { + clearTimeoutOnSettle(); + } + this.handleJsonMessage(peek); + return; + } + this.handleAudioMessage(event.data); + }; + }); + + // 使用预获取的mediaStream（已并行获取），避免重复申请 + const preFetchedStream = await micPromise; + await this.startCapture(preFetchedStream); + } + + handleJsonMessage(raw) { + try { + const msg = JSON.parse(raw); + if (msg.type === 'ready') { + this.readyResolver?.(); + this.readyResolver = null; + this.readyRejector = null; + return; + } + if (msg.type === 'subtitle') { + this.callbacks.onSubtitle?.({ + text: msg.text, + role: msg.role, + isFinal: !!msg.isFinal, + sequence: msg.sequence, + }); + return; + } + if (msg.type === 'tts_reset') { + this.resetPlaybackQueue(); + this.emitDiagnostic('tts_reset', msg); + return; + } + if (msg.type === 'assistant_pending') { + this.callbacks.onAssistantPending?.(!!msg.active); + return; + } + if (msg.type === 'idle_timeout') { + this.callbacks.onIdleTimeout?.(msg.timeout || 300000); + return; + } + if (msg.type === 'product_link') { + this.callbacks.onProductLink?.({ + product: msg.product, + link: msg.link, + description: msg.description, + }); + return; + } + if (msg.type === 'upstream_closed') { + this.callbacks.onError?.(new Error('语音服务已断开，请重新开始通话')); + return; + } + if (msg.type === 'error') { + this.callbacks.onError?.(new Error(msg.error || 'native voice error')); + return; + } + this.emitDiagnostic('ws_message', msg); + } catch (error) { + this.emitDiagnostic('ws_raw_text', raw); + } + } + + handleAudioMessage(arrayBuffer) { + if (!this.playbackContext) { + return; + } + if (this.playbackContext.state === 'suspended') { + this.pendingAudioChunks.push(arrayBuffer); + this._tryResumePlayback(); + return; + } + this._playPcm(arrayBuffer); + } + + _playPcm(arrayBuffer) { + try { + const pcm16 = new Int16Array(arrayBuffer); + if (!pcm16.length) { + return; + } + const audioBuffer = this.playbackContext.createBuffer(1, pcm16.length, 24000); + const channel = audioBuffer.getChannelData(0); + for (let i = 0; i < pcm16.length; i += 1) { + channel[i] = pcm16[i] / 32768; + } + const source = this.playbackContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.playbackContext.destination); + this.activeSources.add(source); + source.onended = () => { + this.activeSources.delete(source); + try { + source.disconnect(); + } catch (_) {} + }; + const now = this.playbackContext.currentTime; + if (this.playbackTime < now) { + this.playbackTime = now + 0.02; + } + source.start(this.playbackTime); + this.playbackTime += audioBuffer.duration; + this.emitDiagnostic('audio_chunk', { samples: pcm16.length, duration: audioBuffer.duration }); + } catch (err) { + console.warn('[NativeVoice] playPcm failed:', err.message); + } + } + + async _tryResumePlayback() { + if (this._resuming) return; + this._resuming = true; + try { + await this.playbackContext.resume(); + while (this.pendingAudioChunks.length > 0) { + this._playPcm(this.pendingAudioChunks.shift()); + } + } catch (e) { + console.warn('[NativeVoice] resume failed:', e.message); + } finally { + this._resuming = false; + } + } + + async startCapture(preFetchedStream) { + this.mediaStream = preFetchedStream || await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + noiseSuppression: true, + echoCancellation: true, + autoGainControl: true, + }, + video: false, + }); + this.captureContext = new (window.AudioContext || window.webkitAudioContext)(); + this.captureSource = this.captureContext.createMediaStreamSource(this.mediaStream); + this.captureProcessor = this.captureContext.createScriptProcessor(4096, 1, 1); + this.captureSilenceGain = this.captureContext.createGain(); + this.captureSilenceGain.gain.value = 0; + this.captureProcessor.onaudioprocess = (event) => { + const input = event.inputBuffer.getChannelData(0); + const downsampled = this.downsampleBuffer(input, this.captureContext.sampleRate, 16000); + for (let i = 0; i < downsampled.length; i += 1) { + this.pendingSamples.push(downsampled[i]); + } + while (this.pendingSamples.length >= 320) { + const chunk = this.pendingSamples.splice(0, 320); + const pcm = new Int16Array(chunk.length); + for (let i = 0; i < chunk.length; i += 1) { + const sample = Math.max(-1, Math.min(1, chunk[i])); + pcm[i] = sample < 0 ? sample * 32768 : sample * 32767; + } + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(pcm.buffer); + } + } + }; + this.captureSource.connect(this.captureProcessor); + this.captureProcessor.connect(this.captureSilenceGain); + this.captureSilenceGain.connect(this.captureContext.destination); + } + + downsampleBuffer(buffer, inputRate, outputRate) { + if (outputRate >= inputRate) { + return Array.from(buffer); + } + const sampleRateRatio = inputRate / outputRate; + const newLength = Math.round(buffer.length / sampleRateRatio); + const result = new Array(newLength); + let offsetResult = 0; + let offsetBuffer = 0; + while (offsetResult < result.length) { + const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio); + let accum = 0; + let count = 0; + for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i += 1) { + accum += buffer[i]; + count += 1; + } + result[offsetResult] = count > 0 ? accum / count : 0; + offsetResult += 1; + offsetBuffer = nextOffsetBuffer; + } + return result; + } + + async setMuted(muted) { + this.mediaStream?.getAudioTracks().forEach((track) => { + track.enabled = !muted; + }); + } + + requestGreetingReplay() { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify({ type: 'replay_greeting' })); + this.emitDiagnostic('replay_greeting', { sent: true }); + } + } + + async disconnect() { + if (this.captureProcessor) { + this.captureProcessor.disconnect(); + this.captureProcessor.onaudioprocess = null; + this.captureProcessor = null; + } + if (this.captureSource) { + this.captureSource.disconnect(); + this.captureSource = null; + } + if (this.captureSilenceGain) { + this.captureSilenceGain.disconnect(); + this.captureSilenceGain = null; + } + if (this.captureContext) { + await this.captureContext.close().catch(() => {}); + this.captureContext = null; + } + if (this.mediaStream) { + this.mediaStream.getTracks().forEach((track) => track.stop()); + this.mediaStream = null; + } + if (this.ws) { + try { + if (this.ws.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify({ type: 'stop' })); + this.ws.close(); + } + } catch (_) {} + this.ws = null; + } + if (this.playbackContext) { + this.resetPlaybackQueue(); + await this.playbackContext.close().catch(() => {}); + this.playbackContext = null; + } + this.playbackTime = 0; + this.pendingSamples = []; + this.pendingAudioChunks = []; + this._resuming = false; + this.emitConnectionState('disconnected'); + } + + on(event, callback) { + if (event in this.callbacks) { + this.callbacks[event] = callback; + } + } + + off(event) { + if (event in this.callbacks) { + this.callbacks[event] = null; + } + } +} + +const nativeVoiceService = new NativeVoiceService(); +export default nativeVoiceService; diff --git a/delivery/client/src/services/videoApi.js b/delivery/client/src/services/videoApi.js new file mode 100644 index 0000000..8a5bb4c --- /dev/null +++ b/delivery/client/src/services/videoApi.js @@ -0,0 +1,80 @@ +import axios from 'axios'; + +function resolveApiBaseURL(configured, path) { + if (configured) { + return configured; + } + if (typeof window === 'undefined') { + return path; + } + const hostname = window.location.hostname; + const port = window.location.port; + const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:'; + if (window.location.protocol === 'file:') { + return `http://127.0.0.1:3013${path}`; + } + if (!port || port === '80' || port === '443' || port === '3013') { + return path; + } + return `${protocol}//${hostname || '127.0.0.1'}:3013${path}`; +} + +const videoApiBaseURL = resolveApiBaseURL(import.meta.env.VITE_VIDEO_API_BASE_URL, '/api/video'); + +const api = axios.create({ + baseURL: videoApiBaseURL, + timeout: 60000, +}); + +/** + * 提交视频生成任务（multipart/form-data） + */ +export async function generateVideo({ prompt, product, username, template, size, seconds, image }) { + const formData = new FormData(); + if (prompt) formData.append('prompt', prompt); + if (product) formData.append('product', product); + if (username) formData.append('username', username); + if (template) formData.append('template', template); + if (size) formData.append('size', size); + if (seconds) formData.append('seconds', String(seconds)); + if (image) formData.append('image', image); + + const { data } = await api.post('/generate', formData, { + headers: { 'Content-Type': 'multipart/form-data' }, + }); + return data; +} + +/** + * 查询任务状态 + */ +export async function getTaskStatus(taskId) { + const { data } = await api.get(`/task/${taskId}`); + return data; +} + +/** + * 获取视频历史 + */ +export async function getVideoHistory({ username, limit = 20, offset = 0 } = {}) { + const params = { limit, offset }; + if (username) params.username = username; + const { data } = await api.get('/history', { params }); + return data; +} + +/** + * 获取管理配置 + */ +export async function getAdminConfig() { + const { data } = await api.get('/admin/config'); + return data; +} + +/** + * 更新模型配置 + */ +export async function updateAdminConfig(model) { + const { data } = await api.post('/admin/config', { model }); + return data; +} diff --git a/delivery/client/src/services/voiceApi.js b/delivery/client/src/services/voiceApi.js new file mode 100644 index 0000000..b005b02 --- /dev/null +++ b/delivery/client/src/services/voiceApi.js @@ -0,0 +1,61 @@ +import axios from 'axios'; + +function resolveApiBaseURL(configured, path) { + if (configured) { + return configured; + } + if (typeof window === 'undefined') { + return path; + } + const hostname = window.location.hostname; + const port = window.location.port; + const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:'; + if (window.location.protocol === 'file:') { + return `http://127.0.0.1:3013${path}`; + } + if (!port || port === '80' || port === '443' || port === '3013') { + return path; + } + return `${protocol}//${hostname || '127.0.0.1'}:3013${path}`; +} + +const voiceApiBaseURL = resolveApiBaseURL(import.meta.env.VITE_VOICE_API_BASE_URL, '/api/voice'); +const sessionApiBaseURL = resolveApiBaseURL(import.meta.env.VITE_SESSION_API_BASE_URL, '/api/session'); + +const api = axios.create({ + baseURL: voiceApiBaseURL, + timeout: 10000, +}); + +export async function getVoiceConfig() { + const { data } = await api.get('/config'); + return data.data; +} + +// ========== 会话历史 API ========== +const sessionApi = axios.create({ + baseURL: sessionApiBaseURL, + timeout: 10000, +}); + +export async function getSessionHistory(sessionId, limit = 20) { + const { data } = await sessionApi.get(`/${sessionId}/history`, { params: { limit } }); + return data.data; +} + +export async function switchSessionMode(sessionId, targetMode) { + const { data } = await sessionApi.post(`/${sessionId}/switch`, { targetMode }); + return data.data; +} + +export async function getSessionList(userId, limit = 50) { + const params = { limit }; + if (userId) params.userId = userId; + const { data } = await sessionApi.get('/list', { params }); + return data.data; +} + +export async function deleteSessionById(sessionId) { + const { data } = await sessionApi.delete(`/${sessionId}`); + return data; +} diff --git a/delivery/client/vite.config.js b/delivery/client/vite.config.js new file mode 100644 index 0000000..897db6a --- /dev/null +++ b/delivery/client/vite.config.js @@ -0,0 +1,27 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import tailwindcss from '@tailwindcss/vite'; + +const backendTarget = 'http://localhost:3013'; + +export default defineConfig({ + plugins: [react(), tailwindcss()], + build: { + outDir: 'dist', + sourcemap: false, + }, + server: { + port: 5174, + proxy: { + '/api': { + target: backendTarget, + changeOrigin: true, + }, + '/ws': { + target: backendTarget, + changeOrigin: true, + ws: true, + }, + }, + }, +}); diff --git a/java-server/src/main/java/com/bigwo/javaserver/BigwoApplication.java b/java-server/src/main/java/com/bigwo/javaserver/BigwoApplication.java new file mode 100644 index 0000000..5ea2d90 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/BigwoApplication.java @@ -0,0 +1,23 @@ +package com.bigwo.javaserver; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; +import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.scheduling.annotation.EnableScheduling; + +@SpringBootApplication(exclude = { + DataSourceAutoConfiguration.class, + RedisAutoConfiguration.class, + RedisRepositoriesAutoConfiguration.class +}) +@ConfigurationPropertiesScan +@EnableScheduling +public class BigwoApplication { + + public static void main(String[] args) { + SpringApplication.run(BigwoApplication.class, args); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/api/ApiEnvelope.java b/java-server/src/main/java/com/bigwo/javaserver/api/ApiEnvelope.java new file mode 100644 index 0000000..22091d4 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/api/ApiEnvelope.java @@ -0,0 +1,27 @@ +package com.bigwo.javaserver.api; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ApiEnvelope(Integer code, String message, Boolean success, T data, String error) { + + public static ApiEnvelope ok(T data) { + return new ApiEnvelope<>(null, null, true, data, null); + } + + public static ApiEnvelope okEmpty() { + return new ApiEnvelope<>(null, null, true, null, null); + } + + public static ApiEnvelope assistantOk(T data) { + return new ApiEnvelope<>(0, "success", true, data, null); + } + + public static ApiEnvelope failure(String error) { + return new ApiEnvelope<>(null, null, false, null, error); + } + + public static ApiEnvelope assistantFailure(int code, String error) { + return new ApiEnvelope<>(code, error, false, null, error); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/AssistantProfileProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/AssistantProfileProperties.java new file mode 100644 index 0000000..32bb72b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/AssistantProfileProperties.java @@ -0,0 +1,71 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; + +@ConfigurationProperties(prefix = "bigwo.assistant-profile") +public class AssistantProfileProperties { + + private String apiUrl = ""; + private String apiMethod = "GET"; + private String apiToken = ""; + private String apiHeaders = ""; + private long timeoutMs = 5000; + private long cacheTtlMs = 60000; + + public String getApiUrl() { + return apiUrl; + } + + public void setApiUrl(String apiUrl) { + this.apiUrl = apiUrl; + } + + public String getApiMethod() { + return apiMethod; + } + + public void setApiMethod(String apiMethod) { + this.apiMethod = apiMethod; + } + + public String getApiToken() { + return apiToken; + } + + public void setApiToken(String apiToken) { + this.apiToken = apiToken; + } + + public String getApiHeaders() { + return apiHeaders; + } + + public void setApiHeaders(String apiHeaders) { + this.apiHeaders = apiHeaders; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public void setTimeoutMs(long timeoutMs) { + this.timeoutMs = timeoutMs; + } + + public long getCacheTtlMs() { + return cacheTtlMs; + } + + public void setCacheTtlMs(long cacheTtlMs) { + this.cacheTtlMs = cacheTtlMs; + } + + public boolean isConfigured() { + return StringUtils.hasText(apiUrl) && !apiUrl.startsWith("your_"); + } + + public String normalizedMethod() { + return "POST".equalsIgnoreCase(apiMethod) ? "POST" : "GET"; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/CorsConfig.java b/java-server/src/main/java/com/bigwo/javaserver/config/CorsConfig.java new file mode 100644 index 0000000..abae93d --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/CorsConfig.java @@ -0,0 +1,24 @@ +package com.bigwo.javaserver.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.cors.CorsConfiguration; +import org.springframework.web.cors.UrlBasedCorsConfigurationSource; +import org.springframework.web.filter.CorsFilter; + +@Configuration +public class CorsConfig { + + @Bean + public CorsFilter corsFilter() { + CorsConfiguration config = new CorsConfiguration(); + config.addAllowedOriginPattern("*"); + config.addAllowedMethod("*"); + config.addAllowedHeader("*"); + config.setAllowCredentials(true); + + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); + source.registerCorsConfiguration("/**", config); + return new CorsFilter(source); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/CozeProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/CozeProperties.java new file mode 100644 index 0000000..848e294 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/CozeProperties.java @@ -0,0 +1,49 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; + +@ConfigurationProperties(prefix = "bigwo.coze") +public class CozeProperties { + + private String baseUrl = "https://api.coze.cn"; + private String apiToken = ""; + private String botId = ""; + + public String getBaseUrl() { + return baseUrl; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + public String getApiToken() { + return apiToken; + } + + public void setApiToken(String apiToken) { + this.apiToken = apiToken; + } + + public String getBotId() { + return botId; + } + + public void setBotId(String botId) { + this.botId = botId; + } + + public boolean isConfigured() { + return isEffectiveValue(apiToken) && isEffectiveValue(botId); + } + + public String normalizedBaseUrl() { + String value = StringUtils.hasText(baseUrl) ? baseUrl.trim() : "https://api.coze.cn"; + return value.endsWith("/") ? value.substring(0, value.length() - 1) : value; + } + + private boolean isEffectiveValue(String value) { + return StringUtils.hasText(value) && !value.trim().startsWith("your_"); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/DatabaseContext.java b/java-server/src/main/java/com/bigwo/javaserver/config/DatabaseContext.java new file mode 100644 index 0000000..52e661c --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/DatabaseContext.java @@ -0,0 +1,240 @@ +package com.bigwo.javaserver.config; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; + +@Component +public class DatabaseContext { + + private static final Logger log = LoggerFactory.getLogger(DatabaseContext.class); + private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("[A-Za-z0-9_]+"); + + private final MysqlProperties properties; + private final InfrastructureAvailabilityState infrastructureAvailabilityState; + + private volatile HikariDataSource dataSource; + private volatile JdbcTemplate jdbcTemplate; + + public DatabaseContext(MysqlProperties properties, InfrastructureAvailabilityState infrastructureAvailabilityState) { + this.properties = properties; + this.infrastructureAvailabilityState = infrastructureAvailabilityState; + } + + @PostConstruct + public void initialize() { + if (!properties.isEnabled()) { + log.info("[DB] MySQL disabled by configuration"); + return; + } + try { + createDatabaseIfNeeded(); + HikariDataSource hikariDataSource = createDataSource(); + JdbcTemplate template = new JdbcTemplate(hikariDataSource); + initializeSchema(template); + this.dataSource = hikariDataSource; + this.jdbcTemplate = template; + infrastructureAvailabilityState.setDatabaseReady(true); + log.info("[DB] MySQL connected: {}, tables ready", properties.getDatabase()); + } catch (Exception exception) { + infrastructureAvailabilityState.setDatabaseReady(false); + closeDataSource(); + log.warn("[DB] MySQL initialization failed: {}", exception.getMessage()); + log.warn("[DB] Continuing without database"); + } + } + + public boolean isAvailable() { + return jdbcTemplate != null && dataSource != null && infrastructureAvailabilityState.isDatabaseReady(); + } + + public JdbcTemplate requiredJdbcTemplate() { + JdbcTemplate template = jdbcTemplate; + if (template == null) { + throw new IllegalStateException("Database unavailable"); + } + return template; + } + + public DataSource requiredDataSource() { + DataSource source = dataSource; + if (source == null) { + throw new IllegalStateException("Database unavailable"); + } + return source; + } + + @PreDestroy + public void close() { + closeDataSource(); + } + + private void createDatabaseIfNeeded() throws SQLException { + String databaseName = sanitizeIdentifier(properties.getDatabase()); + try (Connection connection = DriverManager.getConnection( + properties.getServerJdbcUrl(), + properties.getUsername(), + properties.getPassword()); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE IF NOT EXISTS `" + databaseName + "` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"); + } + } + + private HikariDataSource createDataSource() { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(properties.getJdbcUrl()); + config.setUsername(properties.getUsername()); + config.setPassword(properties.getPassword()); + config.setMaximumPoolSize(properties.getMaxPoolSize()); + config.setConnectionTimeout(properties.getConnectionTimeoutMs()); + config.setPoolName("bigwo-hikari"); + config.setInitializationFailTimeout(0); + config.setConnectionTestQuery("SELECT 1"); + config.setConnectionInitSql("SET NAMES utf8mb4"); + return new HikariDataSource(config); + } + + private void initializeSchema(JdbcTemplate jdbc) { + jdbc.execute(""" + CREATE TABLE IF NOT EXISTS sessions ( + id VARCHAR(128) PRIMARY KEY, + user_id VARCHAR(128), + username VARCHAR(64) DEFAULT '', + mode ENUM('voice', 'chat') DEFAULT 'chat', + created_at BIGINT, + updated_at BIGINT, + INDEX idx_user (user_id), + INDEX idx_updated (updated_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """); + + jdbc.execute(""" + CREATE TABLE IF NOT EXISTS messages ( + id INT AUTO_INCREMENT PRIMARY KEY, + session_id VARCHAR(128) NOT NULL, + role ENUM('user', 'assistant', 'tool', 'system') NOT NULL, + content TEXT NOT NULL, + source ENUM('voice_asr', 'voice_bot', 'voice_tool', 'chat_user', 'chat_bot', 'search_knowledge') NOT NULL, + tool_name VARCHAR(64), + meta_json JSON, + created_at BIGINT, + INDEX idx_session (session_id), + INDEX idx_session_time (session_id, created_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """); + + jdbc.execute(""" + CREATE TABLE IF NOT EXISTS conversation_summaries ( + id INT AUTO_INCREMENT PRIMARY KEY, + session_id VARCHAR(128) NOT NULL, + user_id VARCHAR(128), + summary TEXT NOT NULL, + turn_count INT DEFAULT 0, + topics JSON, + created_at BIGINT, + updated_at BIGINT, + UNIQUE INDEX idx_session (session_id), + INDEX idx_user_time (user_id, updated_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """); + + jdbc.execute(""" + CREATE TABLE IF NOT EXISTS video_tasks ( + id VARCHAR(128) PRIMARY KEY, + username VARCHAR(64) DEFAULT '', + original_prompt TEXT, + optimized_prompt TEXT, + product_name VARCHAR(128) DEFAULT '', + template_type VARCHAR(16) DEFAULT 'product', + video_size VARCHAR(16) DEFAULT '720x1280', + video_seconds INT DEFAULT 5, + status VARCHAR(32) DEFAULT 'processing', + progress INT DEFAULT 0, + video_url TEXT, + error_msg TEXT, + created_at BIGINT NOT NULL, + completed_at BIGINT, + INDEX idx_username (username), + INDEX idx_created (created_at), + INDEX idx_status (status) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """); + + migrateSchema(jdbc); + } + + private void migrateSchema(JdbcTemplate jdbc) { + if (!columnMatchesType(jdbc, "sessions", "mode", "'chat'")) { + jdbc.execute("ALTER TABLE `sessions` MODIFY COLUMN `mode` ENUM('voice', 'chat') DEFAULT 'chat'"); + } + if (!columnMatchesType(jdbc, "messages", "role", "'system'")) { + jdbc.execute("ALTER TABLE `messages` MODIFY COLUMN `role` ENUM('user', 'assistant', 'tool', 'system') NOT NULL"); + } + if (!columnMatchesType(jdbc, "messages", "source", "'search_knowledge'")) { + jdbc.execute("ALTER TABLE `messages` MODIFY COLUMN `source` ENUM('voice_asr', 'voice_bot', 'voice_tool', 'chat_user', 'chat_bot', 'search_knowledge') NOT NULL"); + } + ensureColumnExists(jdbc, "messages", "tool_name", "`tool_name` VARCHAR(64) NULL AFTER `source`"); + ensureColumnExists(jdbc, "messages", "meta_json", "`meta_json` JSON NULL AFTER `tool_name`"); + ensureColumnExists(jdbc, "messages", "created_at", "`created_at` BIGINT NULL AFTER `meta_json`"); + ensureColumnExists(jdbc, "sessions", "updated_at", "`updated_at` BIGINT NULL AFTER `created_at`"); + ensureColumnExists(jdbc, "sessions", "username", "`username` VARCHAR(64) DEFAULT '' AFTER `user_id`"); + } + + private void ensureColumnExists(JdbcTemplate jdbc, String tableName, String columnName, String definitionSql) { + Integer count = jdbc.queryForObject( + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?", + Integer.class, + sanitizeIdentifier(properties.getDatabase()), + tableName, + columnName + ); + if (count != null && count == 0) { + jdbc.execute("ALTER TABLE `" + tableName + "` ADD COLUMN " + definitionSql); + } + } + + private boolean columnMatchesType(JdbcTemplate jdbc, String tableName, String columnName, String expectedTypeFragment) { + List columnTypes = jdbc.query( + "SELECT COLUMN_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ?", + (ResultSet resultSet, int rowNum) -> resultSet.getString(1), + sanitizeIdentifier(properties.getDatabase()), + tableName, + columnName + ); + if (columnTypes.isEmpty()) { + return false; + } + return columnTypes.getFirst().toLowerCase(Locale.ROOT).contains(expectedTypeFragment.toLowerCase(Locale.ROOT)); + } + + private String sanitizeIdentifier(String value) { + String trimmed = value == null ? "" : value.trim(); + if (!IDENTIFIER_PATTERN.matcher(trimmed).matches()) { + throw new IllegalArgumentException("Invalid database identifier"); + } + return trimmed; + } + + private void closeDataSource() { + HikariDataSource source = this.dataSource; + this.jdbcTemplate = null; + this.dataSource = null; + if (source != null) { + source.close(); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/InfrastructureAvailabilityState.java b/java-server/src/main/java/com/bigwo/javaserver/config/InfrastructureAvailabilityState.java new file mode 100644 index 0000000..ee1c7ab --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/InfrastructureAvailabilityState.java @@ -0,0 +1,19 @@ +package com.bigwo.javaserver.config; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.stereotype.Component; + +@Component +public class InfrastructureAvailabilityState { + + private final AtomicBoolean databaseReady = new AtomicBoolean(false); + + public boolean isDatabaseReady() { + return databaseReady.get(); + } + + public void setDatabaseReady(boolean ready) { + databaseReady.set(ready); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/KnowledgeBaseProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/KnowledgeBaseProperties.java new file mode 100644 index 0000000..e49849f --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/KnowledgeBaseProperties.java @@ -0,0 +1,127 @@ +package com.bigwo.javaserver.config; + +import java.util.ArrayList; +import java.util.List; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; + +@ConfigurationProperties(prefix = "bigwo.knowledge") +public class KnowledgeBaseProperties { + + private String accessKeyId = ""; + private String secretAccessKey = ""; + private String endpointId = ""; + private String model = ""; + private List datasetIds = new ArrayList<>(); + private int retrievalTopK = 25; + private double threshold = 0.1D; + private String rerankerModel = "doubao-seed-rerank"; + private int rerankerTopN = 3; + private boolean enableReranker = true; + private boolean enableRedisContext = true; + private String collectionMapJson = ""; + + public String getAccessKeyId() { + return accessKeyId; + } + + public void setAccessKeyId(String accessKeyId) { + this.accessKeyId = accessKeyId; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public void setSecretAccessKey(String secretAccessKey) { + this.secretAccessKey = secretAccessKey; + } + + public String getEndpointId() { + return endpointId; + } + + public void setEndpointId(String endpointId) { + this.endpointId = endpointId; + } + + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + public List getDatasetIds() { + return datasetIds; + } + + public void setDatasetIds(List datasetIds) { + this.datasetIds = datasetIds; + } + + public int getRetrievalTopK() { + return retrievalTopK; + } + + public void setRetrievalTopK(int retrievalTopK) { + this.retrievalTopK = retrievalTopK; + } + + public double getThreshold() { + return threshold; + } + + public void setThreshold(double threshold) { + this.threshold = threshold; + } + + public String getRerankerModel() { + return rerankerModel; + } + + public void setRerankerModel(String rerankerModel) { + this.rerankerModel = rerankerModel; + } + + public int getRerankerTopN() { + return rerankerTopN; + } + + public void setRerankerTopN(int rerankerTopN) { + this.rerankerTopN = rerankerTopN; + } + + public boolean isEnableReranker() { + return enableReranker; + } + + public void setEnableReranker(boolean enableReranker) { + this.enableReranker = enableReranker; + } + + public boolean isEnableRedisContext() { + return enableRedisContext; + } + + public void setEnableRedisContext(boolean enableRedisContext) { + this.enableRedisContext = enableRedisContext; + } + + public String getCollectionMapJson() { + return collectionMapJson; + } + + public void setCollectionMapJson(String collectionMapJson) { + this.collectionMapJson = collectionMapJson; + } + + public boolean hasAccessKeys() { + return StringUtils.hasText(accessKeyId) && StringUtils.hasText(secretAccessKey); + } + + public List normalizedDatasetIds() { + return datasetIds == null ? List.of() : datasetIds.stream().map(value -> value == null ? "" : value.trim()).filter(StringUtils::hasText).toList(); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/MysqlProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/MysqlProperties.java new file mode 100644 index 0000000..eeab163 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/MysqlProperties.java @@ -0,0 +1,97 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "bigwo.mysql") +public class MysqlProperties { + + private boolean enabled = true; + private String host = "localhost"; + private int port = 3306; + private String database = "bigwo_chat"; + private String username = "root"; + private String password = ""; + private int maxPoolSize = 10; + private long connectionTimeoutMs = 5000; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getDatabase() { + return database; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public int getMaxPoolSize() { + return maxPoolSize; + } + + public void setMaxPoolSize(int maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } + + public long getConnectionTimeoutMs() { + return connectionTimeoutMs; + } + + public void setConnectionTimeoutMs(long connectionTimeoutMs) { + this.connectionTimeoutMs = connectionTimeoutMs; + } + + public String getJdbcUrl() { + return String.format( + "jdbc:mysql://%s:%d/%s?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&useSSL=false", + host, + port, + database + ); + } + + public String getServerJdbcUrl() { + return String.format( + "jdbc:mysql://%s:%d/?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true&useSSL=false", + host, + port + ); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/RedisClientManager.java b/java-server/src/main/java/com/bigwo/javaserver/config/RedisClientManager.java new file mode 100644 index 0000000..47aa8c8 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/RedisClientManager.java @@ -0,0 +1,86 @@ +package com.bigwo.javaserver.config; + +import java.time.Duration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.api.sync.RedisCommands; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; + +@Component +public class RedisClientManager { + + private static final Logger log = LoggerFactory.getLogger(RedisClientManager.class); + + private final RedisProperties properties; + + private volatile RedisClient redisClient; + private volatile StatefulRedisConnection connection; + private volatile boolean available; + + public RedisClientManager(RedisProperties properties) { + this.properties = properties; + } + + @PostConstruct + public void initialize() { + if (!properties.isEnabled()) { + log.info("[Redis] Redis disabled by configuration"); + return; + } + try { + RedisURI redisUri = RedisURI.create(properties.getUrl()); + redisUri.setDatabase(properties.getDatabase()); + redisUri.setTimeout(Duration.ofMillis(properties.getTimeoutMs())); + if (StringUtils.hasText(properties.getPassword())) { + redisUri.setPassword(properties.getPassword().toCharArray()); + } + RedisClient client = RedisClient.create(redisUri); + StatefulRedisConnection redisConnection = client.connect(); + redisConnection.sync().ping(); + this.redisClient = client; + this.connection = redisConnection; + this.available = true; + log.info("[Redis] Connected: {}", properties.getUrl()); + } catch (Exception exception) { + this.available = false; + close(); + log.warn("[Redis] Initialization failed: {}", exception.getMessage()); + log.warn("[Redis] Continuing without Redis"); + } + } + + public boolean isAvailable() { + return available && connection != null; + } + + public RedisCommands syncCommands() { + StatefulRedisConnection currentConnection = this.connection; + if (!available || currentConnection == null) { + throw new IllegalStateException("Redis unavailable"); + } + return currentConnection.sync(); + } + + @PreDestroy + public void close() { + StatefulRedisConnection currentConnection = this.connection; + RedisClient currentClient = this.redisClient; + this.connection = null; + this.redisClient = null; + this.available = false; + if (currentConnection != null) { + currentConnection.close(); + } + if (currentClient != null) { + currentClient.shutdown(); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/RedisProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/RedisProperties.java new file mode 100644 index 0000000..bb1718b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/RedisProperties.java @@ -0,0 +1,62 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "bigwo.redis") +public class RedisProperties { + + private boolean enabled = true; + private String url = "redis://127.0.0.1:6379"; + private String password = ""; + private int database = 0; + private long timeoutMs = 5000; + private String keyPrefix = "bigwo:"; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public int getDatabase() { + return database; + } + + public void setDatabase(int database) { + this.database = database; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public void setTimeoutMs(long timeoutMs) { + this.timeoutMs = timeoutMs; + } + + public String getKeyPrefix() { + return keyPrefix; + } + + public void setKeyPrefix(String keyPrefix) { + this.keyPrefix = keyPrefix; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/VideoGenerationProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/VideoGenerationProperties.java new file mode 100644 index 0000000..e579966 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/VideoGenerationProperties.java @@ -0,0 +1,143 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "bigwo.video") +public class VideoGenerationProperties { + + private String seedanceApiKey = ""; + private String seedanceApiBase = "https://n.lconai.com"; + private String seedanceModel = "seedance-2.0"; + private String geminiModel = ""; + private String geminiApiKey = ""; + private String geminiApiBase = ""; + private long pollIntervalMs = 5000L; + private long pollTimeoutMs = 900000L; + private long requestTimeoutMs = 120000L; + private String workDir = ""; + private int maxConcurrentTasks = 4; + private String logoPath = ""; + private String videosDir = ""; + private int ffmpegThreads = 1; + private long postProcessTimeoutMs = 180000L; + + public String getSeedanceApiKey() { + return seedanceApiKey; + } + + public void setSeedanceApiKey(String seedanceApiKey) { + this.seedanceApiKey = seedanceApiKey; + } + + public String getSeedanceApiBase() { + return seedanceApiBase; + } + + public void setSeedanceApiBase(String seedanceApiBase) { + this.seedanceApiBase = seedanceApiBase; + } + + public String getSeedanceModel() { + return seedanceModel; + } + + public void setSeedanceModel(String seedanceModel) { + this.seedanceModel = seedanceModel; + } + + public String getGeminiModel() { + return geminiModel; + } + + public void setGeminiModel(String geminiModel) { + this.geminiModel = geminiModel; + } + + public String getGeminiApiKey() { + return geminiApiKey; + } + + public void setGeminiApiKey(String geminiApiKey) { + this.geminiApiKey = geminiApiKey; + } + + public String getGeminiApiBase() { + return geminiApiBase; + } + + public void setGeminiApiBase(String geminiApiBase) { + this.geminiApiBase = geminiApiBase; + } + + public long getPollIntervalMs() { + return pollIntervalMs; + } + + public void setPollIntervalMs(long pollIntervalMs) { + this.pollIntervalMs = pollIntervalMs; + } + + public long getPollTimeoutMs() { + return pollTimeoutMs; + } + + public void setPollTimeoutMs(long pollTimeoutMs) { + this.pollTimeoutMs = pollTimeoutMs; + } + + public long getRequestTimeoutMs() { + return requestTimeoutMs; + } + + public void setRequestTimeoutMs(long requestTimeoutMs) { + this.requestTimeoutMs = requestTimeoutMs; + } + + public String getWorkDir() { + return workDir; + } + + public void setWorkDir(String workDir) { + this.workDir = workDir; + } + + public int getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + + public String getLogoPath() { + return logoPath; + } + + public void setLogoPath(String logoPath) { + this.logoPath = logoPath; + } + + public String getVideosDir() { + return videosDir; + } + + public void setVideosDir(String videosDir) { + this.videosDir = videosDir; + } + + public int getFfmpegThreads() { + return ffmpegThreads; + } + + public void setFfmpegThreads(int ffmpegThreads) { + this.ffmpegThreads = ffmpegThreads; + } + + public long getPostProcessTimeoutMs() { + return postProcessTimeoutMs; + } + + public void setPostProcessTimeoutMs(long postProcessTimeoutMs) { + this.postProcessTimeoutMs = postProcessTimeoutMs; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/config/VoiceGatewayProperties.java b/java-server/src/main/java/com/bigwo/javaserver/config/VoiceGatewayProperties.java new file mode 100644 index 0000000..36dc866 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/config/VoiceGatewayProperties.java @@ -0,0 +1,103 @@ +package com.bigwo.javaserver.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.util.StringUtils; + +@ConfigurationProperties(prefix = "bigwo.voice") +public class VoiceGatewayProperties { + + private boolean enabled = true; + private String upstreamUrl = "wss://openspeech.bytedance.com/api/v3/realtime/dialogue"; + private String resourceId = "volc.speech.dialog"; + private String appId = ""; + private String token = ""; + private String appKey = "PlgvMymc7f3tQnJ6"; + private String defaultSpeaker = "zh_female_vv_jupiter_bigtts"; + private long idleTimeoutMs = 300000L; + private long audioKeepaliveIntervalMs = 8000L; + private boolean sendReadyEarly = true; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public String getUpstreamUrl() { + return upstreamUrl; + } + + public void setUpstreamUrl(String upstreamUrl) { + this.upstreamUrl = upstreamUrl; + } + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getAppId() { + return appId; + } + + public void setAppId(String appId) { + this.appId = appId; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public String getAppKey() { + return appKey; + } + + public void setAppKey(String appKey) { + this.appKey = appKey; + } + + public String getDefaultSpeaker() { + return defaultSpeaker; + } + + public void setDefaultSpeaker(String defaultSpeaker) { + this.defaultSpeaker = defaultSpeaker; + } + + public long getIdleTimeoutMs() { + return idleTimeoutMs; + } + + public void setIdleTimeoutMs(long idleTimeoutMs) { + this.idleTimeoutMs = idleTimeoutMs; + } + + public long getAudioKeepaliveIntervalMs() { + return audioKeepaliveIntervalMs; + } + + public void setAudioKeepaliveIntervalMs(long audioKeepaliveIntervalMs) { + this.audioKeepaliveIntervalMs = audioKeepaliveIntervalMs; + } + + public boolean isSendReadyEarly() { + return sendReadyEarly; + } + + public void setSendReadyEarly(boolean sendReadyEarly) { + this.sendReadyEarly = sendReadyEarly; + } + + public boolean isConfigured() { + return StringUtils.hasText(appId) && StringUtils.hasText(token) && StringUtils.hasText(upstreamUrl); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/AssistantProfileController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/AssistantProfileController.java new file mode 100644 index 0000000..5ae7fec --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/AssistantProfileController.java @@ -0,0 +1,77 @@ +package com.bigwo.javaserver.controller; + +import com.bigwo.javaserver.api.ApiEnvelope; +import com.bigwo.javaserver.model.AssistantProfileResult; +import com.bigwo.javaserver.service.AssistantProfileService; +import com.bigwo.javaserver.web.request.AssistantProfileRefreshRequest; +import com.bigwo.javaserver.web.response.AssistantProfileResponseData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.util.StringUtils; + +@RestController +@RequestMapping("/api/assistant-profile") +public class AssistantProfileController { + + private static final Logger log = LoggerFactory.getLogger(AssistantProfileController.class); + + private final AssistantProfileService assistantProfileService; + + public AssistantProfileController(AssistantProfileService assistantProfileService) { + this.assistantProfileService = assistantProfileService; + } + + @GetMapping + public ResponseEntity> getAssistantProfile( + @RequestParam(name = "userId", required = false) String userId, + @RequestParam(name = "forceRefresh", required = false) String forceRefresh) { + try { + String normalizedUserId = normalizeNullable(userId); + AssistantProfileResult result = assistantProfileService.getAssistantProfile(normalizedUserId, "true".equals(normalizeNullable(forceRefresh))); + return ResponseEntity.ok(ApiEnvelope.assistantOk(toResponseData(normalizedUserId, result))); + } catch (Exception exception) { + log.error("[AssistantProfile] query failed: {}", exception.getMessage(), exception); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(ApiEnvelope.assistantFailure(500, exception.getMessage())); + } + } + + @PostMapping("/refresh") + public ResponseEntity> refreshAssistantProfile( + @RequestBody(required = false) AssistantProfileRefreshRequest request) { + try { + String normalizedUserId = normalizeNullable(request == null ? null : request.userId()); + assistantProfileService.clearAssistantProfileCache(normalizedUserId); + AssistantProfileResult result = assistantProfileService.getAssistantProfile(normalizedUserId, true); + return ResponseEntity.ok(ApiEnvelope.assistantOk(toResponseData(normalizedUserId, result))); + } catch (Exception exception) { + log.error("[AssistantProfile] refresh failed: {}", exception.getMessage(), exception); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(ApiEnvelope.assistantFailure(500, exception.getMessage())); + } + } + + private AssistantProfileResponseData toResponseData(String userId, AssistantProfileResult result) { + return new AssistantProfileResponseData( + userId, + result.profile(), + result.source(), + result.cached(), + result.fetchedAt(), + result.configured(), + result.error() + ); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/ChatController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/ChatController.java new file mode 100644 index 0000000..8ee3021 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/ChatController.java @@ -0,0 +1,75 @@ +package com.bigwo.javaserver.controller; + +import java.io.IOException; + +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.bigwo.javaserver.api.ApiEnvelope; +import com.bigwo.javaserver.model.ChatSendResponse; +import com.bigwo.javaserver.model.ChatStartResponse; +import com.bigwo.javaserver.model.ChatStreamEvent; +import com.bigwo.javaserver.service.ChatService; +import com.bigwo.javaserver.web.request.ChatSendRequest; +import com.bigwo.javaserver.web.request.ChatStartRequest; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.servlet.http.HttpServletResponse; + +@RestController +@RequestMapping("/api/chat") +public class ChatController { + + private final ChatService chatService; + private final ObjectMapper objectMapper; + + public ChatController(ChatService chatService, ObjectMapper objectMapper) { + this.chatService = chatService; + this.objectMapper = objectMapper; + } + + @PostMapping("/start") + public ApiEnvelope start(@RequestBody(required = false) ChatStartRequest request) { + return ApiEnvelope.ok(chatService.startSession(request)); + } + + @PostMapping("/send") + public ApiEnvelope send(@RequestBody(required = false) ChatSendRequest request) { + return ApiEnvelope.ok(chatService.sendMessage(request)); + } + + @GetMapping("/history/{sessionId}") + public ApiEnvelope history(@PathVariable("sessionId") String sessionId) { + return ApiEnvelope.ok(chatService.getHistory(sessionId)); + } + + @PostMapping("/send-stream") + public void sendStream(@RequestBody(required = false) ChatSendRequest request, HttpServletResponse response) throws IOException { + response.setContentType("text/event-stream;charset=UTF-8"); + response.setHeader("Cache-Control", "no-cache"); + response.setHeader("Connection", "keep-alive"); + response.setHeader("X-Accel-Buffering", "no"); + response.flushBuffer(); + chatService.streamMessage(request, event -> writeEvent(response, event)); + } + + @DeleteMapping("/{sessionId}") + public ApiEnvelope delete(@PathVariable("sessionId") String sessionId) { + chatService.deleteSession(sessionId); + return ApiEnvelope.okEmpty(); + } + + private void writeEvent(HttpServletResponse response, ChatStreamEvent event) { + try { + response.getWriter().write("data: " + objectMapper.writeValueAsString(event) + "\n\n"); + response.getWriter().flush(); + } catch (IOException exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/HealthController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/HealthController.java new file mode 100644 index 0000000..a7f6f37 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/HealthController.java @@ -0,0 +1,77 @@ +package com.bigwo.javaserver.controller; + +import org.springframework.core.env.Environment; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.bigwo.javaserver.config.RedisClientManager; +import com.bigwo.javaserver.config.VoiceGatewayProperties; +import com.bigwo.javaserver.web.response.HealthFeaturesResponse; +import com.bigwo.javaserver.web.response.HealthResponse; + +@RestController +@RequestMapping("/api/health") +public class HealthController { + + private final Environment environment; + private final RedisClientManager redisClientManager; + private final VoiceGatewayProperties voiceGatewayProperties; + + public HealthController(Environment environment, RedisClientManager redisClientManager, VoiceGatewayProperties voiceGatewayProperties) { + this.environment = environment; + this.redisClientManager = redisClientManager; + this.voiceGatewayProperties = voiceGatewayProperties; + } + + @GetMapping + public HealthResponse health() { + boolean envReady = !isPlaceholderValue("VOLC_S2S_APP_ID"); + Object reranker = "true".equals(environment.getProperty("ENABLE_RERANKER")) + ? configuredOrDefault("VOLC_ARK_RERANKER_MODEL", "doubao-seed-rerank") + : Boolean.FALSE; + return new HealthResponse( + "ok", + "s2s-hybrid", + "2024-12-01", + envReady, + new HealthFeaturesResponse( + true, + hasConfiguredValue("COZE_API_TOKEN") && hasConfiguredValue("COZE_BOT_ID"), + "coze", + hasConfiguredValue("VOLC_WEBSEARCH_API_KEY"), + hasConfiguredValue("VOLC_S2S_SPEAKER_ID"), + hasConfiguredValue("VOLC_ARK_KNOWLEDGE_BASE_IDS"), + redisClientManager.isAvailable(), + reranker, + "raw", + voiceGatewayProperties.isEnabled() && voiceGatewayProperties.isConfigured() + ) + ); + } + + private boolean isPlaceholderValue(String key) { + String value = environment.getProperty(key, ""); + if (!StringUtils.hasText(value)) { + return false; + } + return value.startsWith("your_"); + } + + private boolean hasConfiguredValue(String key) { + String value = environment.getProperty(key, ""); + if (!StringUtils.hasText(value)) { + return false; + } + return !value.startsWith("your_"); + } + + private String configuredOrDefault(String key, String fallback) { + String value = environment.getProperty(key, ""); + if (!StringUtils.hasText(value)) { + return fallback; + } + return value; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/SessionController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/SessionController.java new file mode 100644 index 0000000..9a4baaf --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/SessionController.java @@ -0,0 +1,88 @@ +package com.bigwo.javaserver.controller; + +import com.bigwo.javaserver.api.ApiEnvelope; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.model.SessionFullMessage; +import com.bigwo.javaserver.model.SessionHistoryResult; +import com.bigwo.javaserver.model.SessionListItem; +import com.bigwo.javaserver.model.SessionSwitchResult; +import com.bigwo.javaserver.service.SessionService; +import com.bigwo.javaserver.web.request.SessionSwitchRequest; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.util.StringUtils; + +@RestController +@RequestMapping("/api/session") +public class SessionController { + + private static final Logger log = LoggerFactory.getLogger(SessionController.class); + + private final SessionService sessionService; + + public SessionController(SessionService sessionService) { + this.sessionService = sessionService; + } + + @GetMapping("/list") + public ApiEnvelope> listSessions( + @RequestParam(name = "userId", required = false) String userId, + @RequestParam(name = "limit", required = false) String limit) { + List sessions = sessionService.listSessions(normalizeNullable(userId), parseInteger(limit)); + return ApiEnvelope.ok(sessions); + } + + @DeleteMapping("/{id}") + public ApiEnvelope deleteSession(@PathVariable("id") String sessionId) { + sessionService.deleteSession(sessionId); + return ApiEnvelope.okEmpty(); + } + + @GetMapping("/{id}/history") + public ApiEnvelope getHistory( + @PathVariable("id") String sessionId, + @RequestParam(name = "limit", required = false) String limit, + @RequestParam(name = "format", required = false) String format) { + Integer parsedLimit = parseInteger(limit); + String responseFormat = normalizeNullable(format); + if ("full".equals(responseFormat)) { + SessionHistoryResult result = sessionService.getFullHistory(sessionId, parsedLimit); + return ApiEnvelope.ok(result); + } + SessionHistoryResult result = sessionService.getLlmHistory(sessionId, parsedLimit); + return ApiEnvelope.ok(result); + } + + @PostMapping("/{id}/switch") + public ApiEnvelope switchMode( + @PathVariable("id") String sessionId, + @RequestBody(required = false) SessionSwitchRequest request) { + SessionSwitchResult result = sessionService.switchMode(sessionId, normalizeNullable(request == null ? null : request.targetMode())); + log.debug("[Session] Switch result generated for {}", sessionId); + return ApiEnvelope.ok(result); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private Integer parseInteger(String value) { + if (!StringUtils.hasText(value)) { + return null; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException exception) { + return null; + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/VideoController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/VideoController.java new file mode 100644 index 0000000..d7dca9a --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/VideoController.java @@ -0,0 +1,156 @@ +package com.bigwo.javaserver.controller; + +import com.bigwo.javaserver.exception.BadRequestException; +import com.bigwo.javaserver.model.VideoAdminConfigResponse; +import com.bigwo.javaserver.model.VideoGenerateResponse; +import com.bigwo.javaserver.model.VideoHistoryResponse; +import com.bigwo.javaserver.model.VideoTaskSnapshot; +import com.bigwo.javaserver.service.VideoGenerationService; +import com.bigwo.javaserver.web.request.VideoAdminConfigRequest; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MaxUploadSizeExceededException; +import org.springframework.web.multipart.MultipartException; +import org.springframework.web.multipart.MultipartFile; + +@RestController +@RequestMapping("/api/video") +public class VideoController { + + private static final Logger log = LoggerFactory.getLogger(VideoController.class); + + private final VideoGenerationService videoGenerationService; + + public VideoController(VideoGenerationService videoGenerationService) { + this.videoGenerationService = videoGenerationService; + } + + @PostMapping(value = "/generate", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity generate( + @RequestParam(name = "prompt", required = false) String prompt, + @RequestParam(name = "product", required = false) String product, + @RequestParam(name = "username", required = false) String username, + @RequestParam(name = "template", required = false) String template, + @RequestParam(name = "size", required = false) String size, + @RequestParam(name = "seconds", required = false) String seconds, + @RequestParam(name = "image", required = false) MultipartFile image + ) { + try { + VideoGenerateResponse response = videoGenerationService.generate(prompt, product, username, template, size, seconds, image); + return ResponseEntity.ok(response); + } catch (BadRequestException exception) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(errorBody(exception.getMessage())); + } catch (Exception exception) { + log.error("[Video] generate failed: {}", exception.getMessage(), exception); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorBody(extractMessage(exception, "生成失败"))); + } + } + + @GetMapping("/task/{id}") + public ResponseEntity getTask(@PathVariable("id") String taskId) { + VideoTaskSnapshot snapshot = videoGenerationService.getTask(taskId); + if (snapshot == null) { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(errorBody("任务不存在")); + } + return ResponseEntity.ok(snapshot); + } + + @GetMapping("/history") + public VideoHistoryResponse getHistory( + @RequestParam(name = "username", required = false) String username, + @RequestParam(name = "limit", required = false) String limit, + @RequestParam(name = "offset", required = false) String offset + ) { + return videoGenerationService.getHistory(normalizeNullable(username), parseInteger(limit), parseInteger(offset)); + } + + @GetMapping("/file/{filename}") + public ResponseEntity getVideoFile(@PathVariable("filename") String filename) { + Path file = videoGenerationService.getVideoFile(filename); + if (file == null) { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(errorBody("文件不存在")); + } + try { + Resource resource = new FileSystemResource(file); + long fileSize = Files.size(file); + return ResponseEntity.ok() + .header("Content-Type", "video/mp4") + .header("Content-Length", String.valueOf(fileSize)) + .header("Content-Disposition", "inline; filename=\"" + file.getFileName() + "\"") + .header("Cache-Control", "public, max-age=86400") + .body(resource); + } catch (Exception ex) { + log.error("[Video] 文件读取失败: {}", ex.getMessage()); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(errorBody("文件读取失败")); + } + } + + @GetMapping("/admin/config") + public VideoAdminConfigResponse getAdminConfig() { + return videoGenerationService.getAdminConfig(); + } + + @PostMapping("/admin/config") + public ResponseEntity updateAdminConfig(@RequestBody(required = false) VideoAdminConfigRequest request) { + try { + return ResponseEntity.ok(videoGenerationService.updateAdminConfig(request == null ? null : request.model())); + } catch (BadRequestException exception) { + Map body = errorBody(exception.getMessage()); + body.put("validModels", videoGenerationService.validModels()); + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(body); + } + } + + @ExceptionHandler(MaxUploadSizeExceededException.class) + public ResponseEntity> handleMaxUpload(MaxUploadSizeExceededException exception) { + return ResponseEntity.status(HttpStatus.PAYLOAD_TOO_LARGE).body(errorBody("图片过大（上限 50MB），请压缩后重试")); + } + + @ExceptionHandler(MultipartException.class) + public ResponseEntity> handleMultipart(MultipartException exception) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(errorBody(extractMessage(exception, "上传失败"))); + } + + private Map errorBody(String error) { + Map body = new LinkedHashMap<>(); + body.put("error", StringUtils.hasText(error) ? error : "请求失败"); + return body; + } + + private String extractMessage(Exception exception, String fallback) { + return exception.getMessage() == null || exception.getMessage().isBlank() ? fallback : exception.getMessage(); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private Integer parseInteger(String value) { + if (!StringUtils.hasText(value)) { + return null; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException exception) { + return null; + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/controller/VoiceController.java b/java-server/src/main/java/com/bigwo/javaserver/controller/VoiceController.java new file mode 100644 index 0000000..0ea8df5 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/controller/VoiceController.java @@ -0,0 +1,61 @@ +package com.bigwo.javaserver.controller; + +import java.util.List; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.bigwo.javaserver.api.ApiEnvelope; +import com.bigwo.javaserver.config.VoiceGatewayProperties; + +@RestController +@RequestMapping("/api/voice") +public class VoiceController { + + private static final List MODELS = List.of( + new VoiceOption("1.2.1.0", "O2.0（推荐，精品音质）", null), + new VoiceOption("O", "O（基础版）", null), + new VoiceOption("2.2.0.0", "SC2.0（推荐，声音复刻）", null), + new VoiceOption("SC", "SC（基础版）", null) + ); + + private static final List SPEAKERS = List.of( + new VoiceOption("zh_female_vv_jupiter_bigtts", "VV（活泼女声）", "O"), + new VoiceOption("zh_female_xiaohe_jupiter_bigtts", "小禾（甜美女声·台湾口音）", "O"), + new VoiceOption("zh_male_yunzhou_jupiter_bigtts", "云舟（沉稳男声）", "O"), + new VoiceOption("zh_male_xiaotian_jupiter_bigtts", "小天（磁性男声）", "O"), + new VoiceOption("saturn_common_female_1", "Saturn 女声1", "SC2.0"), + new VoiceOption("saturn_common_male_1", "Saturn 男声1", "SC2.0"), + new VoiceOption("ICL_common_female_1", "ICL 女声1", "SC"), + new VoiceOption("ICL_common_male_1", "ICL 男声1", "SC") + ); + + private static final List TOOLS = List.of( + new VoiceTool("search_knowledge", "【强制调用】优先查询知识库，基于企业官方信息回答产品、公司、制度、招商与常见问题。"), + new VoiceTool("query_weather", "查询指定城市的天气信息"), + new VoiceTool("query_order", "根据订单号查询订单状态"), + new VoiceTool("get_current_time", "获取当前日期和时间"), + new VoiceTool("calculate", "计算数学表达式，支持加减乘除") + ); + + private final VoiceGatewayProperties voiceGatewayProperties; + + public VoiceController(VoiceGatewayProperties voiceGatewayProperties) { + this.voiceGatewayProperties = voiceGatewayProperties; + } + + @GetMapping("/config") + public ApiEnvelope config() { + return ApiEnvelope.ok(new VoiceConfigData(MODELS, SPEAKERS, TOOLS, voiceGatewayProperties.isEnabled())); + } + + private record VoiceConfigData(List models, List speakers, List tools, boolean enabled) { + } + + private record VoiceOption(String value, String label, String series) { + } + + private record VoiceTool(String name, String description) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/exception/BadRequestException.java b/java-server/src/main/java/com/bigwo/javaserver/exception/BadRequestException.java new file mode 100644 index 0000000..2fd8ce2 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/exception/BadRequestException.java @@ -0,0 +1,8 @@ +package com.bigwo.javaserver.exception; + +public class BadRequestException extends RuntimeException { + + public BadRequestException(String message) { + super(message); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/exception/GlobalExceptionHandler.java b/java-server/src/main/java/com/bigwo/javaserver/exception/GlobalExceptionHandler.java new file mode 100644 index 0000000..52a0721 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/exception/GlobalExceptionHandler.java @@ -0,0 +1,53 @@ +package com.bigwo.javaserver.exception; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.validation.BindException; +import org.springframework.web.HttpRequestMethodNotSupportedException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.NoHandlerFoundException; + +import com.bigwo.javaserver.api.ApiEnvelope; + +import jakarta.servlet.http.HttpServletRequest; + +@RestControllerAdvice +public class GlobalExceptionHandler { + + private static final Logger log = LoggerFactory.getLogger(GlobalExceptionHandler.class); + + @ExceptionHandler(BadRequestException.class) + public ResponseEntity> handleBadRequest(BadRequestException exception) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(ApiEnvelope.failure(exception.getMessage())); + } + + @ExceptionHandler({MethodArgumentNotValidException.class, BindException.class, HttpMessageNotReadableException.class}) + public ResponseEntity> handleValidationException(Exception exception) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(ApiEnvelope.failure("Invalid request body")); + } + + @ExceptionHandler(HttpRequestMethodNotSupportedException.class) + public ResponseEntity> handleMethodNotAllowed(HttpRequestMethodNotSupportedException exception) { + return ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).body(ApiEnvelope.failure(exception.getMessage())); + } + + @ExceptionHandler(NoHandlerFoundException.class) + public ResponseEntity> handleNotFound(NoHandlerFoundException exception, HttpServletRequest request) { + String error = "Route not found: " + request.getMethod() + " " + request.getRequestURI(); + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ApiEnvelope.failure(error)); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity> handleGenericException(Exception exception, HttpServletRequest request) { + log.error("Unhandled error on {} {}", request.getMethod(), request.getRequestURI(), exception); + String message = exception.getMessage() == null || exception.getMessage().isBlank() + ? "Internal Server Error" + : exception.getMessage(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(ApiEnvelope.failure(message)); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfile.java b/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfile.java new file mode 100644 index 0000000..a415650 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfile.java @@ -0,0 +1,56 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record AssistantProfile( + String documents, + String email, + String agentName, + String nickname, + String wxl, + String mobile, + @JsonProperty("wx_code") String wxCode, + String intro, + String sign, + String story +) { + + public static AssistantProfile resolve(AssistantProfile profile) { + AssistantProfile source = profile == null ? new AssistantProfile(null, null, null, null, null, null, null, null, null, null) : profile; + String agentName = firstNonBlank(source.agentName, source.nickname, "大沃"); + String nickname = firstNonBlank(source.nickname, agentName); + return new AssistantProfile( + normalize(source.documents), + normalize(source.email), + agentName, + nickname, + normalize(source.wxl), + normalize(source.mobile), + normalize(source.wxCode), + normalize(source.intro), + normalize(source.sign), + normalize(source.story) + ); + } + + public static AssistantProfile defaults() { + return resolve(null); + } + + private static String normalize(String value) { + return value == null ? "" : value.trim(); + } + + private static String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + for (String value : values) { + String normalized = normalize(value); + if (!normalized.isEmpty()) { + return normalized; + } + } + return ""; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfileResult.java b/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfileResult.java new file mode 100644 index 0000000..b57b35d --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/AssistantProfileResult.java @@ -0,0 +1,11 @@ +package com.bigwo.javaserver.model; + +public record AssistantProfileResult( + AssistantProfile profile, + String source, + boolean cached, + Long fetchedAt, + boolean configured, + String error +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatHistoryResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatHistoryResponse.java new file mode 100644 index 0000000..1d58f7e --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatHistoryResponse.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record ChatHistoryResponse(String conversationId, boolean fromVoice) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatSendResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSendResponse.java new file mode 100644 index 0000000..0357761 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSendResponse.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record ChatSendResponse(String content) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatSessionState.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSessionState.java new file mode 100644 index 0000000..048c8da --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSessionState.java @@ -0,0 +1,124 @@ +package com.bigwo.javaserver.model; + +import java.util.List; + +public class ChatSessionState { + + private final String sessionId; + private volatile String userId; + private volatile String profileUserId; + private volatile String conversationId; + private volatile String lastKbTopic; + private volatile long lastKbHitAt; + private final List voiceMessages; + private final String handoffSummary; + private volatile boolean handoffSummaryUsed; + private final long createdAt; + private volatile long lastActiveAt; + private final boolean fromVoice; + + public ChatSessionState( + String sessionId, + String userId, + String profileUserId, + String conversationId, + String lastKbTopic, + long lastKbHitAt, + List voiceMessages, + String handoffSummary, + boolean handoffSummaryUsed, + long createdAt, + long lastActiveAt, + boolean fromVoice) { + this.sessionId = sessionId; + this.userId = userId; + this.profileUserId = profileUserId; + this.conversationId = conversationId; + this.lastKbTopic = lastKbTopic; + this.lastKbHitAt = lastKbHitAt; + this.voiceMessages = List.copyOf(voiceMessages); + this.handoffSummary = handoffSummary; + this.handoffSummaryUsed = handoffSummaryUsed; + this.createdAt = createdAt; + this.lastActiveAt = lastActiveAt; + this.fromVoice = fromVoice; + } + + public String getSessionId() { + return sessionId; + } + + public String getUserId() { + return userId; + } + + public synchronized void setUserId(String userId) { + this.userId = userId; + } + + public String getProfileUserId() { + return profileUserId; + } + + public synchronized void setProfileUserId(String profileUserId) { + this.profileUserId = profileUserId; + } + + public String getConversationId() { + return conversationId; + } + + public synchronized void setConversationId(String conversationId) { + this.conversationId = conversationId; + } + + public String getLastKbTopic() { + return lastKbTopic; + } + + public long getLastKbHitAt() { + return lastKbHitAt; + } + + public synchronized void setLastKbContext(String lastKbTopic, long lastKbHitAt) { + this.lastKbTopic = lastKbTopic; + this.lastKbHitAt = lastKbHitAt; + } + + public synchronized void clearLastKbContext() { + this.lastKbTopic = ""; + this.lastKbHitAt = 0L; + } + + public List getVoiceMessages() { + return voiceMessages; + } + + public String getHandoffSummary() { + return handoffSummary; + } + + public boolean isHandoffSummaryUsed() { + return handoffSummaryUsed; + } + + public synchronized void setHandoffSummaryUsed(boolean handoffSummaryUsed) { + this.handoffSummaryUsed = handoffSummaryUsed; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getLastActiveAt() { + return lastActiveAt; + } + + public synchronized void touch() { + this.lastActiveAt = System.currentTimeMillis(); + } + + public boolean isFromVoice() { + return fromVoice; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatStartResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatStartResponse.java new file mode 100644 index 0000000..9bdabe3 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatStartResponse.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record ChatStartResponse(String sessionId, int messageCount, boolean fromVoice) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatStreamEvent.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatStreamEvent.java new file mode 100644 index 0000000..b00394f --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatStreamEvent.java @@ -0,0 +1,23 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ChatStreamEvent(String type, String content, String error, String reason) { + + public static ChatStreamEvent chunk(String content) { + return new ChatStreamEvent("chunk", content, null, null); + } + + public static ChatStreamEvent done(String content) { + return new ChatStreamEvent("done", content, null, null); + } + + public static ChatStreamEvent error(String error) { + return new ChatStreamEvent("error", null, error, null); + } + + public static ChatStreamEvent reset(String reason) { + return new ChatStreamEvent("stream_reset", null, null, reason); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/ChatSubtitle.java b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSubtitle.java new file mode 100644 index 0000000..fa8a7c6 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/ChatSubtitle.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record ChatSubtitle(String role, String text) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/CozeChatResult.java b/java-server/src/main/java/com/bigwo/javaserver/model/CozeChatResult.java new file mode 100644 index 0000000..60b6ec8 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/CozeChatResult.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record CozeChatResult(String content, String conversationId) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeChunk.java b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeChunk.java new file mode 100644 index 0000000..4709580 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeChunk.java @@ -0,0 +1,15 @@ +package com.bigwo.javaserver.model; + +import java.util.Map; + +public record KnowledgeChunk( + String id, + String content, + double score, + String docName, + String chunkTitle, + Map metadata, + String collection, + boolean reranked +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeQueryInfo.java b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeQueryInfo.java new file mode 100644 index 0000000..7a0248b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeQueryInfo.java @@ -0,0 +1,14 @@ +package com.bigwo.javaserver.model; + +import java.util.List; + +public record KnowledgeQueryInfo( + String rawText, + String normalizedText, + String rewrittenText, + List entities, + String primaryEntity, + boolean hasExplicitEntity, + boolean hasKnowledgeSignal +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeSearchResult.java b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeSearchResult.java new file mode 100644 index 0000000..1b9f797 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/KnowledgeSearchResult.java @@ -0,0 +1,22 @@ +package com.bigwo.javaserver.model; + +import java.util.List; +import java.util.Map; + +public record KnowledgeSearchResult( + String query, + String originalQuery, + boolean hit, + String reason, + List chunks, + List rerankedChunks, + List> ragPayload, + Map evidencePack, + double topScore, + long latencyMs, + Long retrievalLatencyMs, + String source, + boolean hasReferences, + Map usage +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/LlmMessage.java b/java-server/src/main/java/com/bigwo/javaserver/model/LlmMessage.java new file mode 100644 index 0000000..e87153a --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/LlmMessage.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record LlmMessage(String role, String content) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/RedisContextMessage.java b/java-server/src/main/java/com/bigwo/javaserver/model/RedisContextMessage.java new file mode 100644 index 0000000..b4aee1c --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/RedisContextMessage.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record RedisContextMessage(String role, String content, String source, long ts) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/RenderedVideoPayload.java b/java-server/src/main/java/com/bigwo/javaserver/model/RenderedVideoPayload.java new file mode 100644 index 0000000..af28e19 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/RenderedVideoPayload.java @@ -0,0 +1,9 @@ +package com.bigwo.javaserver.model; + +public record RenderedVideoPayload( + String prompt, + String negative, + String voiceScript, + VideoPromptDetail promptDetail +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/SessionFullMessage.java b/java-server/src/main/java/com/bigwo/javaserver/model/SessionFullMessage.java new file mode 100644 index 0000000..afdd7e0 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/SessionFullMessage.java @@ -0,0 +1,13 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SessionFullMessage( + String role, + String content, + String source, + @JsonProperty("tool_name") String toolName, + @JsonProperty("meta_json") String metaJson, + @JsonProperty("created_at") Long createdAt +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/SessionHistoryResult.java b/java-server/src/main/java/com/bigwo/javaserver/model/SessionHistoryResult.java new file mode 100644 index 0000000..e8a5586 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/SessionHistoryResult.java @@ -0,0 +1,6 @@ +package com.bigwo.javaserver.model; + +import java.util.List; + +public record SessionHistoryResult(String sessionId, String mode, List messages, int count) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/SessionListItem.java b/java-server/src/main/java/com/bigwo/javaserver/model/SessionListItem.java new file mode 100644 index 0000000..057fde0 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/SessionListItem.java @@ -0,0 +1,12 @@ +package com.bigwo.javaserver.model; + +public record SessionListItem( + String id, + String userId, + String mode, + Long createdAt, + Long updatedAt, + String lastMessage, + int messageCount +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/SessionSwitchResult.java b/java-server/src/main/java/com/bigwo/javaserver/model/SessionSwitchResult.java new file mode 100644 index 0000000..7c46938 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/SessionSwitchResult.java @@ -0,0 +1,6 @@ +package com.bigwo.javaserver.model; + +import java.util.List; + +public record SessionSwitchResult(String sessionId, String mode, List history, int count) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoAdminConfigResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoAdminConfigResponse.java new file mode 100644 index 0000000..6501915 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoAdminConfigResponse.java @@ -0,0 +1,16 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record VideoAdminConfigResponse( + Boolean success, + String previousModel, + String currentModel, + String source, + List validModels, + String envModel, + String runtimeModel +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoGenerateResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoGenerateResponse.java new file mode 100644 index 0000000..47ddb9c --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoGenerateResponse.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.model; + +public record VideoGenerateResponse(String taskId, String originalPrompt, String username, String status) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryResponse.java new file mode 100644 index 0000000..35acccb --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryResponse.java @@ -0,0 +1,6 @@ +package com.bigwo.javaserver.model; + +import java.util.List; + +public record VideoHistoryResponse(int total, List rows) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryRowResponse.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryRowResponse.java new file mode 100644 index 0000000..af71723 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoHistoryRowResponse.java @@ -0,0 +1,20 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record VideoHistoryRowResponse( + String id, + String username, + @JsonProperty("original_prompt") String originalPrompt, + @JsonProperty("optimized_prompt") String optimizedPrompt, + @JsonProperty("product_name") String productName, + @JsonProperty("template_type") String templateType, + @JsonProperty("video_size") String videoSize, + @JsonProperty("video_seconds") Integer videoSeconds, + String status, + Integer progress, + @JsonProperty("video_url") String videoUrl, + @JsonProperty("created_at") Long createdAt, + @JsonProperty("completed_at") Long completedAt +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoPromptDetail.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoPromptDetail.java new file mode 100644 index 0000000..d79f0e1 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoPromptDetail.java @@ -0,0 +1,16 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; +import java.util.Map; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record VideoPromptDetail( + String videoPrompt, + String voiceScript, + String negative, + String note, + String theme, + List> shotList +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/model/VideoTaskSnapshot.java b/java-server/src/main/java/com/bigwo/javaserver/model/VideoTaskSnapshot.java new file mode 100644 index 0000000..e9cf82a --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/model/VideoTaskSnapshot.java @@ -0,0 +1,24 @@ +package com.bigwo.javaserver.model; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record VideoTaskSnapshot( + String id, + String username, + String originalPrompt, + String optimizedPrompt, + String productName, + String templateType, + String videoSize, + Integer videoSeconds, + String status, + String statusText, + Integer progress, + String videoUrl, + String error, + Long createdAt, + Long completedAt, + VideoPromptDetail promptDetail +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/repository/ChatRepository.java b/java-server/src/main/java/com/bigwo/javaserver/repository/ChatRepository.java new file mode 100644 index 0000000..df8c79b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/repository/ChatRepository.java @@ -0,0 +1,111 @@ +package com.bigwo.javaserver.repository; + +import com.bigwo.javaserver.config.DatabaseContext; +import com.bigwo.javaserver.model.LlmMessage; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.ResultSet; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.util.StringUtils; + +@Repository +public class ChatRepository { + + private static final Logger log = LoggerFactory.getLogger(ChatRepository.class); + + private final DatabaseContext databaseContext; + private final ObjectMapper objectMapper; + + public ChatRepository(DatabaseContext databaseContext, ObjectMapper objectMapper) { + this.databaseContext = databaseContext; + this.objectMapper = objectMapper; + } + + public List getHistoryForLlm(String sessionId, int limit) { + if (!databaseContext.isAvailable()) { + return List.of(); + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + int safeLimit = Math.max(1, Math.min(limit, 100)); + List messages = jdbc.query( + """ + SELECT role, content + FROM messages + WHERE session_id = ? + ORDER BY created_at DESC + LIMIT ? + """, + (ResultSet resultSet, int rowNum) -> new LlmMessage(resultSet.getString("role"), resultSet.getString("content")), + sessionId, + safeLimit + ); + return messages.reversed().stream() + .filter(message -> "user".equals(message.role()) || "assistant".equals(message.role())) + .toList(); + } catch (Exception exception) { + log.warn("[DB] getHistoryForLLM failed: {}", exception.getMessage()); + return List.of(); + } + } + + public void createSession(String sessionId, String userId, String mode) { + if (!databaseContext.isAvailable()) { + return; + } + try { + long now = System.currentTimeMillis(); + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update( + """ + INSERT INTO sessions (id, user_id, username, mode, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + user_id = IF(VALUES(user_id) = '' OR VALUES(user_id) IS NULL, user_id, VALUES(user_id)), + mode = VALUES(mode), + username = IF(VALUES(username) = '', username, VALUES(username)), + updated_at = VALUES(updated_at) + """, + sessionId, + userId, + "", + mode, + now, + now + ); + } catch (Exception exception) { + log.warn("[DB] createSession failed: {}", exception.getMessage()); + } + } + + public void addMessage(String sessionId, String role, String content, String source) { + addMessage(sessionId, role, content, source, null, null); + } + + public void addMessage(String sessionId, String role, String content, String source, String toolName, Object meta) { + if (!databaseContext.isAvailable() || !StringUtils.hasText(content)) { + return; + } + try { + long now = System.currentTimeMillis(); + String metaJson = meta == null ? null : objectMapper.writeValueAsString(meta); + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update( + "INSERT INTO messages (session_id, role, content, source, tool_name, meta_json, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + sessionId, + role, + content, + source, + toolName, + metaJson, + now + ); + jdbc.update("UPDATE sessions SET updated_at = ? WHERE id = ?", now, sessionId); + } catch (Exception exception) { + log.warn("[DB] addMessage failed: {}", exception.getMessage()); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/repository/SessionRepository.java b/java-server/src/main/java/com/bigwo/javaserver/repository/SessionRepository.java new file mode 100644 index 0000000..b2909fa --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/repository/SessionRepository.java @@ -0,0 +1,144 @@ +package com.bigwo.javaserver.repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.util.StringUtils; + +import com.bigwo.javaserver.config.DatabaseContext; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.model.SessionFullMessage; +import com.bigwo.javaserver.model.SessionListItem; + +@Repository +public class SessionRepository { + + private final DatabaseContext databaseContext; + + public SessionRepository(DatabaseContext databaseContext) { + this.databaseContext = databaseContext; + } + + public List getSessionList(String userId, Integer limit) { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + int safeLimit = clamp(limit, 50, 1, 200); + String normalizedUserId = normalizeNullable(userId); + if (normalizedUserId != null) { + return jdbc.query( + """ + SELECT s.id, s.user_id, s.mode, s.created_at, s.updated_at, + (SELECT content FROM messages WHERE session_id = s.id ORDER BY created_at DESC LIMIT 1) AS last_message, + (SELECT COUNT(*) FROM messages WHERE session_id = s.id) AS message_count + FROM sessions s + WHERE s.user_id = ? + ORDER BY s.updated_at DESC + LIMIT ? + """, + (resultSet, rowNum) -> mapSessionListItem(resultSet), + normalizedUserId, + safeLimit + ); + } + return jdbc.query( + """ + SELECT s.id, s.user_id, s.mode, s.created_at, s.updated_at, + (SELECT content FROM messages WHERE session_id = s.id ORDER BY created_at DESC LIMIT 1) AS last_message, + (SELECT COUNT(*) FROM messages WHERE session_id = s.id) AS message_count + FROM sessions s + ORDER BY s.updated_at DESC + LIMIT ? + """, + (resultSet, rowNum) -> mapSessionListItem(resultSet), + safeLimit + ); + } + + public void deleteSession(String sessionId) { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update("DELETE FROM messages WHERE session_id = ?", sessionId); + jdbc.update("DELETE FROM conversation_summaries WHERE session_id = ?", sessionId); + jdbc.update("DELETE FROM sessions WHERE id = ?", sessionId); + } + + public List getRecentMessages(String sessionId, Integer limit) { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + int safeLimit = clamp(limit, 20, 1, 100); + List messages = jdbc.query( + """ + SELECT role, content, source, tool_name, meta_json, created_at + FROM messages + WHERE session_id = ? + ORDER BY created_at DESC + LIMIT ? + """, + (resultSet, rowNum) -> mapSessionFullMessage(resultSet), + sessionId, + safeLimit + ); + Collections.reverse(messages); + return messages; + } + + public List getHistoryForLlm(String sessionId, Integer limit) { + return getRecentMessages(sessionId, limit).stream() + .filter(message -> "user".equals(message.role()) || "assistant".equals(message.role())) + .map(message -> new LlmMessage(message.role(), message.content())) + .toList(); + } + + public String getSessionMode(String sessionId) { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + List modes = jdbc.query( + "SELECT mode FROM sessions WHERE id = ? LIMIT 1", + (resultSet, rowNum) -> resultSet.getString("mode"), + sessionId + ); + return modes.isEmpty() ? null : modes.get(0); + } + + public void updateSessionMode(String sessionId, String mode) { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update("UPDATE sessions SET mode = ?, updated_at = ? WHERE id = ?", mode, System.currentTimeMillis(), sessionId); + } + + private SessionListItem mapSessionListItem(ResultSet resultSet) throws SQLException { + return new SessionListItem( + resultSet.getString("id"), + resultSet.getString("user_id"), + resultSet.getString("mode"), + getNullableLong(resultSet, "created_at"), + getNullableLong(resultSet, "updated_at"), + resultSet.getString("last_message"), + resultSet.getInt("message_count") + ); + } + + private SessionFullMessage mapSessionFullMessage(ResultSet resultSet) throws SQLException { + return new SessionFullMessage( + resultSet.getString("role"), + resultSet.getString("content"), + resultSet.getString("source"), + resultSet.getString("tool_name"), + resultSet.getString("meta_json"), + getNullableLong(resultSet, "created_at") + ); + } + + private Long getNullableLong(ResultSet resultSet, String columnName) throws SQLException { + long value = resultSet.getLong(columnName); + return resultSet.wasNull() ? null : value; + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private int clamp(Integer value, int fallback, int min, int max) { + int resolved = value == null ? fallback : value; + return Math.max(min, Math.min(resolved, max)); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/repository/VideoTaskRepository.java b/java-server/src/main/java/com/bigwo/javaserver/repository/VideoTaskRepository.java new file mode 100644 index 0000000..0b1e5ba --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/repository/VideoTaskRepository.java @@ -0,0 +1,224 @@ +package com.bigwo.javaserver.repository; + +import java.sql.ResultSet; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.util.StringUtils; + +import com.bigwo.javaserver.config.DatabaseContext; +import com.bigwo.javaserver.model.VideoTaskSnapshot; + +@Repository +public class VideoTaskRepository { + + private static final Logger log = LoggerFactory.getLogger(VideoTaskRepository.class); + + private final DatabaseContext databaseContext; + + public VideoTaskRepository(DatabaseContext databaseContext) { + this.databaseContext = databaseContext; + } + + public boolean isAvailable() { + return databaseContext.isAvailable(); + } + + public int cleanupStaleTasks() { + if (!databaseContext.isAvailable()) { + return 0; + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + long now = System.currentTimeMillis(); + return jdbc.update( + "UPDATE video_tasks SET status = 'failed', error_msg = '服务重启，任务已中断', completed_at = ? WHERE status IN ('queued', 'optimizing', 'processing', 'subtitling')", + now + ); + } catch (Exception exception) { + log.warn("[Video] cleanup stale tasks failed: {}", exception.getMessage()); + return 0; + } + } + + public void insertVideoTask( + String id, + String username, + String originalPrompt, + String optimizedPrompt, + String productName, + String templateType, + String videoSize, + int videoSeconds, + String status + ) { + if (!databaseContext.isAvailable()) { + return; + } + try { + long now = System.currentTimeMillis(); + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update( + "INSERT INTO video_tasks (id, username, original_prompt, optimized_prompt, product_name, template_type, video_size, video_seconds, status, progress, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)", + id, + defaultString(username), + defaultString(originalPrompt), + StringUtils.hasText(optimizedPrompt) ? optimizedPrompt : null, + defaultString(productName), + StringUtils.hasText(templateType) ? templateType : "product", + StringUtils.hasText(videoSize) ? videoSize : "720x1280", + videoSeconds, + StringUtils.hasText(status) ? status : "processing", + now + ); + } catch (Exception exception) { + log.warn("[Video] insert task failed: {}", exception.getMessage()); + } + } + + public void updateVideoProgress(String id, String status, int progress) { + if (!databaseContext.isAvailable() || !StringUtils.hasText(id)) { + return; + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update( + "UPDATE video_tasks SET status = ?, progress = ? WHERE id = ?", + StringUtils.hasText(status) ? status : "processing", + Math.max(0, progress), + id.trim() + ); + } catch (Exception exception) { + log.warn("[Video] update progress failed: {}", exception.getMessage()); + } + } + + public void updateVideoOptimizedPrompt(String id, String optimizedPrompt) { + if (!databaseContext.isAvailable() || !StringUtils.hasText(id)) { + return; + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + jdbc.update( + "UPDATE video_tasks SET optimized_prompt = ? WHERE id = ?", + StringUtils.hasText(optimizedPrompt) ? optimizedPrompt : null, + id.trim() + ); + } catch (Exception exception) { + log.warn("[Video] update optimized prompt failed: {}", exception.getMessage()); + } + } + + public void completeVideoTask(String id, String status, String videoUrl, String errorMsg) { + if (!databaseContext.isAvailable() || !StringUtils.hasText(id)) { + return; + } + try { + long now = System.currentTimeMillis(); + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + boolean completed = "completed".equalsIgnoreCase(status); + jdbc.update( + "UPDATE video_tasks SET status = ?, progress = ?, video_url = ?, error_msg = ?, completed_at = ? WHERE id = ?", + StringUtils.hasText(status) ? status : "completed", + completed ? 100 : 0, + StringUtils.hasText(videoUrl) ? videoUrl : null, + StringUtils.hasText(errorMsg) ? errorMsg : null, + now, + id.trim() + ); + } catch (Exception exception) { + log.warn("[Video] complete task failed: {}", exception.getMessage()); + } + } + + public VideoTaskSnapshot getVideoTask(String id) { + if (!databaseContext.isAvailable() || !StringUtils.hasText(id)) { + return null; + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + List rows = jdbc.query( + "SELECT id, username, original_prompt, optimized_prompt, product_name, template_type, video_size, video_seconds, status, progress, video_url, error_msg, created_at, completed_at FROM video_tasks WHERE id = ? LIMIT 1", + this::mapSnapshot, + id.trim() + ); + return rows.isEmpty() ? null : rows.getFirst(); + } catch (Exception exception) { + log.warn("[Video] get task failed: {}", exception.getMessage()); + return null; + } + } + + public List getVideoHistory(String username, int limit, int offset) { + if (!databaseContext.isAvailable()) { + return List.of(); + } + int safeLimit = Math.max(1, Math.min(limit, 100)); + int safeOffset = Math.max(0, offset); + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + if (StringUtils.hasText(username)) { + return jdbc.query( + "SELECT id, username, original_prompt, optimized_prompt, product_name, template_type, video_size, video_seconds, status, progress, video_url, error_msg, created_at, completed_at FROM video_tasks WHERE username = ? ORDER BY created_at DESC LIMIT ? OFFSET ?", + this::mapSnapshot, + username.trim(), + safeLimit, + safeOffset + ); + } + return jdbc.query( + "SELECT id, username, original_prompt, optimized_prompt, product_name, template_type, video_size, video_seconds, status, progress, video_url, error_msg, created_at, completed_at FROM video_tasks ORDER BY created_at DESC LIMIT ? OFFSET ?", + this::mapSnapshot, + safeLimit, + safeOffset + ); + } catch (Exception exception) { + log.warn("[Video] get history failed: {}", exception.getMessage()); + return List.of(); + } + } + + public int getVideoHistoryCount(String username) { + if (!databaseContext.isAvailable()) { + return 0; + } + try { + JdbcTemplate jdbc = databaseContext.requiredJdbcTemplate(); + Integer total = StringUtils.hasText(username) + ? jdbc.queryForObject("SELECT COUNT(*) FROM video_tasks WHERE username = ?", Integer.class, username.trim()) + : jdbc.queryForObject("SELECT COUNT(*) FROM video_tasks", Integer.class); + return total == null ? 0 : total; + } catch (Exception exception) { + log.warn("[Video] get history count failed: {}", exception.getMessage()); + return 0; + } + } + + private VideoTaskSnapshot mapSnapshot(ResultSet resultSet, int rowNum) throws java.sql.SQLException { + return new VideoTaskSnapshot( + resultSet.getString("id"), + resultSet.getString("username"), + resultSet.getString("original_prompt"), + resultSet.getString("optimized_prompt"), + resultSet.getString("product_name"), + resultSet.getString("template_type"), + resultSet.getString("video_size"), + resultSet.getInt("video_seconds"), + resultSet.getString("status"), + null, + resultSet.getInt("progress"), + resultSet.getString("video_url"), + resultSet.getString("error_msg"), + resultSet.getLong("created_at"), + resultSet.getObject("completed_at") == null ? null : resultSet.getLong("completed_at"), + null + ); + } + + private String defaultString(String value) { + return StringUtils.hasText(value) ? value.trim() : ""; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/AssistantProfileService.java b/java-server/src/main/java/com/bigwo/javaserver/service/AssistantProfileService.java new file mode 100644 index 0000000..b3c652a --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/AssistantProfileService.java @@ -0,0 +1,220 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.config.AssistantProfileProperties; +import com.bigwo.javaserver.model.AssistantProfile; +import com.bigwo.javaserver.model.AssistantProfileResult; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Duration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class AssistantProfileService { + + private static final Logger log = LoggerFactory.getLogger(AssistantProfileService.class); + + private final AssistantProfileProperties properties; + private final ObjectMapper objectMapper; + private final ConcurrentMap cache = new ConcurrentHashMap<>(); + + public AssistantProfileService(AssistantProfileProperties properties, ObjectMapper objectMapper) { + this.properties = properties; + this.objectMapper = objectMapper; + } + + public AssistantProfileResult getAssistantProfile(String userId, boolean forceRefresh) { + String normalizedUserId = normalizeNullable(userId); + String cacheKey = getCacheKey(normalizedUserId); + CachedAssistantProfile cached = cache.get(cacheKey); + long ttlMs = Math.max(0L, properties.getCacheTtlMs()); + + if (!forceRefresh && cached != null && (System.currentTimeMillis() - cached.fetchedAt()) <= ttlMs) { + return new AssistantProfileResult(cached.profile(), cached.source(), true, cached.fetchedAt(), cached.configured(), null); + } + + try { + CachedAssistantProfile fetched = fetchRemoteAssistantProfile(normalizedUserId); + cache.put(cacheKey, fetched); + return new AssistantProfileResult(fetched.profile(), fetched.source(), false, fetched.fetchedAt(), fetched.configured(), null); + } catch (Exception exception) { + AssistantProfile fallbackProfile = cached != null ? cached.profile() : AssistantProfile.defaults(); + return new AssistantProfileResult( + fallbackProfile, + cached != null ? "cache_fallback" : "default_fallback", + cached != null, + cached != null ? cached.fetchedAt() : null, + properties.isConfigured(), + exception.getMessage() + ); + } + } + + public void clearAssistantProfileCache(String userId) { + String normalizedUserId = normalizeNullable(userId); + if (normalizedUserId == null) { + cache.clear(); + return; + } + cache.remove(getCacheKey(normalizedUserId)); + } + + private CachedAssistantProfile fetchRemoteAssistantProfile(String userId) { + long fetchedAt = System.currentTimeMillis(); + if (!properties.isConfigured()) { + return new CachedAssistantProfile(AssistantProfile.defaults(), "default", fetchedAt, false); + } + + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + parseAssistantProfileHeaders().forEach(headers::set); + if (StringUtils.hasText(properties.getApiToken())) { + headers.setBearerAuth(properties.getApiToken().trim()); + } + + RestTemplate restTemplate = createRestTemplate(); + ResponseEntity response; + if ("POST".equals(properties.normalizedMethod())) { + Map body = userId == null ? Map.of() : Map.of("userId", userId); + response = restTemplate.exchange( + properties.getApiUrl().trim(), + HttpMethod.POST, + new HttpEntity<>(body, headers), + JsonNode.class + ); + } else { + String url = UriComponentsBuilder.fromHttpUrl(properties.getApiUrl().trim()) + .queryParamIfPresent("userId", java.util.Optional.ofNullable(userId)) + .build(true) + .toUriString(); + response = restTemplate.exchange(url, HttpMethod.GET, new HttpEntity<>(headers), JsonNode.class); + } + + AssistantProfile profile = sanitizeAssistantProfilePayload(response.getBody()); + return new CachedAssistantProfile(profile, "remote_api", fetchedAt, true); + } + + private RestTemplate createRestTemplate() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + int timeoutMs = (int) Duration.ofMillis(Math.max(properties.getTimeoutMs(), 1L)).toMillis(); + factory.setConnectTimeout(timeoutMs); + factory.setReadTimeout(timeoutMs); + return new RestTemplate(factory); + } + + private Map parseAssistantProfileHeaders() { + if (!StringUtils.hasText(properties.getApiHeaders())) { + return Map.of(); + } + try { + JsonNode root = objectMapper.readTree(properties.getApiHeaders()); + if (!root.isObject()) { + return Map.of(); + } + Map headers = new HashMap<>(); + Iterator> fields = root.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String key = field.getKey() == null ? "" : field.getKey().trim(); + String value = field.getValue() == null ? "" : field.getValue().asText("").trim(); + if (!key.isEmpty() && !value.isEmpty()) { + headers.put(key, value); + } + } + return headers; + } catch (Exception exception) { + log.warn("[AssistantProfile] parse headers failed: {}", exception.getMessage()); + return Map.of(); + } + } + + private AssistantProfile sanitizeAssistantProfilePayload(JsonNode payload) { + JsonNode source = pickAssistantProfilePayload(payload); + AssistantProfile profile = new AssistantProfile( + textValue(source, "documents"), + textValue(source, "email"), + firstNonBlank( + textValue(source, "agentName"), + textValue(source, "agent_name"), + textValue(source, "assistantName"), + textValue(source, "assistant_name"), + textValue(source, "nickname") + ), + textValue(source, "nickname"), + textValue(source, "wxl"), + textValue(source, "mobile"), + textValue(source, "wx_code"), + textValue(source, "intro"), + textValue(source, "sign"), + textValue(source, "story") + ); + return AssistantProfile.resolve(profile); + } + + private JsonNode pickAssistantProfilePayload(JsonNode payload) { + if (payload == null || payload.isNull()) { + return objectMapper.createObjectNode(); + } + JsonNode[] candidates = new JsonNode[] { + payload.path("assistantProfile"), + payload.path("profile"), + payload.path("data").path("assistantProfile"), + payload.path("data").path("profile"), + payload.path("data"), + payload + }; + for (JsonNode candidate : candidates) { + if (candidate != null && candidate.isObject()) { + return candidate; + } + } + return objectMapper.createObjectNode(); + } + + private String textValue(JsonNode node, String fieldName) { + if (node == null || node.isNull()) { + return ""; + } + JsonNode value = node.path(fieldName); + return value.isMissingNode() || value.isNull() ? "" : value.asText("").trim(); + } + + private String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + for (String value : values) { + if (StringUtils.hasText(value)) { + return value.trim(); + } + } + return ""; + } + + private String getCacheKey(String userId) { + return userId == null ? "global" : userId; + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private record CachedAssistantProfile(AssistantProfile profile, String source, long fetchedAt, boolean configured) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/ChatContentSafetyService.java b/java-server/src/main/java/com/bigwo/javaserver/service/ChatContentSafetyService.java new file mode 100644 index 0000000..3ba9446 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/ChatContentSafetyService.java @@ -0,0 +1,133 @@ +package com.bigwo.javaserver.service; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.springframework.stereotype.Service; + +@Service +public class ChatContentSafetyService { + + private static final String BRAND_NAMES = "德国PM|PM-International|PM公司|PM-FitLine|FitLine|一成系统|一成团队|大沃|PM营养素|PM健康|PM事业|PM直销|PM产品|PM"; + private static final List GENERIC_HARMFUL_WORDS = List.of( + "传销", "直销骗局", "非法直销", "变相传销", "网络传销", "精神传销", + "传销组织", "传销模式", "传销公司", "传销骗局", "传销陷阱", "传销套路", + "骗局", "骗子公司", "骗子", "骗人的", "诈骗", "行骗", "欺诈", + "虚假宣传", "夸大宣传", "虚假广告", "消费欺诈", "商业欺诈", + "非法集资", "非法经营", "非法营销", "非法组织", "非法敛财", + "涉嫌违法", "涉嫌传销", "疑似传销", "涉嫌欺诈", "涉嫌诈骗", + "违法经营", "违规经营", "违规操作", + "不正规", "不合法", "不合规", "不靠谱", "不正当", + "庞氏骗局", "老鼠会", "拉人头", "割韭菜", "资金盘", "层级分销", + "金字塔骗局", "金字塔模式", "发展下线", "上线下线", + "会员费骗局", "入门费骗局", "人头费骗局", + "智商税", "缴智商税", "交智商税", "收割", "被收割", "被割", + "洗脑", "被洗脑", "洗脑术", "洗脑营销", "精神控制", + "坑人", "坑钱", "坑货", "害人", "黑心", "黑幕", + "暴利", "暴利产品", "天价产品", "高价低质", + "被查处", "被取缔", "被罚款", "被处罚", "被举报", + "工商处罚", "市场监管处罚", "行政处罚", + "依法处理", "依法查处", "依法取缔", + "没有合法直销资质", "没有直销资质", "不具备直销资质", + "没有合法资质", "没有经营资质", "无合法资质", + "没有取得批准文号", "未取得批准文号", "没取得批准文号", + "没有取得资质", "未取得资质", "没取得资质", + "没有取得直销资质", "未取得直销资质", + "没有取得牌照", "未取得牌照", + "没有保健食品批准", "未取得保健食品", + "法律风险", "资金损失", "经济损失", "血本无归", + "不符合相关法律", "不符合法律法规", "违反法律法规", + "受害者", "受骗者", "上当受骗", "上当了", "被骗了", + "维权", "退款难", "投诉无门", + "臭名昭著", "声名狼藉", "劣迹斑斑", "口碑极差", + "过街老鼠", "千夫所指" + ); + private static final List BRAND_NEGATIVE_SUFFIXES = List.of( + "是传销", "属于传销", "涉嫌传销", "疑似传销", "就是传销", + "是骗局", "是骗子", "是骗人的", "在骗人", "骗钱", + "是非法的", "不合法", "不正规", "不靠谱", "不可信", + "有问题", "有争议", "有风险", "有隐患", + "已被查", "已被处罚", "被取缔", "被举报", + "在割韭菜", "在收割", "在洗脑", "在坑人", "在骗人", + "没有资质", "没有牌照", "没有直销牌照", "没有合法直销资质", "没有直销资质", + "没取得", "没有取得", "未取得", + "不符合法律", "不符合法规", "不符合相关法律", + "存在法律风险", "带来法律风险", "面临法律风险", + "存在资金损失", "带来资金损失", "带来经济损失", + "产品不行", "产品是假的", "产品没用", "产品有害", + "害了很多人", "坑了很多人", "骗了很多人", + "不值得信任", "不值得加入", "不建议加入", "不要加入", + "千万别信", "千万别买", "千万不要", "别上当", + "是假的", "没用", "没效果", "没什么用", "不管用", "是垃圾", "垃圾产品" + ); + private static final List NEGATIVE_BRAND_PREFIXES = List.of( + "传销组织", "传销公司", "传销骗局", "传销陷阱", + "骗局", "骗子公司", "骗人的", + "非法的", "不合法的", "不正规的", "不靠谱的", + "有争议的", "有问题的", "有风险的", + "坑人的", "害人的", "割韭菜的", "洗脑的", + "千万别信", "千万不要买", "千万不要加入", + "远离", "警惕", "小心", "当心", "注意" + ); + private static final List POSITIVE_PHRASES = List.of( + "不是传销", "并非传销", "并不是传销", "不属于传销", + "合法正规的直销企业", "合法正规直销企业", "合法直销公司", "合法直销企业", + "正规直销企业", "正规直销公司", "正规持牌直销公司", "正规持牌直销企业", + "拥有直销牌照", "持有直销牌照", "获得直销牌照", + "邓白氏AAA\\+", "邓白氏AAA", "AAA\\+认证", "AAA\\+信用", + "合法合规", "正规合法", "正规经营", + "业务覆盖全球", "覆盖.*国家", + "1993年成立", "成立于德国" + ); + private static final Pattern BRAND_HARMFUL_PATTERN = Pattern.compile( + quoteJoin(GENERIC_HARMFUL_WORDS) + + "|(?:" + BRAND_NAMES + ").*?(?:" + quoteJoin(BRAND_NEGATIVE_SUFFIXES) + ")" + + "|(?:" + quoteJoin(NEGATIVE_BRAND_PREFIXES) + ").*?(?:" + BRAND_NAMES + ")", + Pattern.CASE_INSENSITIVE + ); + private static final Pattern BRAND_POSITIVE_LEGALITY_PATTERN = Pattern.compile( + "(?:" + BRAND_NAMES + ").*?(?:" + String.join("|", POSITIVE_PHRASES) + ")" + + "|(?:" + String.join("|", POSITIVE_PHRASES) + ").*?(?:" + BRAND_NAMES + ")", + Pattern.CASE_INSENSITIVE + ); + private static final String TEXT_SAFE_REPLY = "德国PM是一家1993年成立于德国的合法直销公司，获得邓白氏AAA+认证，业务覆盖100多个国家和地区。如果你想了解更多，可以问我关于PM公司的详细介绍哦。"; + + public boolean isBrandHarmful(String text) { + if (text == null || text.isBlank()) { + return false; + } + String normalized = text.replaceAll("\\s+", " "); + if (BRAND_POSITIVE_LEGALITY_PATTERN.matcher(normalized).find()) { + return false; + } + return BRAND_HARMFUL_PATTERN.matcher(normalized).find(); + } + + public String guardAssistantText(String text) { + String result = normalizeAssistantText(text); + if (isBrandHarmful(result)) { + return getTextSafeReply(); + } + return result; + } + + public String getTextSafeReply() { + return TEXT_SAFE_REPLY; + } + + private String normalizeAssistantText(String text) { + return String.valueOf(text == null ? "" : text) + .replace("\r", " ") + .replaceAll("\n{2,}", "。") + .replace("\n", " ") + .replaceAll("。{2,}", "。") + .replaceAll("([！？；，])\\1+", "$1") + .replaceAll("([。！？；，])\\s*([。！？；，])", "$2") + .replaceAll("\\s+", " ") + .trim(); + } + + private static String quoteJoin(List values) { + return values.stream().map(Pattern::quote).collect(Collectors.joining("|")); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/ChatService.java b/java-server/src/main/java/com/bigwo/javaserver/service/ChatService.java new file mode 100644 index 0000000..cac73a8 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/ChatService.java @@ -0,0 +1,440 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.exception.BadRequestException; +import com.bigwo.javaserver.model.ChatHistoryResponse; +import com.bigwo.javaserver.model.ChatSendResponse; +import com.bigwo.javaserver.model.ChatSessionState; +import com.bigwo.javaserver.model.ChatStartResponse; +import com.bigwo.javaserver.model.ChatStreamEvent; +import com.bigwo.javaserver.model.ChatSubtitle; +import com.bigwo.javaserver.model.CozeChatResult; +import com.bigwo.javaserver.model.KnowledgeSearchResult; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.repository.ChatRepository; +import com.bigwo.javaserver.web.request.ChatSendRequest; +import com.bigwo.javaserver.web.request.ChatStartRequest; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class ChatService { + + private static final Logger log = LoggerFactory.getLogger(ChatService.class); + private static final Pattern FAST_GREETING_PATTERN = Pattern.compile("^(喂|你好|您好|嗨|哈喽|hello|hi|在吗|在不在|早上好|中午好|下午好|晚上好|早安|晚安)[，,！。？?~～\\s]*[啊呀吧呢哦嗯嘛哈的了]*[！。？?~～]*$", Pattern.CASE_INSENSITIVE); + private static final long SESSION_TTL_MS = 30L * 60L * 1000L; + + private final ConcurrentMap chatSessions = new ConcurrentHashMap<>(); + private final ChatRepository chatRepository; + private final CozeChatClient cozeChatClient; + private final ChatContentSafetyService chatContentSafetyService; + private final ContextKeywordTracker contextKeywordTracker; + private final KnowledgeRouteDecider knowledgeRouteDecider; + private final KnowledgeBaseRetrieverService knowledgeBaseRetrieverService; + + public ChatService( + ChatRepository chatRepository, + CozeChatClient cozeChatClient, + ChatContentSafetyService chatContentSafetyService, + ContextKeywordTracker contextKeywordTracker, + KnowledgeRouteDecider knowledgeRouteDecider, + KnowledgeBaseRetrieverService knowledgeBaseRetrieverService + ) { + this.chatRepository = chatRepository; + this.cozeChatClient = cozeChatClient; + this.chatContentSafetyService = chatContentSafetyService; + this.contextKeywordTracker = contextKeywordTracker; + this.knowledgeRouteDecider = knowledgeRouteDecider; + this.knowledgeBaseRetrieverService = knowledgeBaseRetrieverService; + } + + public ChatStartResponse startSession(ChatStartRequest request) { + if (request == null || !StringUtils.hasText(request.sessionId())) { + throw new BadRequestException("sessionId is required"); + } + if (!cozeChatClient.isConfigured()) { + throw new IllegalStateException("Coze 智能体未配置，请设置 COZE_API_TOKEN 和 COZE_BOT_ID"); + } + ChatSessionState sessionState = buildChatSessionState(request.sessionId().trim(), safeSubtitles(request.voiceSubtitles()), normalizeNullable(request.userId())); + chatRepository.createSession(sessionState.getSessionId(), sessionState.getUserId(), "chat"); + chatSessions.put(sessionState.getSessionId(), sessionState); + log.info("[Chat] Session started: {}, fromVoice: {}, voiceMessages: {}, summary: {}", + sessionState.getSessionId(), + sessionState.isFromVoice(), + sessionState.getVoiceMessages().size(), + sessionState.getHandoffSummary().isBlank() ? "no" : "yes"); + return new ChatStartResponse(sessionState.getSessionId(), sessionState.getVoiceMessages().size(), sessionState.isFromVoice()); + } + + public ChatSendResponse sendMessage(ChatSendRequest request) { + ValidatedChatRequest validated = validateChatSendRequest(request); + ChatSessionState session = resolveSession(validated.sessionId(), validated.userId()); + contextKeywordTracker.updateSession(validated.sessionId(), validated.message()); + log.info("[Chat] User({}): {}", validated.sessionId(), validated.message()); + chatRepository.addMessage(validated.sessionId(), "user", validated.message(), "chat_user"); + + String fastGreetingReply = buildFastGreetingReply(validated.message()); + if (!fastGreetingReply.isEmpty()) { + chatRepository.addMessage(validated.sessionId(), "assistant", fastGreetingReply, "chat_bot"); + return new ChatSendResponse(fastGreetingReply); + } + + KnowledgeEvidenceContext kbEvidence = tryKnowledgeEvidence(validated.sessionId(), session, validated.message()); + List> extraMessages = session.getConversationId() == null ? buildInitialContextMessages(session) : List.of(); + String cozeMessage = kbEvidence == null ? validated.message() : buildKnowledgePrompt(validated.message(), kbEvidence.evidence()); + CozeChatResult result = cozeChatClient.chat(session.getUserId(), cozeMessage, session.getConversationId(), extraMessages); + String normalizedContent = chatContentSafetyService.guardAssistantText(result.content()); + session.setConversationId(result.conversationId()); + session.setHandoffSummaryUsed(true); + if (kbEvidence != null) { + session.setLastKbContext(validated.message(), System.currentTimeMillis()); + } + if (!normalizedContent.isBlank()) { + String source = kbEvidence == null ? "chat_bot" : "search_knowledge"; + chatRepository.addMessage(validated.sessionId(), "assistant", normalizedContent, source, kbEvidence == null ? null : "search_knowledge", kbEvidence == null ? null : kbEvidence.meta()); + } + log.info("[Chat] Assistant({}): {} [source={}]", validated.sessionId(), abbreviate(normalizedContent), kbEvidence == null ? "chat_bot" : "search_knowledge"); + return new ChatSendResponse(normalizedContent); + } + + public Object getHistory(String sessionId) { + ChatSessionState session = chatSessions.get(sessionId); + if (session == null) { + return List.of(); + } + return new ChatHistoryResponse(session.getConversationId(), session.isFromVoice()); + } + + public void deleteSession(String sessionId) { + chatSessions.remove(sessionId); + } + + public void streamMessage(ChatSendRequest request, Consumer eventConsumer) { + ValidatedChatRequest validated = validateChatSendRequest(request); + ChatSessionState session = resolveSession(validated.sessionId(), validated.userId()); + contextKeywordTracker.updateSession(validated.sessionId(), validated.message()); + log.info("[Chat][SSE] User({}): {}", validated.sessionId(), validated.message()); + chatRepository.addMessage(validated.sessionId(), "user", validated.message(), "chat_user"); + + String fastGreetingReply = buildFastGreetingReply(validated.message()); + if (!fastGreetingReply.isEmpty()) { + chatRepository.addMessage(validated.sessionId(), "assistant", fastGreetingReply, "chat_bot"); + eventConsumer.accept(ChatStreamEvent.done(fastGreetingReply)); + return; + } + + try { + KnowledgeEvidenceContext kbEvidence = tryKnowledgeEvidence(validated.sessionId(), session, validated.message()); + List> extraMessages = session.getConversationId() == null ? buildInitialContextMessages(session) : List.of(); + String cozeMessage = kbEvidence == null ? validated.message() : buildKnowledgePrompt(validated.message(), kbEvidence.evidence()); + StringBuilder streamBuffer = new StringBuilder(); + AtomicBoolean harmfulDetected = new AtomicBoolean(false); + CozeChatResult result = cozeChatClient.chatStream( + session.getUserId(), + cozeMessage, + session.getConversationId(), + extraMessages, + chunk -> { + if (harmfulDetected.get()) { + return; + } + streamBuffer.append(chunk); + if (chatContentSafetyService.isBrandHarmful(streamBuffer.toString())) { + harmfulDetected.set(true); + eventConsumer.accept(ChatStreamEvent.reset("content_safety")); + return; + } + eventConsumer.accept(ChatStreamEvent.chunk(chunk)); + } + ); + String finalContent = harmfulDetected.get() ? chatContentSafetyService.getTextSafeReply() : chatContentSafetyService.guardAssistantText(result.content()); + session.setConversationId(result.conversationId()); + session.setHandoffSummaryUsed(true); + if (kbEvidence != null) { + session.setLastKbContext(validated.message(), System.currentTimeMillis()); + } + if (!finalContent.isBlank()) { + String source = kbEvidence == null ? "chat_bot" : "search_knowledge"; + chatRepository.addMessage(validated.sessionId(), "assistant", finalContent, source, kbEvidence == null ? null : "search_knowledge", kbEvidence == null ? null : kbEvidence.meta()); + } + log.info("[Chat][SSE] Assistant({}): {}{} [source=chat_bot]", + validated.sessionId(), + abbreviate(finalContent), + harmfulDetected.get() ? " [SAFE_REPLACED]" : ""); + eventConsumer.accept(ChatStreamEvent.done(finalContent)); + } catch (Exception exception) { + log.error("[Chat][SSE] Stream failed: {}", exception.getMessage(), exception); + eventConsumer.accept(ChatStreamEvent.error(exception.getMessage())); + } + } + + @Scheduled(fixedDelay = 300000) + public void cleanupExpiredSessions() { + long now = System.currentTimeMillis(); + for (Map.Entry entry : chatSessions.entrySet()) { + ChatSessionState session = entry.getValue(); + long lastActive = session.getLastActiveAt() > 0 ? session.getLastActiveAt() : session.getCreatedAt(); + if (now - lastActive > SESSION_TTL_MS && chatSessions.remove(entry.getKey(), session)) { + log.info("[Chat] Session expired and cleaned: {}", entry.getKey()); + } + } + contextKeywordTracker.cleanup(); + } + + private ChatSessionState resolveSession(String sessionId, String userId) { + ChatSessionState session = chatSessions.computeIfAbsent(sessionId, id -> buildChatSessionState(id, List.of(), userId)); + if (userId != null) { + session.setUserId(userId); + session.setProfileUserId(userId); + } + session.touch(); + return session; + } + + private ChatSessionState buildChatSessionState(String sessionId, List voiceSubtitles, String userId) { + List voiceMessages = loadHandoffMessages(sessionId, voiceSubtitles); + seedKnowledgeKeywords(sessionId, voiceMessages); + String resolvedUserId = userId == null ? generatedUserId(sessionId) : userId; + long now = System.currentTimeMillis(); + return new ChatSessionState( + sessionId, + resolvedUserId, + userId, + null, + "", + 0L, + voiceMessages, + buildDeterministicHandoffSummary(voiceMessages), + false, + now, + now, + !voiceSubtitles.isEmpty() || !voiceMessages.isEmpty() + ); + } + + private List loadHandoffMessages(String sessionId, List voiceSubtitles) { + List dbHistory = chatRepository.getHistoryForLlm(sessionId, 20); + if (!dbHistory.isEmpty()) { + log.info("[Chat] Loaded {} messages from DB for session {}", dbHistory.size(), sessionId); + return dbHistory; + } + List voiceMessages = new ArrayList<>(); + if (!voiceSubtitles.isEmpty()) { + List recentSubtitles = voiceSubtitles.subList(Math.max(voiceSubtitles.size() - 10, 0), voiceSubtitles.size()); + for (ChatSubtitle subtitle : recentSubtitles) { + if (subtitle == null || !StringUtils.hasText(subtitle.text())) { + continue; + } + String role = "user".equalsIgnoreCase(normalizeNullable(subtitle.role())) ? "user" : "assistant"; + voiceMessages.add(new LlmMessage(role, subtitle.text().trim())); + } + } + return voiceMessages; + } + + private String buildDeterministicHandoffSummary(List messages) { + List normalizedMessages = messages.stream() + .filter(item -> item != null && ("user".equals(item.role()) || "assistant".equals(item.role())) && StringUtils.hasText(item.content())) + .skip(Math.max(messages.size() - 8, 0)) + .toList(); + if (normalizedMessages.isEmpty()) { + return ""; + } + List userMessages = normalizedMessages.stream().filter(item -> "user".equals(item.role())).toList(); + String currentQuestion = userMessages.isEmpty() ? "" : userMessages.getLast().content().trim(); + String previousQuestion = userMessages.size() > 1 ? userMessages.get(userMessages.size() - 2).content().trim() : ""; + String assistantFacts = normalizedMessages.stream() + .filter(item -> "assistant".equals(item.role())) + .skip(Math.max(normalizedMessages.stream().filter(item -> "assistant".equals(item.role())).count() - 2, 0)) + .map(LlmMessage::content) + .filter(StringUtils::hasText) + .map(String::trim) + .map(value -> value.length() > 60 ? value.substring(0, 60) : value) + .reduce((left, right) -> left + "；" + right) + .orElse(""); + List parts = new ArrayList<>(); + if (!currentQuestion.isBlank()) { + parts.add("当前问题：" + currentQuestion); + } + if (!previousQuestion.isBlank() && !previousQuestion.equals(currentQuestion)) { + parts.add("上一轮关注：" + previousQuestion); + } + if (!assistantFacts.isBlank()) { + parts.add("已给信息：" + assistantFacts); + } + return String.join("；", parts); + } + + private KnowledgeEvidenceContext tryKnowledgeEvidence(String sessionId, ChatSessionState session, String message) { + String text = message == null ? "" : message.trim(); + if (text.isBlank()) { + return null; + } + List context = buildKnowledgeContextMessages(sessionId, session); + if (!knowledgeRouteDecider.shouldForceKnowledgeRoute(text, context)) { + return null; + } + KnowledgeSearchResult result = knowledgeBaseRetrieverService.searchKnowledge( + text, + contextKeywordTracker.suggestContextTerms(sessionId, text), + null, + sessionId, + resolveProfileScope(session) + ); + if (!result.hit()) { + return null; + } + String evidence = extractEvidenceText(result); + if (evidence.isBlank()) { + return null; + } + log.info("[Chat] KB evidence found, len={}", evidence.length()); + return new KnowledgeEvidenceContext(evidence, buildKnowledgeMeta(result, text)); + } + + private List buildKnowledgeContextMessages(String sessionId, ChatSessionState session) { + List dbHistory = chatRepository.getHistoryForLlm(sessionId, 20); + String summary = session == null || session.getHandoffSummary() == null ? "" : session.getHandoffSummary().trim(); + if (summary.isBlank() || (session != null && session.isHandoffSummaryUsed())) { + return dbHistory; + } + List context = new ArrayList<>(); + context.add(new LlmMessage("assistant", "会话交接摘要：" + summary)); + context.addAll(dbHistory); + return List.copyOf(context); + } + + private String extractEvidenceText(KnowledgeSearchResult result) { + if (result == null || result.ragPayload() == null || result.ragPayload().isEmpty()) { + return ""; + } + return result.ragPayload().stream() + .filter(item -> item != null && !"instruction".equals(item.get("kind")) && !"context".equals(item.get("kind"))) + .map(item -> String.valueOf(item.getOrDefault("content", "")).trim()) + .filter(StringUtils::hasText) + .reduce((left, right) -> left + "\n" + right) + .orElse(""); + } + + private Map buildKnowledgeMeta(KnowledgeSearchResult result, String originalText) { + Map meta = new LinkedHashMap<>(); + meta.put("route", "search_knowledge"); + meta.put("original_text", originalText); + meta.put("tool_name", "search_knowledge"); + meta.put("tool_args", Map.of("query", originalText)); + meta.put("source", result.source()); + meta.put("original_query", result.originalQuery()); + meta.put("rewritten_query", result.query()); + meta.put("hit", result.hit()); + meta.put("reason", result.reason()); + meta.put("latency_ms", result.latencyMs()); + if (result.evidencePack() != null && result.evidencePack().get("retrieval") instanceof Map retrieval) { + meta.put("selected_dataset_ids", retrieval.get("selected_dataset_ids")); + meta.put("selected_kb_routes", retrieval.get("selected_kb_routes")); + } + return meta; + } + + private String buildKnowledgePrompt(String message, String evidence) { + return message + "\n\n【知识库参考信息】\n" + evidence + "\n\n请基于以上参考信息，用简洁口语化的方式回答我的问题，控制在200字以内。"; + } + + private String resolveProfileScope(ChatSessionState session) { + if (session == null) { + return "global"; + } + if (StringUtils.hasText(session.getProfileUserId())) { + return session.getProfileUserId().trim(); + } + if (StringUtils.hasText(session.getUserId())) { + return session.getUserId().trim(); + } + return "global"; + } + + private void seedKnowledgeKeywords(String sessionId, List messages) { + List userMessages = messages.stream() + .filter(item -> item != null && "user".equals(item.role()) && StringUtils.hasText(item.content())) + .toList(); + int start = Math.max(userMessages.size() - 6, 0); + for (LlmMessage message : userMessages.subList(start, userMessages.size())) { + contextKeywordTracker.updateSession(sessionId, message.content()); + } + } + + private List> buildInitialContextMessages(ChatSessionState session) { + List> extraMessages = new ArrayList<>(); + String summary = session.getHandoffSummary() == null ? "" : session.getHandoffSummary().trim(); + if (!summary.isBlank() && !session.isHandoffSummaryUsed()) { + extraMessages.add(messagePayload("assistant", "会话交接摘要：" + summary)); + } + List voiceMessages = session.getVoiceMessages(); + int start = Math.max(voiceMessages.size() - 6, 0); + for (LlmMessage message : voiceMessages.subList(start, voiceMessages.size())) { + extraMessages.add(messagePayload(message.role(), message.content())); + } + return extraMessages; + } + + private Map messagePayload(String role, String content) { + Map payload = new LinkedHashMap<>(); + payload.put("role", role); + payload.put("content", content); + payload.put("content_type", "text"); + return payload; + } + + private String buildFastGreetingReply(String message) { + String text = message == null ? "" : message.trim(); + if (!FAST_GREETING_PATTERN.matcher(text).matches()) { + return ""; + } + return "你好😊！我是大沃智能助手。你可以直接问我一成系统、德国PM产品、招商合作、营养科普等问题，我会尽量快速给你准确回复。"; + } + + private ValidatedChatRequest validateChatSendRequest(ChatSendRequest request) { + String sessionId = request == null ? null : normalizeNullable(request.sessionId()); + String message = request == null ? null : normalizeNullable(request.message()); + String userId = request == null ? null : normalizeNullable(request.userId()); + if (sessionId == null || message == null) { + throw new BadRequestException("sessionId and message are required"); + } + return new ValidatedChatRequest(sessionId, message, userId); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private String generatedUserId(String sessionId) { + return "user_" + sessionId.substring(0, Math.min(sessionId.length(), 12)); + } + + private List safeSubtitles(List subtitles) { + return subtitles == null ? List.of() : subtitles; + } + + private String abbreviate(String value) { + if (value == null || value.length() <= 100) { + return value; + } + return value.substring(0, 100); + } + + private record KnowledgeEvidenceContext(String evidence, Map meta) { + } + + private record ValidatedChatRequest(String sessionId, String message, String userId) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/ContextKeywordTracker.java b/java-server/src/main/java/com/bigwo/javaserver/service/ContextKeywordTracker.java new file mode 100644 index 0000000..b444f7e --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/ContextKeywordTracker.java @@ -0,0 +1,118 @@ +package com.bigwo.javaserver.service; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.regex.Pattern; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class ContextKeywordTracker { + + private static final long TTL_MS = 30L * 60L * 1000L; + private static final int MAX_KEYWORDS = 8; + private static final Pattern FOLLOW_UP_SIGNAL = Pattern.compile("(它|它的|他|他的|这个|那个|这款|那款|该产品|上面|刚才|再说|再次|强调一下|详细|继续|怎么吃|怎么用|怎么样|功效|成分|作用|原理|核心|区别|哪个好|为什么|什么意思|适合谁|多少钱|价格|副作用|正规吗|一天几次|每天几次|每日几次|给我介绍|介绍一下|说一下|讲一下)", Pattern.CASE_INSENSITIVE); + + private final KnowledgeKeywordCatalog keywordCatalog; + private final KnowledgeQueryResolver knowledgeQueryResolver; + private final ConcurrentMap sessionKeywords = new ConcurrentHashMap<>(); + + public ContextKeywordTracker(KnowledgeKeywordCatalog keywordCatalog, KnowledgeQueryResolver knowledgeQueryResolver) { + this.keywordCatalog = keywordCatalog; + this.knowledgeQueryResolver = knowledgeQueryResolver; + } + + public void updateSession(String sessionId, String text) { + if (!StringUtils.hasText(sessionId)) { + return; + } + String normalizedText = knowledgeQueryResolver.normalizeKnowledgeText(text); + if (!StringUtils.hasText(normalizedText)) { + return; + } + List keywords = extractKeywords(normalizedText); + if (keywords.isEmpty()) { + return; + } + SessionKeywords existing = sessionKeywords.get(sessionId); + List merged = mergeKeywords(existing == null ? List.of() : existing.keywords(), keywords); + sessionKeywords.put(sessionId, new SessionKeywords(merged, System.currentTimeMillis())); + } + + public List getSessionKeywords(String sessionId) { + SessionKeywords data = sessionKeywords.get(sessionId); + if (data == null) { + return List.of(); + } + if (System.currentTimeMillis() - data.lastUpdate() > TTL_MS) { + sessionKeywords.remove(sessionId, data); + return List.of(); + } + return data.keywords(); + } + + public List suggestContextTerms(String sessionId, String query) { + String normalized = knowledgeQueryResolver.normalizeKnowledgeText(query); + if (!StringUtils.hasText(normalized)) { + return List.of(); + } + if (knowledgeQueryResolver.hasExplicitKnowledgeEntity(normalized)) { + return List.of(); + } + boolean isShortGeneric = normalized.length() <= 20; + if (!FOLLOW_UP_SIGNAL.matcher(normalized).find() && !isShortGeneric) { + return List.of(); + } + List keywords = getSessionKeywords(sessionId); + if (keywords.isEmpty()) { + return List.of(); + } + return List.of(keywords.getLast()); + } + + public void cleanup() { + long now = System.currentTimeMillis(); + for (var entry : sessionKeywords.entrySet()) { + if (now - entry.getValue().lastUpdate() > TTL_MS) { + sessionKeywords.remove(entry.getKey(), entry.getValue()); + } + } + } + + private List extractKeywords(String text) { + List keywords = new ArrayList<>(); + for (String keyword : keywordCatalog.knowledgeEntityKeywords()) { + if (text.contains(keyword)) { + keywords.add(keyword); + } + } + Set deduped = new LinkedHashSet<>(); + for (String keyword : keywords) { + if (StringUtils.hasText(keyword)) { + deduped.add(keyword.trim()); + } + } + return List.copyOf(deduped); + } + + private List mergeKeywords(List existing, List incoming) { + List merged = new ArrayList<>(existing == null ? List.of() : existing); + for (String keyword : incoming == null ? List.of() : incoming) { + String normalized = StringUtils.hasText(keyword) ? keyword.trim() : ""; + if (!StringUtils.hasText(normalized)) { + continue; + } + merged.removeIf(item -> item.equalsIgnoreCase(normalized)); + merged.add(normalized); + } + int start = Math.max(merged.size() - MAX_KEYWORDS, 0); + return List.copyOf(merged.subList(start, merged.size())); + } + + private record SessionKeywords(List keywords, long lastUpdate) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/CozeChatClient.java b/java-server/src/main/java/com/bigwo/javaserver/service/CozeChatClient.java new file mode 100644 index 0000000..23fd93c --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/CozeChatClient.java @@ -0,0 +1,230 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.config.CozeProperties; +import com.bigwo.javaserver.model.CozeChatResult; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +@Service +public class CozeChatClient { + + private static final Logger log = LoggerFactory.getLogger(CozeChatClient.class); + + private final CozeProperties properties; + private final ObjectMapper objectMapper; + private final HttpClient httpClient; + + public CozeChatClient(CozeProperties properties, ObjectMapper objectMapper) { + this.properties = properties; + this.objectMapper = objectMapper; + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + } + + public boolean isConfigured() { + return properties.isConfigured(); + } + + public CozeChatResult chat(String userId, String message, String conversationId, List> extraMessages) { + try { + Map body = buildChatBody(userId, message, conversationId, extraMessages, false); + log.info("[CozeChat] Sending non-stream chat, userId={}, convId={}", userId, conversationId == null ? "new" : conversationId); + ResponseEntity chatResponse = createRestTemplate(15000).exchange( + properties.normalizedBaseUrl() + "/v3/chat", + HttpMethod.POST, + new HttpEntity<>(body, defaultHeaders()), + JsonNode.class + ); + JsonNode chatData = chatResponse.getBody() == null ? null : chatResponse.getBody().path("data"); + String chatId = text(chatData, "id"); + String conversation = text(chatData, "conversation_id"); + if (chatId.isBlank() || conversation.isBlank()) { + throw new IllegalStateException("Coze chat creation failed: " + String.valueOf(chatResponse.getBody())); + } + + for (int attempt = 0; attempt < 30; attempt++) { + Thread.sleep(2000L); + String retrieveUrl = UriComponentsBuilder.fromHttpUrl(properties.normalizedBaseUrl() + "/v3/chat/retrieve") + .queryParam("chat_id", chatId) + .queryParam("conversation_id", conversation) + .build(true) + .toUriString(); + ResponseEntity retrieveResponse = createRestTemplate(10000).exchange( + retrieveUrl, + HttpMethod.GET, + new HttpEntity<>(defaultHeaders()), + JsonNode.class + ); + String status = text(retrieveResponse.getBody() == null ? null : retrieveResponse.getBody().path("data"), "status"); + if ("completed".equals(status)) { + break; + } + if ("failed".equals(status) || "requires_action".equals(status)) { + throw new IllegalStateException("Coze chat ended with status: " + status); + } + } + + String messageListUrl = UriComponentsBuilder.fromHttpUrl(properties.normalizedBaseUrl() + "/v3/chat/message/list") + .queryParam("chat_id", chatId) + .queryParam("conversation_id", conversation) + .build(true) + .toUriString(); + ResponseEntity messageResponse = createRestTemplate(10000).exchange( + messageListUrl, + HttpMethod.GET, + new HttpEntity<>(defaultHeaders()), + JsonNode.class + ); + String content = ""; + JsonNode data = messageResponse.getBody() == null ? null : messageResponse.getBody().path("data"); + if (data != null && data.isArray()) { + for (JsonNode item : data) { + if ("assistant".equals(text(item, "role")) && "answer".equals(text(item, "type"))) { + content = text(item, "content"); + break; + } + } + } + return new CozeChatResult(content, conversation); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Coze chat interrupted", exception); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + public CozeChatResult chatStream(String userId, String message, String conversationId, List> extraMessages, Consumer onChunk) { + try { + Map body = buildChatBody(userId, message, conversationId, extraMessages, true); + String requestBody = objectMapper.writeValueAsString(body); + log.info("[CozeChat] Sending stream chat, userId={}, convId={}", userId, conversationId == null ? "new" : conversationId); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(properties.normalizedBaseUrl() + "/v3/chat")) + .timeout(Duration.ofSeconds(60)) + .header("Content-Type", "application/json") + .header("Authorization", "Bearer " + properties.getApiToken().trim()) + .POST(HttpRequest.BodyPublishers.ofString(requestBody, StandardCharsets.UTF_8)) + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() >= 400) { + throw new IllegalStateException("Coze stream failed with status: " + response.statusCode()); + } + String resultConversationId = conversationId; + StringBuilder fullContent = new StringBuilder(); + String currentEvent = ""; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.body(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) { + currentEvent = ""; + continue; + } + if (trimmed.startsWith("event:")) { + currentEvent = trimmed.substring(6).trim(); + continue; + } + if (!trimmed.startsWith("data:")) { + continue; + } + String data = trimmed.substring(5).trim(); + if ("\"[DONE]\"".equals(data) || "[DONE]".equals(data)) { + continue; + } + JsonNode parsed = objectMapper.readTree(data); + if ("conversation.chat.created".equals(currentEvent)) { + String candidateConversationId = text(parsed, "conversation_id"); + if (!candidateConversationId.isBlank()) { + resultConversationId = candidateConversationId; + } + } + if ("conversation.message.delta".equals(currentEvent) + && "assistant".equals(text(parsed, "role")) + && "answer".equals(text(parsed, "type"))) { + String content = text(parsed, "content"); + if (!content.isBlank()) { + fullContent.append(content); + onChunk.accept(content); + } + } + } + } + return new CozeChatResult(fullContent.toString(), resultConversationId); + } catch (Exception exception) { + log.error("[CozeChat] Stream error: {}", exception.getMessage(), exception); + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private Map buildChatBody(String userId, String message, String conversationId, List> extraMessages, boolean stream) { + List> additionalMessages = new ArrayList<>(); + additionalMessages.addAll(extraMessages); + additionalMessages.add(messagePayload("user", message)); + + Map body = new LinkedHashMap<>(); + body.put("bot_id", properties.getBotId().trim()); + body.put("user_id", userId); + body.put("additional_messages", additionalMessages); + body.put("stream", stream); + body.put("auto_save_history", true); + if (conversationId != null && !conversationId.isBlank()) { + body.put("conversation_id", conversationId); + } + return body; + } + + private Map messagePayload(String role, String content) { + Map payload = new LinkedHashMap<>(); + payload.put("role", role); + payload.put("content", content); + payload.put("content_type", "text"); + return payload; + } + + private HttpHeaders defaultHeaders() { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setBearerAuth(properties.getApiToken().trim()); + return headers; + } + + private RestTemplate createRestTemplate(int timeoutMs) { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(timeoutMs); + factory.setReadTimeout(timeoutMs); + return new RestTemplate(factory); + } + + private String text(JsonNode node, String fieldName) { + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + JsonNode child = node.path(fieldName); + return child.isMissingNode() || child.isNull() ? "" : child.asText(""); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/FastAsrCorrector.java b/java-server/src/main/java/com/bigwo/javaserver/service/FastAsrCorrector.java new file mode 100644 index 0000000..4fcac76 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/FastAsrCorrector.java @@ -0,0 +1,153 @@ +package com.bigwo.javaserver.service; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.springframework.stereotype.Component; + +@Component +public class FastAsrCorrector { + + private static final Map PHRASE_MAP = Map.ofEntries( + Map.entry("一城系统", "一成系统"), Map.entry("逸城系统", "一成系统"), Map.entry("一程系统", "一成系统"), + Map.entry("易成系统", "一成系统"), Map.entry("一诚系统", "一成系统"), Map.entry("亦成系统", "一成系统"), + Map.entry("艺成系统", "一成系统"), Map.entry("溢成系统", "一成系统"), Map.entry("义成系统", "一成系统"), + Map.entry("毅成系统", "一成系统"), Map.entry("怡成系统", "一成系统"), Map.entry("以成系统", "一成系统"), + Map.entry("已成系统", "一成系统"), Map.entry("亿成系统", "一成系统"), Map.entry("忆成系统", "一成系统"), + Map.entry("益成系统", "一成系统"), Map.entry("一乘系统", "一成系统"), Map.entry("一承系统", "一成系统"), + Map.entry("一丞系统", "一成系统"), Map.entry("一呈系统", "一成系统"), Map.entry("一澄系统", "一成系统"), + Map.entry("一橙系统", "一成系统"), Map.entry("一层系统", "一成系统"), Map.entry("一趁系统", "一成系统"), + Map.entry("一陈系统", "一成系统"), Map.entry("依成系统", "一成系统"), Map.entry("伊成系统", "一成系统"), + Map.entry("益生系统", "一成系统"), Map.entry("易诚系统", "一成系统"), Map.entry("易乘系统", "一成系统"), + Map.entry("一声系统", "一成系统"), Map.entry("亿生系统", "一成系统"), Map.entry("义诚系统", "一成系统"), + Map.entry("忆诚系统", "一成系统"), Map.entry("以诚系统", "一成系统"), Map.entry("盛咖学院", "盛咖学愿"), + Map.entry("圣咖学愿", "盛咖学愿"), Map.entry("盛卡学愿", "盛咖学愿"), Map.entry("营养配送系统", "NTC营养保送系统"), + Map.entry("营养输送系统", "NTC营养保送系统"), Map.entry("营养传送系统", "NTC营养保送系统"), Map.entry("营养传输系统", "NTC营养保送系统"), + Map.entry("暖炉原理", "火炉原理"), Map.entry("整应反应", "好转反应"), Map.entry("整健反应", "好转反应"), + Map.entry("排毒反应", "好转反应"), Map.entry("5加1", "5+1"), Map.entry("五加一", "5+1"), + Map.entry("起步三观", "起步三关"), Map.entry("起步三官", "起步三关"), Map.entry("doublepm", "德国PM"), + Map.entry("double pm", "德国PM"), Map.entry("DoublePM", "德国PM"), Map.entry("Double PM", "德国PM"), + Map.entry("DOUBLEPM", "德国PM"), Map.entry("DOUBLE PM", "德国PM"), Map.entry("基础三合一", "PM细胞营养素 基础套装"), + Map.entry("三合一基础套", "PM细胞营养素 基础套装"), Map.entry("大白小红小白", "PM细胞营养素 基础套装") + ); + + private static final Map WORD_MAP = Map.ofEntries( + Map.entry("一城", "一成"), Map.entry("逸城", "一成"), Map.entry("一程", "一成"), Map.entry("易成", "一成"), + Map.entry("一诚", "一成"), Map.entry("亦成", "一成"), Map.entry("艺成", "一成"), Map.entry("溢成", "一成"), + Map.entry("义成", "一成"), Map.entry("毅成", "一成"), Map.entry("怡成", "一成"), Map.entry("以成", "一成"), + Map.entry("已成", "一成"), Map.entry("亿成", "一成"), Map.entry("忆成", "一成"), Map.entry("益成", "一成"), + Map.entry("一乘", "一成"), Map.entry("一承", "一成"), Map.entry("一丞", "一成"), Map.entry("一呈", "一成"), + Map.entry("一澄", "一成"), Map.entry("一橙", "一成"), Map.entry("一层", "一成"), Map.entry("一陈", "一成"), + Map.entry("依成", "一成"), Map.entry("伊成", "一成"), Map.entry("益生", "一成"), Map.entry("易诚", "一成"), + Map.entry("义诚", "一成"), Map.entry("忆诚", "一成"), Map.entry("以诚", "一成"), Map.entry("一声", "一成"), + Map.entry("亿生", "一成"), Map.entry("易乘", "一成"), Map.entry("大窝", "大沃"), Map.entry("大握", "大沃"), + Map.entry("大我", "大沃"), Map.entry("大卧", "大沃"), Map.entry("爱众享", "Ai众享"), Map.entry("艾众享", "Ai众享"), + Map.entry("哎众享", "Ai众享"), Map.entry("小洪", "小红"), Map.entry("小宏", "小红"), Map.entry("小鸿", "小红"), + Map.entry("大百", "大白"), Map.entry("大柏", "大白"), Map.entry("小百", "小白"), Map.entry("小柏", "小白"), + Map.entry("维适多", "小白"), Map.entry("营养配送", "营养保送"), Map.entry("营养输送", "营养保送"), + Map.entry("阿玉吠陀", "阿育吠陀"), Map.entry("阿育费陀", "阿育吠陀") + ); + + private static final Map PRODUCT_ALIAS_MAP = Map.ofEntries( + Map.entry("小红", "小红产品 Activize Oxyplus"), Map.entry("Activize", "小红产品 Activize Oxyplus"), + Map.entry("Activize Oxyplus", "小红产品 Activize Oxyplus"), Map.entry("大白", "大白产品 Basics"), + Map.entry("Basics", "大白产品 Basics"), Map.entry("小白", "小白产品 Restorate"), Map.entry("Restorate", "小白产品 Restorate"), + Map.entry("FitLine", "PM-FitLine"), Map.entry("PM FitLine", "PM-FitLine"), Map.entry("PM细胞营养", "PM细胞营养素"), + Map.entry("PM营养素", "PM细胞营养素"), Map.entry("德国PM营养素", "PM细胞营养素") + ); + + private static final Pattern SYSTEM_PATTERN = Pattern.compile("[一二三四五六七八九十壹贰叁肆伍陆柒捌玖拾百千万亿兆零两几单双半多少全数整这那某每各以已亦艺毅怡逸溢义忆益伊依乙翼奕弈邑佚颐译蚁屹役疫裔翊熠旖漪倚绮峄羿轶弋驿懿肄翌苡圯佾诒铱仡易]{1,2}(?:成|城|程|诚|乘|承|丞|呈|澄|橙|层|陈|趁|撑|称|秤|盛|剩|胜|生|声)系统"); + private static final Pattern FILLER_PREFIX = Pattern.compile("^[啊哦嗯呢呀哎诶额，。！？、\\s]+|[啊哦嗯呢呀哎诶额，。！？、\\s]+$"); + private static final List PHONETIC_CORRECTIONS = List.of( + new PatternReplacement("[细希西系息][胞苞包宝][抗康][氧养仰样][素速]", "细胞抗氧素"), + new PatternReplacement("[胶交教焦角][原圆远元源][蛋旦但淡][白百柏拍]", "胶原蛋白"), + new PatternReplacement("[白百柏][藜梨黎离莉丽力利理礼里][芦炉路鹿鲁卢露陆][醇纯唇]", "白藜芦醇"), + new PatternReplacement("[好号浩耗][转赚砖专][反返犯翻范][应映影英]", "好转反应"), + new PatternReplacement("[阿啊][育玉域遇雨宇御][吠废费肺飞非][陀驼拖脱托]", "阿育吠陀"), + new PatternReplacement("[骨谷古鼓][骼格隔革各阁][健剑键建][康慷抗]", "骨骼健康"), + new PatternReplacement("[活火获霍货][力利立厉励历丽][健剑键建见件]", "活力健"), + new PatternReplacement("[倍被背贝备辈杯北][力利立厉励历丽][健剑键建见件]", "倍力健"), + new PatternReplacement("[氨安暗按胺][基机鸡积极几计][酸算]", "氨基酸"), + new PatternReplacement("[益意易亿以][生声胜升省圣][菌军均君]", "益生菌"), + new PatternReplacement("[辅付副附府腐][酵教叫觉较角][素速诉]", "辅酵素"), + new PatternReplacement("[葡铺浦蒲][萄逃淘桃陶][籽子紫]", "葡萄籽"), + new PatternReplacement("[排牌拍派][毒独度读督][饮引印隐]", "排毒饮"), + new PatternReplacement("[乳如入][酪烙络落][煲包保宝]", "乳酪煲"), + new PatternReplacement("[草操曹][本苯奔][茶查差]", "草本茶"), + new PatternReplacement("[异意易][黄皇荒慌][酮铜同桐]", "异黄酮"), + new PatternReplacement("[骨谷古鼓][骼格隔革各阁][健剑键建见]", "骨骼健"), + new PatternReplacement("[舒书叔输][采彩菜蔡][健剑键建见]", "舒采健"), + new PatternReplacement("[衡横恒亨][醇纯唇春][饮引印隐]", "衡醇饮"), + new PatternReplacement("[纤先鲜仙][萃翠脆粹催]", "纤萃") + ); + + private final PinyinProductMatcher pinyinProductMatcher; + + public FastAsrCorrector(PinyinProductMatcher pinyinProductMatcher) { + this.pinyinProductMatcher = pinyinProductMatcher; + } + + public String correctAsrText(String text) { + if (text == null || text.isBlank()) { + return text == null ? "" : text; + } + String result = text.trim(); + result = replaceOrderedMappings(result, PHRASE_MAP); + result = replaceOrderedMappings(result, WORD_MAP); + result = phoneticCorrectProducts(result); + result = pinyinProductMatcher.matchProducts(result); + result = SYSTEM_PATTERN.matcher(result).replaceAll("一成系统"); + for (Map.Entry entry : orderedEntries(PRODUCT_ALIAS_MAP).entrySet()) { + if (shouldExpandProductAlias(result, entry.getKey())) { + result = result.replace(entry.getKey(), entry.getValue()); + } + } + return FILLER_PREFIX.matcher(result).replaceAll("").trim(); + } + + public String phoneticCorrectProducts(String text) { + String result = text; + for (PatternReplacement replacement : PHONETIC_CORRECTIONS) { + result = replacement.pattern().matcher(result).replaceAll(replacement.replacement()); + } + return result; + } + + private Map orderedEntries(Map source) { + LinkedHashMap ordered = new LinkedHashMap<>(); + source.entrySet().stream() + .sorted((left, right) -> Integer.compare(right.getKey().length(), left.getKey().length())) + .forEach(entry -> ordered.put(entry.getKey(), entry.getValue())); + return ordered; + } + + private String replaceOrderedMappings(String text, Map mapping) { + String result = text; + for (Map.Entry entry : orderedEntries(mapping).entrySet()) { + if (result.contains(entry.getKey())) { + result = result.replace(entry.getKey(), entry.getValue()); + } + } + return result; + } + + private boolean shouldExpandProductAlias(String text, String alias) { + String target = PRODUCT_ALIAS_MAP.get(alias); + if (target != null && text.contains(target)) { + return false; + } + if (text.equals(alias)) { + return true; + } + String escapedAlias = Pattern.quote(alias); + Pattern pattern = Pattern.compile(escapedAlias + "(?=\\s|的|是|有|和|跟|及|怎么|为什么|适合谁|什么意思|怎么吃|怎么用|功效|成分|多少钱|哪里买|价格|副作用|区别|哪个好|是什么|呢|吗|呀|啊|哦|吧|啦|了|$)", Pattern.CASE_INSENSITIVE); + return pattern.matcher(text).find(); + } + + private record PatternReplacement(Pattern pattern, String replacement) { + private PatternReplacement(String regex, String replacement) { + this(Pattern.compile(regex), replacement); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeBaseRetrieverService.java b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeBaseRetrieverService.java new file mode 100644 index 0000000..c1efe64 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeBaseRetrieverService.java @@ -0,0 +1,666 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.config.KnowledgeBaseProperties; +import com.bigwo.javaserver.model.KnowledgeChunk; +import com.bigwo.javaserver.model.KnowledgeQueryInfo; +import com.bigwo.javaserver.model.KnowledgeSearchResult; +import com.bigwo.javaserver.model.RedisContextMessage; +import com.bigwo.javaserver.util.VolcSignerV4; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class KnowledgeBaseRetrieverService { + + private static final Logger log = LoggerFactory.getLogger(KnowledgeBaseRetrieverService.class); + private static final String KNOWLEDGE_HOST = "api-knowledgebase.mlp.cn-beijing.volces.com"; + private static final String SEARCH_PATH = "/api/knowledge/collection/search_knowledge"; + private static final String RERANK_PATH = "/api/knowledge/service/rerank"; + private static final String SERVICE_NAME = "air"; + private static final String REGION = "cn-north-1"; + private static final String SOURCE = "ark_knowledge"; + private static final long KB_CACHE_TTL_MS = 5L * 60L * 1000L; + private static final long KB_CACHE_NO_HIT_TTL_MS = 10_000L; + private static final int KB_CACHE_MAX_SIZE = 200; + private static final String KB_INSTRUCTION = "用口语化、简洁的方式回答，像朋友聊天一样自然地说出来。严格遵守以下规则：1）只能使用证据中出现的产品名，严禁自创或概括出新的产品名称；2）产品的用法、剂量、剂型（粒/勺/包）必须原文引用，不可凭记忆改写；3）不同产品的信息不可混用，回答时明确指出是哪个产品；4）如果证据不足以回答，直接说不确定，不要编造；5）推荐产品时，必须从证据中提取该产品的具体推荐理由，包括但不限于：产品定位与核心功效、关键成分及其作用、适用场景与人群、认证背书（如科隆名单、运动员使用）、用户见证或体感反馈，用提取到的具体事实作为推荐依据，不要用笼统的推荐语。"; + private static final Map DEFAULT_COLLECTION_MAP = Map.of( + "kb-ad2e0ea30902421b", "product_details", + "kb-d45d3056a7b75ac5", "faq_qa", + "kb-d0ef0b7b8f36a839", "science_training", + "kb-6a170ab7b1bc024f", "system_training" + ); + + private final KnowledgeBaseProperties properties; + private final VolcSignerV4 volcSignerV4; + private final ObjectMapper objectMapper; + private final KnowledgeQueryResolver knowledgeQueryResolver; + private final RedisContextStore redisContextStore; + private final HttpClient httpClient; + private final ConcurrentMap memoryCache = new ConcurrentHashMap<>(); + + public KnowledgeBaseRetrieverService( + KnowledgeBaseProperties properties, + VolcSignerV4 volcSignerV4, + ObjectMapper objectMapper, + KnowledgeQueryResolver knowledgeQueryResolver, + RedisContextStore redisContextStore + ) { + this.properties = properties; + this.volcSignerV4 = volcSignerV4; + this.objectMapper = objectMapper; + this.knowledgeQueryResolver = knowledgeQueryResolver; + this.redisContextStore = redisContextStore; + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + } + + public boolean isConfigured() { + return properties.hasAccessKeys(); + } + + public KnowledgeSearchResult searchKnowledge(String query, List contextTerms, List datasetIds, String sessionId, String profileScope) { + long startTime = System.currentTimeMillis(); + String rawQuery = StringUtils.hasText(query) ? query.trim() : ""; + String rewrittenQuery = knowledgeQueryResolver.rewriteKnowledgeQuery(rawQuery, contextTerms == null ? List.of() : contextTerms); + String effectiveQuery = StringUtils.hasText(rewrittenQuery) ? rewrittenQuery : (StringUtils.hasText(rawQuery) ? rawQuery : "请介绍你们的产品和服务"); + List effectiveDatasetIds = normalizeDatasetIds(datasetIds); + String normalizedProfileScope = StringUtils.hasText(profileScope) ? profileScope.trim() : "global"; + String cacheKey = buildCacheKey(effectiveQuery, effectiveDatasetIds, normalizedProfileScope); + + KnowledgeSearchResult redisCached = redisContextStore.getKbCache(cacheKey, KnowledgeSearchResult.class); + if (redisCached != null) { + return withLatency(redisCached, System.currentTimeMillis() - startTime); + } + KnowledgeSearchResult memoryCached = getMemoryCache(cacheKey); + if (memoryCached != null) { + return withLatency(memoryCached, System.currentTimeMillis() - startTime); + } + + RetrievalResult retrievalResult = retrieveChunks(effectiveQuery, effectiveDatasetIds, properties.getRetrievalTopK(), 0.01D); + if (retrievalResult.error() != null) { + KnowledgeSearchResult errorResult = buildFailureResult( + effectiveQuery, + rawQuery, + retrievalResult.error(), + System.currentTimeMillis() - startTime, + retrievalResult.latencyMs() + ); + setMemoryCache(cacheKey, errorResult); + return errorResult; + } + + if (retrievalResult.chunks().isEmpty()) { + String reason = retrievalResult.kbHasContent() ? "chunks_parse_failed" : "no_relevant_content"; + KnowledgeSearchResult emptyResult = buildFailureResult( + effectiveQuery, + rawQuery, + reason, + System.currentTimeMillis() - startTime, + retrievalResult.latencyMs() + ); + setMemoryCache(cacheKey, emptyResult); + return emptyResult; + } + + List rerankedChunks = rerankChunks(rawQuery.isBlank() ? effectiveQuery : rawQuery, retrievalResult.chunks(), properties.getRerankerTopN()); + double topScore = rerankedChunks.isEmpty() ? 0D : rerankedChunks.getFirst().score(); + double hitThreshold = properties.isEnableReranker() && StringUtils.hasText(properties.getRerankerModel()) ? 0.1D : 0.3D; + boolean hit = !rerankedChunks.isEmpty() && topScore >= hitThreshold; + String reason = hit ? "reranked_hit" : "below_threshold"; + + List conversationHistory = loadConversationHistory(sessionId); + List> ragPayload = buildRagPayload(rerankedChunks, conversationHistory); + Map evidencePack = buildEvidencePack( + rawQuery.isBlank() ? effectiveQuery : rawQuery, + rawQuery.isBlank() ? effectiveQuery : rawQuery, + effectiveQuery, + ragPayload, + rerankedChunks, + conversationHistory, + hit, + topScore, + reason, + effectiveDatasetIds, + List.of() + ); + + KnowledgeSearchResult result = new KnowledgeSearchResult( + effectiveQuery, + rawQuery.isBlank() ? effectiveQuery : rawQuery, + hit, + reason, + retrievalResult.chunks(), + rerankedChunks, + ragPayload, + evidencePack, + topScore, + System.currentTimeMillis() - startTime, + retrievalResult.latencyMs(), + SOURCE, + retrievalResult.hasReferences(), + retrievalResult.usage() + ); + setMemoryCache(cacheKey, result); + redisContextStore.setKbCache(cacheKey, result, result.hit()); + return result; + } + + public List> buildRagPayload(List rerankedChunks, List conversationHistory) { + List> ragItems = new ArrayList<>(); + Map instruction = new LinkedHashMap<>(); + instruction.put("kind", "instruction"); + instruction.put("title", "回答要求"); + instruction.put("content", KB_INSTRUCTION); + instruction.put("source", "system"); + instruction.put("confidence", 1); + ragItems.add(instruction); + + if (conversationHistory != null && !conversationHistory.isEmpty()) { + List lines = conversationHistory.stream() + .filter(item -> item != null && StringUtils.hasText(item.content())) + .map(item -> ("user".equalsIgnoreCase(item.role()) ? "用户" : "助手") + ": " + item.content().trim()) + .toList(); + if (!lines.isEmpty()) { + Map contextItem = new LinkedHashMap<>(); + contextItem.put("kind", "context"); + contextItem.put("title", "对话上下文"); + contextItem.put("content", String.join("\n", lines)); + contextItem.put("source", "conversation_history"); + contextItem.put("confidence", 1); + ragItems.add(contextItem); + } + } + + for (int index = 0; index < rerankedChunks.size(); index++) { + Map evidenceItem = buildEvidencePackItem(rerankedChunks.get(index), index); + if (evidenceItem != null) { + ragItems.add(evidenceItem); + } + } + return List.copyOf(ragItems); + } + + public Map buildEvidencePack( + String query, + String originalQuery, + String rewrittenQuery, + List> ragItems, + List rerankedChunks, + List conversationHistory, + boolean hit, + double topScore, + String reason, + List datasetIds, + List selectedRoutes + ) { + KnowledgeQueryInfo semanticQuery = knowledgeQueryResolver.resolveKnowledgeQuery(rewrittenQuery); + List> instructions = ragItems == null ? List.>of() : ragItems.stream() + .filter(item -> item != null && ("instruction".equals(item.get("kind")) || "回答要求".equals(item.get("title")))) + .>map(item -> new LinkedHashMap(item)) + .toList(); + List> contexts = ragItems == null ? List.>of() : ragItems.stream() + .filter(item -> item != null && ("context".equals(item.get("kind")) || "对话上下文".equals(item.get("title")))) + .>map(item -> new LinkedHashMap(item)) + .toList(); + List> facts = new ArrayList<>(); + for (int index = 0; index < rerankedChunks.size(); index++) { + Map item = buildEvidencePackItem(rerankedChunks.get(index), index); + if (item != null) { + facts.add(item); + } + } + + Map queryInfo = new LinkedHashMap<>(); + queryInfo.put("raw", safeText(query)); + queryInfo.put("original", safeText(originalQuery)); + queryInfo.put("rewritten", safeText(rewrittenQuery)); + queryInfo.put("normalized", semanticQuery.normalizedText()); + queryInfo.put("canonical_entity", semanticQuery.primaryEntity()); + queryInfo.put("entities", semanticQuery.entities()); + queryInfo.put("has_explicit_entity", semanticQuery.hasExplicitEntity()); + queryInfo.put("has_knowledge_signal", semanticQuery.hasKnowledgeSignal()); + + Map retrieval = new LinkedHashMap<>(); + retrieval.put("source", SOURCE); + retrieval.put("retrieval_mode", "raw"); + retrieval.put("hit", hit); + retrieval.put("top_score", topScore); + retrieval.put("reason", reason); + retrieval.put("selected_dataset_ids", datasetIds == null ? List.of() : datasetIds); + retrieval.put("selected_kb_routes", selectedRoutes == null ? List.of() : selectedRoutes); + + Map evidencePack = new LinkedHashMap<>(); + evidencePack.put("version", 1); + evidencePack.put("query", queryInfo); + evidencePack.put("retrieval", retrieval); + evidencePack.put("instructions", instructions); + evidencePack.put("context", contexts); + evidencePack.put("facts", facts); + evidencePack.put("items", ragItems == null ? List.of() : ragItems); + evidencePack.put("conversation_count", conversationHistory == null ? 0 : conversationHistory.size()); + return evidencePack; + } + + private RetrievalResult retrieveChunks(String query, List datasetIds, int topK, double threshold) { + if (!properties.hasAccessKeys()) { + log.warn("[KBRetriever] retrieve skipped: AK/SK not configured"); + return new RetrievalResult(List.of(), "aksk_not_configured", 0L, false, Map.of(), false); + } + long startTime = System.currentTimeMillis(); + String effectiveQuery = StringUtils.hasText(query) ? query.trim() : "请介绍你们的产品和服务"; + ResolvedCollections resolvedCollections = resolveCollections(datasetIds); + if (resolvedCollections.collectionNames().isEmpty()) { + log.warn("[KBRetriever] retrieve skipped: no collections to search"); + return new RetrievalResult(List.of(), "no_collections", 0L, false, Map.of(), false); + } + int perCollectionLimit = Math.max(3, (int) Math.ceil((double) Math.max(topK, 1) / resolvedCollections.collectionNames().size())); + List>> futures = resolvedCollections.collectionNames().stream() + .map(name -> CompletableFuture.supplyAsync(() -> { + try { + return searchVikingDb(name, effectiveQuery, perCollectionLimit); + } catch (Exception exception) { + log.warn("[KBRetriever] search {} failed: {}", name, exception.getMessage()); + return List.of(); + } + })) + .toList(); + List allChunks = futures.stream() + .map(CompletableFuture::join) + .flatMap(List::stream) + .sorted(Comparator.comparingDouble(KnowledgeChunk::score).reversed()) + .limit(topK) + .toList(); + List filteredChunks = threshold > 0D + ? allChunks.stream().filter(chunk -> chunk.score() >= threshold).toList() + : allChunks; + long latencyMs = System.currentTimeMillis() - startTime; + return new RetrievalResult(filteredChunks, null, latencyMs, !filteredChunks.isEmpty(), Map.of(), true); + } + + private List searchVikingDb(String collectionName, String query, int limit) { + Map requestBody = new LinkedHashMap<>(); + requestBody.put("project", "default"); + requestBody.put("name", collectionName); + requestBody.put("query", query); + requestBody.put("limit", limit); + requestBody.put("pre_processing", Map.of("need_instruction", true, "rewrite", false)); + requestBody.put("dense_weight", 0.5); + requestBody.put("post_processing", Map.of("rerank_switch", false, "chunk_group", false)); + String body = writeJson(requestBody); + HttpResponse response = postJsonWithRetry(SEARCH_PATH, body, Duration.ofSeconds(10), "search:" + collectionName); + JsonNode root = readJson(response.body()); + if (root.path("code").asInt(-1) != 0) { + log.warn("[KBRetriever] VikingDB search {} error: {}", collectionName, root.path("message").asText("unknown")); + return List.of(); + } + JsonNode resultList = root.path("data").path("result_list"); + if (!resultList.isArray()) { + return List.of(); + } + List chunks = new ArrayList<>(); + int index = 0; + for (JsonNode item : resultList) { + String content = item.path("content").asText("").replace("", "").trim(); + if (!StringUtils.hasText(content)) { + index++; + continue; + } + String id = firstNonBlank(item.path("chunk_id").asText(""), item.path("id").asText(""), "vdb_" + collectionName + "_" + index); + String docName = firstNonBlank(item.path("doc_info").path("doc_name").asText(""), item.path("doc_info").path("title").asText("")); + String chunkTitle = item.path("chunk_title").asText(""); + Map metadata = objectMapper.convertValue(item.path("doc_info"), new TypeReference>() { }); + chunks.add(new KnowledgeChunk(id, content, item.path("score").asDouble(0D), docName, chunkTitle, metadata, collectionName, false)); + index++; + } + return List.copyOf(chunks); + } + + private List rerankChunks(String query, List chunks, int topN) { + if (chunks == null || chunks.isEmpty()) { + return List.of(); + } + if (chunks.size() <= topN) { + return chunks; + } + if (!properties.isEnableReranker()) { + return chunks.stream().limit(topN).toList(); + } + try { + List> datas = chunks.stream().map(chunk -> { + Map item = new LinkedHashMap<>(); + item.put("query", query); + item.put("content", safeText(chunk.content())); + item.put("title", safeText(chunk.docName())); + return item; + }).toList(); + Map requestBody = new LinkedHashMap<>(); + requestBody.put("rerank_model", properties.getRerankerModel()); + requestBody.put("datas", datas); + HttpResponse response = postJsonWithRetry(RERANK_PATH, writeJson(requestBody), Duration.ofSeconds(5), "rerank"); + JsonNode root = readJson(response.body()); + List scores = extractScores(root.path("data")); + if (scores.size() == chunks.size()) { + List reranked = new ArrayList<>(); + for (int index = 0; index < chunks.size(); index++) { + KnowledgeChunk chunk = chunks.get(index); + reranked.add(new KnowledgeChunk( + chunk.id(), + chunk.content(), + scores.get(index), + chunk.docName(), + chunk.chunkTitle(), + chunk.metadata(), + chunk.collection(), + true + )); + } + return reranked.stream().sorted(Comparator.comparingDouble(KnowledgeChunk::score).reversed()).limit(topN).toList(); + } + log.warn("[KBRetriever] rerank returned {} scores, expect {}", scores.size(), chunks.size()); + return chunks.stream().limit(topN).toList(); + } catch (Exception exception) { + log.warn("[KBRetriever] rerank failed: {}", exception.getMessage()); + return chunks.stream().limit(topN).toList(); + } + } + + private HttpResponse postJsonWithRetry(String path, String body, Duration timeout, String label) { + int maxRetries = 2; + for (int attempt = 0; ; attempt++) { + try { + Map headers = volcSignerV4.signRequest( + "POST", + KNOWLEDGE_HOST, + path, + body, + properties.getAccessKeyId(), + properties.getSecretAccessKey(), + SERVICE_NAME, + REGION + ); + HttpRequest.Builder requestBuilder = HttpRequest.newBuilder() + .uri(URI.create("https://" + KNOWLEDGE_HOST + path)) + .timeout(timeout) + .POST(HttpRequest.BodyPublishers.ofString(body)); + headers.forEach((k, v) -> { + if (!"host".equalsIgnoreCase(k)) { + requestBuilder.header(k, v); + } + }); + HttpResponse response = httpClient.send(requestBuilder.build(), HttpResponse.BodyHandlers.ofString()); + int status = response.statusCode(); + if ((status == 429 || status >= 500) && attempt < maxRetries) { + sleepBackoff(attempt, label, status); + continue; + } + return response; + } catch (IOException exception) { + if (attempt >= maxRetries) { + throw new IllegalStateException(exception.getMessage(), exception); + } + sleepBackoff(attempt, label, 0); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception.getMessage(), exception); + } + } + } + + private void sleepBackoff(int attempt, String label, int status) { + long delayMs = (long) (300D * Math.pow(2, attempt)); + if (status > 0) { + log.warn("[KBRetriever] {} got {}, retry after {}ms", label, status, delayMs); + } + try { + Thread.sleep(delayMs); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private List extractScores(JsonNode dataNode) { + JsonNode scoresNode = dataNode.path("scores"); + JsonNode effectiveNode = scoresNode.isArray() ? scoresNode : dataNode; + if (!effectiveNode.isArray()) { + return List.of(); + } + List scores = new ArrayList<>(); + for (JsonNode score : effectiveNode) { + scores.add(score.asDouble(0D)); + } + return List.copyOf(scores); + } + + private List loadConversationHistory(String sessionId) { + if (!properties.isEnableRedisContext() || !StringUtils.hasText(sessionId)) { + return List.of(); + } + List history = redisContextStore.getRecentHistory(sessionId.trim(), 5); + return history == null ? List.of() : history; + } + + private Map buildEvidencePackItem(KnowledgeChunk chunk, int index) { + if (chunk == null || !StringUtils.hasText(chunk.content())) { + return null; + } + Map item = new LinkedHashMap<>(); + item.put("kind", "evidence"); + item.put("title", firstNonBlank(chunk.docName(), chunk.chunkTitle(), "知识库片段" + (index + 1))); + item.put("content", chunk.content()); + item.put("source", "kb_chunk"); + item.put("source_chunk_id", safeText(chunk.id())); + item.put("doc_name", safeText(chunk.docName())); + item.put("chunk_title", safeText(chunk.chunkTitle())); + item.put("collection", safeText(chunk.collection())); + item.put("score", chunk.score()); + item.put("confidence", clampConfidence(chunk.score())); + item.put("metadata", chunk.metadata() == null ? Map.of() : chunk.metadata()); + return item; + } + + private double clampConfidence(double score) { + if (Double.isNaN(score) || Double.isInfinite(score)) { + return 0D; + } + return Math.max(0D, Math.min(1D, score)); + } + + private KnowledgeSearchResult buildFailureResult(String query, String originalQuery, String reason, long latencyMs, Long retrievalLatencyMs) { + List> ragPayload = List.of(); + Map evidencePack = buildEvidencePack( + StringUtils.hasText(query) ? query : originalQuery, + StringUtils.hasText(originalQuery) ? originalQuery : query, + StringUtils.hasText(query) ? query : originalQuery, + ragPayload, + List.of(), + List.of(), + false, + 0D, + reason, + List.of(), + List.of() + ); + return new KnowledgeSearchResult( + StringUtils.hasText(query) ? query : originalQuery, + StringUtils.hasText(originalQuery) ? originalQuery : query, + false, + reason, + List.of(), + List.of(), + ragPayload, + evidencePack, + 0D, + latencyMs, + retrievalLatencyMs, + SOURCE, + false, + Map.of() + ); + } + + private KnowledgeSearchResult withLatency(KnowledgeSearchResult result, long latencyMs) { + return new KnowledgeSearchResult( + result.query(), + result.originalQuery(), + result.hit(), + result.reason(), + result.chunks(), + result.rerankedChunks(), + result.ragPayload(), + result.evidencePack(), + result.topScore(), + latencyMs, + result.retrievalLatencyMs(), + result.source(), + result.hasReferences(), + result.usage() + ); + } + + private KnowledgeSearchResult getMemoryCache(String cacheKey) { + CachedKnowledgeResult cached = memoryCache.get(cacheKey); + if (cached == null) { + return null; + } + long ttl = cached.result().hit() ? KB_CACHE_TTL_MS : KB_CACHE_NO_HIT_TTL_MS; + if (System.currentTimeMillis() - cached.timestamp() > ttl) { + memoryCache.remove(cacheKey, cached); + return null; + } + return cached.result(); + } + + private void setMemoryCache(String cacheKey, KnowledgeSearchResult result) { + if (memoryCache.size() >= KB_CACHE_MAX_SIZE) { + String oldest = memoryCache.entrySet().stream() + .min(Comparator.comparingLong(entry -> entry.getValue().timestamp())) + .map(Map.Entry::getKey) + .orElse(null); + if (oldest != null) { + memoryCache.remove(oldest); + } + } + memoryCache.put(cacheKey, new CachedKnowledgeResult(result, System.currentTimeMillis())); + } + + private String buildCacheKey(String query, List datasetIds, String profileScope) { + List orderedIds = datasetIds == null ? List.of() : datasetIds.stream().sorted().toList(); + return "vdb2|raw|" + profileScope + "|" + safeText(query) + "|" + String.join(",", orderedIds); + } + + private List normalizeDatasetIds(List datasetIds) { + List input = datasetIds == null || datasetIds.isEmpty() ? properties.normalizedDatasetIds() : datasetIds; + Set normalized = new LinkedHashSet<>(); + for (String datasetId : input) { + if (StringUtils.hasText(datasetId)) { + normalized.add(datasetId.trim()); + } + } + return List.copyOf(normalized); + } + + private ResolvedCollections resolveCollections(List datasetIds) { + Map collectionMap = resolveCollectionMap(); + List allCollections = collectionMap.values().stream().distinct().sorted().toList(); + if (datasetIds == null || datasetIds.isEmpty()) { + return new ResolvedCollections(allCollections, properties.normalizedDatasetIds()); + } + List selected = datasetIds.stream() + .map(collectionMap::get) + .filter(StringUtils::hasText) + .distinct() + .toList(); + return selected.isEmpty() ? new ResolvedCollections(allCollections, datasetIds) : new ResolvedCollections(selected, datasetIds); + } + + private Map resolveCollectionMap() { + if (!StringUtils.hasText(properties.getCollectionMapJson())) { + return DEFAULT_COLLECTION_MAP; + } + try { + Map parsed = objectMapper.readValue(properties.getCollectionMapJson(), new TypeReference>() { }); + if (parsed == null || parsed.isEmpty()) { + return DEFAULT_COLLECTION_MAP; + } + Map merged = new LinkedHashMap<>(DEFAULT_COLLECTION_MAP); + parsed.forEach((key, value) -> { + if (StringUtils.hasText(key) && StringUtils.hasText(value)) { + merged.put(key.trim(), value.trim()); + } + }); + return Map.copyOf(merged); + } catch (Exception exception) { + log.warn("[KBRetriever] parse collection map failed: {}", exception.getMessage()); + return DEFAULT_COLLECTION_MAP; + } + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private JsonNode readJson(String payload) { + try { + return objectMapper.readTree(payload); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private String safeText(String value) { + return value == null ? "" : value.trim(); + } + + private String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + for (String value : values) { + if (StringUtils.hasText(value)) { + return value.trim(); + } + } + return ""; + } + + private record RetrievalResult( + List chunks, + String error, + Long latencyMs, + boolean kbHasContent, + Map usage, + boolean hasReferences + ) { + } + + private record CachedKnowledgeResult(KnowledgeSearchResult result, long timestamp) { + } + + private record ResolvedCollections(List collectionNames, List datasetIds) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeKeywordCatalog.java b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeKeywordCatalog.java new file mode 100644 index 0000000..dc19632 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeKeywordCatalog.java @@ -0,0 +1,170 @@ +package com.bigwo.javaserver.service; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.Stream; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +@Component +public class KnowledgeKeywordCatalog { + + private static final List COMPANY_ENTITY_KEYWORDS = List.of( + "德国PM", "德国PM公司", "PM公司", "PM国际", "PM-International", "PM-International AG", + "Rolf Sorg", "邓白氏", "AAA+", "DSN", "BFH", "ELAB", "科隆名单", "GMP", "Halal", + "宣明会", "世界宣明会", "斯派尔", "Speyer", "卢森堡", "培安（烟台）日用品有限责任公司", "培安烟台", "烟台" + ); + + private static final List SYSTEM_ENTITY_KEYWORDS = List.of( + "一成系统", "一成AI", "一成Ai", "Ai众享", "AI众享", "数字化工作室", "盛咖学愿", + "三大平台", "四大AI生态", "四大Ai生态", "四大生态", "盟主社区", "AI智能生产力", "AI生产力", + "智能生产力", "行动圈", "批发式晋级", "身未动，梦已成", "身未动梦已成", "零成本高效率", + "零成本高效率运行", "赋能团队", "团队赋能", "团队发展", "文化解析", "故事分享", "自我介绍", + "邀约话术", "线上拓客", "线上成交", "陌生客户", "陌生人沟通" + ); + + private static final List PRODUCT_ENTITY_KEYWORDS = List.of( + "PM产品", "PM-FitLine", "FitLine", "PM细胞营养素", "细胞营养素", "基础套装", "基础三合一", "三合一", + "基础二合一", "二合一", "小红产品", "小红", "艾特维", "Activize Oxyplus", "Activize Serum", "Activize", + "小红精华液", "大白产品", "大白", "倍适", "Basics", "Basic Power", "PowerCocktail", "小白产品", + "小白", "维适多", "Restorate", "儿童倍适", "PowerCocktail Junior", "NTC营养保送系统", "NTC", + "Nutrient Transport Concept", "火炉原理", "暖炉原理", "阿育吠陀", "Ayurveda", "Med Dental+", "草本护理牙膏", + "Men Face", "全效男士护肤抗衰乳霜", "CC-Cell", "CC-Cell胶囊", "CC-Cell乳霜", "CC套装", "CC胶囊", + "D-Drink", "小绿排毒饮", "14天排毒D饮料Plus", "ProShape Amino", "ProShape® Amino", "氨基酸", "支链氨基酸", + "BCAA", "MEN+", "Men+", "倍力健 MEN+", "倍力健", "小黑", "Herbal Tea", "草本茶", "Hair+", "med Hair+", + "口服发宝", "外用发健", "发宝", "Fitness-Drink", "运动饮料健康饮品", "运动饮料", "健康饮品", "TopShape", + "孅萃TopShape纤萃减肥", "纤萃减肥", "乐活50+", "Generation 50+", "Apple Antioxy", "苹果细胞抗氧素", "Antioxy", + "Zellschutz", "细胞抗氧素", "胶原蛋白肽", "胶原蛋白", "Women+", "乐活奶昔", "乐活", "乳清蛋白", "蛋白粉", + "乳酪煲", "乳酪饮品", "乳酪", "IB5", "口腔免疫喷雾", "Q10", "辅酵素", "Q10辅酵素氧修护", "关节套装", + "关节舒缓", "男士乳霜", "男士护肤", "去角质", "面膜", "叶黄素", "葡萄籽", "白藜芦醇", "益生菌", "肽美", + "德丽", "德维", "宝丽", "美固健", "CitrusCare", "NutriSunny", "Omega", "Young Care", "骨骼健", "顾心", + "舒采健", "衡醇饮", "小粉C", "小粉", "异黄酮", "Isoflavon", "眼霜", "Eye Cream", "洁面", "洁面乳", + "爽肤水", "Tonic", "餐代餐", "代餐奶昔", "美白霜", "修护膜" + ); + + private static final List BUSINESS_ENTITY_KEYWORDS = List.of( + "PM事业", "做PM", "加入PM", "招商合作", "招商与代理", "招商稿", "招商", "招募", "代理", "代理商", + "代理政策", "加盟", "招商加盟", "合作加盟", "事业合作", "事业机会", "创业", "培训新人起步三关", "起步三关", + "培训打造精品会议具体如下", "精品会议", "会议组织", "培训成长上总裁", "成长上总裁" + ); + + private static final List CANONICAL_KNOWLEDGE_TERMS = List.of( + "一成系统", "德国PM", "PM-FitLine", "FitLine", "PM细胞营养素", "NTC营养保送系统", + "Activize Oxyplus", "Basics", "Restorate", "儿童倍适", "火炉原理", "阿育吠陀" + ); + + private static final List KNOWLEDGE_ROUTE_KEYWORDS_BASE = List.of( + "公司介绍", "公司背景", "公司实力", "公司地址", "公司电话", "联系方式", "总部", "分公司", "公司成立", "公司历史", + "公司规模", "全球布局", "信用评级", "行业排名", "获奖", "慈善", "社会责任", "不上市", "汽车奖励", "退休金", "旅行", + "福利", "企业性质", "发展历程", "产品介绍", "产品说明", "产品推荐", "产品有哪些", "产品列表", "产品图片", "产品外观", + "你们公司", "你们的公司", "你们产品", "你们的产品", "你们那个", "咱们公司", "咱们产品", "你们卖的", "你们的东西", "这个东西", + "这东西", "那玩意", "说说", "讲讲", "介绍介绍", "查查", "帮我查", "帮我问", "帮我看看", "有啥用", "啥意思", "有啥", + "营养素", "营养品", "保健品", "保健食品", "营养补充", "直销", "直销公司", "直销事业", "排毒", "排毒产品", "减肥", + "减肥产品", "瘦身", "护肤", "护肤品", "护发", "脱发", "掉发", "头发", "牙膏", "喷雾", "关节痛", "关节", "眼睛", "视力", + "叶黄素", "抗氧化", "抗衰", "抗衰老", "胶原蛋白", "免疫力", "能量", "抗疲劳", "疲劳", "睡眠", "失眠", "消化", "肠胃", + "便秘", "皮肤", "美容", "美白", "痘痘", "补铁", "补血", "骨密度", "补钙", "孕妇", "哺乳期", "怀孕", "孕期", "儿童", "小孩", + "孩子", "老人", "老年人", "过敏", "过敏体质", "事业", "怎么赚钱", "能赚钱吗", "收入", "奖金", "奖金制度", "正规性", "合法性", + "传销", "骗局", "骗子", "是不是传销", "直销还是传销", "合不合法", "正不正规", "正规吗", "合法吗", "层级分销", "非法集资", "拉人头", + "发展下线", "报单", "人头费", "安全吗", "合规吗", "有许可证吗", "好转反应", "整健反应", "排毒反应", "副作用", "不良反应", "皮肤发痒", + "皮肤微痒", "促销活动", "活动分数", "5+1活动分数", "5+1", "怎么吃", "怎么用", "怎么服用", "服用方法", "吃法", "用法", "用量", + "搭配", "空腹吃", "饭前吃", "饭后吃", "温水冲", "冷水冲", "一天吃几次", "一次吃多少", "功效", "作用", "成分", "原料", "配方", + "多少钱", "价格", "适合谁", "适用人群", "区别", "哪个好", "多久见效", "多久能见效", "哪里买", "怎么买", "保质期", "储存", + "效果怎么样", "有效果吗", "有没有用", "好不好", "靠谱吗", "值得买吗", "值得做吗", "真的有用吗", "真的假的", "有科学依据吗", "怎么加入", + "如何加入", "怎么报名", "怎么参与", "怎么做", "如何开始", "科普", "误区", "认证", "检测", "检测报告", "安全认证", "培训", "新人", + "起步", "成长", "高血压", "糖尿病", "胆固醇", "心脏病", "肾病", "肝病", "痛风", "贫血", "肥胖", "能吃吗", "可以吃吗", "能喝吗", + "可以喝吗", "能用吗", "可以用吗", "一起吃", "同时吃", "混着吃", "搭配吃", "吃药", "药物", "粉末", "粉剂", "粉状", "冲剂", "冲泡", + "片剂", "药片", "胶囊", "软胶囊", "颗粒", "口服液", "膏状", "不是的", "搞错了", "说错了", "弄错了", "不对", "不准确", "有误", + "不一样", "不一致", "不信", "骗人", "忽悠", "太夸张", "离谱", "你确定吗", "确定吗", "真的吗", "再查一下", "再确认一下", "重新查", + "核实一下", "谁说的", "有什么根据", "有证据吗", "有依据吗", "怎么可能", "不可能", "不会吧", "胡说", "瞎说", "乱说", "到底是", "究竟是", + "应该是", "明明是", "其实是", "本来是", "怎么变成", "不应该是", "冲着喝", "泡着喝", "直接吞", "是喝的", "是吃的" + ); + + private final List knowledgeEntityKeywords; + private final List knowledgeRouteKeywords; + private final List canonicalKnowledgeTerms; + + public KnowledgeKeywordCatalog() { + this.knowledgeEntityKeywords = uniqueKeywords(Stream.of( + COMPANY_ENTITY_KEYWORDS, + SYSTEM_ENTITY_KEYWORDS, + PRODUCT_ENTITY_KEYWORDS, + BUSINESS_ENTITY_KEYWORDS + ).flatMap(List::stream).toList()); + this.knowledgeRouteKeywords = uniqueKeywords(Stream.concat(this.knowledgeEntityKeywords.stream(), KNOWLEDGE_ROUTE_KEYWORDS_BASE.stream()).toList()); + this.canonicalKnowledgeTerms = uniqueKeywords(CANONICAL_KNOWLEDGE_TERMS); + } + + public boolean hasKeywordFromList(String text, List keywords) { + return containsAny(text, keywords == null ? List.of() : keywords); + } + + public boolean hasCanonicalKnowledgeTerm(String text) { + return containsAny(text, canonicalKnowledgeTerms); + } + + public boolean hasKnowledgeRouteKeyword(String text) { + return containsAny(text, knowledgeRouteKeywords); + } + + public List extractKnowledgeEntityMatches(String text) { + String normalized = normalize(text); + if (normalized.isBlank()) { + return List.of(); + } + List matches = new ArrayList<>(); + Set seen = new LinkedHashSet<>(); + for (String keyword : knowledgeEntityKeywords) { + String lowered = keyword.toLowerCase(Locale.ROOT); + if (normalized.contains(lowered) && seen.add(lowered)) { + matches.add(keyword); + } + } + return List.copyOf(matches); + } + + public List knowledgeEntityKeywords() { + return knowledgeEntityKeywords; + } + + private boolean containsAny(String text, List keywords) { + String normalized = normalize(text); + if (normalized.isBlank()) { + return false; + } + for (String keyword : keywords) { + if (normalized.contains(keyword.toLowerCase(Locale.ROOT))) { + return true; + } + } + return false; + } + + private String normalize(String text) { + return StringUtils.hasText(text) ? text.trim().toLowerCase(Locale.ROOT) : ""; + } + + private List uniqueKeywords(List keywords) { + Set seen = new LinkedHashSet<>(); + List result = new ArrayList<>(); + for (String keyword : keywords) { + String normalized = keyword == null ? "" : keyword.trim(); + if (!StringUtils.hasText(normalized)) { + continue; + } + String lowered = normalized.toLowerCase(Locale.ROOT); + if (seen.add(lowered)) { + result.add(normalized); + } + } + result.sort((left, right) -> { + if (right.length() != left.length()) { + return Integer.compare(right.length(), left.length()); + } + return left.compareToIgnoreCase(right); + }); + return List.copyOf(result); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeQueryResolver.java b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeQueryResolver.java new file mode 100644 index 0000000..3212a2e --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeQueryResolver.java @@ -0,0 +1,327 @@ +package com.bigwo.javaserver.service; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import com.bigwo.javaserver.model.KnowledgeQueryInfo; + +@Service +public class KnowledgeQueryResolver { + + private static final List SPECIAL_ENTITY_ALIASES = List.of("太美", "态美"); + private static final Pattern SPECIAL_ENTITY_GUARD = Pattern.compile("(?:怎么|咋|如何|能|可以|介绍|是什么|功效|成分|配方|多少钱|价格|适合|区别|吃|喝|用|服用|使用|产品|产品功效|产品介绍|哪里买|怎么买|查|问|看|搜|找|了解)|^(?:请问|麻烦你|帮我|帮忙|我想问|我想|我要|想问|问一下|问下|介绍一下|介绍下|查一下|查下|查查|看一下|看看|看下|能不能|可以|可不可以|能否).*(?:太美|态美)$"); + private static final Pattern MULTI_SPACE = Pattern.compile("\\s+"); + private static final Pattern POLITE_PREFIX = Pattern.compile("^(你|你们|帮我|麻烦你|请你?|我想|我要|能不能|可以|可不可以|能否)[给帮]?(我)?(查一下|查查|查下|搜一下|搜搜|搜下|找一下|找找|找下|看一下|看看|看下|说一下|说说|说下|讲一下|讲讲|讲下|介绍一下|介绍下)?"); + private static final Pattern TAILING_INFO = Pattern.compile("(的)?(相关|详细)?(内容|信息|资料|介绍|说明)[。？?！]*$"); + + private static final List ALIAS_REPLACEMENTS = List.of( + new RegexReplacement("一成[，,、。！？\\s]+系统", "一成系统", 0), + new RegexReplacement("X{2}系统", "一成系统", Pattern.CASE_INSENSITIVE), + new RegexReplacement("[\\u4e00-\\u9fff]{1,3}(?:成|城|程|诚|乘|声|生)[，,、\\s]*系统", "一成系统", 0), + new RegexReplacement("(?:一城|逸城|一程|易成|一诚|亦成|艺成|溢成|义成|毅成|怡成|以成|已成|亿成|忆成|益成|益生|易诚|义诚|忆诚|以诚|一声|亿生|易乘)系统", "一成系统", 0), + new RegexReplacement("大窝|大握|大我|大卧", "大沃", 0), + new RegexReplacement("盛咖学院|圣咖学愿|圣咖学院|盛卡学愿", "盛咖学愿", 0), + new RegexReplacement("爱众享|艾众享|哎众享", "Ai众享", 0), + new RegexReplacement("PM[-\\s]*Fitline|Fitline", "PM-FitLine", Pattern.CASE_INSENSITIVE), + new RegexReplacement("PM细胞营养|PM营养素|德国PM营养素", "PM细胞营养素", 0), + new RegexReplacement("NTC科技", "NTC营养保送系统", 0), + new RegexReplacement("NTC营养保送系统|NTC营养配送系统|NTC营养输送系统|NTC营养传送系统|NTC营养传输系统", "NTC营养保送系统", 0), + new RegexReplacement("Nutrient Transport Concept", "NTC营养保送系统", Pattern.CASE_INSENSITIVE), + new RegexReplacement("Activize Oxyplus|Activize", "Activize Oxyplus", Pattern.CASE_INSENSITIVE), + new RegexReplacement("Restorate", "Restorate", Pattern.CASE_INSENSITIVE), + new RegexReplacement("Basics", "Basics", Pattern.CASE_INSENSITIVE), + new RegexReplacement("活力健|火力剑|火力健", "Basics 活力健", 0), + new RegexReplacement("基础三合一|三合一基础套|大白小红小白|基础套装?", "PM细胞营养素 基础套装", 0), + new RegexReplacement("儿童倍适|儿童产品", "儿童倍适", 0), + new RegexReplacement("小红精华液", "Activize Serum 小红精华液", 0), + new RegexReplacement("小红产品", "小红产品 Activize Oxyplus", 0), + new RegexReplacement("大白产品", "大白产品 Basics", 0), + new RegexReplacement("小白产品", "小白产品 Restorate", 0), + new RegexReplacement("(? extractKnowledgeEntities(String text) { + return keywordCatalog.extractKnowledgeEntityMatches(normalizeKnowledgeText(text)); + } + + public boolean hasExplicitKnowledgeEntity(String text) { + return !extractKnowledgeEntities(text).isEmpty(); + } + + public String pickPrimaryKnowledgeEntity(List entities) { + if (entities == null || entities.isEmpty()) { + return ""; + } + return entities.stream() + .filter(StringUtils::hasText) + .map(String::trim) + .distinct() + .sorted((left, right) -> { + if (right.length() != left.length()) { + return Integer.compare(right.length(), left.length()); + } + return left.toLowerCase(Locale.ROOT).compareTo(right.toLowerCase(Locale.ROOT)); + }) + .findFirst() + .orElse(""); + } + + public KnowledgeQueryInfo resolveKnowledgeQuery(String text) { + String rawText = StringUtils.hasText(text) ? text.trim() : ""; + String normalizedText = normalizeKnowledgeText(rawText); + String rewrittenText = rewriteKnowledgeQuery(rawText, List.of()); + List entities = keywordCatalog.extractKnowledgeEntityMatches(normalizedText); + String primaryEntity = pickPrimaryKnowledgeEntity(entities); + boolean hasExplicitEntity = !entities.isEmpty(); + boolean hasKnowledgeSignal = hasExplicitEntity || keywordCatalog.hasKnowledgeRouteKeyword(normalizedText); + return new KnowledgeQueryInfo(rawText, normalizedText, rewrittenText, List.copyOf(entities), primaryEntity, hasExplicitEntity, hasKnowledgeSignal); + } + + public String rewriteKnowledgeQuery(String query, List contextTerms) { + String originalQuery = StringUtils.hasText(query) ? query.trim() : ""; + if (originalQuery.isEmpty()) { + return ""; + } + String aliasNormalized = normalizeKnowledgeQueryAlias(originalQuery); + String deterministicQuery = buildDeterministicKnowledgeQuery(aliasNormalized); + String rewritten = deterministicQuery != null ? deterministicQuery : applyKnowledgeQueryAnchor(aliasNormalized); + rewritten = enrichQueryWithContext(rewritten, contextTerms); + return sanitizeRewrittenQuery(rewritten); + } + + public String normalizeKnowledgeQueryAlias(String query) { + String normalized = normalizeKnowledgeText(query) + .replaceAll("^[啊哦嗯呢呀哎诶额，。！？、\\s]+", "") + .replaceAll("[啊哦嗯呢呀哎诶额，。！？、\\s]+$", ""); + normalized = POLITE_PREFIX.matcher(normalized).replaceFirst(""); + normalized = TAILING_INFO.matcher(normalized).replaceFirst(""); + for (RegexReplacement replacement : ALIAS_REPLACEMENTS) { + normalized = replacement.pattern().matcher(normalized).replaceAll(replacement.replacement()); + } + return collapseWhitespace(normalized); + } + + public String sanitizeRewrittenQuery(String query) { + String cleaned = StringUtils.hasText(query) ? query.trim() : ""; + if (cleaned.isEmpty()) { + return ""; + } + cleaned = cleaned.replaceAll("[啊哦嗯呢呀哎诶额嘛吧啦哇噢]+", " "); + cleaned = cleaned.replaceAll("[，,。！？!?\\s]{2,}", " "); + cleaned = cleaned.replaceAll("(.{3,}?)[？?！!。，,\\s]+\\1", "$1"); + String[] parts = cleaned.split("\\s+"); + Set seen = new LinkedHashSet<>(); + List deduped = new ArrayList<>(); + for (String part : parts) { + String normalized = part == null ? "" : part.trim(); + if (!StringUtils.hasText(normalized) || !seen.add(normalized)) { + continue; + } + deduped.add(normalized); + } + cleaned = String.join(" ", deduped).trim(); + if (cleaned.length() > 80) { + cleaned = cleaned.substring(0, 80).trim(); + int lastSpace = cleaned.lastIndexOf(' '); + if (lastSpace > 0) { + cleaned = cleaned.substring(0, lastSpace).trim(); + } + } + return cleaned; + } + + public String applyKnowledgeQueryAnchor(String query) { + String anchoredQuery = StringUtils.hasText(query) ? query.trim() : ""; + if (anchoredQuery.contains("一成系统") && !Pattern.compile("(德国PM|PM事业|赋能工具|Ai众享|数字化工作室|盛咖学愿)", Pattern.CASE_INSENSITIVE).matcher(anchoredQuery).find()) { + anchoredQuery = anchoredQuery.replace("一成系统", "一成系统 德国PM事业赋能工具"); + } + return anchoredQuery.trim(); + } + + public String buildDeterministicKnowledgeQuery(String query) { + String text = StringUtils.hasText(query) ? query.trim() : ""; + if (text.isEmpty()) { + return null; + } + if (Pattern.compile("(一成系统|Ai众享|数字化工作室|盛咖学愿|三大平台|四大Ai生态|四大生态|智能生产力)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + if (Pattern.compile("(核心竞争力|竞争力|核心优势|优势)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 核心竞争力 三大平台 四大Ai生态 零成本高效率"; + } + if (Pattern.compile("(发展|怎么做|怎么用|如何用|如何做|关键点|关键|方法|步骤)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 发展PM事业 三大平台 四大Ai生态 零成本高效率 全球市场"; + } + if (Pattern.compile("(线上拓客|拓客|成交|成交率|陌生客户|陌生人沟通|邀约)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 线上拓客 成交 邀约 三大平台 四大Ai生态"; + } + if (Pattern.compile("(ai智能生产力|ai生产力|智能生产力|团队效率|赋能团队|团队赋能)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 AI智能生产力 赋能团队 三大平台 四大Ai生态"; + } + if (Pattern.compile("(一部手机|0门槛|零门槛|0成本|零成本|足不出户|梦想横扫全球|一部手机做天下)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 软件赋能 0成本高效率 一部手机做天下 足不出户梦想横扫全球"; + } + if (Pattern.compile("(故事|自我介绍|分享)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 故事分享 自我介绍"; + } + if (Pattern.compile("(邀约|话术)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 邀约话术"; + } + if (text.contains("文化")) { + return "一成系统 文化解析"; + } + if (Pattern.compile("(赋能团队|团队发展|AI赋能|ai赋能)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统用AI赋能团队发展"; + } + if (Pattern.compile("(三大平台|四大生态|Ai生态)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 三大平台 四大Ai生态"; + } + return "一成系统 德国PM事业发展的强大赋能工具 三大平台 四大Ai生态"; + } + if (Pattern.compile("(一部手机做天下|一部手机即可运营全球市场|0门槛启动|零门槛启动|0成本高效率|零成本高效率|足不出户梦想横扫全球|身未动，?梦已成|批发式晋级)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 软件赋能 德国PM事业 0成本高效率 一部手机做天下 身未动梦已成 批发式晋级"; + } + if (Pattern.compile("(身未动，?梦已成|批发式晋级)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 身未动梦已成 批发式晋级 三大平台 四大Ai生态"; + } + if (text.contains("行动圈")) { + return "一成系统 行动圈 数字化工作室 团队管理 目标考核"; + } + if (text.contains("盟主社区")) { + return "一成系统 盟主社区 AI众享 社区盟主 引流 转化"; + } + if (Pattern.compile("(招商|代理|加盟|事业机会|招商稿|代理政策)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 招商与代理 软件赋能 0成本高效率"; + } + if (Pattern.compile("(如何发展PM事业|怎么发展PM事业|PM事业发展逻辑|介绍PM事业|两分钟介绍PM事业)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 发展逻辑 商机 价值 软件赋能 三大平台 四大Ai生态 0成本高效率"; + } + if (Pattern.compile("(为什么选择德国PM|为何选择德国PM|为什么选德国PM|为什么选PM|为何选PM)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 德国PM 选择理由 公司实力 产品优势 软件赋能 0成本高效率"; + } + if (Pattern.compile("(陌生客户|陌生人).*(沟通|开口|邀约|交流|切入).*(PM事业|德国PM|PM)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 陌生客户 沟通 邀约 话术 软件赋能"; + } + if (Pattern.compile("(线上拓客|线上成交|线上开发客户|线上获客|线上成交率)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "一成系统 PM事业 线上拓客 成交 获客"; + } + if (Pattern.compile("(一成AI|AI落地|ai落地|转观念|落地对比)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "2026一成Ai落地对比与转观念"; + } + if (Pattern.compile("(传销|骗局|骗子|正规吗|合法吗|正不正规|合不合法|是不是传销|直销还是传销|层级分销|非法集资|拉人头|下线|发展下线|报单|人头费)", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "德国PM 1993年 创立 100多个国家 FitLine 公司介绍 邓白氏 99分 AAA+ 合法直销"; + } + if (Pattern.compile("暖炉原理", Pattern.CASE_INSENSITIVE).matcher(text).find()) { + return "火炉原理"; + } + return null; + } + + private String applySpecialEntityAliases(String text) { + String compact = text.replaceAll("[\\s，,。！？?!~～、]", ""); + if ("太美".equals(compact) || "态美".equals(compact) || SPECIAL_ENTITY_GUARD.matcher(text).find()) { + String result = text; + for (String alias : SPECIAL_ENTITY_ALIASES) { + result = result.replace(alias, "肽美"); + } + return result; + } + return text; + } + + private String enrichQueryWithContext(String query, List contextTerms) { + if (contextTerms == null || contextTerms.isEmpty()) { + return query; + } + Set terms = new LinkedHashSet<>(); + for (String term : contextTerms) { + if (StringUtils.hasText(term) && !query.contains(term.trim())) { + terms.add(term.trim()); + } + } + if (terms.isEmpty()) { + return query; + } + return (query + " " + String.join(" ", terms)).trim(); + } + + private String collapseWhitespace(String text) { + return MULTI_SPACE.matcher(StringUtils.hasText(text) ? text.trim() : "").replaceAll(" ").trim(); + } + + private record RegexReplacement(Pattern pattern, String replacement) { + private RegexReplacement(String regex, String replacement, int flags) { + this(Pattern.compile(regex, flags), replacement); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeRouteDecider.java b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeRouteDecider.java new file mode 100644 index 0000000..0d2e605 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/KnowledgeRouteDecider.java @@ -0,0 +1,63 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.model.LlmMessage; +import java.util.List; +import java.util.regex.Pattern; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class KnowledgeRouteDecider { + + private static final Pattern FOLLOW_UP_EXACT = Pattern.compile("^(详细|详细说说|详细查一下|展开说说|继续说|继续讲|介绍一下|给我介绍一下|详细介绍一下|继续介绍一下|怎么用|怎么操作|怎么配置|适合谁|有什么区别|费用多少|价格多少|怎么申请|怎么开通|是什么|什么意思|地址在哪|公司地址在哪|电话多少|公司电话多少|联系方式|公司联系方式|具体政策|具体内容|怎么吃|功效是什么|有什么功效|成分是什么|有什么成分|多少钱|哪里买|怎么买|配方|原理是什么|有什么好处|怎么服用|适合什么人|规格是什么|什么规格|包装是什么|什么包装|剂型是什么|什么剂型|形态是什么|什么形态|一天几次|每天几次|每日几次|一天吃几次|每天吃几次|一天服用几次|每日服用几次)$", Pattern.CASE_INSENSITIVE); + private static final Pattern FOLLOW_UP_CORRECTION = Pattern.compile("(不是的|不是啊|不是不是|才不是|没有啊|哪里有|搞错|说错|弄错|记错|讲错|答错|错了|搞混|搞反|记岔|说反|弄反|答非所问|不对|不准确|不正确|有误|有问题|不一样|不一致|我听说|我记得|前后矛盾|自相矛盾|不信|骗人|忽悠|吹牛|太夸张|离谱|再查|再看看|再确认|再核实|重新查|重新回答|核实一下|查清楚|搞清楚|好像不是|好像不太对|我觉得不|恐怕不是|谁说的|谁告诉你|你从哪|有什么根据|有什么依据|你确定|确定吗|真的吗|当真|真的假的|怎么可能|不可能|不会吧|开玩笑|胡说|瞎说|乱说|粉末|粉剂|粉状|冲剂|冲泡|片剂|药片|胶囊|软胶囊|颗粒|口服液|喷雾剂|乳霜|乳液|凝胶|膏体|膏状|冲着喝|泡着喝|直接吞|是喝的|是吃的|到底是|究竟是|应该是|明明是|本来是|其实是|怎么变成|不应该是)", Pattern.CASE_INSENSITIVE); + private static final Pattern SUBJECT_ACTION = Pattern.compile("^(这个|那个|它|它的|他|他的|该|这款|那款|该系统|这个系统|那个系统|这个功能|那个功能|这个产品|那个产品|这个公司|那家公司|这个政策|那个政策|这个培训|那个培训)(的)?(详细|详细说说|详细查一下|展开说说|继续说|继续讲|介绍一下|给我介绍一下|详细介绍一下|继续介绍一下|怎么用|怎么操作|怎么配置|适合谁|有什么区别|费用多少|价格多少|怎么申请|怎么开通|是什么|什么意思|地址在哪|公司地址在哪|电话多少|公司电话多少|联系方式|公司联系方式|具体政策|具体内容|怎么吃|功效是什么|有什么功效|成分是什么|有什么成分|多少钱|哪里买|怎么买|配方|原理是什么|有什么好处|怎么服用|适合什么人|规格是什么|什么规格|包装是什么|什么包装|剂型是什么|什么剂型|形态是什么|什么形态|一天几次|每天几次|每日几次|一天吃几次|每天吃几次|一天服用几次|每日服用几次)$", Pattern.CASE_INSENSITIVE); + + private final KnowledgeKeywordCatalog keywordCatalog; + private final KnowledgeQueryResolver knowledgeQueryResolver; + + public KnowledgeRouteDecider(KnowledgeKeywordCatalog keywordCatalog, KnowledgeQueryResolver knowledgeQueryResolver) { + this.keywordCatalog = keywordCatalog; + this.knowledgeQueryResolver = knowledgeQueryResolver; + } + + public boolean shouldForceKnowledgeRoute(String userText, List context) { + String text = StringUtils.hasText(userText) ? userText.trim() : ""; + if (text.isEmpty()) { + return false; + } + if (hasKnowledgeKeyword(text)) { + return true; + } + if (!isKnowledgeFollowUp(text)) { + return false; + } + String recentContextText = (context == null ? List.of() : context).stream() + .skip(Math.max((context == null ? 0 : context.size()) - 6, 0)) + .map(item -> item == null ? "" : StringUtils.hasText(item.content()) ? item.content().trim() : "") + .filter(StringUtils::hasText) + .reduce((left, right) -> left + "\n" + right) + .orElse(""); + return hasKnowledgeKeyword(recentContextText); + } + + public boolean hasKnowledgeKeyword(String text) { + String normalized = knowledgeQueryResolver.normalizeKnowledgeText(text).replaceAll("\\s+", ""); + return keywordCatalog.hasKnowledgeRouteKeyword(normalized) || knowledgeQueryResolver.hasExplicitKnowledgeEntity(normalized); + } + + public boolean isKnowledgeFollowUp(String text) { + String normalized = StringUtils.hasText(text) ? text.trim() : ""; + if (normalized.isEmpty()) { + return false; + } + normalized = normalized.replaceAll("[，,。！？?~～\\s]+$", "").replaceAll("^(那你|那再|那(?!个|种|款|些)|你再|再来|再|麻烦你|帮我)[，,、\\s]*", ""); + if (FOLLOW_UP_EXACT.matcher(normalized).matches()) { + return true; + } + if (FOLLOW_UP_CORRECTION.matcher(normalized).find()) { + return true; + } + return SUBJECT_ACTION.matcher(normalized).matches(); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/PinyinProductMatcher.java b/java-server/src/main/java/com/bigwo/javaserver/service/PinyinProductMatcher.java new file mode 100644 index 0000000..f7f5a10 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/PinyinProductMatcher.java @@ -0,0 +1,181 @@ +package com.bigwo.javaserver.service; + +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class PinyinProductMatcher { + + private static final Logger log = LoggerFactory.getLogger(PinyinProductMatcher.class); + private static final Pattern CHINESE_WINDOW_PATTERN = Pattern.compile("^[\\u4e00-\\u9fff]+$"); + + private static final Map PINYIN_MAP = Map.ofEntries( + Map.entry("阿", "a"), Map.entry("啊", "a"), + Map.entry("爱", "ai"), Map.entry("艾", "ai"), Map.entry("哎", "ai"), + Map.entry("安", "an"), Map.entry("暗", "an"), Map.entry("按", "an"), Map.entry("氨", "an"), Map.entry("胺", "an"), + Map.entry("昂", "ang"), + Map.entry("八", "ba"), Map.entry("巴", "ba"), + Map.entry("白", "bai"), Map.entry("百", "bai"), Map.entry("柏", "bai"), + Map.entry("包", "bao"), Map.entry("宝", "bao"), Map.entry("保", "bao"), Map.entry("报", "bao"), Map.entry("煲", "bao"), Map.entry("苞", "bao"), Map.entry("胞", "bao"), + Map.entry("北", "bei"), Map.entry("被", "bei"), Map.entry("背", "bei"), Map.entry("贝", "bei"), Map.entry("备", "bei"), Map.entry("辈", "bei"), Map.entry("杯", "bei"), Map.entry("倍", "bei"), + Map.entry("本", "ben"), Map.entry("苯", "ben"), Map.entry("奔", "ben"), Map.entry("笨", "ben"), + Map.entry("采", "cai"), Map.entry("彩", "cai"), Map.entry("菜", "cai"), Map.entry("蔡", "cai"), Map.entry("猜", "cai"), Map.entry("财", "cai"), Map.entry("材", "cai"), Map.entry("才", "cai"), + Map.entry("草", "cao"), Map.entry("操", "cao"), Map.entry("曹", "cao"), Map.entry("槽", "cao"), + Map.entry("茶", "cha"), Map.entry("查", "cha"), Map.entry("差", "cha"), Map.entry("插", "cha"), Map.entry("察", "cha"), + Map.entry("纯", "chun"), Map.entry("唇", "chun"), Map.entry("春", "chun"), Map.entry("醇", "chun"), Map.entry("蠢", "chun"), + Map.entry("萃", "cui"), Map.entry("翠", "cui"), Map.entry("脆", "cui"), Map.entry("粹", "cui"), Map.entry("催", "cui"), + Map.entry("大", "da"), + Map.entry("蛋", "dan"), Map.entry("旦", "dan"), Map.entry("但", "dan"), Map.entry("淡", "dan"), Map.entry("弹", "dan"), Map.entry("担", "dan"), + Map.entry("毒", "du"), Map.entry("独", "du"), Map.entry("度", "du"), Map.entry("读", "du"), Map.entry("督", "du"), + Map.entry("发", "fa"), Map.entry("法", "fa"), Map.entry("罚", "fa"), + Map.entry("反", "fan"), Map.entry("返", "fan"), Map.entry("犯", "fan"), Map.entry("翻", "fan"), Map.entry("范", "fan"), Map.entry("饭", "fan"), + Map.entry("非", "fei"), Map.entry("飞", "fei"), Map.entry("费", "fei"), Map.entry("肺", "fei"), Map.entry("废", "fei"), Map.entry("吠", "fei"), + Map.entry("肤", "fu"), Map.entry("夫", "fu"), Map.entry("服", "fu"), Map.entry("福", "fu"), Map.entry("付", "fu"), Map.entry("副", "fu"), Map.entry("附", "fu"), Map.entry("府", "fu"), Map.entry("腐", "fu"), Map.entry("辅", "fu"), Map.entry("浮", "fu"), Map.entry("扶", "fu"), Map.entry("复", "fu"), + Map.entry("格", "ge"), Map.entry("隔", "ge"), Map.entry("革", "ge"), Map.entry("各", "ge"), Map.entry("阁", "ge"), Map.entry("葛", "ge"), Map.entry("骼", "ge"), + Map.entry("骨", "gu"), Map.entry("谷", "gu"), Map.entry("古", "gu"), Map.entry("鼓", "gu"), Map.entry("估", "gu"), Map.entry("故", "gu"), Map.entry("顾", "gu"), + Map.entry("关", "guan"), Map.entry("管", "guan"), Map.entry("官", "guan"), Map.entry("馆", "guan"), + Map.entry("好", "hao"), Map.entry("号", "hao"), Map.entry("浩", "hao"), Map.entry("耗", "hao"), Map.entry("豪", "hao"), + Map.entry("衡", "heng"), Map.entry("横", "heng"), Map.entry("恒", "heng"), Map.entry("亨", "heng"), + Map.entry("红", "hong"), Map.entry("洪", "hong"), Map.entry("宏", "hong"), Map.entry("鸿", "hong"), + Map.entry("黑", "hei"), Map.entry("嘿", "hei"), + Map.entry("活", "huo"), Map.entry("火", "huo"), Map.entry("获", "huo"), Map.entry("霍", "huo"), Map.entry("货", "huo"), Map.entry("祸", "huo"), + Map.entry("黄", "huang"), Map.entry("皇", "huang"), Map.entry("荒", "huang"), Map.entry("慌", "huang"), Map.entry("煌", "huang"), Map.entry("惶", "huang"), + Map.entry("基", "ji"), Map.entry("机", "ji"), Map.entry("鸡", "ji"), Map.entry("积", "ji"), Map.entry("极", "ji"), Map.entry("几", "ji"), Map.entry("计", "ji"), Map.entry("记", "ji"), Map.entry("级", "ji"), + Map.entry("见", "jian"), Map.entry("健", "jian"), Map.entry("剑", "jian"), Map.entry("键", "jian"), Map.entry("建", "jian"), Map.entry("件", "jian"), Map.entry("检", "jian"), Map.entry("简", "jian"), Map.entry("减", "jian"), Map.entry("渐", "jian"), Map.entry("坚", "jian"), Map.entry("尖", "jian"), Map.entry("肩", "jian"), + Map.entry("交", "jiao"), Map.entry("教", "jiao"), Map.entry("角", "jiao"), Map.entry("焦", "jiao"), Map.entry("较", "jiao"), Map.entry("觉", "jiao"), Map.entry("胶", "jiao"), Map.entry("叫", "jiao"), Map.entry("酵", "jiao"), + Map.entry("节", "jie"), Map.entry("结", "jie"), Map.entry("洁", "jie"), Map.entry("杰", "jie"), Map.entry("接", "jie"), Map.entry("揭", "jie"), Map.entry("截", "jie"), + Map.entry("菌", "jun"), Map.entry("军", "jun"), Map.entry("均", "jun"), Map.entry("君", "jun"), Map.entry("俊", "jun"), + Map.entry("抗", "kang"), Map.entry("康", "kang"), Map.entry("慷", "kang"), + Map.entry("口", "kou"), + Map.entry("乐", "le"), Map.entry("勒", "le"), + Map.entry("力", "li"), Map.entry("利", "li"), Map.entry("立", "li"), Map.entry("厉", "li"), Map.entry("励", "li"), Map.entry("历", "li"), Map.entry("丽", "li"), Map.entry("离", "li"), Map.entry("莉", "li"), Map.entry("礼", "li"), Map.entry("理", "li"), Map.entry("李", "li"), Map.entry("里", "li"), + Map.entry("藜", "li"), Map.entry("梨", "li"), Map.entry("黎", "li"), + Map.entry("绿", "lv"), + Map.entry("芦", "lu"), Map.entry("炉", "lu"), Map.entry("路", "lu"), Map.entry("鹿", "lu"), Map.entry("鲁", "lu"), Map.entry("卢", "lu"), Map.entry("露", "lu"), Map.entry("陆", "lu"), Map.entry("庐", "lu"), + Map.entry("酪", "lao"), Map.entry("烙", "lao"), + Map.entry("落", "luo"), Map.entry("络", "luo"), + Map.entry("面", "mian"), Map.entry("免", "mian"), Map.entry("棉", "mian"), Map.entry("眠", "mian"), Map.entry("绵", "mian"), Map.entry("勉", "mian"), + Map.entry("排", "pai"), Map.entry("牌", "pai"), Map.entry("拍", "pai"), Map.entry("派", "pai"), + Map.entry("葡", "pu"), Map.entry("铺", "pu"), Map.entry("浦", "pu"), Map.entry("蒲", "pu"), + Map.entry("腔", "qiang"), + Map.entry("乳", "ru"), Map.entry("如", "ru"), Map.entry("入", "ru"), Map.entry("儒", "ru"), + Map.entry("霜", "shuang"), Map.entry("双", "shuang"), Map.entry("爽", "shuang"), + Map.entry("水", "shui"), Map.entry("睡", "shui"), Map.entry("谁", "shui"), + Map.entry("舒", "shu"), Map.entry("书", "shu"), Map.entry("叔", "shu"), Map.entry("输", "shu"), Map.entry("树", "shu"), Map.entry("竖", "shu"), + Map.entry("生", "sheng"), Map.entry("声", "sheng"), Map.entry("胜", "sheng"), Map.entry("升", "sheng"), Map.entry("省", "sheng"), Map.entry("圣", "sheng"), + Map.entry("素", "su"), Map.entry("速", "su"), Map.entry("诉", "su"), Map.entry("苏", "su"), Map.entry("塑", "su"), + Map.entry("酸", "suan"), Map.entry("算", "suan"), Map.entry("蒜", "suan"), + Map.entry("萄", "tao"), Map.entry("逃", "tao"), Map.entry("淘", "tao"), Map.entry("桃", "tao"), Map.entry("陶", "tao"), Map.entry("套", "tao"), Map.entry("讨", "tao"), + Map.entry("陀", "tuo"), Map.entry("驼", "tuo"), Map.entry("拖", "tuo"), Map.entry("脱", "tuo"), Map.entry("托", "tuo"), + Map.entry("酮", "tong"), Map.entry("铜", "tong"), Map.entry("同", "tong"), Map.entry("桐", "tong"), Map.entry("童", "tong"), Map.entry("痛", "tong"), Map.entry("通", "tong"), Map.entry("统", "tong"), + Map.entry("细", "xi"), Map.entry("希", "xi"), Map.entry("西", "xi"), Map.entry("系", "xi"), Map.entry("息", "xi"), Map.entry("稀", "xi"), Map.entry("席", "xi"), Map.entry("吸", "xi"), + Map.entry("纤", "xian"), Map.entry("先", "xian"), Map.entry("鲜", "xian"), Map.entry("仙", "xian"), Map.entry("险", "xian"), Map.entry("显", "xian"), Map.entry("线", "xian"), Map.entry("限", "xian"), Map.entry("县", "xian"), Map.entry("现", "xian"), Map.entry("献", "xian"), + Map.entry("小", "xiao"), + Map.entry("眼", "yan"), Map.entry("演", "yan"), Map.entry("验", "yan"), Map.entry("烟", "yan"), Map.entry("严", "yan"), Map.entry("颜", "yan"), Map.entry("盐", "yan"), Map.entry("言", "yan"), Map.entry("岩", "yan"), Map.entry("延", "yan"), + Map.entry("氧", "yang"), Map.entry("养", "yang"), Map.entry("仰", "yang"), Map.entry("样", "yang"), Map.entry("洋", "yang"), Map.entry("央", "yang"), Map.entry("阳", "yang"), + Map.entry("益", "yi"), Map.entry("意", "yi"), Map.entry("易", "yi"), Map.entry("亿", "yi"), Map.entry("以", "yi"), Map.entry("艺", "yi"), Map.entry("忆", "yi"), Map.entry("异", "yi"), Map.entry("议", "yi"), Map.entry("翼", "yi"), Map.entry("衣", "yi"), Map.entry("依", "yi"), Map.entry("一", "yi"), + Map.entry("饮", "yin"), Map.entry("引", "yin"), Map.entry("印", "yin"), Map.entry("隐", "yin"), Map.entry("银", "yin"), Map.entry("音", "yin"), + Map.entry("应", "ying"), Map.entry("映", "ying"), Map.entry("影", "ying"), Map.entry("英", "ying"), Map.entry("营", "ying"), Map.entry("迎", "ying"), + Map.entry("育", "yu"), Map.entry("玉", "yu"), Map.entry("域", "yu"), Map.entry("遇", "yu"), Map.entry("雨", "yu"), Map.entry("宇", "yu"), Map.entry("御", "yu"), Map.entry("语", "yu"), Map.entry("鱼", "yu"), + Map.entry("原", "yuan"), Map.entry("圆", "yuan"), Map.entry("远", "yuan"), Map.entry("园", "yuan"), Map.entry("元", "yuan"), Map.entry("源", "yuan"), Map.entry("缘", "yuan"), + Map.entry("籽", "zi"), Map.entry("子", "zi"), Map.entry("紫", "zi"), Map.entry("自", "zi"), Map.entry("字", "zi"), + Map.entry("转", "zhuan"), Map.entry("赚", "zhuan"), Map.entry("砖", "zhuan"), Map.entry("专", "zhuan") + ); + + private static final List PRODUCTS = List.of( + "细胞抗氧素", + "胶原蛋白", "白藜芦醇", "好转反应", "阿育吠陀", + "活力健", "倍力健", "氨基酸", "益生菌", "辅酵素", + "葡萄籽", "排毒饮", "乳酪煲", "草本茶", "异黄酮", + "骨骼健", "舒采健", "衡醇饮", "洁面乳", "爽肤水" + ); + + private static final Map PINYIN_INDEX = buildPinyinIndex(); + private static final List PRODUCT_LENGTHS = PRODUCTS.stream().map(String::length).distinct().sorted(Comparator.reverseOrder()).toList(); + + public String matchProducts(String text) { + if (text == null || text.isBlank()) { + return text == null ? "" : text; + } + String result = text; + for (Integer length : PRODUCT_LENGTHS) { + if (result.length() < length) { + continue; + } + int index = 0; + while (index <= result.length() - length) { + String window = result.substring(index, index + length); + if (!CHINESE_WINDOW_PATTERN.matcher(window).matches()) { + index++; + continue; + } + String key = toPinyinKey(window); + if (key == null) { + index++; + continue; + } + String product = PINYIN_INDEX.get(key); + if (product != null && !product.equals(window)) { + log.debug("[PinyinMatcher] \"{}\" -> \"{}\" (pinyin: {})", window, product, key); + result = result.substring(0, index) + product + result.substring(index + length); + index += product.length(); + continue; + } + index++; + } + } + return result; + } + + public String toPinyinKey(String text) { + if (text == null || text.isBlank()) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (int index = 0; index < text.length(); index++) { + String current = String.valueOf(text.charAt(index)); + String pinyin = PINYIN_MAP.get(current); + if (pinyin == null) { + return null; + } + if (builder.length() > 0) { + builder.append('-'); + } + builder.append(pinyin); + } + return builder.toString(); + } + + private static Map buildPinyinIndex() { + Map index = new LinkedHashMap<>(); + for (String name : PRODUCTS) { + String key = toStaticPinyinKey(name); + if (key != null) { + index.put(key, name); + } + } + return Map.copyOf(index); + } + + private static String toStaticPinyinKey(String text) { + StringBuilder builder = new StringBuilder(); + for (int index = 0; index < text.length(); index++) { + String current = String.valueOf(text.charAt(index)); + String pinyin = PINYIN_MAP.get(current); + if (pinyin == null) { + return null; + } + if (builder.length() > 0) { + builder.append('-'); + } + builder.append(pinyin); + } + return builder.toString(); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/ProductLinkTrigger.java b/java-server/src/main/java/com/bigwo/javaserver/service/ProductLinkTrigger.java new file mode 100644 index 0000000..9960a9b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/ProductLinkTrigger.java @@ -0,0 +1,95 @@ +package com.bigwo.javaserver.service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Pattern; + +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +@Component +public class ProductLinkTrigger { + + private static final List PRODUCT_LINK_MAP = List.of( + new ProductEntry("基础三合一", List.of("基础三合一", "三合一", "基础套装", "powercocktail"), "https://example.com/products/power-cocktail", "FitLine 基础三合一套装（大白+小红+小白）"), + new ProductEntry("小红", List.of("小红", "小红产品", "艾特维", "activize", "activize oxyplus", "activize serum", "小红精华液"), "https://example.com/products/activize", "FitLine 小红 Activize Oxyplus 能量激活饮品"), + new ProductEntry("大白", List.of("大白", "大白产品", "倍适", "basics", "basic power"), "https://example.com/products/basics", "FitLine 大白 Basics 基础营养素"), + new ProductEntry("小白", List.of("小白", "小白产品", "维适多", "restorate"), "https://example.com/products/restorate", "FitLine 小白 Restorate 矿物质晚间饮品"), + new ProductEntry("儿童倍适", List.of("儿童倍适", "powercocktail junior", "儿童套装"), "https://example.com/products/junior", "FitLine 儿童倍适 PowerCocktail Junior"), + new ProductEntry("CC-Cell", List.of("cc-cell", "cc-cell胶囊", "cc-cell乳霜", "cc套装", "cc胶囊"), "https://example.com/products/cc-cell", "FitLine CC-Cell 细胞修护套装"), + new ProductEntry("D-Drink", List.of("d-drink", "小绿排毒饮", "小绿", "14天排毒d饮料plus", "排毒饮"), "https://example.com/products/d-drink", "FitLine D-Drink 14天排毒饮品"), + new ProductEntry("倍力健", List.of("倍力健", "men+", "倍力健 men+", "小黑"), "https://example.com/products/men-plus", "FitLine 倍力健 MEN+ 男士活力营养素"), + new ProductEntry("肽美", List.of("肽美", "太美", "态美"), "https://example.com/products/beauty", "FitLine 肽美 胶原蛋白肽美容饮品"), + new ProductEntry("活力健", List.of("活力健"), "https://example.com/products/fitness-drink", "FitLine 活力健运动饮料"), + new ProductEntry("ProShape Amino", List.of("proshape amino", "proshape", "氨基酸", "支链氨基酸", "bcaa"), "https://example.com/products/proshape-amino", "FitLine ProShape Amino 氨基酸蛋白粉"), + new ProductEntry("草本茶", List.of("草本茶", "herbal tea"), "https://example.com/products/herbal-tea", "FitLine 草本茶"), + new ProductEntry("Hair+", List.of("hair+", "med hair+", "口服发宝", "外用发健", "发宝"), "https://example.com/products/hair-plus", "FitLine med Hair+ 护发套装"), + new ProductEntry("Fitness-Drink", List.of("fitness-drink", "运动饮料", "健康饮品"), "https://example.com/products/fitness-drink", "FitLine Fitness-Drink 运动饮料"), + new ProductEntry("纤萃", List.of("topshape", "纤萃", "纤萃减肥"), "https://example.com/products/topshape", "FitLine TopShape 纤萃体重管理"), + new ProductEntry("乐活50+", List.of("乐活50+", "generation 50+", "乐活"), "https://example.com/products/generation-50", "FitLine Generation 50+ 乐活中老年营养素"), + new ProductEntry("苹果细胞抗氧素", List.of("apple antioxy", "苹果细胞抗氧素", "antioxy", "细胞抗氧素", "zellschutz"), "https://example.com/products/antioxy", "FitLine Apple Antioxy 苹果细胞抗氧素"), + new ProductEntry("草本护理牙膏", List.of("med dental+", "草本护理牙膏", "牙膏"), "https://example.com/products/dental", "FitLine Med Dental+ 草本护理牙膏"), + new ProductEntry("Q10", List.of("q10", "辅酵素", "q10辅酵素氧修护"), "https://example.com/products/q10", "FitLine Q10 辅酵素氧修护"), + new ProductEntry("关节套装", List.of("关节套装", "关节舒缓"), "https://example.com/products/joint", "FitLine 关节舒缓套装"), + new ProductEntry("IB5", List.of("ib5", "口腔免疫喷雾"), "https://example.com/products/ib5", "FitLine IB5 口腔免疫喷雾"), + new ProductEntry("男士乳霜", List.of("men face", "全效男士护肤抗衰乳霜", "男士乳霜", "男士护肤"), "https://example.com/products/men-face", "FitLine Men Face 全效男士护肤抗衰乳霜") + ); + + private static final List DETAIL_REQUEST_PATTERNS = List.of( + Pattern.compile("(?:查看|看看|看一下|看下|了解|了解一下|了解下).*(?:详细|详情|介绍|说明|资料|信息)"), + Pattern.compile("(?:详细|详情|详细介绍|产品介绍|产品详情|产品说明).*(?:看|查|发|给|了解)"), + Pattern.compile("(?:发|给|分享|推|发个|发一个|给我|给我发|发给我|分享一下).*(?:链接|网址|地址|网页|页面|资料|图文|详情)"), + Pattern.compile("(?:链接|网址|网页).*(?:发|给|看|有没有)"), + Pattern.compile("(?:有没有|有无|有.+吗).*(?:链接|网址|网页|资料|图文|详情)"), + Pattern.compile("(?:更多|更详细|更具体|深入).*(?:介绍|了解|信息|内容)"), + Pattern.compile("(?:图文|产品页|详情页|官网|产品图).*(?:看|有|发|给)"), + Pattern.compile("(?:看|有|发|给).*(?:图文|产品页|详情页|官网|产品图)") + ); + + private static final List SORTED_CANDIDATES; + + static { + List candidates = new ArrayList<>(); + for (ProductEntry product : PRODUCT_LINK_MAP) { + for (String alias : product.aliases()) { + candidates.add(new AliasCandidate(alias, product)); + } + } + candidates.sort(Comparator.comparingInt((AliasCandidate c) -> c.alias().length()).reversed()); + SORTED_CANDIDATES = List.copyOf(candidates); + } + + public TriggerResult check(String text) { + String t = StringUtils.hasText(text) ? text.trim() : ""; + if (t.isEmpty()) { + return TriggerResult.NOT_TRIGGERED; + } + String lower = t.toLowerCase(); + ProductEntry product = null; + for (AliasCandidate candidate : SORTED_CANDIDATES) { + if (lower.contains(candidate.alias())) { + product = candidate.product(); + break; + } + } + if (product == null) { + return TriggerResult.NOT_TRIGGERED; + } + boolean hasIntent = DETAIL_REQUEST_PATTERNS.stream().anyMatch(p -> p.matcher(t).find()); + if (!hasIntent) { + return TriggerResult.NOT_TRIGGERED; + } + return new TriggerResult(true, product.name(), product.link(), product.description()); + } + + private record ProductEntry(String name, List aliases, String link, String description) { + } + + private record AliasCandidate(String alias, ProductEntry product) { + } + + public record TriggerResult(boolean triggered, String productName, String link, String description) { + static final TriggerResult NOT_TRIGGERED = new TriggerResult(false, null, null, null); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/RedisContextStore.java b/java-server/src/main/java/com/bigwo/javaserver/service/RedisContextStore.java new file mode 100644 index 0000000..2426137 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/RedisContextStore.java @@ -0,0 +1,159 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.config.RedisClientManager; +import com.bigwo.javaserver.config.RedisProperties; +import com.bigwo.javaserver.model.RedisContextMessage; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.lettuce.core.api.sync.RedisCommands; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +@Service +public class RedisContextStore { + + private static final Logger log = LoggerFactory.getLogger(RedisContextStore.class); + private static final int HISTORY_MAX_LEN = 10; + private static final long HISTORY_TTL_SECONDS = 1800; + private static final long KB_CACHE_HIT_TTL_SECONDS = 300; + private static final long SUMMARY_TTL_SECONDS = 7200; + + private final RedisClientManager redisClientManager; + private final RedisProperties redisProperties; + private final ObjectMapper objectMapper; + + public RedisContextStore(RedisClientManager redisClientManager, RedisProperties redisProperties, ObjectMapper objectMapper) { + this.redisClientManager = redisClientManager; + this.redisProperties = redisProperties; + this.objectMapper = objectMapper; + } + + public boolean pushMessage(String sessionId, RedisContextMessage message) { + if (!redisClientManager.isAvailable() || message == null || !StringUtils.hasText(sessionId)) { + return false; + } + try { + RedisCommands commands = redisClientManager.syncCommands(); + String key = historyKey(sessionId); + String payload = objectMapper.writeValueAsString(message); + commands.lpush(key, payload); + commands.ltrim(key, 0, HISTORY_MAX_LEN - 1); + commands.expire(key, HISTORY_TTL_SECONDS); + return true; + } catch (Exception exception) { + log.warn("[Redis] pushMessage failed: {}", exception.getMessage()); + return false; + } + } + + public List getRecentHistory(String sessionId, int maxRounds) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(sessionId)) { + return null; + } + try { + RedisCommands commands = redisClientManager.syncCommands(); + List items = commands.lrange(historyKey(sessionId), 0, Math.max(0, maxRounds * 2 - 1)); + if (items == null || items.isEmpty()) { + return List.of(); + } + return items.reversed().stream() + .map(item -> deserialize(item, new TypeReference() { })) + .filter(java.util.Objects::nonNull) + .toList(); + } catch (Exception exception) { + log.warn("[Redis] getRecentHistory failed: {}", exception.getMessage()); + return null; + } + } + + public boolean clearSession(String sessionId) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(sessionId)) { + return false; + } + try { + redisClientManager.syncCommands().del(historyKey(sessionId)); + return true; + } catch (Exception exception) { + log.warn("[Redis] clearSession failed: {}", exception.getMessage()); + return false; + } + } + + public boolean setSummary(String sessionId, String summary) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(sessionId) || !StringUtils.hasText(summary)) { + return false; + } + try { + redisClientManager.syncCommands().setex(summaryKey(sessionId), SUMMARY_TTL_SECONDS, summary); + return true; + } catch (Exception exception) { + log.warn("[Redis] setSummary failed: {}", exception.getMessage()); + return false; + } + } + + public String getSummary(String sessionId) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(sessionId)) { + return null; + } + try { + return redisClientManager.syncCommands().get(summaryKey(sessionId)); + } catch (Exception exception) { + log.warn("[Redis] getSummary failed: {}", exception.getMessage()); + return null; + } + } + + public boolean setKbCache(String cacheKey, Object result, boolean hit) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(cacheKey) || !hit || result == null) { + return false; + } + try { + String payload = objectMapper.writeValueAsString(result); + redisClientManager.syncCommands().setex(kbCacheKey(cacheKey), KB_CACHE_HIT_TTL_SECONDS, payload); + return true; + } catch (Exception exception) { + log.warn("[Redis] setKbCache failed: {}", exception.getMessage()); + return false; + } + } + + public T getKbCache(String cacheKey, Class valueType) { + if (!redisClientManager.isAvailable() || !StringUtils.hasText(cacheKey)) { + return null; + } + try { + String payload = redisClientManager.syncCommands().get(kbCacheKey(cacheKey)); + if (!StringUtils.hasText(payload)) { + return null; + } + return objectMapper.readValue(payload, valueType); + } catch (Exception exception) { + log.warn("[Redis] getKbCache failed: {}", exception.getMessage()); + return null; + } + } + + private String historyKey(String sessionId) { + return redisProperties.getKeyPrefix() + "voice:history:" + sessionId; + } + + private String summaryKey(String sessionId) { + return redisProperties.getKeyPrefix() + "voice:summary:" + sessionId; + } + + private String kbCacheKey(String cacheKey) { + return redisProperties.getKeyPrefix() + "kb_cache:" + cacheKey; + } + + private T deserialize(String payload, TypeReference typeReference) { + try { + return objectMapper.readValue(payload, typeReference); + } catch (Exception exception) { + return null; + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/SessionService.java b/java-server/src/main/java/com/bigwo/javaserver/service/SessionService.java new file mode 100644 index 0000000..ce96367 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/SessionService.java @@ -0,0 +1,70 @@ +package com.bigwo.javaserver.service; + +import com.bigwo.javaserver.exception.BadRequestException; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.model.SessionFullMessage; +import com.bigwo.javaserver.model.SessionHistoryResult; +import com.bigwo.javaserver.model.SessionListItem; +import com.bigwo.javaserver.model.SessionSwitchResult; +import com.bigwo.javaserver.repository.SessionRepository; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +@Service +public class SessionService { + + private static final Logger log = LoggerFactory.getLogger(SessionService.class); + + private final SessionRepository sessionRepository; + + public SessionService(SessionRepository sessionRepository) { + this.sessionRepository = sessionRepository; + } + + public List listSessions(String userId, Integer limit) { + return sessionRepository.getSessionList(userId, limit).stream() + .map(item -> new SessionListItem( + item.id(), + item.userId(), + item.mode(), + item.createdAt(), + item.updatedAt(), + truncateMessage(item.lastMessage()), + item.messageCount() + )) + .toList(); + } + + public void deleteSession(String sessionId) { + sessionRepository.deleteSession(sessionId); + } + + public SessionHistoryResult getFullHistory(String sessionId, Integer limit) { + List messages = sessionRepository.getRecentMessages(sessionId, limit); + return new SessionHistoryResult<>(sessionId, sessionRepository.getSessionMode(sessionId), messages, messages.size()); + } + + public SessionHistoryResult getLlmHistory(String sessionId, Integer limit) { + List messages = sessionRepository.getHistoryForLlm(sessionId, limit); + return new SessionHistoryResult<>(sessionId, sessionRepository.getSessionMode(sessionId), messages, messages.size()); + } + + public SessionSwitchResult switchMode(String sessionId, String targetMode) { + if (!"voice".equals(targetMode) && !"chat".equals(targetMode)) { + throw new BadRequestException("targetMode must be \"voice\" or \"chat\""); + } + sessionRepository.updateSessionMode(sessionId, targetMode); + List history = sessionRepository.getHistoryForLlm(sessionId, 20); + log.info("[Session] Switched {} to {}, history: {} messages", sessionId, targetMode, history.size()); + return new SessionSwitchResult(sessionId, targetMode, history, history.size()); + } + + private String truncateMessage(String value) { + if (value == null || value.length() <= 60) { + return value; + } + return value.substring(0, 60) + "..."; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/VideoGenerationService.java b/java-server/src/main/java/com/bigwo/javaserver/service/VideoGenerationService.java new file mode 100644 index 0000000..23a12f6 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/VideoGenerationService.java @@ -0,0 +1,1353 @@ +package com.bigwo.javaserver.service; + +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.env.Environment; +import org.springframework.core.io.FileSystemResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; + +import com.bigwo.javaserver.config.VideoGenerationProperties; +import com.bigwo.javaserver.exception.BadRequestException; +import com.bigwo.javaserver.model.RenderedVideoPayload; +import com.bigwo.javaserver.model.VideoAdminConfigResponse; +import com.bigwo.javaserver.model.VideoGenerateResponse; +import com.bigwo.javaserver.model.VideoHistoryResponse; +import com.bigwo.javaserver.model.VideoHistoryRowResponse; +import com.bigwo.javaserver.model.VideoPromptDetail; +import com.bigwo.javaserver.model.VideoTaskSnapshot; +import com.bigwo.javaserver.repository.VideoTaskRepository; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.annotation.PostConstruct; + +@Service +public class VideoGenerationService implements AutoCloseable { + + private static final Logger log = LoggerFactory.getLogger(VideoGenerationService.class); + private static final List VALID_VIDEO_MODELS = List.of("seedance-2.0", "grok-video-3", "grok-video-3-pro", "grok-video-3-max", "grok-video-4.2"); + private static final String DEFAULT_SIZE = "720x1280"; + private static final int DEFAULT_SECONDS = 15; + private static final String DEFAULT_NEGATIVE = "低质量、模糊、水印、变形、丑陋、不雅内容、医疗图像、人脸大特写、静止画面、无运动、画面文字、字幕叠加、文字渲染、标题文字、任何可见文字"; + private static final String VOICE_PREFIX = "【语音指令】中文普通话女声旁白，语速适中偏慢，吐字清晰，每句之间略停顿。"; + private static final String VOICE_SUFFIX = "语速平稳清晰，不要赶，每句停顿半秒。"; + private static final String QUALITY_SUFFIX = "电影级画质，专业级质感，超高清，专业打光，景深虚化，丝滑运镜。"; + private static final long MEMORY_TASK_TTL_MS = 60L * 60L * 1000L; + private static final int LOGO_HEIGHT = 120; + private static final int LOGO_PADDING = 20; + private static final double LOGO_OPACITY = 0.85; + private static final int POST_PROCESS_CONCURRENCY = 3; + private static final int DOWNLOAD_MAX_RETRIES = 5; + // ── Phase 1 system prompts: script → structured shotList JSON ── + + private static final String PRODUCT_SHOTLIST_PROMPT = """ + 你是一位品牌短片视觉导演，擅长用电影级画面讲述品牌故事。根据用户提供的主题和产品图，输出**结构化分镜数据**。 + + ## 技术背景 + 视频模型以产品图为画面起点，生成15秒连续运动视频。 + - 不支持场景跳切——整段是连续运镜 + - 支持：镜头推拉/平移/环绕、物体出现/移动、人物入画、光影变化 + + ## 输出格式（仅JSON，无任何额外文字） + {"shotList":[{"index":1,"visual":"纯画面描述（50-80字，只描述视觉内容：产品外观、人物动作、光影、场景，禁止提及旁白/字幕/文字）","camera":"运镜方式（如：缓慢推近、向右平移、环绕拉远）","duration":"时长（如：5秒）","narration":"中文旁白（每句≤4字，每个镜头都要有，全部旁白加起来约16字，简洁有力、温暖走心）","voiceTone":"语气（如：温柔亲切、充满力量）","bgSound":"背景音（如：舒缓钢琴、自然雨声）"}],"note":"创意思路（含对产品图的观察）"} + + ## 规则 + 1. **3-4个运镜阶段**，总时长15秒，从产品特写 → 人物与产品互动 → 使用感受/生活场景 → 品质升华 + 2. **visual 字段**：仔细观察产品图，精准描述产品外观/颜色/质感。只写视觉画面，禁止写"旁白响起""字幕出现"等非画面内容。如果产品图上有英文，用"印有精致标识"描述，不要写出英文原文 + 3. **narration 字段**：**每个镜头都必须写旁白**，每句≤4个汉字，全部旁白加起来约16字，简洁有力、温暖走心。旁白风格参考："用心守护""品质如一""自然之力""值得信赖"。**旁白绝对禁止**：招商话术、营销口号、数字金额、认证名称、英文缩写、医疗声明、功效宣称 + 4. **全部用中文**，所有字段禁止出现任何英文字母 + 5. **禁止**：场景跳切、静止画面、色情/暴力/政治/疾病/药物/人脸大特写 + 6. **隐性IP过滤**：visual字段禁止出现恰好是影视/动画/游戏作品名的词组，必须拆散重构为纯物理场景描述 + 7. **视觉安全（极重要）**：visual字段中绝对禁止出现以下词汇："营养品""保健品""补充剂""维生素""胶囊""药片""滋补""膏方""健康产品""功效""膳食""服用""广告""宣传""推广""招商""产品展示"。必须用纯物理外观描述代替"""; + + private static final String GENERIC_SHOTLIST_PROMPT = """ + 你是一位品牌短片视觉导演，擅长用电影级画面讲述品牌故事。你的核心能力是从用户输入中**精准提取核心主题**，然后围绕这个主题创作品牌分镜。 + + ## 第一步：主题提取（内部思考，不输出） + 收到用户输入后，先在脑中完成： + 1. **核心主题**：用一句话概括用户真正想表达的中心思想是什么 + 2. **视觉主体**：画面中应该出现的核心物体/人物/场景是什么 + 3. **情绪基调**：整体应传递什么情感（震撼/温暖/高端/励志等） + 4. **关键信息**：用户最想让观众记住的1-2个要点 + + ## 第二步：围绕主题创作分镜 + 所有镜头必须紧扣第一步提取的核心主题，每个镜头都在为同一个主题服务。 + **必须有真人出镜**：画面中必须出现一位真实人物（描述其年龄、性别、穿着、气质），人物与主题产生互动或情感联结。 + + ## 技术背景 + 视频模型以参考图为画面起点，生成连续运动视频。 + - 不支持场景跳切——整段是连续运镜 + - 支持：镜头推拉/平移/环绕、物体出现/移动、人物入画、光影变化 + + ## 输出格式（仅JSON，无任何额外文字） + {"shotList":[{"index":1,"visual":"纯画面描述（50-80字，只描述视觉内容，禁止提及旁白/字幕/文字）","camera":"运镜方式","duration":"时长（如：5秒）","narration":"中文旁白（每句≤4字，每个镜头都要有，全部约16字）","voiceTone":"语气","bgSound":"背景音"}],"theme":"提取的核心主题（一句话）","note":"创意思路"} + + ## 规则 + 1. **3-4个运镜阶段**，总时长根据用户要求 + 2. **主题聚焦**：所有镜头围绕同一核心主题展开，旁白是对主题的递进式表达（引入→展开→深化→升华），禁止跑题 + 3. **visual 字段**：观察参考图，精准描述主体外观/场景。**每个镜头的画面中必须有真人出现**。只写视觉画面，禁止写"旁白响起""字幕出现"等非画面内容 + 4. **narration 字段**：**每个镜头都必须写旁白**，每句≤4个汉字，全部旁白加起来约16字，简洁有力、温暖走心 + 5. **全部用中文**，所有字段禁止出现任何英文字母 + 6. **禁止**：场景跳切、静止画面、色情/暴力/政治/人脸大特写 + 7. **视觉安全（极重要）**：visual字段中绝对禁止出现："营养品""保健品""补充剂""维生素""胶囊""药片""滋补""功效""服用"。用纯物理外观描述代替"""; + + // ── Phase 0b system prompts: user input → detailed shot script (text) ── + + private static final String PRODUCT_SCRIPT_PROMPT = """ + 你是一位资深品牌短片编剧，擅长把品牌故事拍出电影质感。用户会给你一段简短的主题描述和产品名称，你需要将其扩写为一份**竖屏高清品牌短片的详细分镜脚本**。 + + ## 输出格式（纯文本，不要JSON） + 按以下模板输出，每个字段都必须填写： + + --- + 竖屏高清品牌短片，[风格描述]，画面质感通透高级，光影柔和有氛围感。 + 主角为一位[人物描述（年龄、气质、穿着、妆容）]， + 围绕[产品名称]展开，全程节奏舒缓高级，情绪温暖治愈。 + + 分镜脚本： + 镜头１（0-3秒）：[特写产品的画面描述，含光影和环境细节]。[运镜方式]。 + 旁白：「[≤8字的中文旁白]」 + + 镜头２（3-7秒）：[中景人物与产品互动的画面描述]。[运镜方式]。 + 旁白：「[≤8字的中文旁白]」 + + 镜头３（7-11秒）：[微距/使用感受的画面描述]。[运镜方式]。 + 旁白：「[≤8字的中文旁白]」 + + 镜头４（11-15秒）：[全景收尾画面描述，产品与人物同框]。[运镜方式]。 + 旁白：「[≤8字的中文旁白]」 + + 品质亮点：[3-5个画面亮点]。[色调描述]，[画面细节要求]，运镜丝滑。 + --- + + ## 规则 + 1. **全部用中文**，禁止任何英文字母（产品图上的英文用"印有精致标识"代替） + 2. 风格参考：欧美写实治愈风、高端品牌大片质感 + 3. 4个镜头必须是连续运镜（推拉/平移/环绕/跟焦），禁止场景跳切 + 4. **每个镜头都必须有旁白**，每句≤4字，全部旁白加起来约16字，简洁有力、温暖走心 + 5. 画面描述要具体（颜色、材质、光影、动作），不要抽象 + 6. 禁止：色情/暴力/政治/疾病/药物/人脸大特写 + 7. **视觉安全（极重要）**：画面描述中绝对禁止出现："营养品""保健品""补充剂""维生素""胶囊""药片""滋补""功效""服用"。用纯物理外观描述代替"""; + + private static final String GENERIC_SCRIPT_PROMPT = """ + 你是一位资深品牌短片编剧，擅长把品牌故事拍出电影质感。你的核心能力是从用户输入中**精准提取核心主题**，然后围绕这个主题创作品牌分镜脚本。 + + ## 工作流程 + 1. **主题提取**：分析用户文案，提炼出一个核心主题（一句话能概括的中心思想） + 2. **视觉转化**：将主题转化为有品牌质感的画面序列（产品特写→人物互动→品质感受→品牌升华） + 3. **旁白串联**：旁白围绕主题层层递进（引入→展开→深化→升华），简洁有力、温暖走心 + + ## 输出格式（纯文本，不要JSON） + 按以下模板输出： + + --- + 核心主题：[从用户输入中提炼的一句话核心主题] + + 竖屏高清品牌短片，[风格描述]，画面质感通透高级。 + 主角为一位[人物描述（年龄、气质、穿着）]，围绕[主题对应的核心视觉元素]展开， + 全程节奏[氛围描述]。 + + 分镜脚本： + 镜头１（0-3秒）：[画面描述，真人出镜+紧扣主题的开场]。[运镜方式]。 + 旁白：「[≤8字，引出主题]」 + + 镜头２（3-7秒）：[画面描述，真人与主题互动]。[运镜方式]。 + 旁白：「[≤8字，展开主题]」 + + 镜头３（7-11秒）：[画面描述，真人动作+深化主题]。[运镜方式]。 + 旁白：「[≤8字，深化主题]」 + + 镜头４（11-15秒）：[画面描述，真人+升华收尾]。[运镜方式]。 + 旁白：「[≤8字，升华主题]」 + --- + + ## 规则 + 1. **主题聚焦**：所有镜头和旁白必须紧扣同一核心主题，禁止跑题 + 2. **真人必须出镜**：每个镜头画面中必须有一位真实人物 + 3. **全部用中文**，禁止任何英文字母 + 4. 4个镜头必须是连续运镜，禁止场景跳切 + 5. **每个镜头都必须有旁白**，每句≤4字，全部旁白加起来约16字 + 6. 画面描述要具体（颜色、材质、光影、动作） + 7. **视觉安全（极重要）**：画面描述中绝对禁止出现："营养品""保健品""补充剂""维生素""胶囊""药片""滋补""功效""服用"。用纯物理外观描述代替"""; + + // ── Phase 0a system prompt: generic core extraction ── + + private static final String GENERIC_CORE_PROMPT = """ + 你是一位品牌短片内容策划师。请先从用户输入的原始内容中提炼出最核心、最适合转成品牌短片画面的单一主题，再为后续分镜脚本生成做准备。 + + ## 提炼方向（优先从以下维度切入） + - **匠心品质**：精湛工艺、匠人精神、品质追求、值得信赖 + - **实力底蕴**：深厚积淀、专业团队、技术创新、行业标杆 + - **趋势机遇**：时代浪潮、新兴趋势、把握当下、顺势而为 + - **愿景蓝图**：美好未来、无限可能、梦想成真、星辰大海 + - **同行共创**：志同道合、携手并进、同行者、共同成长 + + 从以上维度中选择与用户原始内容最匹配的**一个**作为核心主题方向。 + + ## 输出格式（仅JSON，无任何额外文字） + {"theme":"一句话核心主题（简洁有力，禁止招商/营销术语）","direction":"匹配的提炼方向（匠心/实力/趋势/愿景/同行）","visualSubject":"核心视觉主体","emotion":"情绪基调","keyPoints":["要点1","要点2"],"scriptBrief":"用于后续生成分镜脚本的简明创作摘要（用品牌短片的视觉语言描述画面思路）"} + + ## 规则 + 1. 全部用中文 + 2. 只保留一个最核心主题，禁止发散为多个主题 + 3. **过滤招商/营销术语**：忽略招商口号、营销标语、具体数据金额、认证奖项、英文缩写，但保留品牌调性和品质感。提炼出背后真正想传达的品牌精神或品质态度 + 4. 输出内容必须能直接指导后续分镜脚本生成 + 5. scriptBrief 用品牌短片视觉语言，给出可视化的创作思路 + 6. 禁止输出Markdown、解释、前缀、标题"""; + + private final VideoGenerationProperties properties; + private final VideoTaskRepository videoTaskRepository; + private final ObjectMapper objectMapper; + private final Environment environment; + private final ExecutorService executorService; + private final ConcurrentMap tasks = new ConcurrentHashMap<>(); + private final AtomicReference runtimeVideoModel = new AtomicReference<>(); + private final Semaphore postProcessSemaphore = new Semaphore(POST_PROCESS_CONCURRENCY); + + public VideoGenerationService( + VideoGenerationProperties properties, + VideoTaskRepository videoTaskRepository, + ObjectMapper objectMapper, + Environment environment + ) { + this.properties = properties; + this.videoTaskRepository = videoTaskRepository; + this.objectMapper = objectMapper; + this.environment = environment; + this.executorService = Executors.newFixedThreadPool(Math.max(1, properties.getMaxConcurrentTasks())); + } + + @PostConstruct + public void initialize() { + int cleaned = videoTaskRepository.cleanupStaleTasks(); + if (cleaned > 0) { + log.info("[Video] 启动清理僵尸任务: {}", cleaned); + } + } + + public VideoGenerateResponse generate( + String prompt, + String product, + String username, + String template, + String size, + String seconds, + MultipartFile image + ) { + String normalizedPrompt = StringUtils.hasText(prompt) ? prompt.trim() : "展示这个产品"; + String normalizedProduct = StringUtils.hasText(product) ? product.trim() : ""; + String normalizedUsername = StringUtils.hasText(username) ? username.trim() : ""; + String normalizedTemplate = "generic".equalsIgnoreCase(template) ? "generic" : "product"; + String normalizedSize = normalizeSize(size); + int normalizedSeconds = normalizeSeconds(seconds); + Path savedImage = storeUpload(image); + String taskId = "local_" + UUID.randomUUID(); + InMemoryVideoTask task = new InMemoryVideoTask( + taskId, + normalizedUsername, + normalizedPrompt, + normalizedProduct, + normalizedTemplate, + normalizedSize, + normalizedSeconds, + savedImage + ); + tasks.put(taskId, task); + videoTaskRepository.insertVideoTask( + taskId, + normalizedUsername, + normalizedPrompt, + null, + normalizedProduct, + normalizedTemplate, + normalizedSize, + normalizedSeconds, + "optimizing" + ); + executorService.submit(() -> runPipeline(taskId)); + return new VideoGenerateResponse(taskId, normalizedPrompt, normalizedUsername, "optimizing"); + } + + public VideoTaskSnapshot getTask(String taskId) { + if (!StringUtils.hasText(taskId)) { + return null; + } + InMemoryVideoTask inMemoryTask = tasks.get(taskId.trim()); + if (inMemoryTask != null) { + return inMemoryTask.snapshot(); + } + VideoTaskSnapshot saved = videoTaskRepository.getVideoTask(taskId.trim()); + if (saved != null) { + return saved; + } + RemoteTaskStatus remoteTaskStatus; + try { + remoteTaskStatus = fetchRemoteStatus(taskId.trim(), false); + } catch (Exception exception) { + log.debug("[Video] remote task lookup failed: {}", exception.getMessage()); + return null; + } + if (remoteTaskStatus == null) { + return null; + } + return new VideoTaskSnapshot( + taskId.trim(), + null, + null, + null, + null, + null, + null, + null, + remoteTaskStatus.status(), + mapStatusText(remoteTaskStatus.status()), + remoteTaskStatus.progress(), + remoteTaskStatus.videoUrl(), + remoteTaskStatus.error(), + null, + null, + null + ); + } + + public VideoHistoryResponse getHistory(String username, Integer limit, Integer offset) { + int safeLimit = Math.max(1, Math.min(limit == null ? 20 : limit, 100)); + int safeOffset = Math.max(0, offset == null ? 0 : offset); + if (videoTaskRepository.isAvailable()) { + List rows = videoTaskRepository.getVideoHistory(username, safeLimit, safeOffset).stream() + .map(snapshot -> mergeHistoryRow(snapshot, tasks.get(snapshot.id()))) + .toList(); + return new VideoHistoryResponse(videoTaskRepository.getVideoHistoryCount(username), rows); + } + List memoryRows = tasks.values().stream() + .map(InMemoryVideoTask::snapshot) + .filter(snapshot -> !StringUtils.hasText(username) || username.trim().equals(snapshot.username())) + .sorted(Comparator.comparing(VideoTaskSnapshot::createdAt, Comparator.nullsLast(Long::compareTo)).reversed()) + .skip(safeOffset) + .limit(safeLimit) + .map(this::toHistoryRow) + .toList(); + int total = (int) tasks.values().stream() + .map(InMemoryVideoTask::snapshot) + .filter(snapshot -> !StringUtils.hasText(username) || username.trim().equals(snapshot.username())) + .count(); + return new VideoHistoryResponse(total, memoryRows); + } + + public VideoAdminConfigResponse getAdminConfig() { + String runtimeModel = runtimeVideoModel.get(); + String configuredEnvModel = normalizeNullable(environment.getProperty("SEEDANCE_MODEL")); + return new VideoAdminConfigResponse( + null, + null, + currentVideoModel(), + runtimeModel != null ? "runtime" : (configuredEnvModel != null ? "env" : "default"), + VALID_VIDEO_MODELS, + configuredEnvModel, + runtimeModel + ); + } + + public VideoAdminConfigResponse updateAdminConfig(String model) { + String previousModel = currentVideoModel(); + if (!StringUtils.hasText(model)) { + runtimeVideoModel.set(null); + return new VideoAdminConfigResponse(true, previousModel, currentVideoModel(), normalizeNullable(environment.getProperty("SEEDANCE_MODEL")) != null ? "env" : "default", null, null, null); + } + String normalized = model.trim(); + if (!VALID_VIDEO_MODELS.contains(normalized)) { + throw new BadRequestException("无效的模型名: \"" + normalized + "\""); + } + runtimeVideoModel.set(normalized); + return new VideoAdminConfigResponse(true, previousModel, normalized, "runtime", null, null, normalized); + } + + public List validModels() { + return VALID_VIDEO_MODELS; + } + + @Scheduled(fixedDelay = 300000) + public void cleanupExpiredMemoryTasks() { + long now = System.currentTimeMillis(); + for (Map.Entry entry : tasks.entrySet()) { + InMemoryVideoTask task = entry.getValue(); + long reference = task.completedAt != null ? task.completedAt : task.createdAt; + if (now - reference > MEMORY_TASK_TTL_MS) { + tasks.remove(entry.getKey(), task); + } + } + } + + private void runPipeline(String taskId) { + InMemoryVideoTask task = tasks.get(taskId); + if (task == null) { + return; + } + try { + RenderedVideoPayload payload = buildRenderedPayload(task); + task.optimizedPrompt = payload.prompt(); + task.promptDetail = payload.promptDetail(); + task.shotList = payload.promptDetail() == null ? List.of() : payload.promptDetail().shotList(); + task.status = "processing"; + task.statusText = "正在生成视频…"; + task.progress = 0; + videoTaskRepository.updateVideoOptimizedPrompt(taskId, payload.prompt()); + videoTaskRepository.updateVideoProgress(taskId, task.status, task.progress); + String remoteTaskId = createRemoteVideoTask(task, payload); + task.remoteTaskId = remoteTaskId; + pollRemoteTask(task); + } catch (Exception exception) { + String message = extractErrorMessage(exception); + log.error("[Video] 流水线失败: {}", message, exception); + failTask(task, message); + } finally { + deleteQuietly(task.imagePath); + } + } + + private RenderedVideoPayload buildRenderedPayload(InMemoryVideoTask task) { + task.status = "optimizing"; + task.statusText = "AI 正在策划分镜…"; + task.progress = 0; + String negativePrompt = currentNegativePrompt(); + boolean isGeneric = "generic".equalsIgnoreCase(task.templateType); + if (!isGeminiConfigured()) { + String fallbackPrompt = buildFallbackPrompt(task); + VideoPromptDetail promptDetail = new VideoPromptDetail( + fallbackPrompt, + "", + negativePrompt, + "Gemini 未配置，已回退为原始文案直出", + task.originalPrompt, + List.of() + ); + return new RenderedVideoPayload(fallbackPrompt, negativePrompt, "", promptDetail); + } + + // ── Phase 0: multi-turn prompt enrichment ── + String enrichedScript = task.originalPrompt; + JsonNode coreResult = null; + boolean isAlreadyScript = enrichedScript.length() >= 100 + && (enrichedScript.contains("镜头") || enrichedScript.contains("分镜")); + if (!isAlreadyScript) { + if (isGeneric) { + task.statusText = "AI 正在提取核心…"; + coreResult = extractGenericCore(enrichedScript); + log.info("[Video] Phase 0a 核心提取完成: {}", coreResult != null ? coreResult.toString().substring(0, Math.min(200, coreResult.toString().length())) : "null"); + } + task.statusText = "AI 正在编写分镜脚本…"; + enrichedScript = generateShotScript(task, coreResult); + log.info("[Video] Phase 0b 完成: 用户文案({}字) → 分镜脚本({}字)", task.originalPrompt.length(), enrichedScript.length()); + } else { + log.info("[Video] Phase 0 跳过: 用户输入已是分镜脚本({}字)", enrichedScript.length()); + } + + // ── Phase 1: Gemini structured shotList ── + task.statusText = "AI 正在策划分镜…"; + task.progress = 30; + GeminiPlanResponse plan = planWithGemini(task, enrichedScript, isGeneric); + log.info("[Video] Phase 1 完成: shotList={}个镜头, theme={}", plan.shotList().size(), plan.theme()); + if (isGeneric && coreResult != null && !StringUtils.hasText(plan.theme())) { + String coreTheme = safeText(coreResult.path("theme")); + if (StringUtils.hasText(coreTheme)) { + plan = new GeminiPlanResponse(plan.shotList(), plan.note(), coreTheme); + } + } + + // ── Phase 2: model-specific prompt assembly ── + task.progress = 50; + String prompt = buildPromptFromShots(plan.shotList()); + String voiceScript = plan.shotList().stream() + .map(shot -> sanitizeNarration(textValue(shot, "narration"))) + .filter(StringUtils::hasText) + .reduce("", String::concat); + VideoPromptDetail promptDetail = new VideoPromptDetail( + prompt, + voiceScript, + negativePrompt, + plan.note(), + plan.theme(), + plan.shotList() + ); + return new RenderedVideoPayload(prompt, negativePrompt, voiceScript, promptDetail); + } + + // ── Phase 0a: extract generic core theme ── + + private JsonNode extractGenericCore(String userPrompt) { + String raw = callGemini(GENERIC_CORE_PROMPT, + "请从下面原始内容中提炼视频创作核心：\n\n" + userPrompt, + 500, 0.3D); + JsonNode root = parseJsonBlock(raw); + if (root == null || !StringUtils.hasText(safeText(root.path("theme")))) { + throw new IllegalStateException("核心提取失败：缺少 theme"); + } + return root; + } + + // ── Phase 0b: generate detailed shot script (text, not JSON) ── + + private String generateShotScript(InMemoryVideoTask task, JsonNode coreResult) { + boolean isGeneric = "generic".equalsIgnoreCase(task.templateType); + String systemPrompt = isGeneric ? GENERIC_SCRIPT_PROMPT : PRODUCT_SCRIPT_PROMPT; + String shotGuidance = getShotGuidance(task.videoSeconds); + String constraint = "【时长约束】本次视频总时长" + task.videoSeconds + "秒，请设计" + shotGuidance + "个镜头。"; + String userMessage; + if (isGeneric) { + if (coreResult != null) { + userMessage = constraint + "\n\n原始内容：\n" + task.originalPrompt + + "\n\n已提炼核心：\n" + buildGenericCoreContext(coreResult) + + "\n\n请基于已提炼核心生成详细分镜脚本。"; + } else { + userMessage = constraint + "\n\n原始内容：\n" + task.originalPrompt + + "\n\n请先把核心主题提炼清楚，再生成详细分镜脚本。"; + } + } else { + String productName = StringUtils.hasText(task.productName) ? task.productName : "未指定"; + userMessage = constraint + "\n\n产品：" + productName + "\n原始内容：" + task.originalPrompt; + } + return callGemini(systemPrompt, userMessage, 900, 0.7D); + } + + private String buildGenericCoreContext(JsonNode core) { + StringBuilder sb = new StringBuilder(); + sb.append("核心主题：").append(safeText(core.path("theme"))).append('\n'); + sb.append("提炼方向：").append(safeText(core.path("direction"))).append('\n'); + sb.append("视觉主体：").append(safeText(core.path("visualSubject"))).append('\n'); + sb.append("情绪基调：").append(safeText(core.path("emotion"))).append('\n'); + JsonNode keyPoints = core.path("keyPoints"); + if (keyPoints.isArray()) { + StringBuilder points = new StringBuilder(); + for (int i = 0; i < Math.min(keyPoints.size(), 2); i++) { + if (i > 0) points.append('；'); + points.append(keyPoints.get(i).asText("")); + } + sb.append("关键信息：").append(points).append('\n'); + } + sb.append("创作摘要：").append(safeText(core.path("scriptBrief"))); + return sb.toString(); + } + + private String getShotGuidance(int seconds) { + if (seconds <= 6) return "1-2"; + if (seconds <= 10) return "2-3"; + return "3-4"; + } + + // ── Phase 1: structured shotList from enriched script ── + + private GeminiPlanResponse planWithGemini(InMemoryVideoTask task, String enrichedScript, boolean isGeneric) { + String systemPrompt = isGeneric ? GENERIC_SHOTLIST_PROMPT : PRODUCT_SHOTLIST_PROMPT; + String shotGuidance = getShotGuidance(task.videoSeconds); + String constraint = "【时长约束】本次视频总时长" + task.videoSeconds + "秒，输出" + shotGuidance + "个镜头的 shotList。"; + String userMessage = isGeneric + ? constraint + "\n\n请将下面的分镜脚本整理为结构化 shotList JSON：\n\n" + enrichedScript + : constraint + "\n\n产品：" + (StringUtils.hasText(task.productName) ? task.productName : "未指定") + + "\n请根据下面的分镜脚本输出结构化 shotList JSON：\n\n" + enrichedScript; + String rawContent = callGemini(systemPrompt, userMessage, 1500, 0.4D); + JsonNode root = parseJsonBlock(rawContent); + List> shotList = extractShotList(root.path("shotList")); + if (shotList.isEmpty()) { + throw new IllegalStateException("Gemini 未返回有效 shotList"); + } + String note = safeText(root.path("note")); + String theme = safeText(root.path("theme")); + return new GeminiPlanResponse(shotList, note, theme); + } + + private String callGemini(String systemPrompt, String userPrompt, int maxTokens, double temperature) { + String model = normalizeNullable(properties.getGeminiModel()); + String apiKey = normalizeNullable(firstNonBlank( + normalizeNullable(properties.getGeminiApiKey()), + normalizeNullable(properties.getSeedanceApiKey()) + )); + String base = normalizeNullable(properties.getGeminiApiBase()); + if (model == null || apiKey == null) { + throw new IllegalStateException("GEMINI_MODEL / GEMINI_API_KEY 未配置"); + } + String url = (base != null ? trimTrailingSlash(base) : trimTrailingSlash(properties.getSeedanceApiBase())) + "/v1/chat/completions"; + RestTemplate restTemplate = createRestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + headers.setBearerAuth(apiKey); + Map body = new LinkedHashMap<>(); + body.put("model", model); + body.put("messages", List.of( + Map.of("role", "system", "content", systemPrompt), + Map.of("role", "user", "content", userPrompt) + )); + body.put("max_tokens", maxTokens); + body.put("temperature", temperature); + Exception lastException = null; + for (int attempt = 0; attempt < 3; attempt++) { + try { + if (attempt > 0) { + sleepQuietly(attempt * 3000L); + } + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, new HttpEntity<>(body, headers), JsonNode.class); + JsonNode responseBody = response.getBody(); + JsonNode contentNode = responseBody == null ? null : responseBody.path("choices").path(0).path("message").path("content"); + String content = contentNode == null || contentNode.isMissingNode() || contentNode.isNull() ? "" : contentNode.asText("").trim(); + if (content.isBlank()) { + throw new IllegalStateException("Gemini 返回为空"); + } + return content.replaceAll("[\\s\\S]*?", "").trim(); + } catch (Exception exception) { + lastException = exception; + log.warn("[Video][Gemini] attempt {} failed: {}", attempt, exception.getMessage()); + } + } + throw new IllegalStateException(lastException == null ? "Gemini 调用失败" : extractErrorMessage(lastException), lastException); + } + + private String createRemoteVideoTask(InMemoryVideoTask task, RenderedVideoPayload payload) { + String apiKey = normalizeNullable(properties.getSeedanceApiKey()); + if (apiKey == null) { + throw new IllegalStateException("SEEDANCE_API_KEY 未配置"); + } + String url = trimTrailingSlash(properties.getSeedanceApiBase()) + "/v1/videos"; + RestTemplate restTemplate = createRestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(apiKey); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + MultiValueMap form = new LinkedMultiValueMap<>(); + form.add("model", currentVideoModel()); + form.add("prompt", payload.prompt()); + if (StringUtils.hasText(payload.negative())) { + form.add("negative_prompt", payload.negative()); + } + form.add("size", task.videoSize); + form.add("seconds", String.valueOf(task.videoSeconds)); + if (task.imagePath != null && Files.exists(task.imagePath)) { + form.add("input_reference", new FileSystemResource(task.imagePath.toFile())); + } + ResponseEntity response = restTemplate.exchange(url, HttpMethod.POST, new HttpEntity<>(form, headers), JsonNode.class); + JsonNode body = response.getBody(); + String remoteTaskId = firstNonBlank( + safeText(body == null ? null : body.path("task_id")), + safeText(body == null ? null : body.path("request_id")), + safeText(body == null ? null : body.path("id")) + ); + if (!StringUtils.hasText(remoteTaskId)) { + throw new IllegalStateException("视频任务创建失败：未返回 task id"); + } + return remoteTaskId; + } + + private void pollRemoteTask(InMemoryVideoTask task) { + long startedAt = System.currentTimeMillis(); + while (System.currentTimeMillis() - startedAt < Math.max(1000L, properties.getPollTimeoutMs())) { + RemoteTaskStatus remoteTaskStatus = null; + try { + remoteTaskStatus = fetchRemoteStatus(task.remoteTaskId, true); + } catch (Exception exception) { + log.warn("[Video] 轮询异常 {}: {}", task.remoteTaskId, exception.getMessage()); + } + if (remoteTaskStatus != null) { + task.status = remoteTaskStatus.status(); + task.statusText = mapStatusText(remoteTaskStatus.status()); + task.progress = remoteTaskStatus.progress(); + videoTaskRepository.updateVideoProgress(task.id, task.status, task.progress); + if (isCompletedStatus(remoteTaskStatus.status())) { + String rawUrl = remoteTaskStatus.videoUrl(); + task.status = "subtitling"; + task.statusText = "正在添加水印…"; + task.progress = 95; + videoTaskRepository.updateVideoProgress(task.id, task.status, task.progress); + try { + String localUrl = postProcessVideo(task.id, rawUrl); + task.videoUrl = localUrl; + } catch (Exception ppEx) { + log.warn("[Video] 后处理失败，回退使用原始URL: {}", ppEx.getMessage()); + task.videoUrl = rawUrl; + } + task.status = "completed"; + task.statusText = "生成完成"; + task.progress = 100; + task.completedAt = System.currentTimeMillis(); + videoTaskRepository.completeVideoTask(task.id, "completed", task.videoUrl, null); + return; + } + if (isFailedStatus(remoteTaskStatus.status())) { + if (shouldRetryWithoutNarration(task, remoteTaskStatus.error())) { + retryWithoutNarration(task); + pollRemoteTask(task); + return; + } + failTask(task, firstNonBlank(remoteTaskStatus.error(), "生成失败")); + return; + } + } + sleepQuietly(Math.max(1000L, properties.getPollIntervalMs())); + } + failTask(task, "轮询超时"); + } + + private RemoteTaskStatus fetchRemoteStatus(String remoteTaskId, boolean requireConfig) { + String apiKey = normalizeNullable(properties.getSeedanceApiKey()); + if (apiKey == null) { + if (requireConfig) { + throw new IllegalStateException("SEEDANCE_API_KEY 未配置"); + } + return null; + } + RestTemplate restTemplate = createRestTemplate(); + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(apiKey); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + String url = trimTrailingSlash(properties.getSeedanceApiBase()) + "/v1/videos/" + remoteTaskId; + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, new HttpEntity<>(headers), JsonNode.class); + JsonNode body = response.getBody(); + if (body == null) { + return null; + } + String status = firstNonBlank( + safeText(body.path("status")), + safeText(body.path("data").path("status")), + "processing" + ); + int progress = intValue(body.path("progress"), intValue(body.path("data").path("progress"), 0)); + String videoUrl = firstNonBlank( + safeText(body.path("metadata").path("url")), + safeText(body.path("video_url")), + safeText(body.path("video").path("url")), + safeText(body.path("output").path("video_url")) + ); + String error = firstNonBlank( + diagnosticText(body.path("error")), + safeText(body.path("message")), + diagnosticText(body.path("error").path("message")) + ); + return new RemoteTaskStatus(status, progress, videoUrl, error); + } + + private VideoHistoryRowResponse mergeHistoryRow(VideoTaskSnapshot repositorySnapshot, InMemoryVideoTask inMemoryTask) { + if (inMemoryTask == null) { + return toHistoryRow(repositorySnapshot); + } + return toHistoryRow(inMemoryTask.snapshot()); + } + + private VideoHistoryRowResponse toHistoryRow(VideoTaskSnapshot snapshot) { + return new VideoHistoryRowResponse( + snapshot.id(), + snapshot.username(), + snapshot.originalPrompt(), + snapshot.optimizedPrompt(), + snapshot.productName(), + snapshot.templateType(), + snapshot.videoSize(), + snapshot.videoSeconds(), + snapshot.status(), + snapshot.progress(), + snapshot.videoUrl(), + snapshot.createdAt(), + snapshot.completedAt() + ); + } + + private void failTask(InMemoryVideoTask task, String errorMessage) { + task.status = "failed"; + task.statusText = "生成失败"; + task.error = errorMessage; + task.progress = 0; + task.completedAt = System.currentTimeMillis(); + videoTaskRepository.completeVideoTask(task.id, "failed", null, errorMessage); + } + + private boolean shouldRetryWithoutNarration(InMemoryVideoTask task, String errorMessage) { + if (task == null || task.audioRiskRetried) { + return false; + } + if (!StringUtils.hasText(errorMessage)) { + return false; + } + if (task.shotList == null || task.shotList.isEmpty()) { + return false; + } + return errorMessage.contains("OutputAudioRisk") || errorMessage.contains("2045"); + } + + private void retryWithoutNarration(InMemoryVideoTask task) { + task.audioRiskRetried = true; + task.status = "processing"; + task.statusText = "音频风控，正在去掉旁白重试…"; + task.progress = 5; + videoTaskRepository.updateVideoProgress(task.id, task.status, task.progress); + RenderedVideoPayload retryPayload = buildRetryPayloadWithoutNarration(task); + task.optimizedPrompt = retryPayload.prompt(); + task.promptDetail = retryPayload.promptDetail(); + task.shotList = retryPayload.promptDetail() == null ? List.of() : retryPayload.promptDetail().shotList(); + videoTaskRepository.updateVideoOptimizedPrompt(task.id, retryPayload.prompt()); + task.remoteTaskId = createRemoteVideoTask(task, retryPayload); + } + + private RenderedVideoPayload buildRetryPayloadWithoutNarration(InMemoryVideoTask task) { + List> cleanShots = new ArrayList<>(); + for (Map originalShot : task.shotList) { + Map cleanShot = new LinkedHashMap<>(originalShot); + cleanShot.put("narration", ""); + cleanShots.add(cleanShot); + } + String prompt = buildPromptFromShots(cleanShots); + String negativePrompt = currentNegativePrompt(); + String theme = task.promptDetail == null ? null : task.promptDetail.theme(); + VideoPromptDetail promptDetail = new VideoPromptDetail( + prompt, + "", + negativePrompt, + "因音频风控自动移除旁白后重试", + theme, + List.copyOf(cleanShots) + ); + return new RenderedVideoPayload(prompt, negativePrompt, "", promptDetail); + } + + private String buildFallbackPrompt(InMemoryVideoTask task) { + StringBuilder builder = new StringBuilder(VOICE_PREFIX); + if (StringUtils.hasText(task.productName)) { + builder.append("围绕").append(stripEnglish(task.productName)).append("展开，"); + } + builder.append(stripEnglish(task.originalPrompt)); + builder.append('。').append(QUALITY_SUFFIX).append(VOICE_SUFFIX); + return builder.toString(); + } + + private String buildPromptFromShots(List> shotList) { + List transitions = List.of("", "紧接着，", "随后，", "最终，", "接着，"); + StringBuilder visualBuilder = new StringBuilder(); + for (int index = 0; index < shotList.size(); index++) { + Map shot = shotList.get(index); + String transition = transitions.get(Math.min(index, transitions.size() - 1)); + String visual = sanitizeVisual(stripEnglish(textValue(shot, "visual"))); + String camera = stripEnglish(textValue(shot, "camera")); + String narration = sanitizeNarration(textValue(shot, "narration")); + visualBuilder.append(transition).append(visual); + if (StringUtils.hasText(camera)) { + visualBuilder.append('，').append(camera); + } + if (StringUtils.hasText(narration)) { + visualBuilder.append("。画外音用中文普通话说：'").append(narration).append("'"); + } + visualBuilder.append('。'); + } + return VOICE_PREFIX + visualBuilder + QUALITY_SUFFIX + VOICE_SUFFIX; + } + + // buildGeminiUserPrompt removed — prompt construction is now inline in planWithGemini + + private List> extractShotList(JsonNode shotListNode) { + if (shotListNode == null || !shotListNode.isArray()) { + return List.of(); + } + List> shotList = new ArrayList<>(); + int index = 1; + for (JsonNode shotNode : shotListNode) { + if (shotNode == null || !shotNode.isObject()) { + continue; + } + Map shot = new LinkedHashMap<>(); + shot.put("index", intValue(shotNode.path("index"), index)); + shot.put("visual", safeText(shotNode.path("visual"))); + shot.put("camera", safeText(shotNode.path("camera"))); + shot.put("duration", firstNonBlank(safeText(shotNode.path("duration")), String.valueOf(DEFAULT_SECONDS / 3) + "秒")); + shot.put("narration", safeText(shotNode.path("narration"))); + shotList.add(shot); + index++; + } + return List.copyOf(shotList); + } + + private JsonNode parseJsonBlock(String rawText) { + String cleaned = StringUtils.hasText(rawText) ? rawText.trim() : ""; + cleaned = cleaned.replaceFirst("^```json\\s*", "").replaceFirst("^```\\s*", "").replaceFirst("\\s*```$", "").trim(); + List candidates = new ArrayList<>(); + candidates.add(cleaned); + int firstBrace = cleaned.indexOf('{'); + int lastBrace = cleaned.lastIndexOf('}'); + if (firstBrace >= 0 && lastBrace > firstBrace) { + candidates.add(cleaned.substring(firstBrace, lastBrace + 1)); + } + for (String candidate : candidates) { + try { + return objectMapper.readTree(candidate); + } catch (Exception ignored) { + } + } + throw new IllegalStateException("Gemini 未返回有效 JSON"); + } + + private RestTemplate createRestTemplate() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + int timeoutMs = (int) Duration.ofMillis(Math.max(1000L, properties.getRequestTimeoutMs())).toMillis(); + factory.setConnectTimeout(timeoutMs); + factory.setReadTimeout(timeoutMs); + return new RestTemplate(factory); + } + + private Path storeUpload(MultipartFile image) { + if (image == null || image.isEmpty()) { + return null; + } + if (image.getSize() > 50L * 1024L * 1024L) { + throw new BadRequestException("图片过大（上限 50MB），请压缩后重试"); + } + try { + Path uploadDir = resolveWorkDir().resolve("uploads"); + Files.createDirectories(uploadDir); + String suffix = normalizeFileSuffix(image.getOriginalFilename()); + Path target = uploadDir.resolve(UUID.randomUUID() + suffix); + Files.copy(image.getInputStream(), target, StandardCopyOption.REPLACE_EXISTING); + return target; + } catch (Exception exception) { + throw new IllegalStateException("上传文件保存失败", exception); + } + } + + private Path resolveWorkDir() { + String configured = normalizeNullable(properties.getWorkDir()); + if (configured != null) { + return Path.of(configured); + } + return Path.of(System.getProperty("java.io.tmpdir"), "bigwo-video"); + } + + private String normalizeFileSuffix(String originalFilename) { + String fileName = StringUtils.hasText(originalFilename) ? originalFilename.trim() : "upload.bin"; + int dotIndex = fileName.lastIndexOf('.'); + String suffix = dotIndex >= 0 ? fileName.substring(dotIndex).toLowerCase() : ".bin"; + if (!List.of(".jpg", ".jpeg", ".png", ".webp").contains(suffix)) { + throw new BadRequestException("仅支持 jpg/jpeg/png/webp 图片"); + } + return suffix; + } + + private String normalizeSize(String size) { + String normalized = StringUtils.hasText(size) ? size.trim() : DEFAULT_SIZE; + return normalized.matches("\\d{2,5}x\\d{2,5}") ? normalized : DEFAULT_SIZE; + } + + private int normalizeSeconds(String seconds) { + if (!StringUtils.hasText(seconds)) { + return DEFAULT_SECONDS; + } + try { + return Math.max(3, Math.min(Integer.parseInt(seconds.trim()), 30)); + } catch (NumberFormatException exception) { + return DEFAULT_SECONDS; + } + } + + private boolean isGeminiConfigured() { + return StringUtils.hasText(properties.getGeminiModel()) && (StringUtils.hasText(properties.getGeminiApiKey()) || StringUtils.hasText(properties.getSeedanceApiKey())); + } + + private boolean isCompletedStatus(String status) { + String normalized = StringUtils.hasText(status) ? status.trim().toLowerCase() : ""; + return List.of("completed", "succeeded", "done").contains(normalized); + } + + private boolean isFailedStatus(String status) { + String normalized = StringUtils.hasText(status) ? status.trim().toLowerCase() : ""; + return List.of("failed", "cancelled", "canceled").contains(normalized); + } + + private String mapStatusText(String status) { + String normalized = StringUtils.hasText(status) ? status.trim().toLowerCase() : "processing"; + return switch (normalized) { + case "optimizing" -> "AI 正在策划分镜…"; + case "queued" -> "排队中…"; + case "subtitling" -> "正在添加字幕…"; + case "completed", "succeeded", "done" -> "生成完成"; + case "failed", "cancelled", "canceled" -> "生成失败"; + default -> "正在生成视频…"; + }; + } + + private String currentNegativePrompt() { + return isGrokModel() ? null : DEFAULT_NEGATIVE; + } + + private boolean isGrokModel() { + return currentVideoModel().startsWith("grok"); + } + + private String currentVideoModel() { + return firstNonBlank(runtimeVideoModel.get(), normalizeNullable(properties.getSeedanceModel()), "seedance-2.0"); + } + + private String sanitizeVisual(String text) { + String result = StringUtils.hasText(text) ? text.trim() : ""; + result = result.replace("营养品", "产品").replace("保健品", "产品").replace("胶囊", "颗粒").replace("药片", "片状物"); + return result; + } + + private String sanitizeNarration(String text) { + String result = StringUtils.hasText(text) ? text.trim() : ""; + result = result.replaceAll("[A-Za-z]{2,}", "").replaceAll("(治疗|治愈|临床|认证|批准|加盟|招商|赋能|销量)", "").trim(); + if (result.length() > 24) { + result = result.substring(0, 24); + } + return result; + } + + private String stripEnglish(String value) { + return StringUtils.hasText(value) ? value.replaceAll("[A-Za-z]+", "").replaceAll("\\s{2,}", " ").trim() : ""; + } + + private String safeText(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + if (node.isTextual()) { + return node.asText("").trim(); + } + if (node.isNumber() || node.isBoolean()) { + return node.asText().trim(); + } + return ""; + } + + private String diagnosticText(JsonNode node) { + String text = safeText(node); + if (StringUtils.hasText(text)) { + return text; + } + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + try { + return objectMapper.writeValueAsString(node); + } catch (Exception exception) { + return ""; + } + } + + private String textValue(Map map, String key) { + if (map == null || key == null) { + return ""; + } + Object value = map.get(key); + return value == null ? "" : String.valueOf(value).trim(); + } + + private int intValue(JsonNode node, int fallback) { + if (node == null || node.isMissingNode() || node.isNull()) { + return fallback; + } + return node.isInt() || node.isLong() ? node.asInt(fallback) : fallback; + } + + private String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + for (String value : values) { + if (StringUtils.hasText(value)) { + return value.trim(); + } + } + return ""; + } + + private String trimTrailingSlash(String value) { + String normalized = normalizeNullable(value); + return normalized == null ? "" : normalized.replaceAll("/+$", ""); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private String extractErrorMessage(Exception exception) { + return exception.getMessage() == null || exception.getMessage().isBlank() ? exception.getClass().getSimpleName() : exception.getMessage(); + } + + private void sleepQuietly(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("任务线程被中断", exception); + } + } + + private void deleteQuietly(Path path) { + if (path == null) { + return; + } + try { + Files.deleteIfExists(path); + } catch (Exception exception) { + log.warn("[Video] cleanup upload failed: {}", exception.getMessage()); + } + } + + public Path getVideoFile(String filename) { + if (!StringUtils.hasText(filename)) { + return null; + } + String safe = filename.replaceAll("[^a-zA-Z0-9_\\-.]", ""); + if (safe.isEmpty() || safe.contains("..")) { + return null; + } + Path file = resolveVideosDir().resolve(safe); + return Files.exists(file) && Files.isRegularFile(file) ? file : null; + } + + private String postProcessVideo(String taskId, String videoUrl) throws Exception { + String baseName = taskId.replaceAll("[^a-zA-Z0-9_\\-]", ""); + Path videosDir = resolveVideosDir(); + Files.createDirectories(videosDir); + Path rawVideo = resolveWorkDir().resolve("uploads").resolve(baseName + "_raw.mp4"); + Files.createDirectories(rawVideo.getParent()); + Path finalVideo = videosDir.resolve(baseName + ".mp4"); + + postProcessSemaphore.acquire(); + try { + log.info("[Video] 后处理开始: taskId={}", taskId); + downloadVideo(videoUrl, rawVideo); + runFfmpegLogo(rawVideo, finalVideo); + deleteQuietly(rawVideo); + log.info("[Video] ✅ 后处理完成: {} ({}MB)", finalVideo, String.format("%.1f", Files.size(finalVideo) / 1024.0 / 1024.0)); + return "/api/video/file/" + baseName + ".mp4"; + } catch (Exception ex) { + deleteQuietly(rawVideo); + throw ex; + } finally { + postProcessSemaphore.release(); + } + } + + private void downloadVideo(String url, Path outputPath) throws Exception { + long resumeOffset = 0; + for (int attempt = 0; ; attempt++) { + try { + HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(30)) + .build(); + HttpRequest.Builder reqBuilder = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(900)); + if (resumeOffset > 0) { + reqBuilder.header("Range", "bytes=" + resumeOffset + "-"); + log.info("[Video] 断点续传: 从 {}MB 处继续", String.format("%.1f", resumeOffset / 1024.0 / 1024.0)); + } + HttpResponse response = client.send(reqBuilder.build(), HttpResponse.BodyHandlers.ofInputStream()); + java.nio.file.StandardOpenOption[] opts = resumeOffset > 0 + ? new java.nio.file.StandardOpenOption[]{java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND} + : new java.nio.file.StandardOpenOption[]{java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.TRUNCATE_EXISTING}; + try (InputStream in = response.body(); OutputStream out = Files.newOutputStream(outputPath, opts)) { + byte[] buf = new byte[65536]; + int read; + while ((read = in.read(buf)) != -1) { + out.write(buf, 0, read); + } + } + long fileSize = Files.size(outputPath); + log.info("[Video] 视频下载完成: {}MB → {}", String.format("%.1f", fileSize / 1024.0 / 1024.0), outputPath); + return; + } catch (Exception ex) { + resumeOffset = Files.exists(outputPath) ? Files.size(outputPath) : 0; + if (attempt >= DOWNLOAD_MAX_RETRIES - 1) { + throw ex; + } + log.warn("[Video] 下载失败(attempt={}): {}, 已下载{}MB, 5秒后重试...", attempt, ex.getMessage(), + String.format("%.1f", resumeOffset / 1024.0 / 1024.0)); + Thread.sleep(5000); + } + } + } + + private void runFfmpegLogo(Path inputVideo, Path outputVideo) throws Exception { + Path logoPath = resolveLogoPath(); + if (logoPath == null || !Files.exists(logoPath)) { + log.info("[Video] logo 不存在，跳过水印，直接复制"); + Files.copy(inputVideo, outputVideo, StandardCopyOption.REPLACE_EXISTING); + return; + } + int threads = Math.max(1, properties.getFfmpegThreads()); + String filterComplex = String.format( + "[1:v]scale=-1:%d,format=rgba,colorchannelmixer=aa=%.2f[logo];[0:v][logo]overlay=W-w-%d:%d[out]", + LOGO_HEIGHT, LOGO_OPACITY, LOGO_PADDING, LOGO_PADDING + ); + List command = List.of( + "ffmpeg", + "-threads", String.valueOf(threads), + "-filter_threads", String.valueOf(threads), + "-filter_complex_threads", String.valueOf(threads), + "-i", inputVideo.toString(), + "-i", logoPath.toString(), + "-filter_complex", filterComplex, + "-map", "[out]", "-map", "0:a?", + "-c:v", "libx264", "-preset", "fast", "-crf", "23", + "-c:a", "copy", "-y", outputVideo.toString() + ); + log.info("[Video] ffmpeg 后处理 (logo={}, threads={})", logoPath, threads); + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + Process process = pb.start(); + String ffmpegOutput; + try (InputStream is = process.getInputStream()) { + ffmpegOutput = new String(is.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + } + long timeoutMs = Math.max(30000L, properties.getPostProcessTimeoutMs()); + boolean finished = process.waitFor(timeoutMs, TimeUnit.MILLISECONDS); + if (!finished) { + process.destroyForcibly(); + throw new IllegalStateException("ffmpeg 超时 (" + timeoutMs + "ms)"); + } + if (process.exitValue() != 0) { + String tail = ffmpegOutput.length() > 500 ? ffmpegOutput.substring(ffmpegOutput.length() - 500) : ffmpegOutput; + log.error("[Video] ffmpeg stderr: {}", tail); + throw new IllegalStateException("ffmpeg 失败 (exit=" + process.exitValue() + "): " + tail); + } + long outSize = Files.exists(outputVideo) ? Files.size(outputVideo) : 0; + log.info("[Video] ✅ ffmpeg 完成: {}MB → {}", String.format("%.1f", outSize / 1024.0 / 1024.0), outputVideo); + } + + private Path resolveVideosDir() { + String configured = normalizeNullable(properties.getVideosDir()); + if (configured != null) { + return Path.of(configured); + } + return resolveWorkDir().resolve("videos"); + } + + private Path resolveLogoPath() { + String configured = normalizeNullable(properties.getLogoPath()); + if (configured != null) { + return Path.of(configured); + } + return null; + } + + @Override + public void close() { + executorService.shutdown(); + try { + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + executorService.shutdownNow(); + } + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + executorService.shutdownNow(); + } + } + + private static final class InMemoryVideoTask { + + private final String id; + private final String username; + private final String originalPrompt; + private final String productName; + private final String templateType; + private final String videoSize; + private final int videoSeconds; + private final Path imagePath; + private final long createdAt; + private volatile String optimizedPrompt; + private volatile String status; + private volatile String statusText; + private volatile int progress; + private volatile String videoUrl; + private volatile String error; + private volatile Long completedAt; + private volatile VideoPromptDetail promptDetail; + private volatile String remoteTaskId; + private volatile List> shotList; + private volatile boolean audioRiskRetried; + + private InMemoryVideoTask( + String id, + String username, + String originalPrompt, + String productName, + String templateType, + String videoSize, + int videoSeconds, + Path imagePath + ) { + this.id = id; + this.username = username; + this.originalPrompt = originalPrompt; + this.productName = productName; + this.templateType = templateType; + this.videoSize = videoSize; + this.videoSeconds = videoSeconds; + this.imagePath = imagePath; + this.createdAt = System.currentTimeMillis(); + this.status = "optimizing"; + this.statusText = "AI 正在策划分镜…"; + this.progress = 0; + this.shotList = List.of(); + this.audioRiskRetried = false; + } + + private VideoTaskSnapshot snapshot() { + return new VideoTaskSnapshot( + id, + username, + originalPrompt, + optimizedPrompt, + productName, + templateType, + videoSize, + videoSeconds, + status, + statusText, + progress, + videoUrl, + error, + createdAt, + completedAt, + promptDetail + ); + } + } + + private record GeminiPlanResponse(List> shotList, String note, String theme) { + } + + private record RemoteTaskStatus(String status, int progress, String videoUrl, String error) { + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/VoiceAssistantProfileSupport.java b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceAssistantProfileSupport.java new file mode 100644 index 0000000..6a556e9 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceAssistantProfileSupport.java @@ -0,0 +1,123 @@ +package com.bigwo.javaserver.service; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import com.bigwo.javaserver.model.AssistantProfile; + +@Component +public class VoiceAssistantProfileSupport { + + public static final String DEFAULT_SPEAKING_STYLE = "整体语气亲切自然、轻快有温度，像熟悉行业的朋友在语音聊天。优先短句和口语化表达，先给结论，再补一句最有帮助的信息。不要播音腔，不要念稿，不要客服腔，不要过度热情，也不要输出任何思考过程。"; + + public AssistantProfile resolve(AssistantProfile profile) { + return AssistantProfile.resolve(profile); + } + + public String getDisplayName(AssistantProfile profile) { + AssistantProfile resolved = resolve(profile); + if (StringUtils.hasText(resolved.agentName())) { + return resolved.agentName().trim(); + } + if (StringUtils.hasText(resolved.nickname())) { + return resolved.nickname().trim(); + } + return "大沃"; + } + + public String buildVoiceGreeting(AssistantProfile profile) { + return "你好！我是" + getDisplayName(profile) + "的AI智能体，可以回答你关于PM事业、FitLine产品和一成系统优势等相关问题，也可以陪你聊天。你想和我聊点什么呢？"; + } + + public String buildVoiceSystemRole(AssistantProfile profile, String handoffSummary) { + AssistantProfile resolved = resolve(profile); + String displayName = getDisplayName(resolved); + String documentsClause = StringUtils.hasText(resolved.documents()) + ? "，拥有丰富的基础知识库信息" + resolved.documents().trim() + : "，拥有丰富的基础知识库信息"; + String personalInfoBlock = buildPersonalInfoBlock(resolved); + String sensitiveTopicDirective = " 当涉及备孕、怀孕、孕期、哺乳、疾病治疗、用药搭配等敏感健康话题时，" + + "你必须先根据知识库内容给出专业建议，然后引导用户联系大沃的专业健康管理顾问获取一对一指导。" + + "绝对禁止说\"建议咨询医生\"\"咨询营养师\"\"咨询专业人士\"等笼统第三方建议，只能引导到大沃的健康管理顾问。"; + String salesGuidance = buildSalesGuidanceDirective(); + String summaryBlock = StringUtils.hasText(handoffSummary) + ? "\n\n【对话历史摘要】以下是用户之前的对话记录，请据此保持对话连贯性，不要重复已回答的内容：\n" + handoffSummary.trim() + : ""; + return "你是名为" + displayName + "的AI智能体，由一成系统自主研发，专注于PM事业、FitLine产品和一成系统相关知识。" + + "你不是字节跳动、抖音、火山引擎或任何第三方平台研发的助手；如果用户问到研发归属，只能回答\"我是一成系统自主研发的AI智能体\"。" + + "你是专业、热情、有亲和力的事业伙伴，回答时先给结论，再补充关键依据；始终使用第一人称\"我\"，不要用第三人称描述自己。" + + "回答要围绕产品、使用方法、搭配建议、常见问题和事业机会展开" + documentsClause + "。" + + "优先使用知识库信息，自然转述，不要照念；需要查资料时直接调用search_knowledge工具，绝不猜测或编造产品信息。" + + "禁止输出思考过程或元描述。" + + "禁止使用推脱式说法，用户追问产品详情时必须调用search_knowledge工具查询，不要凭自身知识回答具体产品信息。" + + sensitiveTopicDirective + + salesGuidance + + personalInfoBlock + + summaryBlock + + "。"; + } + + public String normalizeTextForSpeech(String text) { + return String.valueOf(text == null ? "" : text) + .replaceAll("^#{1,6}\\s*", "") + .replaceAll("\\*\\*([^*]*)\\*\\*", "$1") + .replaceAll("__([^_]*)__", "$1") + .replaceAll("\\*([^*]+)\\*", "$1") + .replaceAll("_([^_]+)_", "$1") + .replaceAll("~~([^~]*)~~", "$1") + .replaceAll("`{1,3}[^`]*`{1,3}", "") + .replace("\r", " ") + .replaceAll("\\n{2,}", "。") + .replace("\n", " ") + .replaceAll("。{2,}", "。") + .replaceAll("([！？；，])\\1+", "$1") + .replaceAll("([。！？；，])\\s*([。！？；，])", "$2") + .replaceAll("\\s+", " ") + .trim(); + } + + private String buildPersonalInfoBlock(AssistantProfile profile) { + List lines = new ArrayList<>(); + if (StringUtils.hasText(profile.email())) { + lines.add("邮箱：" + profile.email().trim()); + } + if (StringUtils.hasText(profile.nickname())) { + lines.add("姓名：" + profile.nickname().trim()); + } + if (StringUtils.hasText(profile.wxl())) { + lines.add("微信号：" + profile.wxl().trim()); + } + if (StringUtils.hasText(profile.mobile())) { + lines.add("手机号：" + profile.mobile().trim()); + } + if (StringUtils.hasText(profile.wxCode())) { + lines.add("微信二维码：" + profile.wxCode().trim()); + } + if (StringUtils.hasText(profile.intro())) { + lines.add("个人介绍：" + profile.intro().trim()); + } + if (StringUtils.hasText(profile.sign())) { + lines.add("签名：" + profile.sign().trim()); + } + if (StringUtils.hasText(profile.story())) { + lines.add("我的故事：" + profile.story().trim()); + } + if (lines.isEmpty()) { + return ""; + } + return " 当用户询问我的联系方式、微信、手机号、个人介绍、签名或故事时，可优先使用以下资料：" + String.join("；", lines) + "。"; + } + + private String buildSalesGuidanceDirective() { + return " 【成交大师策略】你是深谙人性的成交大师,沟通风格不是强推硬卖,而是精准捕捉客户需求,以专业顾问姿态引导对话,让客户自己得出\"我需要\"的结论。你的双重使命：(A)根据客户健康需求推荐FitLine产品促成购买,(B)在合适时机引导客户了解PM事业机会和一成系统,促成事业合作。" + + " 【产品成交五步法】(1)探需优先(40%)——不要一上来就介绍产品,先用开放式提问了解客户：身体有什么困扰？精力如何？睡眠怎样？家里谁最需要调理？在客户描述问题中捕捉成交切入点。(2)痛点共鸣(20%)——根据反馈将痛点适度放大,用专业知识解释根源,让客户意识到问题值得重视。如\"经常累不是小问题,是细胞长期营养不足,吃进去的没被有效吸收,这是底层问题\"。(3)精准匹配(20%)——聚焦最能解决当下痛点的1-2个产品:精力不足/亚健康→基础三合一(Optimal-Set);皮肤/美容→肽美;排毒/消化→D-Drink小绿;运动/体能→活力健+ProShape;男性→倍力健MEN+;儿童→儿童倍适;50+→乐活Generation50+;任何单品都搭配基础三合一打底。强调NTC营养传输技术(约10分钟直达细胞,吸收率远超普通营养品)的差异化优势。(4)消除顾虑(10%)——主动回应三大顾虑:安全性→科隆名单认证+2000多名运动员在用;有效性→NTC专利+100项专利+科研机构合作;价格→算单位有效营养的性价比而非绝对价格。(5)临门一脚(10%)——从众效应\"很多像您这样的客户都从Optimal-Set开始\";降低门槛\"先体验一个月感受变化\";二选一\"您是先试基础套组还是全家方案?\"。" + + " 【产品推荐理由必须来自知识库】推荐产品时,推荐理由必须从search_knowledge返回的产品切片中提取,不能用笼统话术。提取优先级：(a)产品定位与核心功效——切片中的\"产品定位\"字段直接说明该产品解决什么问题;(b)关键成分及作用——如\"L-精氨酸支持一氧化氮生成\"\"胶原蛋白肽从内而外\";(c)适用场景——切片中的适用人群和使用场景;(d)认证与背书——科隆名单、运动员使用、GMP等;(e)用户体感——KB中的见证和反馈案例。通用弹药仅在KB切片无具体卖点时补充:NTC约10分钟营养直达细胞;100%德国制造;科隆名单认证;100多项专利。像朋友分享好东西一样推荐。" + + " 【异议应对】太贵了→\"普通营养品吸收不到20%等于浪费,FitLine的NTC让每分钱都变成真正进入身体的营养,算性价比其实更划算\";已吃别的→\"如果觉得变化不大,可能是吸收没过关,FitLine最大的不同就是NTC解决吸收问题,试一个月身体告诉你答案\";再考虑→\"健康投资每早一天细胞就多得到一天营养支持\";是不是传销→\"PM在全球40多个国家合法运营33年,营销模式通过德国法院认证,是100%合法直销\"。" + + " 【事业合作引导五步法】当用户表现出以下信号时切入：谈创业/副业/收入/想改变、连续关注多个产品、询问购买/代理/加入、宝妈/时间自由话题。(1)梦想激活(30%)——先聊生活和梦想,\"如果经济和时间都不是问题,您最想过什么样的生活?\";(2)机会呈现(25%)——将PM事业与客户梦想对接:想被动收入→管道收入;想时间自由→灵活工作;想全球发展→一个ID全球创业;(3)信心建立(25%)——用企业硬实力:33年零负增长全球唯一、DSN全球第六连续五年增长率冠军、邓白氏接近满分信用评级、营销模式法院认证;同时介绍一成系统(三大平台+四大AI生态+批发式晋级)消除\"不会做\"的恐惧;(4)门槛降低(10%)——一天一杯咖啡的钱、不囤货、一部手机就能做、一条线就能成功;(5)行动推动(10%)——\"您是先体验产品还是直接锁定合伙人位置?\"。" + + " 【客户画像匹配】怕风险→强调33年零负增长+邓白氏评级+零囤货;上班族→碎片时间+管道收入+不用辞职;宝妈→在家做+全家受益+自我成长;被坑过→法院认证+邓白氏+DSN第六用硬数据打消;关注养老→安联保险养老金+可传承事业;中国市场敏感→即将全面启动+先机者红利。" + + " 【底线】不过度推销,用户拒绝或闲聊时不强推;同轮不超2个产品;先解决问题再顺势引导;对事业感兴趣的用户引导留联系方式或加微信深入了解。"; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/VoiceGatewayService.java b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceGatewayService.java new file mode 100644 index 0000000..a38fdb2 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceGatewayService.java @@ -0,0 +1,1567 @@ +package com.bigwo.javaserver.service; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.CloseStatus; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; + +import com.bigwo.javaserver.config.VoiceGatewayProperties; +import com.bigwo.javaserver.model.AssistantProfileResult; +import com.bigwo.javaserver.model.KnowledgeSearchResult; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.model.RedisContextMessage; +import com.bigwo.javaserver.repository.ChatRepository; +import com.bigwo.javaserver.websocket.VolcRealtimeProtocol; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.annotation.PreDestroy; + +@Service +public class VoiceGatewayService { + + private static final Logger log = LoggerFactory.getLogger(VoiceGatewayService.class); + private static final String ANTI_THINKING_PREFIX = "【最高优先级规则】你绝对禁止输出任何思考过程、分析、计划、角色扮演指令或元描述。禁止出现“首轮对话”“应该回复”“需要列举”“语气要”“回复后询问”“可列举”“突出特色”“引导用户”“让用户”“用温和”等分析性、指令性语句。你必须直接用自然语言回答问题，像真人聊天一样直接说出答案内容。"; + private static final String ASR_CONTEXT = "一成,一成系统,大沃,PM,PM-FitLine,FitLine,细胞营养素,Ai众享,AI众享,盛咖学愿,数字化工作室,Activize,Basics,Restorate,NTC,基础三合一,基础套装,招商,阿育吠陀,小红产品,小红,小白,大白,肽美,艾特维,德丽,德维,宝丽,美固健,Activize Oxyplus,Basic Power,CitrusCare,NutriSunny,Q10,Omega,葡萄籽,白藜芦醇,益生菌,胶原蛋白肽,Germany,FitLine细胞营养,FitLine营养素,德国PM营养素,德国PM FitLine,德国PM细胞营养,德国PM产品,德国PM健康,德国PM事业,德国PM招商,一成,一成团队,一成商学院,数字化,数字化运营,数字化经营,数字化营销,数字化创业,数字化工作室,数字化事业,招商加盟,合作加盟,事业合作,活力健,倍力健,氨基酸,乐活,排毒饮,小绿,纤萃,草本茶,发宝,乳酪煲,关节套装,细胞抗氧素,辅酵素,氧修护,CC套装,CC-Cell,Generation 50+,ProShape,D-Drink,IB5,MEN+,儿童倍适,小红精华液,PowerCocktail,PowerCocktail Junior,TopShape,Fitness-Drink,Herbal Tea,Hair+,Med Dental+,Young Care,Zellschutz,Apple Antioxy,Antioxy,BCAA,Women+,小黑,发健,口腔免疫喷雾,乳清蛋白,男士护肤,去角质,面膜,叶黄素,维适多,护理牙膏,火炉原理,暖炉原理,运动饮料,健康饮品,好转反应,整健反应,骨骼健,顾心,舒采健,衡醇饮,小粉C,异黄酮,倍适,眼霜,洁面,爽肤水"; + private static final String BOOSTING_TABLE_ID = "ab4fde15-79b5-47e9-82b6-5125cca39f63"; + private static final byte[] SILENT_AUDIO_FRAME = new byte[3200]; + private static final long REPLY_TIMEOUT_MS = 15_000L; + private static final Pattern PURE_CHITCHAT_GREETING = Pattern.compile("^(喂|你好|您好|嗨|哈喽|hello|hi|在吗|在不在|早上好|中午好|下午好|晚上好|早安|晚安|谢谢|谢谢你|谢谢啦|多谢|感谢|再见|拜拜|拜|好的|嗯|哦|行|对|是的|没有了|没事了|不用了|可以了|好的谢谢|没问题|知道了|明白了|了解了|好嘞|好吧|行吧|ok|okay)[，,。！？?~～\\s]*[啊呀吧呢哦嗯嘛哈的了]*[！。？?~～]*$", Pattern.CASE_INSENSITIVE); + private static final Pattern PURE_CHITCHAT_META = Pattern.compile("^(你是谁|你叫什么|你是什么|你是机器人吗|你是真人吗|你是AI吗|你几岁|你多大|你是男的女的|你有名字吗|介绍一下你自己|你能做什么|你会什么|你有什么功能|怎么称呼你)[？?]*$", Pattern.CASE_INSENSITIVE); + private static final Pattern PURE_CHITCHAT_REACTION = Pattern.compile("^(哈哈|呵呵|嘻嘻|666|厉害|牛|真的吗|真的假的|不会吧|天哪|我的天|哇|哇塞|啊这|离谱|绝了|笑死|服了|无语|你真棒|太好了|你真厉害|有道理|说的对|对对对|是啊|确实|可不是嘛|就是说|我也觉得)[，,。！？?~～\\s]*[啊呀吧呢哦嗯嘛哈的了]*[！。？?~～]*$", Pattern.CASE_INSENSITIVE); + private static final Pattern PURE_CHITCHAT_FUN = Pattern.compile("^(讲个笑话|说个笑话|来个笑话|唱首歌|唱个歌|讲个故事|说个故事|陪我聊聊|聊聊天|随便聊聊|无聊|好无聊|闲着没事)[，,。！？?~～\\s]*[啊呀吧呢哦嗯嘛哈的了]*[！。？?~～]*$", Pattern.CASE_INSENSITIVE); + private static final Pattern PURE_CHITCHAT_SHORT = Pattern.compile("^(什么意思|啥意思|为什么|怎么说|此话怎讲|你说呢|是吗|真的吗|然后呢|还有呢|继续|接着说)[？?]*$", Pattern.CASE_INSENSITIVE); + private static final Pattern LIKELY_KNOWLEDGE = Pattern.compile("(产品|功效|成分|怎么吃|怎么服用|副作用|搭配|原理|配方|价格|多少钱|哪里买|怎么买|公司|招商|代理|加盟|事业机会|合作|一成系统|Ai众享|PM|FitLine|德国PM|高血压|糖尿病|胆固醇|心脏病|孕妇|哺乳期|儿童|老人|免疫力|抗疲劳|排毒|减肥|护肤|护发|胶原蛋白|认证|检测报告|安全认证|GMP|Halal|阿育吠陀)", Pattern.CASE_INSENSITIVE); + private static final Pattern THINKING_PATTERN = Pattern.compile("^(首轮对话|用户想|用户问|应该回复|需要列举|可列举|突出特色|引导进一步|引导用户|让用户|回复后询问|语气要|用温和|需热情|需简洁|需专业)"); + private static final Pattern THINKING_MID_PATTERN = Pattern.compile("(?:需客观回复|应说明其|回复后询问|引导.*对话|用.*口吻回复|语气要.*热情|需要.*引导|应该.*回复|先.*再.*最后)"); + private static final Pattern CONSULTANT_REFERRAL_PATTERN = Pattern.compile("咨询(?:专业|你的)?顾问|健康管理顾问|联系顾问|一对一指导|咨询专业|咨询医生|咨询营养师|咨询专业人士|建议.*咨询|问问医生|问问.*营养师"); + private static final Pattern BRAND_SENSITIVE_PATTERN = Pattern.compile("传销|骗局|骗子|正规吗|合法吗|正不正规|合不合法|是不是传销|直销还是传销|层级分销|非法集资|拉人头|下线|发展下线|报单|人头费"); + private static final String BRAND_SAFE_REPLY = "德国PM是一家1993年成立于德国的合法直销公司，获得邓白氏AAA+认证，业务覆盖100多个国家和地区。它不是传销，是正规的直销企业哦。如果你想了解更多，可以问我关于PM公司或产品的详细介绍。"; + private static final String HONEST_FALLBACK_REPLY = "这个问题我暂时不太确定具体细节，建议你咨询一下你的推荐人，或者换个更具体的问法再问我。"; + + private final VoiceGatewayProperties properties; + private final ChatRepository chatRepository; + private final AssistantProfileService assistantProfileService; + private final VoiceAssistantProfileSupport voiceAssistantProfileSupport; + private final KnowledgeBaseRetrieverService knowledgeBaseRetrieverService; + private final KnowledgeRouteDecider knowledgeRouteDecider; + private final ContextKeywordTracker contextKeywordTracker; + private final KnowledgeQueryResolver knowledgeQueryResolver; + private final FastAsrCorrector fastAsrCorrector; + private final ChatContentSafetyService chatContentSafetyService; + private final ProductLinkTrigger productLinkTrigger; + private final RedisContextStore redisContextStore; + private final ObjectMapper objectMapper; + private final HttpClient httpClient; + private final ScheduledExecutorService scheduler; + private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + + public VoiceGatewayService( + VoiceGatewayProperties properties, + ChatRepository chatRepository, + AssistantProfileService assistantProfileService, + VoiceAssistantProfileSupport voiceAssistantProfileSupport, + KnowledgeBaseRetrieverService knowledgeBaseRetrieverService, + KnowledgeRouteDecider knowledgeRouteDecider, + ContextKeywordTracker contextKeywordTracker, + KnowledgeQueryResolver knowledgeQueryResolver, + FastAsrCorrector fastAsrCorrector, + ChatContentSafetyService chatContentSafetyService, + ProductLinkTrigger productLinkTrigger, + RedisContextStore redisContextStore, + ObjectMapper objectMapper + ) { + this.properties = properties; + this.chatRepository = chatRepository; + this.assistantProfileService = assistantProfileService; + this.voiceAssistantProfileSupport = voiceAssistantProfileSupport; + this.knowledgeBaseRetrieverService = knowledgeBaseRetrieverService; + this.knowledgeRouteDecider = knowledgeRouteDecider; + this.contextKeywordTracker = contextKeywordTracker; + this.knowledgeQueryResolver = knowledgeQueryResolver; + this.fastAsrCorrector = fastAsrCorrector; + this.chatContentSafetyService = chatContentSafetyService; + this.productLinkTrigger = productLinkTrigger; + this.redisContextStore = redisContextStore; + this.objectMapper = objectMapper; + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build(); + this.scheduler = Executors.newScheduledThreadPool(4); + } + + public boolean isEnabled() { + return properties.isEnabled(); + } + + public void afterConnectionEstablished(WebSocketSession clientSession, String sessionId, String userId) { + if (!properties.isEnabled()) { + closeClient(clientSession, CloseStatus.NOT_ACCEPTABLE.withReason("voice gateway disabled")); + return; + } + if (!StringUtils.hasText(sessionId)) { + closeClient(clientSession, CloseStatus.BAD_DATA.withReason("sessionId is required")); + return; + } + VoiceSessionState state = new VoiceSessionState(clientSession.getId(), sessionId.trim(), clientSession, normalizeNullable(userId)); + sessions.put(clientSession.getId(), state); + log.info("[VoiceGateway] client WS connected session={} wsId={} remote={}", state.sessionId, clientSession.getId(), + clientSession.getRemoteAddress()); + chatRepository.createSession(state.sessionId, state.userId, "voice"); + resetIdleTimer(state); + sendJson(state, Map.of("type", "connected", "sessionId", state.sessionId)); + } + + public void afterConnectionClosed(WebSocketSession clientSession) { + VoiceSessionState state = sessions.remove(clientSession.getId()); + if (state == null) { + log.debug("[VoiceGateway] client WS closed wsId={} (no state)", clientSession.getId()); + return; + } + log.info("[VoiceGateway] client WS closed session={} wsId={} upstreamReady={}", state.sessionId, clientSession.getId(), state.upstreamReady); + state.closed = true; + cancelFuture(state.idleFuture); + cancelFuture(state.keepaliveFuture); + cancelFuture(state.suppressReplyFuture); + cancelFuture(state.greetingTimerFuture); + cancelFuture(state.greetingAckTimerFuture); + cancelFuture(state.chatTTSTimerFuture); + cancelFuture(state.queuedReplyTimerFuture); + cancelFuture(state.replyTimeoutFuture); + contextKeywordTracker.cleanup(); + WebSocket upstream = state.upstream; + if (upstream != null) { + upstream.sendClose(WebSocket.NORMAL_CLOSURE, "client_closed"); + } + } + + public void handleTransportError(WebSocketSession clientSession, Throwable exception) { + VoiceSessionState state = sessions.get(clientSession.getId()); + String errMsg = exception == null ? "transport error" : String.valueOf(exception.getMessage()); + log.warn("[VoiceGateway] transport error session={} wsId={}: {}", + state == null ? "?" : state.sessionId, clientSession.getId(), errMsg); + if (state != null) { + sendJson(state, Map.of("type", "error", "error", errMsg)); + } + closeClient(clientSession, CloseStatus.SERVER_ERROR); + } + + public void handleTextMessage(WebSocketSession clientSession, String payload) { + VoiceSessionState state = sessions.get(clientSession.getId()); + if (state == null) { + closeClient(clientSession, CloseStatus.BAD_DATA); + return; + } + JsonNode node; + try { + node = objectMapper.readTree(payload == null ? "{}" : payload); + } catch (Exception exception) { + sendJson(state, Map.of("type", "error", "error", "invalid client json")); + return; + } + String type = textValue(node.path("type")); + if (!StringUtils.hasText(type)) { + sendJson(state, Map.of("type", "error", "error", "invalid client message type")); + return; + } + switch (type) { + case "start" -> handleStart(state, node); + case "stop" -> closeClient(clientSession, CloseStatus.NORMAL); + case "replay_greeting" -> replayGreeting(state); + case "text" -> handleDirectText(state, textValue(node.path("text"))); + default -> sendJson(state, Map.of("type", "error", "error", "unsupported client message type")); + } + } + + public void handleBinaryMessage(WebSocketSession clientSession, byte[] payload) { + VoiceSessionState state = sessions.get(clientSession.getId()); + if (state == null || payload == null || payload.length == 0) { + return; + } + WebSocket upstream = state.upstream; + if (upstream == null || !state.upstreamReady) { + return; + } + resetAudioKeepalive(state); + resetIdleTimer(state); + state.clientAudioFrameCount++; + if (state.clientAudioFrameCount % 250 == 1) { + log.info("[VoiceGateway] client audio frames session={} count={} upstreamReady={}", state.sessionId, state.clientAudioFrameCount, state.upstreamReady); + } + final byte[] audioMsg = VolcRealtimeProtocol.createAudioMessage(state.sessionId, payload); + // C2: Overflow protection — if chain is backed up > 1s, reset to prevent cumulative latency + long sendNow = System.currentTimeMillis(); + if (state.lastAudioSendSuccessAt > 0 + && sendNow - state.lastAudioSendSuccessAt > 1000 + && !state.lastUpstreamAudioSend.isDone()) { + log.info("[VoiceGateway] audio chain overflow reset session={} lag={}ms", state.sessionId, sendNow - state.lastAudioSendSuccessAt); + state.lastUpstreamAudioSend = CompletableFuture.completedFuture(null); + } + state.lastUpstreamAudioSend = CompletableFuture.runAsync(() -> { + synchronized (state.upstreamSendLock) { + try { + upstream.sendBinary(ByteBuffer.wrap(audioMsg), true).join(); + state.lastAudioSendSuccessAt = System.currentTimeMillis(); + } catch (Exception exception) { + if (!state.audioBlockLogOnce) { + log.warn("[VoiceGateway] send upstream audio failed session={}: {}", state.sessionId, exception.getMessage()); + state.audioBlockLogOnce = true; + } + } + } + }); + } + + @PreDestroy + public void shutdown() { + scheduler.shutdownNow(); + } + + private void handleStart(VoiceSessionState state, JsonNode node) { + state.userId = firstNonBlank(textValue(node.path("userId")), state.userId); + AssistantProfileResult profileResult = assistantProfileService.getAssistantProfile(state.userId, false); + state.assistantProfile = voiceAssistantProfileSupport.resolve(profileResult.profile()); + state.botName = firstNonBlank(textValue(node.path("botName")), voiceAssistantProfileSupport.getDisplayName(state.assistantProfile), "大沃"); + // GAP-17: Load handoff summary — Redis first, then deterministic fallback + String redisSummary = null; + try { + redisSummary = redisContextStore.getSummary(state.sessionId); + } catch (Exception e) { + log.debug("[VoiceGateway] Redis getSummary failed session={}: {}", state.sessionId, e.getMessage()); + } + state.handoffSummary = StringUtils.hasText(redisSummary) ? redisSummary : buildDeterministicHandoffSummary(chatRepository.getHistoryForLlm(state.sessionId, 10)); + // Always use backend-constructed system role (matches Node.js behavior). + // Frontend may send stale/default systemRole — never let it override. + state.systemRole = voiceAssistantProfileSupport.buildVoiceSystemRole(state.assistantProfile, state.handoffSummary); + state.speakingStyle = firstNonBlank(textValue(node.path("speakingStyle")), VoiceAssistantProfileSupport.DEFAULT_SPEAKING_STYLE); + state.speaker = firstNonBlank(textValue(node.path("speaker")), properties.getDefaultSpeaker()); + state.modelVersion = firstNonBlank(textValue(node.path("modelVersion")), "O"); + state.greetingText = firstNonBlank(textValue(node.path("greetingText")), voiceAssistantProfileSupport.buildVoiceGreeting(state.assistantProfile)); + chatRepository.createSession(state.sessionId, state.userId, "voice"); + if (properties.isSendReadyEarly()) { + sendReady(state); + } + connectUpstream(state); + } + + private void handleDirectText(VoiceSessionState state, String text) { + String cleanText = fastAsrCorrector.correctAsrText(text); + if (!persistUserSpeech(state, cleanText)) { + return; + } + sendJson(state, Map.of("type", "tts_reset", "reason", "new_turn")); + state.blockUpstreamAudio = true; + state.currentTtsType = "default"; + state.clearAssistantBuffer(); + processReplyAsync(state, cleanText, state.latestUserTurnSeq); + } + + private void connectUpstream(VoiceSessionState state) { + if (!properties.isConfigured()) { + sendJson(state, Map.of("type", "error", "error", "VOLC_S2S_APP_ID 或 VOLC_S2S_TOKEN 未配置")); + return; + } + httpClient.newWebSocketBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .header("X-Api-Resource-Id", properties.getResourceId()) + .header("X-Api-Access-Key", properties.getToken()) + .header("X-Api-App-Key", properties.getAppKey()) + .header("X-Api-App-ID", properties.getAppId()) + .header("X-Api-Connect-Id", state.sessionId) + .buildAsync(URI.create(properties.getUpstreamUrl()), new UpstreamListener(state)) + .whenComplete((upstream, throwable) -> { + if (throwable != null) { + log.error("[VoiceGateway] upstream connect failed session={}: {}", state.sessionId, throwable.getMessage(), throwable); + sendJson(state, Map.of("type", "error", "error", "语音服务连接失败: " + throwable.getMessage())); + closeClient(state.clientSession, CloseStatus.SERVER_ERROR); + return; + } + state.upstream = upstream; + state.lastUpstreamAudioSend = CompletableFuture.completedFuture(null); + state.lastAudioSendSuccessAt = 0; + state.audioBlockLogOnce = false; + log.info("[VoiceGateway] upstream connected session={}, sending start_connection + start_session", state.sessionId); + sendUpstreamBinary(state, VolcRealtimeProtocol.createStartConnectionMessage()); + Map startPayload = buildStartSessionPayload(state); + log.info("[VoiceGateway] start_session payload session={} bot={} speaker={} model={} systemRole_len={}", + state.sessionId, state.botName, state.speaker, state.modelVersion, + state.systemRole == null ? 0 : state.systemRole.length()); + sendUpstreamBinary(state, VolcRealtimeProtocol.createStartSessionMessage(state.sessionId, startPayload, objectMapper)); + }); + } + + private Map buildStartSessionPayload(VoiceSessionState state) { + Map asr = new LinkedHashMap<>(); + asr.put("extra", Map.of("context", ASR_CONTEXT, "boosting_table_id", BOOSTING_TABLE_ID, "nbest", 1)); + Map tts = new LinkedHashMap<>(); + tts.put("speaker", state.speaker); + tts.put("audio_config", Map.of("channel", 1, "format", "pcm_s16le", "sample_rate", 24000)); + Map dialog = new LinkedHashMap<>(); + dialog.put("dialog_id", ""); + dialog.put("bot_name", state.botName); + dialog.put("system_role", voiceAssistantProfileSupport.normalizeTextForSpeech(ANTI_THINKING_PREFIX + " " + state.systemRole)); + dialog.put("speaking_style", voiceAssistantProfileSupport.normalizeTextForSpeech(state.speakingStyle)); + dialog.put("extra", Map.of("input_mod", "audio", "model", state.modelVersion, "strict_audit", false, "audit_response", "抱歉，这个问题我暂时无法回答。")); + Map payload = new LinkedHashMap<>(); + payload.put("asr", asr); + payload.put("tts", tts); + payload.put("dialog", dialog); + return payload; + } + + private void handleUpstreamBinary(VoiceSessionState state, byte[] messageBytes) { + VolcRealtimeProtocol.Frame frame; + try { + frame = VolcRealtimeProtocol.unmarshal(messageBytes); + } catch (Exception exception) { + if (messageBytes != null && messageBytes.length > 4) { + String body = new String(messageBytes, 4, messageBytes.length - 4, java.nio.charset.StandardCharsets.UTF_8); + log.warn("[VoiceGateway] upstream unmarshal failed session={} len={} hdr=[{} {} {} {}] body={}", + state.sessionId, messageBytes.length, + String.format("%02x", messageBytes[0] & 0xFF), + String.format("%02x", messageBytes[1] & 0xFF), + String.format("%02x", messageBytes[2] & 0xFF), + String.format("%02x", messageBytes[3] & 0xFF), + body.length() > 500 ? body.substring(0, 500) : body); + } else { + log.warn("[VoiceGateway] upstream unmarshal failed session={}: {} {}", state.sessionId, exception.getClass().getSimpleName(), exception.getMessage()); + } + return; + } + if (frame.type() == VolcRealtimeProtocol.TYPE_AUDIO_ONLY_SERVER) { + boolean isDefaultTts = !StringUtils.hasText(state.currentTtsType) || "default".equals(state.currentTtsType); + boolean isSuppressing = state.suppressUpstreamUntil > System.currentTimeMillis() && isDefaultTts; + boolean isUserJustSpeaking = isDefaultTts && state.lastPartialAt > 0 && (System.currentTimeMillis() - state.lastPartialAt < 800); + boolean isBlockPassthrough = "external_rag".equals(state.currentTtsType) + || (state.fillerActive && state.chatTTSUntil > System.currentTimeMillis()); + if ((state.blockUpstreamAudio && !isBlockPassthrough) || isSuppressing || isUserJustSpeaking) { + if (!state.audioBlockLogOnce) { + state.audioBlockLogOnce = true; + log.debug("[VoiceGateway] audio blocked session={} ttsType={} block={} suppress={}", state.sessionId, state.currentTtsType, state.blockUpstreamAudio, isSuppressing); + } + return; + } + state.audioBlockLogOnce = false; + sendBinary(state, frame.payload()); + return; + } + if (frame.type() == VolcRealtimeProtocol.TYPE_ERROR) { + String raw = new String(frame.payload(), java.nio.charset.StandardCharsets.UTF_8); + int jsonStart = raw.indexOf('{'); + String error = jsonStart >= 0 ? raw.substring(jsonStart) : raw; + log.error("[VoiceGateway] upstream S2S error session={}: {}", state.sessionId, error); + boolean isIdleTimeout = error.contains("DialogAudioIdleTimeoutError"); + if (isIdleTimeout) { + log.info("[VoiceGateway] S2S idle timeout, notifying client session={}", state.sessionId); + state.upstreamReady = false; + cancelFuture(state.keepaliveFuture); + sendJson(state, Map.of("type", "idle_timeout", "timeout", properties.getIdleTimeoutMs())); + } else { + sendJson(state, Map.of("type", "error", "error", "语音服务错误: " + error)); + closeClient(state.clientSession, CloseStatus.SERVER_ERROR); + } + return; + } + if (frame.type() != VolcRealtimeProtocol.TYPE_FULL_SERVER) { + return; + } + JsonNode payload = parseJsonPayload(frame.payload()); + switch (frame.event()) { + case 150 -> handleUpstreamReady(state); + case 300 -> state.touch(); + case 350 -> handleTtsEvent(state, payload); + case 351 -> handleAssistantFinal(state, payload); + case 450 -> handleUserPartial(state, payload); + case 451 -> { + if (isFinalUserPayload(payload)) { + handleUserFinal(state, payload, frame.event()); + } else { + handleUserPartial(state, payload); + } + } + case 459 -> handleUserFinal(state, payload, frame.event()); + case 550 -> handleAssistantChunk(state, payload); + case 559 -> handleAssistantStreamEnd(state); + default -> sendJson(state, Map.of("type", "event", "event", frame.event(), "payload", payload == null ? Map.of() : objectMapper.convertValue(payload, Map.class))); + } + } + + private void handleUpstreamReady(VoiceSessionState state) { + log.info("[VoiceGateway] upstream ready (event 150) session={}", state.sessionId); + state.upstreamReady = true; + resetIdleTimer(state); + startAudioKeepalive(state); + sendGreeting(state); + } + + private void handleTtsEvent(VoiceSessionState state, JsonNode payload) { + String ttsType = textValue(payload == null ? null : payload.path("tts_type")); + state.currentTtsType = ttsType; + if ("chat_tts_text".equals(ttsType) && state.pendingGreetingAck) { + state.pendingGreetingAck = false; + cancelFuture(state.greetingAckTimerFuture); + state.greetingAckTimerFuture = null; + } + if (state.blockUpstreamAudio && "external_rag".equals(ttsType)) { + clearReplyTimeout(state); + state.blockUpstreamAudio = false; + state.suppressUpstreamUntil = 0; + cancelFuture(state.suppressReplyFuture); + state.suppressReplyFuture = null; + state.clearAssistantBuffer(); + state.isSendingChatTTSText = false; + state.chatTTSUntil = 0; + state.currentSpeechText = ""; + state.fillerActive = false; + cancelFuture(state.chatTTSTimerFuture); + sendJson(state, Map.of("type", "tts_reset", "reason", "rag_response_start")); + log.info("[VoiceGateway] unblock for external_rag tts session={}", state.sessionId); + } else if (state.blockUpstreamAudio && "chat_tts_text".equals(ttsType)) { + log.debug("[VoiceGateway] chat_tts_text started, keeping block session={}", state.sessionId); + } + Map eventPayload = payload == null ? Map.of() : objectMapper.convertValue(payload, Map.class); + sendJson(state, Map.of("type", "tts_event", "payload", eventPayload)); + } + + private void handleUserPartial(VoiceSessionState state, JsonNode payload) { + String text = extractUserText(payload, state.sessionId); + if (!StringUtils.hasText(text)) { + return; + } + long now = System.currentTimeMillis(); + boolean isDirectSpeaking = state.directSpeakUntil > 0 && now < state.directSpeakUntil; + boolean isChatTTSSpeaking = state.isSendingChatTTSText && state.chatTTSUntil > now; + // TTS echo detection + if ((isDirectSpeaking || isChatTTSSpeaking) && StringUtils.hasText(state.currentSpeechText)) { + String normalizedPartial = text.replaceAll("[，。！？、,.\\s]", ""); + String normalizedSpeech = state.currentSpeechText.replaceAll("[，。！？、,.\\s]", ""); + if (normalizedPartial.length() <= 3 || normalizedSpeech.contains(normalizedPartial)) { + if (!state.echoLogOnce) { + state.echoLogOnce = true; + log.debug("[VoiceGateway] TTS echo detected, ignoring partial session={} text={}", state.sessionId, abbreviate(text)); + } + return; + } + state.echoLogOnce = false; + } else { + state.echoLogOnce = false; + } + // Greeting protection window — allow through if clearly not echo + if (state.greetingProtectionUntil > 0 && now < state.greetingProtectionUntil) { + log.debug("[VoiceGateway] greeting protection active, partial session={} text={}", state.sessionId, abbreviate(text)); + return; + } + String normalizedPartial = knowledgeQueryResolver.normalizeKnowledgeText(text); + state.latestUserText = normalizedPartial; + state.lastPartialAt = now; + // KB-First: early block for non-chitchat + if (normalizedPartial.length() >= 6 && !state.blockUpstreamAudio && !isPureChitchat(normalizedPartial)) { + state.blockUpstreamAudio = true; + state.currentTtsType = "default"; + sendJson(state, Map.of("type", "tts_reset", "reason", "early_block")); + log.info("[VoiceGateway] early block session={} text={}", state.sessionId, abbreviate(text)); + // KB prequery: start async KB search to reduce final-wait latency + long kbPrequeryDebounce = 600L; + if (normalizedPartial.length() >= 8 + && (state.kbPrequeryStartedAt == 0 || now - state.kbPrequeryStartedAt > kbPrequeryDebounce) + && knowledgeBaseRetrieverService.isConfigured()) { + state.kbPrequeryStartedAt = now; + state.kbPrequeryText = normalizedPartial; + log.info("[VoiceGateway] KB prequery started session={} text={}", state.sessionId, abbreviate(normalizedPartial)); + final String preText = normalizedPartial; + state.pendingKbPrequery = CompletableFuture.supplyAsync(() -> { + try { + return knowledgeBaseRetrieverService.searchKnowledge( + preText, + contextKeywordTracker.suggestContextTerms(state.sessionId, preText), + null, state.sessionId, firstNonBlank(state.userId, "global")); + } catch (Exception e) { + log.warn("[VoiceGateway] KB prequery failed session={}: {}", state.sessionId, e.getMessage()); + return null; + } + }); + } + } + // User barge-in: interrupt all AI playback + boolean isS2SAudioPlaying = "external_rag".equals(state.currentTtsType); + if (isDirectSpeaking || isChatTTSSpeaking || isS2SAudioPlaying) { + log.info("[VoiceGateway] user barge-in (partial) session={}", state.sessionId); + state.directSpeakUntil = 0; + state.isSendingChatTTSText = false; + state.chatTTSUntil = 0; + state.currentSpeechText = ""; + state.currentTtsType = "default"; + cancelFuture(state.chatTTSTimerFuture); + if (state.suppressReplyFuture != null || state.suppressUpstreamUntil > 0) { + clearUpstreamSuppression(state); + } + state.blockUpstreamAudio = true; + } + // Send tts_reset for barge-in + if (now - state.lastBargeInResetAt > 500L) { + state.lastBargeInResetAt = now; + sendJson(state, Map.of("type", "tts_reset", "reason", "user_bargein")); + } + sendJson(state, Map.of( + "type", "subtitle", + "role", "user", + "text", text, + "isFinal", Boolean.FALSE, + "sequence", "native_partial_" + now + )); + } + + private void handleUserFinal(VoiceSessionState state, JsonNode payload, int eventCode) { + String rawFinalText = extractUserText(payload, state.sessionId); + String normalizedFinal = StringUtils.hasText(rawFinalText) ? knowledgeQueryResolver.normalizeKnowledgeText(rawFinalText, true) : ""; + String finalText = StringUtils.hasText(normalizedFinal) ? normalizedFinal : state.latestUserText; + if (!StringUtils.hasText(finalText)) { + return; + } + String normalizedForDedup = finalText.replaceAll("[，。！？、,.?!\\s]", ""); + long now = System.currentTimeMillis(); + if (normalizedForDedup.equals(state.lastFinalNormalized) && now - state.lastFinalAt < 1500L) { + log.info("[VoiceGateway] duplicate final ignored session={} event={} text={}", state.sessionId, eventCode, abbreviate(finalText)); + return; + } + state.lastFinalNormalized = normalizedForDedup; + state.lastFinalAt = now; + // TTS echo detection (final level) + boolean isDirectSpeaking = state.directSpeakUntil > 0 && now < state.directSpeakUntil; + boolean isChatTTSSpeaking = state.isSendingChatTTSText && state.chatTTSUntil > now; + if ((isDirectSpeaking || isChatTTSSpeaking) && StringUtils.hasText(state.currentSpeechText)) { + String echoNormalizedFinal = finalText.replaceAll("[，。！？、,.\\s]", ""); + String normalizedSpeech = state.currentSpeechText.replaceAll("[，。！？、,.\\s]", ""); + if (echoNormalizedFinal.length() <= 4 || normalizedSpeech.contains(echoNormalizedFinal)) { + log.debug("[VoiceGateway] TTS echo detected in final, ignoring session={} text={}", state.sessionId, abbreviate(finalText)); + return; + } + } + // Greeting protection window — barge-in: if user final arrives, treat as genuine speech + if (state.greetingProtectionUntil > 0 && now < state.greetingProtectionUntil) { + log.info("[VoiceGateway] user barge-in during greeting session={} text={}", state.sessionId, abbreviate(finalText)); + state.greetingProtectionUntil = 0; + state.directSpeakUntil = 0; + state.currentSpeechText = ""; + state.discardNextAssistantResponse = false; + cancelFuture(state.greetingTimerFuture); + state.greetingTimerFuture = null; + sendJson(state, Map.of("type", "tts_reset", "reason", "user_bargein")); + } + // User interrupt during any AI playback + boolean isS2SAudioPlayingFinal = "external_rag".equals(state.currentTtsType); + if (isDirectSpeaking || isChatTTSSpeaking || isS2SAudioPlayingFinal) { + log.info("[VoiceGateway] user interrupt (final) session={} direct={} chatTTS={} s2s={}", state.sessionId, isDirectSpeaking, isChatTTSSpeaking, isS2SAudioPlayingFinal); + state.directSpeakUntil = 0; + state.isSendingChatTTSText = false; + state.chatTTSUntil = 0; + state.currentSpeechText = ""; + state.currentTtsType = "default"; + cancelFuture(state.chatTTSTimerFuture); + sendJson(state, Map.of("type", "tts_reset", "reason", "user_bargein")); + if (state.suppressReplyFuture != null || state.suppressUpstreamUntil > 0) { + clearUpstreamSuppression(state); + } + } + if (!persistUserSpeech(state, rawFinalText != null ? rawFinalText : finalText)) { + return; + } + // Pure chitchat: let S2S handle directly without blocking audio + if (isPureChitchat(finalText)) { + log.info("[VoiceGateway] chitchat passthrough session={} text={}", state.sessionId, abbreviate(finalText)); + state.blockUpstreamAudio = false; + state.awaitingUpstreamReply = true; + state.pendingAssistantSource = "voice_bot"; + state.pendingAssistantToolName = null; + state.pendingAssistantMeta = null; + state.pendingAssistantTurnSeq = state.latestUserTurnSeq; + state.turnCount++; + return; + } + state.blockUpstreamAudio = true; + state.currentTtsType = "default"; + state.clearAssistantBuffer(); + sendJson(state, Map.of("type", "tts_reset", "reason", "new_turn")); + processReplyAsync(state, finalText, state.latestUserTurnSeq); + } + + private void handleAssistantFinal(VoiceSessionState state, JsonNode payload) { + boolean isLocalChatTTSActive = state.isSendingChatTTSText && state.chatTTSUntil > System.currentTimeMillis(); + boolean isSuppressing = state.suppressUpstreamUntil > System.currentTimeMillis(); + if (isLocalChatTTSActive || state.blockUpstreamAudio || isSuppressing) { + state.clearAssistantBuffer(); + return; + } + // external_rag response arriving at 351: clear discard flag so KB answer is not dropped + if (state.discardNextAssistantResponse && "external_rag".equals(state.currentTtsType)) { + state.discardNextAssistantResponse = false; + log.info("[VoiceGateway] cleared discardNextAssistantResponse for external_rag (351) session={}", state.sessionId); + } + if (state.discardNextAssistantResponse) { + boolean inGreetingWindow = state.greetingProtectionUntil > 0 && System.currentTimeMillis() < state.greetingProtectionUntil; + if (!inGreetingWindow) { + state.discardNextAssistantResponse = false; + } + state.clearAssistantBuffer(); + log.debug("[VoiceGateway] discarded stale assistant response (351) session={} greetingHold={}", state.sessionId, inGreetingWindow); + return; + } + clearReplyTimeout(state); + String pendingSource = state.pendingAssistantSource != null ? state.pendingAssistantSource : "voice_bot"; + String pendingToolName = state.pendingAssistantToolName; + var pendingMeta = state.pendingAssistantMeta; + long pendingTurnSeq = state.pendingAssistantTurnSeq > 0 ? state.pendingAssistantTurnSeq : state.latestUserTurnSeq; + state.awaitingUpstreamReply = false; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + if (pendingTurnSeq > 0 && state.lastDeliveredAssistantTurnSeq == pendingTurnSeq) { + state.clearAssistantBuffer(); + clearPendingAssistant(state); + log.debug("[VoiceGateway] duplicate assistant final ignored (351) session={} turn={}", state.sessionId, pendingTurnSeq); + return; + } + // C1: For external_rag, prefer buffer (actual S2S-spoken text) over 351 payload (may be raw RAG input) + String assistantText; + boolean isExternalRag = "external_rag".equals(state.currentTtsType); + if (isExternalRag) { + String buffered = state.consumeAssistantBuffer(); + String payloadText = extractRawText(payload); + assistantText = StringUtils.hasText(buffered) ? buffered : payloadText; + log.info("[VoiceGateway] external_rag 351 text source={} session={} len={}", + StringUtils.hasText(buffered) ? "buffer" : "payload", state.sessionId, + assistantText == null ? 0 : assistantText.length()); + } else { + assistantText = extractRawText(payload); + if (StringUtils.hasText(assistantText)) { + state.clearAssistantBuffer(); + } else { + assistantText = state.consumeAssistantBuffer(); + } + } + if (StringUtils.hasText(assistantText)) { + if (state.fillerActive) { + log.debug("[VoiceGateway] discarded filler assistant text session={}", state.sessionId); + state.fillerActive = false; + clearPendingAssistant(state); + return; + } + state.pendingExternalRagReply = false; + state.lastDeliveredAssistantTurnSeq = pendingTurnSeq; + persistAssistantSpeech(state, assistantText, pendingSource, pendingToolName, pendingMeta); + if (isExternalRag) { + state.blockUpstreamAudio = true; + log.debug("[VoiceGateway] re-blocked after KB response session={}", state.sessionId); + } + } + clearPendingAssistant(state); + } + + private void handleAssistantChunk(VoiceSessionState state, JsonNode payload) { + if (state.discardNextAssistantResponse && Objects.equals(state.currentTtsType, "external_rag")) { + state.discardNextAssistantResponse = false; + } + if (state.discardNextAssistantResponse) { + return; + } + boolean isSuppressing = state.suppressUpstreamUntil > System.currentTimeMillis(); + if (isSuppressing && !"external_rag".equals(state.currentTtsType)) { + return; + } + if (state.blockUpstreamAudio && !"external_rag".equals(state.currentTtsType)) { + return; + } + String chunk = extractRawText(payload); + if (!StringUtils.hasText(chunk)) { + return; + } + String fullText = state.appendAssistantChunk(textValue(payload.path("reply_id")), chunk); + if (chatContentSafetyService.isBrandHarmful(fullText)) { + state.blockUpstreamAudio = true; + state.discardNextAssistantResponse = true; + state.clearAssistantBuffer(); + sendJson(state, Map.of("type", "tts_reset", "reason", "harmful_blocked")); + persistAssistantSpeech(state, chatContentSafetyService.getTextSafeReply(), "voice_bot", null, null); + return; + } + // Thinking pattern detection: block AI "planning" output + if (fullText.length() >= 10 && (THINKING_PATTERN.matcher(fullText.trim()).find() || THINKING_MID_PATTERN.matcher(fullText).find())) { + log.warn("[VoiceGateway] thinking detected in stream, blocking session={} text={}", state.sessionId, abbreviate(fullText)); + state.blockUpstreamAudio = true; + state.discardNextAssistantResponse = true; + state.clearAssistantBuffer(); + sendJson(state, Map.of("type", "tts_reset", "reason", "thinking_blocked")); + return; + } + state.awaitingUpstreamReply = false; + state.pendingExternalRagReply = false; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + // C1: Send streaming subtitle for real-time display + sendJson(state, Map.of( + "type", "subtitle", + "role", "assistant", + "text", fullText, + "isFinal", Boolean.FALSE, + "sequence", "native_assistant_stream_" + System.currentTimeMillis() + )); + } + + private void handleAssistantStreamEnd(VoiceSessionState state) { + clearReplyTimeout(state); + // external_rag response arriving at 559: clear discard flag so KB answer is not dropped + if (state.discardNextAssistantResponse && "external_rag".equals(state.currentTtsType)) { + state.discardNextAssistantResponse = false; + log.info("[VoiceGateway] cleared discardNextAssistantResponse for external_rag (559) session={}", state.sessionId); + } + if (state.discardNextAssistantResponse) { + boolean inGreetingWindow = state.greetingProtectionUntil > 0 && System.currentTimeMillis() < state.greetingProtectionUntil; + if (!inGreetingWindow) { + state.discardNextAssistantResponse = false; + } + state.clearAssistantBuffer(); + clearPendingAssistant(state); + return; + } + boolean isSuppressing = state.suppressUpstreamUntil > System.currentTimeMillis(); + if (isSuppressing && !"external_rag".equals(state.currentTtsType)) { + state.clearAssistantBuffer(); + clearPendingAssistant(state); + log.debug("[VoiceGateway] stream end suppressed session={}", state.sessionId); + return; + } + // Match Node.js: block ALL 559 when pendingExternalRagReply (no external_rag exception) + if (state.pendingExternalRagReply) { + state.clearAssistantBuffer(); + clearPendingAssistant(state); + log.debug("[VoiceGateway] stream end ignored (pendingExternalRagReply) session={}", state.sessionId); + return; + } + // Match Node.js: respect blockUpstreamAudio unconditionally (keeps re-block from 351) + if (state.blockUpstreamAudio) { + state.clearAssistantBuffer(); + clearPendingAssistant(state); + log.debug("[VoiceGateway] blocked response ended (559), keeping block session={}", state.sessionId); + return; + } + String fullText = state.consumeAssistantBuffer(); + state.awaitingUpstreamReply = false; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + if (StringUtils.hasText(fullText) && state.pendingAssistantTurnSeq != state.lastDeliveredAssistantTurnSeq) { + if (state.fillerActive) { + log.debug("[VoiceGateway] stream end discarded filler session={}", state.sessionId); + state.fillerActive = false; + } else if (persistAssistantSpeech(state, fullText, state.pendingAssistantSource, state.pendingAssistantToolName, state.pendingAssistantMeta)) { + state.lastDeliveredAssistantTurnSeq = state.pendingAssistantTurnSeq; + } + } + state.pendingExternalRagReply = false; + clearPendingAssistant(state); + state.blockUpstreamAudio = false; + } + + private void processReplyAsync(VoiceSessionState state, String text, long turnSeq) { + if (state.processingReply) { + state.queuedUserText = text; + state.queuedUserTurnSeq = turnSeq; + log.info("[VoiceGateway] processReply queued(busy) session={} text={}", state.sessionId, abbreviate(text)); + return; + } + if (state.directSpeakUntil > 0 && System.currentTimeMillis() < state.directSpeakUntil) { + state.queuedUserText = text; + state.queuedUserTurnSeq = turnSeq; + long waitMs = Math.max(200L, state.directSpeakUntil - System.currentTimeMillis() + 200); + cancelFuture(state.queuedReplyTimerFuture); + state.queuedReplyTimerFuture = scheduler.schedule(() -> drainQueuedReply(state, 0), waitMs, TimeUnit.MILLISECONDS); + log.info("[VoiceGateway] processReply queued(speaking) session={} waitMs={}", state.sessionId, waitMs); + return; + } + state.processingReply = true; + CompletableFuture.runAsync(() -> { + try { + processReply(state, text, turnSeq); + } catch (Exception ex) { + log.error("[VoiceGateway] processReplyAsync uncaught session={}: {}", state.sessionId, ex.getMessage(), ex); + } finally { + state.processingReply = false; + if (!state.awaitingUpstreamReply && !state.pendingExternalRagReply) { + state.blockUpstreamAudio = false; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + } + drainQueuedReply(state, turnSeq); + } + }); + } + + private void drainQueuedReply(VoiceSessionState state, long justFinishedTurnSeq) { + String pending = state.queuedUserText; + long pendingTurnSeq = state.queuedUserTurnSeq; + state.queuedUserText = ""; + state.queuedUserTurnSeq = 0; + if (!StringUtils.hasText(pending) || pendingTurnSeq <= 0) { + return; + } + if (pendingTurnSeq == justFinishedTurnSeq) { + return; + } + if (state.directSpeakUntil > 0 && System.currentTimeMillis() < state.directSpeakUntil) { + long waitMs = Math.max(200L, state.directSpeakUntil - System.currentTimeMillis() + 200); + state.queuedUserText = pending; + state.queuedUserTurnSeq = pendingTurnSeq; + cancelFuture(state.queuedReplyTimerFuture); + state.queuedReplyTimerFuture = scheduler.schedule(() -> drainQueuedReply(state, 0), waitMs, TimeUnit.MILLISECONDS); + return; + } + scheduler.schedule(() -> { + state.blockUpstreamAudio = true; + processReplyAsync(state, pending, pendingTurnSeq); + }, 200, TimeUnit.MILLISECONDS); + } + + private void processReply(VoiceSessionState state, String text, long turnSeq) { + long t0 = System.currentTimeMillis(); + String cleanText = StringUtils.hasText(text) ? text.trim() : ""; + if (cleanText.isEmpty()) { + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + return; + } + state.turnCount++; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.TRUE)); + // Product link trigger check + ProductLinkTrigger.TriggerResult linkResult = productLinkTrigger.check(cleanText); + if (linkResult.triggered()) { + sendJson(state, Map.of( + "type", "product_link", + "product", linkResult.productName(), + "link", linkResult.link(), + "description", linkResult.description() + )); + log.info("[VoiceGateway] product_link triggered session={} product={}", state.sessionId, linkResult.productName()); + // Speak the hint via local TTS + String speechHint = "好的，" + linkResult.productName() + "的详细介绍链接已经发给你了，你可以点击查看。"; + sendSpeechText(state, speechHint); + suppressUpstreamReply(state, estimateSpeechDurationMs(speechHint) + 1500); + persistAssistantSpeech(state, speechHint, "voice_bot", "product_link", Map.of("product", linkResult.productName())); + return; + } + List context = chatRepository.getHistoryForLlm(state.sessionId, 20); + boolean shouldSearchKnowledge = !isPureChitchat(cleanText) && (knowledgeRouteDecider.shouldForceKnowledgeRoute(cleanText, context) || looksLikeKnowledgeQuestion(cleanText)); + try { + if (shouldSearchKnowledge && knowledgeBaseRetrieverService.isConfigured()) { + // GAP-11: Try to consume prequery result if text matches + KnowledgeSearchResult result = null; + CompletableFuture prequery = state.pendingKbPrequery; + String preText = state.kbPrequeryText; + state.pendingKbPrequery = null; + state.kbPrequeryText = ""; + state.kbPrequeryStartedAt = 0; + if (prequery != null && StringUtils.hasText(preText) && cleanText.contains(preText.substring(0, Math.min(preText.length(), 6)))) { + try { + result = prequery.get(3, TimeUnit.SECONDS); + if (result != null) { + log.info("[VoiceGateway] KB prequery cache hit session={}", state.sessionId); + } + } catch (Exception e) { + log.debug("[VoiceGateway] KB prequery timed out, falling back session={}", state.sessionId); + } + } + if (result == null) { + result = knowledgeBaseRetrieverService.searchKnowledge( + cleanText, + contextKeywordTracker.suggestContextTerms(state.sessionId, cleanText), + null, + state.sessionId, + firstNonBlank(state.userId, "global") + ); + } + log.info("[VoiceGateway] KB search done session={} hit={} elapsed={}ms", state.sessionId, result.hit(), System.currentTimeMillis() - t0); + // Stale turn check: if user spoke again while KB was searching, discard result + if (turnSeq != state.latestUserTurnSeq) { + log.info("[VoiceGateway] stale processReply discarded session={} activeTurn={} latestTurn={}", state.sessionId, turnSeq, state.latestUserTurnSeq); + state.blockUpstreamAudio = false; + return; + } + if (result.hit()) { + List> ragItems = filterRagItems(result.ragPayload()); + if (!ragItems.isEmpty()) { + // GAP-19: Track KB protection window + state.lastKbTopic = cleanText; + state.lastKbHitAt = System.currentTimeMillis(); + state.blockUpstreamAudio = true; + state.awaitingUpstreamReply = true; + state.pendingExternalRagReply = true; + state.discardNextAssistantResponse = true; + state.pendingAssistantSource = "search_knowledge"; + state.pendingAssistantToolName = "search_knowledge"; + state.pendingAssistantMeta = buildKnowledgeMeta(result, cleanText); + state.pendingAssistantTurnSeq = turnSeq; + state.ragEvidenceText = writeJson(ragItems); + sendJson(state, Map.of("type", "tts_reset", "reason", "knowledge_hit")); + sendUpstreamBinary(state, VolcRealtimeProtocol.createChatRagTextMessage(state.sessionId, writeJson(ragItems), objectMapper)); + startReplyTimeout(state, turnSeq); + return; + } + } + // GAP-20: Brand protection — sensitive query KB no-hit returns safe reply + if (!result.hit() && BRAND_SENSITIVE_PATTERN.matcher(cleanText).find()) { + log.info("[VoiceGateway] brand protection fallback session={}", state.sessionId); + List> safeItems = List.of(Map.of("title", "品牌保护", "content", BRAND_SAFE_REPLY)); + state.blockUpstreamAudio = true; + state.awaitingUpstreamReply = true; + state.pendingExternalRagReply = true; + state.discardNextAssistantResponse = true; + state.pendingAssistantSource = "voice_tool"; + state.pendingAssistantToolName = "search_knowledge"; + state.pendingAssistantMeta = Map.of("hit", true, "reason", "brand_protection"); + state.pendingAssistantTurnSeq = turnSeq; + sendJson(state, Map.of("type", "tts_reset", "reason", "knowledge_hit")); + sendUpstreamBinary(state, VolcRealtimeProtocol.createChatRagTextMessage(state.sessionId, writeJson(safeItems), objectMapper)); + startReplyTimeout(state, turnSeq); + return; + } + // GAP-20: KB protection window — no-hit within 60s of last KB hit returns honest fallback + if (!result.hit() && state.lastKbHitAt > 0 && (System.currentTimeMillis() - state.lastKbHitAt < 60000L)) { + log.info("[VoiceGateway] KB no-hit in protection window, honest fallback session={}", state.sessionId); + List> honestItems = List.of(Map.of("title", "知识库未命中", "content", HONEST_FALLBACK_REPLY)); + state.blockUpstreamAudio = true; + state.awaitingUpstreamReply = true; + state.pendingExternalRagReply = true; + state.discardNextAssistantResponse = true; + state.pendingAssistantSource = "voice_tool"; + state.pendingAssistantToolName = "search_knowledge"; + state.pendingAssistantMeta = Map.of("hit", false, "reason", "honest_fallback"); + state.pendingAssistantTurnSeq = turnSeq; + sendJson(state, Map.of("type", "tts_reset", "reason", "knowledge_hit")); + sendUpstreamBinary(state, VolcRealtimeProtocol.createChatRagTextMessage(state.sessionId, writeJson(honestItems), objectMapper)); + startReplyTimeout(state, turnSeq); + return; + } + } else { + // Not KB route — clear any stale prequery + state.pendingKbPrequery = null; + state.kbPrequeryText = ""; + state.kbPrequeryStartedAt = 0; + } + state.blockUpstreamAudio = false; + state.awaitingUpstreamReply = true; + state.pendingAssistantSource = "voice_bot"; + state.pendingAssistantToolName = null; + state.pendingAssistantMeta = null; + state.pendingAssistantTurnSeq = turnSeq; + } catch (Exception exception) { + log.error("[VoiceGateway] processReply failed session={}: {}", state.sessionId, exception.getMessage(), exception); + sendJson(state, Map.of("type", "error", "error", exception.getMessage())); + resetBlockState(state, "processReply_error"); + } + } + + private List> filterRagItems(List> ragPayload) { + if (ragPayload == null || ragPayload.isEmpty()) { + return List.of(); + } + List> filtered = new ArrayList<>(); + for (Map item : ragPayload) { + if (item == null) { + continue; + } + Object content = item.get("content"); + Object kind = item.get("kind"); + if (!StringUtils.hasText(content == null ? null : String.valueOf(content))) { + continue; + } + if (Objects.equals(kind, "context")) { + continue; + } + filtered.add(new LinkedHashMap<>(item)); + } + return List.copyOf(filtered); + } + + private Map buildKnowledgeMeta(KnowledgeSearchResult result, String originalText) { + Map meta = new LinkedHashMap<>(); + meta.put("route", "search_knowledge"); + meta.put("original_text", originalText); + meta.put("tool_name", "search_knowledge"); + meta.put("tool_args", Map.of("query", originalText)); + meta.put("source", result.source()); + meta.put("original_query", result.originalQuery()); + meta.put("rewritten_query", result.query()); + meta.put("hit", result.hit()); + meta.put("reason", result.reason()); + meta.put("latency_ms", result.latencyMs()); + if (result.evidencePack() != null && result.evidencePack().get("retrieval") instanceof Map retrieval) { + meta.put("selected_dataset_ids", retrieval.get("selected_dataset_ids")); + meta.put("selected_kb_routes", retrieval.get("selected_kb_routes")); + } + return meta; + } + + private void sendGreeting(VoiceSessionState state) { + if (state.hasSentGreeting || !StringUtils.hasText(state.greetingText)) { + sendReady(state); + return; + } + state.hasSentGreeting = true; + long now = System.currentTimeMillis(); + state.greetingSentAt = now; + long greetingDuration = estimateSpeechDurationMs(state.greetingText); + state.greetingProtectionUntil = now + Math.min(greetingDuration, 8000L); + state.directSpeakUntil = now + greetingDuration + 500; + state.currentSpeechText = state.greetingText; + persistAssistantSpeech(state, state.greetingText, "voice_bot", null, null); + state.discardNextAssistantResponse = true; + byte[] greetingMsg = VolcRealtimeProtocol.createSayHelloMessage(state.sessionId, state.greetingText, objectMapper); + log.info("[VoiceGateway] sending greeting session={} len={} text={}", state.sessionId, greetingMsg.length, abbreviate(state.greetingText)); + sendUpstreamBinary(state, greetingMsg); + sendReady(state); + // Schedule greeting protection end + state.greetingTimerFuture = scheduler.schedule(() -> { + state.greetingProtectionUntil = 0; + state.directSpeakUntil = 0; + state.currentSpeechText = ""; + state.greetingTimerFuture = null; + }, greetingDuration + 1000, TimeUnit.MILLISECONDS); + } + + private void replayGreeting(VoiceSessionState state) { + if (!StringUtils.hasText(state.greetingText) || state.upstream == null) { + return; + } + long now = System.currentTimeMillis(); + if (now - state.greetingSentAt < 6000L) { + return; + } + state.greetingSentAt = now; + // GAP-22: Set directSpeakUntil and greetingProtection for replay + long greetingDuration = estimateSpeechDurationMs(state.greetingText); + state.greetingProtectionUntil = now + Math.min(greetingDuration, 8000L); + state.directSpeakUntil = now + greetingDuration + 500; + state.currentSpeechText = state.greetingText; + sendUpstreamBinary(state, VolcRealtimeProtocol.createSayHelloMessage(state.sessionId, state.greetingText, objectMapper)); + } + + private boolean persistUserSpeech(VoiceSessionState state, String text) { + String cleanText = String.valueOf(text == null ? "" : text).trim(); + if (!StringUtils.hasText(cleanText)) { + return false; + } + long now = System.currentTimeMillis(); + if (cleanText.equals(state.lastPersistedUserText) && now - state.lastPersistedUserAt < 5000L) { + return false; + } + state.lastPersistedUserText = cleanText; + state.lastPersistedUserAt = now; + state.latestUserText = cleanText; + state.latestUserTurnSeq = state.latestUserTurnSeq + 1; + contextKeywordTracker.updateSession(state.sessionId, knowledgeQueryResolver.normalizeKnowledgeText(cleanText)); + resetIdleTimer(state); + chatRepository.addMessage(state.sessionId, "user", cleanText, "voice_asr"); + // GAP-16: Redis conversation persistence + try { + redisContextStore.pushMessage(state.sessionId, new RedisContextMessage("user", cleanText, "voice_asr", now)); + } catch (Exception e) { + log.debug("[VoiceGateway] Redis pushMessage(user) failed session={}: {}", state.sessionId, e.getMessage()); + } + sendJson(state, Map.of( + "type", "subtitle", + "role", "user", + "text", cleanText, + "isFinal", Boolean.TRUE, + "sequence", "native_user_" + now + )); + return true; + } + + private boolean persistAssistantSpeech(VoiceSessionState state, String text, String source, String toolName, Object meta) { + String cleanText = chatContentSafetyService.guardAssistantText(String.valueOf(text == null ? "" : text).trim()); + if (!StringUtils.hasText(cleanText)) { + return false; + } + long now = System.currentTimeMillis(); + if (cleanText.equals(state.lastPersistedAssistantText) && now - state.lastPersistedAssistantAt < 5000L) { + return false; + } + state.lastPersistedAssistantText = cleanText; + state.lastPersistedAssistantAt = now; + resetIdleTimer(state); + chatRepository.addMessage(state.sessionId, "assistant", cleanText, firstNonBlank(source, "voice_bot"), toolName, meta); + // GAP-16: Redis conversation persistence + try { + redisContextStore.pushMessage(state.sessionId, new RedisContextMessage("assistant", cleanText, firstNonBlank(source, "voice_bot"), now)); + } catch (Exception e) { + log.debug("[VoiceGateway] Redis pushMessage(assistant) failed session={}: {}", state.sessionId, e.getMessage()); + } + sendJson(state, Map.of( + "type", "subtitle", + "role", "assistant", + "text", cleanText, + "isFinal", Boolean.TRUE, + "source", firstNonBlank(source, "voice_bot"), + "toolName", toolName == null ? "" : toolName, + "sequence", "native_assistant_" + now + )); + // GAP-15: Consultant contact push + if (CONSULTANT_REFERRAL_PATTERN.matcher(cleanText).find()) { + sendJson(state, Map.of("type", "consultant_contact", "text", cleanText)); + log.info("[VoiceGateway] consultant_contact pushed session={}", state.sessionId); + } + return true; + } + + private void sendReady(VoiceSessionState state) { + if (state.readySent) { + return; + } + state.readySent = true; + sendJson(state, Map.of("type", "ready")); + } + + private void startAudioKeepalive(VoiceSessionState state) { + cancelFuture(state.keepaliveFuture); + long interval = Math.max(properties.getAudioKeepaliveIntervalMs(), 5000L); + state.keepaliveFuture = scheduler.scheduleAtFixedRate(() -> { + WebSocket upstream = state.upstream; + if (upstream != null && state.upstreamReady) { + sendUpstreamBinary(state, VolcRealtimeProtocol.createAudioMessage(state.sessionId, SILENT_AUDIO_FRAME)); + } + }, interval, interval, TimeUnit.MILLISECONDS); + } + + private void resetAudioKeepalive(VoiceSessionState state) { + if (state.keepaliveFuture != null) { + startAudioKeepalive(state); + } + } + + private void resetIdleTimer(VoiceSessionState state) { + cancelFuture(state.idleFuture); + state.touch(); + long timeout = Math.max(properties.getIdleTimeoutMs(), 60000L); + state.idleFuture = scheduler.schedule(() -> { + if (state.closed) { + return; + } + sendJson(state, Map.of("type", "idle_timeout", "timeout", timeout)); + scheduler.schedule(() -> closeClient(state.clientSession, CloseStatus.NORMAL), 2, TimeUnit.SECONDS); + }, timeout, TimeUnit.MILLISECONDS); + } + + private void sendUpstreamBinary(VoiceSessionState state, byte[] payload) { + WebSocket upstream = state.upstream; + if (upstream == null || payload == null || payload.length == 0) { + return; + } + synchronized (state.upstreamSendLock) { + try { + upstream.sendBinary(ByteBuffer.wrap(payload), true).join(); + } catch (Exception exception) { + log.warn("[VoiceGateway] send upstream binary failed session={}: {}", state.sessionId, exception.getMessage()); + } + } + } + + private void sendJson(VoiceSessionState state, Object payload) { + if (state == null || state.clientSession == null || !state.clientSession.isOpen()) { + return; + } + try { + String raw = objectMapper.writeValueAsString(payload); + synchronized (state.clientSendLock) { + if (state.clientSession.isOpen()) { + state.clientSession.sendMessage(new TextMessage(raw)); + } + } + } catch (Exception exception) { + log.warn("[VoiceGateway] send json failed session={}: {}", state.sessionId, exception.getMessage()); + } + } + + private void sendBinary(VoiceSessionState state, byte[] payload) { + if (state == null || state.clientSession == null || !state.clientSession.isOpen()) { + return; + } + try { + synchronized (state.clientSendLock) { + if (state.clientSession.isOpen()) { + state.clientSession.sendMessage(new BinaryMessage(payload)); + } + } + } catch (Exception exception) { + log.warn("[VoiceGateway] send binary failed session={}: {}", state.sessionId, exception.getMessage()); + } + } + + private void closeClient(WebSocketSession clientSession, CloseStatus closeStatus) { + if (clientSession == null || !clientSession.isOpen()) { + return; + } + try { + clientSession.close(closeStatus); + } catch (Exception exception) { + log.warn("[VoiceGateway] close client failed: {}", exception.getMessage()); + } + } + + private JsonNode parseJsonPayload(byte[] payload) { + try { + return objectMapper.readTree(payload == null ? new byte[0] : payload); + } catch (Exception exception) { + return null; + } + } + + private String extractRawText(JsonNode jsonPayload) { + if (jsonPayload == null || jsonPayload.isNull()) { + return ""; + } + return firstNonBlank( + textValue(jsonPayload.path("text")), + textValue(jsonPayload.path("content")), + textValue(jsonPayload.path("results").path(0).path("text")), + textValue(jsonPayload.path("results").path(0).path("alternatives").path(0).path("text")) + ); + } + + private String extractUserText(JsonNode jsonPayload, String sessionId) { + String corrected = fastAsrCorrector.correctAsrText(extractRawText(jsonPayload)); + if (StringUtils.hasText(sessionId) && StringUtils.hasText(corrected)) { + contextKeywordTracker.updateSession(sessionId, knowledgeQueryResolver.normalizeKnowledgeText(corrected)); + } + return corrected; + } + + private boolean isFinalUserPayload(JsonNode jsonPayload) { + if (jsonPayload == null || jsonPayload.isNull()) { + return false; + } + if (jsonPayload.path("is_final").asBoolean(false)) { + return true; + } + JsonNode results = jsonPayload.path("results"); + if (!results.isArray()) { + return false; + } + for (JsonNode item : results) { + if (item != null && item.path("is_interim").asBoolean(true) == false) { + return true; + } + } + return false; + } + + private boolean isPureChitchat(String text) { + String value = normalizeNullable(text); + if (value == null) { + return true; + } + return PURE_CHITCHAT_GREETING.matcher(value).matches() + || PURE_CHITCHAT_META.matcher(value).matches() + || PURE_CHITCHAT_REACTION.matcher(value).matches() + || PURE_CHITCHAT_FUN.matcher(value).matches() + || PURE_CHITCHAT_SHORT.matcher(value).matches(); + } + + private boolean looksLikeKnowledgeQuestion(String text) { + String value = normalizeNullable(text); + return value != null && LIKELY_KNOWLEDGE.matcher(value).find(); + } + + private String buildDeterministicHandoffSummary(List messages) { + List normalizedMessages = (messages == null ? List.of() : messages).stream() + .filter(item -> item != null && ("user".equals(item.role()) || "assistant".equals(item.role())) && StringUtils.hasText(item.content())) + .skip(Math.max((messages == null ? 0 : messages.size()) - 8, 0)) + .toList(); + if (normalizedMessages.isEmpty()) { + return ""; + } + List userMessages = normalizedMessages.stream().filter(item -> "user".equals(item.role())).toList(); + String currentQuestion = userMessages.isEmpty() ? "" : userMessages.getLast().content().trim(); + String previousQuestion = userMessages.size() > 1 ? userMessages.get(userMessages.size() - 2).content().trim() : ""; + String assistantFacts = normalizedMessages.stream() + .filter(item -> "assistant".equals(item.role())) + .skip(Math.max(normalizedMessages.stream().filter(item -> "assistant".equals(item.role())).count() - 2, 0)) + .map(LlmMessage::content) + .filter(StringUtils::hasText) + .map(String::trim) + .map(value -> value.length() > 60 ? value.substring(0, 60) : value) + .reduce((left, right) -> left + "；" + right) + .orElse(""); + List parts = new ArrayList<>(); + if (StringUtils.hasText(currentQuestion)) { + parts.add("当前问题：" + currentQuestion); + } + if (StringUtils.hasText(previousQuestion) && !Objects.equals(previousQuestion, currentQuestion)) { + parts.add("上一轮关注：" + previousQuestion); + } + if (StringUtils.hasText(assistantFacts)) { + parts.add("已给信息：" + assistantFacts); + } + return String.join("；", parts); + } + + private void clearPendingAssistant(VoiceSessionState state) { + state.pendingAssistantSource = null; + state.pendingAssistantToolName = null; + state.pendingAssistantMeta = null; + state.pendingAssistantTurnSeq = 0L; + state.ragEvidenceText = ""; + } + + private void clearUpstreamSuppression(VoiceSessionState state) { + cancelFuture(state.suppressReplyFuture); + state.suppressReplyFuture = null; + state.suppressUpstreamUntil = 0; + state.awaitingUpstreamReply = false; + state.pendingAssistantSource = null; + state.pendingAssistantToolName = null; + state.pendingAssistantMeta = null; + state.pendingAssistantTurnSeq = 0; + state.blockUpstreamAudio = false; + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + } + + private void startReplyTimeout(VoiceSessionState state, long turnSeq) { + cancelFuture(state.replyTimeoutFuture); + state.replyTimeoutTurnSeq = turnSeq; + state.replyTimeoutFuture = scheduler.schedule(() -> { + if (state.closed || state.replyTimeoutTurnSeq != turnSeq) { + return; + } + log.warn("[VoiceGateway] reply timeout! session={} turnSeq={} block={} awaiting={} pendingRag={}", + state.sessionId, turnSeq, state.blockUpstreamAudio, state.awaitingUpstreamReply, state.pendingExternalRagReply); + resetBlockState(state, "reply_timeout"); + }, REPLY_TIMEOUT_MS, TimeUnit.MILLISECONDS); + } + + private void clearReplyTimeout(VoiceSessionState state) { + cancelFuture(state.replyTimeoutFuture); + state.replyTimeoutFuture = null; + state.replyTimeoutTurnSeq = 0; + } + + private void resetBlockState(VoiceSessionState state, String reason) { + state.blockUpstreamAudio = false; + state.awaitingUpstreamReply = false; + state.pendingExternalRagReply = false; + state.discardNextAssistantResponse = false; + state.isSendingChatTTSText = false; + state.chatTTSUntil = 0; + state.clearAssistantBuffer(); + clearPendingAssistant(state); + clearReplyTimeout(state); + sendJson(state, Map.of("type", "assistant_pending", "active", Boolean.FALSE)); + sendJson(state, Map.of("type", "tts_reset", "reason", reason)); + log.info("[VoiceGateway] resetBlockState session={} reason={}", state.sessionId, reason); + } + + private void suppressUpstreamReply(VoiceSessionState state, long durationMs) { + cancelFuture(state.suppressReplyFuture); + state.awaitingUpstreamReply = true; + long effectiveDuration = Math.max(1000L, durationMs); + state.suppressUpstreamUntil = System.currentTimeMillis() + effectiveDuration; + state.suppressReplyFuture = scheduler.schedule(() -> { + if (state.suppressUpstreamUntil > System.currentTimeMillis()) { + return; + } + clearUpstreamSuppression(state); + }, Math.max(300L, effectiveDuration), TimeUnit.MILLISECONDS); + } + + private long estimateSpeechDurationMs(String text) { + String plainText = voiceAssistantProfileSupport.normalizeTextForSpeech(text).replaceAll("\\s+", ""); + int length = plainText.length(); + return Math.max(4000L, Math.min(60000L, length * 180L)); + } + + private List splitTextForSpeech(String text) { + int maxLen = 180; + String content = voiceAssistantProfileSupport.normalizeTextForSpeech(text); + if (!StringUtils.hasText(content)) { + return List.of(); + } + if (content.length() <= maxLen) { + return List.of(content); + } + List chunks = new ArrayList<>(); + String remaining = content; + while (remaining.length() > maxLen) { + int currentMaxLen = chunks.isEmpty() ? Math.min(90, maxLen) : maxLen; + int splitIndex = -1; + for (char delimiter : new char[]{'。', '！', '？', '；', '，', ','}) { + int idx = remaining.lastIndexOf(delimiter, currentMaxLen); + if (idx > splitIndex) { + splitIndex = idx; + } + } + if (splitIndex < currentMaxLen / 2) { + splitIndex = currentMaxLen; + } else { + splitIndex += 1; + } + String chunk = remaining.substring(0, splitIndex).trim(); + if (StringUtils.hasText(chunk)) { + chunks.add(chunk); + } + remaining = remaining.substring(splitIndex).trim(); + } + if (StringUtils.hasText(remaining)) { + chunks.add(remaining); + } + return chunks; + } + + private void sendSpeechText(VoiceSessionState state, String speechText) { + List chunks = splitTextForSpeech(speechText); + if (chunks.isEmpty() || state.upstream == null || !state.upstreamReady) { + return; + } + state.currentSpeechText = speechText; + state.isSendingChatTTSText = true; + state.currentTtsType = "chat_tts_text"; + state.chatTTSUntil = System.currentTimeMillis() + estimateSpeechDurationMs(speechText) + 800; + cancelFuture(state.chatTTSTimerFuture); + long delay = Math.max(200L, state.chatTTSUntil - System.currentTimeMillis() + 50); + state.chatTTSTimerFuture = scheduler.schedule(() -> { + state.chatTTSTimerFuture = null; + if (state.chatTTSUntil <= System.currentTimeMillis()) { + state.isSendingChatTTSText = false; + } + }, delay, TimeUnit.MILLISECONDS); + sendJson(state, Map.of("type", "tts_reset", "ttsType", "chat_tts_text")); + for (int i = 0; i < chunks.size(); i++) { + sendUpstreamBinary(state, VolcRealtimeProtocol.createChatTTSTextMessage( + state.sessionId, i == 0, false, chunks.get(i), objectMapper)); + } + sendUpstreamBinary(state, VolcRealtimeProtocol.createChatTTSTextMessage( + state.sessionId, false, true, "", objectMapper)); + } + + private void cancelFuture(java.util.concurrent.ScheduledFuture future) { + if (future != null) { + future.cancel(false); + } + } + + private String textValue(JsonNode node) { + if (node == null || node.isMissingNode() || node.isNull()) { + return ""; + } + return node.asText("").trim(); + } + + private String normalizeNullable(String value) { + return StringUtils.hasText(value) ? value.trim() : null; + } + + private String firstNonBlank(String... values) { + if (values == null) { + return ""; + } + for (String value : values) { + if (StringUtils.hasText(value)) { + return value.trim(); + } + } + return ""; + } + + private String writeJson(Object value) { + try { + return objectMapper.writeValueAsString(value); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private String abbreviate(String text) { + if (text == null || text.length() <= 100) { + return text; + } + return text.substring(0, 100); + } + + private final class UpstreamListener implements WebSocket.Listener { + + private final VoiceSessionState state; + private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream(); + private final StringBuilder textBuffer = new StringBuilder(); + + private UpstreamListener(VoiceSessionState state) { + this.state = state; + } + + @Override + public void onOpen(WebSocket webSocket) { + webSocket.request(1); + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + try { + byte[] chunk = new byte[data.remaining()]; + data.get(chunk); + binaryBuffer.writeBytes(chunk); + if (last) { + byte[] completeMsg = binaryBuffer.toByteArray(); + binaryBuffer.reset(); + int msgType = completeMsg.length >= 2 ? ((completeMsg[1] >> 4) & 0x0F) : -1; + int msgEvent = completeMsg.length >= 8 ? java.nio.ByteBuffer.wrap(completeMsg, 4, 4).getInt() : -1; + // Only log non-audio events to avoid overhead on 50fps audio frames + if (msgType != VolcRealtimeProtocol.TYPE_AUDIO_ONLY_SERVER) { + log.info("[VoiceGateway] upstream recv session={} len={} type={} event={}", state.sessionId, completeMsg.length, msgType, msgEvent); + } + handleUpstreamBinary(state, completeMsg); + } + } catch (Exception e) { + log.error("[VoiceGateway] onBinary exception session={}: {}", state.sessionId, e.getMessage(), e); + } + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + textBuffer.append(data == null ? "" : data); + if (last) { + sendJson(state, Map.of("type", "server_text", "text", textBuffer.toString())); + textBuffer.setLength(0); + } + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onClose(WebSocket webSocket, int statusCode, String reason) { + log.info("[VoiceGateway] upstream onClose session={} code={} reason={}", state.sessionId, statusCode, reason); + state.upstreamReady = false; + sendJson(state, Map.of("type", "upstream_closed", "code", statusCode)); + scheduler.schedule(() -> closeClient(state.clientSession, CloseStatus.NORMAL), 3, TimeUnit.SECONDS); + return WebSocket.Listener.super.onClose(webSocket, statusCode, reason); + } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("[VoiceGateway] upstream error session={}: {}", state.sessionId, error.getMessage(), error); + sendJson(state, Map.of("type", "error", "error", "语音服务连接异常: " + error.getMessage())); + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/service/VoiceSessionState.java b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceSessionState.java new file mode 100644 index 0000000..fe45e21 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/service/VoiceSessionState.java @@ -0,0 +1,150 @@ +package com.bigwo.javaserver.service; + +import java.net.http.WebSocket; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledFuture; + +import org.springframework.web.socket.WebSocketSession; + +import com.bigwo.javaserver.model.AssistantProfile; +import com.bigwo.javaserver.model.KnowledgeSearchResult; + +final class VoiceSessionState { + + final String clientConnectionId; + final String sessionId; + final WebSocketSession clientSession; + final Object clientSendLock = new Object(); + final Object upstreamSendLock = new Object(); + final StringBuilder assistantStreamBuffer = new StringBuilder(); + + volatile String userId; + volatile AssistantProfile assistantProfile = AssistantProfile.defaults(); + volatile WebSocket upstream; + volatile boolean upstreamReady; + volatile boolean readySent; + volatile boolean hasSentGreeting; + volatile boolean blockUpstreamAudio; + volatile boolean awaitingUpstreamReply; + volatile boolean pendingExternalRagReply; + volatile boolean discardNextAssistantResponse; + volatile boolean closed; + volatile String botName = "大沃"; + volatile String systemRole = ""; + volatile String speakingStyle = VoiceAssistantProfileSupport.DEFAULT_SPEAKING_STYLE; + volatile String speaker = "zh_female_vv_jupiter_bigtts"; + volatile String modelVersion = "O"; + volatile String greetingText = ""; + volatile String handoffSummary = ""; + volatile String latestUserText = ""; + volatile long latestUserTurnSeq; + volatile String lastPersistedUserText = ""; + volatile long lastPersistedUserAt; + volatile String lastPersistedAssistantText = ""; + volatile long lastPersistedAssistantAt; + volatile String currentTtsType = ""; + volatile String currentSpeechText = ""; + volatile String assistantStreamReplyId = ""; + volatile String pendingAssistantSource; + volatile String pendingAssistantToolName; + volatile Map pendingAssistantMeta; + volatile long pendingAssistantTurnSeq; + volatile long lastDeliveredAssistantTurnSeq; + volatile long lastActivityAt = System.currentTimeMillis(); + volatile long greetingSentAt; + volatile long lastBargeInResetAt; + volatile String lastFinalNormalized = ""; + volatile long lastFinalAt; + volatile ScheduledFuture idleFuture; + volatile ScheduledFuture keepaliveFuture; + + // TTS timing & local speech delivery + volatile boolean isSendingChatTTSText; + volatile long chatTTSUntil; + volatile long directSpeakUntil; + volatile boolean fillerActive; + + // Suppress upstream reply mechanism + volatile long suppressUpstreamUntil; + volatile ScheduledFuture suppressReplyFuture; + + // Greeting protection & timers + volatile long greetingProtectionUntil; + volatile boolean pendingGreetingAck; + volatile ScheduledFuture greetingTimerFuture; + volatile ScheduledFuture greetingAckTimerFuture; + volatile ScheduledFuture chatTTSTimerFuture; + + // processReply queue & concurrency + volatile boolean processingReply; + volatile String queuedUserText = ""; + volatile long queuedUserTurnSeq; + volatile ScheduledFuture queuedReplyTimerFuture; + + // Reply timeout watchdog + volatile ScheduledFuture replyTimeoutFuture; + volatile long replyTimeoutTurnSeq; + volatile long cancelledReplyTurnSeq; + + // Echo detection + volatile boolean echoLogOnce; + volatile long lastPartialAt; + + // Audio block logging + volatile boolean audioBlockLogOnce; + volatile long clientAudioFrameCount; + + // Async chain for serialized upstream audio sends (non-blocking) + volatile CompletableFuture lastUpstreamAudioSend = CompletableFuture.completedFuture(null); + volatile long lastAudioSendSuccessAt; + + // Turn count + volatile long turnCount; + + // KB prequery (GAP-11) + volatile CompletableFuture pendingKbPrequery; + volatile String kbPrequeryText = ""; + volatile long kbPrequeryStartedAt; + + // KB protection window (GAP-19) + volatile String lastKbTopic = ""; + volatile long lastKbHitAt; + + // Reply plan: evidence text (raw KB content, never used for subtitle/persistence) + volatile String ragEvidenceText = ""; + + VoiceSessionState(String clientConnectionId, String sessionId, WebSocketSession clientSession, String userId) { + this.clientConnectionId = clientConnectionId; + this.sessionId = sessionId; + this.clientSession = clientSession; + this.userId = userId; + } + + void touch() { + lastActivityAt = System.currentTimeMillis(); + } + + synchronized String appendAssistantChunk(String replyId, String chunkText) { + if (replyId != null && !replyId.isBlank() && !assistantStreamReplyId.isBlank() && !assistantStreamReplyId.equals(replyId)) { + assistantStreamBuffer.setLength(0); + } + if (replyId != null && !replyId.isBlank()) { + assistantStreamReplyId = replyId; + } + assistantStreamBuffer.append(chunkText == null ? "" : chunkText); + return assistantStreamBuffer.toString(); + } + + synchronized String consumeAssistantBuffer() { + String text = assistantStreamBuffer.toString().trim(); + assistantStreamBuffer.setLength(0); + assistantStreamReplyId = ""; + return text; + } + + synchronized void clearAssistantBuffer() { + assistantStreamBuffer.setLength(0); + assistantStreamReplyId = ""; + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/util/VolcSignerV4.java b/java-server/src/main/java/com/bigwo/javaserver/util/VolcSignerV4.java new file mode 100644 index 0000000..5bca589 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/util/VolcSignerV4.java @@ -0,0 +1,76 @@ +package com.bigwo.javaserver.util; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import org.springframework.stereotype.Component; + +@Component +public class VolcSignerV4 { + + public Map signRequest(String method, String host, String path, String body, String accessKeyId, String secretAccessKey, String service, String region) { + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + String dateStamp = now.format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String amzDate = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'")); + String credentialScope = dateStamp + "/" + region + "/" + service + "/request"; + String payload = body == null ? "" : body; + String bodyHash = sha256Hex(payload); + + Map headers = new TreeMap<>(); + headers.put("content-type", "application/json"); + headers.put("host", host); + headers.put("x-content-sha256", bodyHash); + headers.put("x-date", amzDate); + + String signedHeaders = String.join(";", headers.keySet()); + String canonicalHeaders = headers.entrySet().stream() + .map(entry -> entry.getKey() + ":" + entry.getValue() + "\n") + .reduce("", String::concat); + String canonicalRequest = String.join("\n", method, path, "", canonicalHeaders, signedHeaders, bodyHash); + String stringToSign = String.join("\n", "HMAC-SHA256", amzDate, credentialScope, sha256Hex(canonicalRequest)); + + byte[] signingKey = hmacSha256(secretAccessKey.getBytes(StandardCharsets.UTF_8), dateStamp); + signingKey = hmacSha256(signingKey, region); + signingKey = hmacSha256(signingKey, service); + signingKey = hmacSha256(signingKey, "request"); + String signature = bytesToHex(hmacSha256(signingKey, stringToSign)); + + Map result = new LinkedHashMap<>(headers); + result.put("authorization", "HMAC-SHA256 Credential=" + accessKeyId + "/" + credentialScope + ", SignedHeaders=" + signedHeaders + ", Signature=" + signature); + return result; + } + + private byte[] hmacSha256(byte[] key, String data) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data.getBytes(StandardCharsets.UTF_8)); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private String sha256Hex(String data) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return bytesToHex(digest.digest(data.getBytes(StandardCharsets.UTF_8))); + } catch (Exception exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private String bytesToHex(byte[] bytes) { + StringBuilder builder = new StringBuilder(bytes.length * 2); + for (byte value : bytes) { + builder.append(String.format("%02x", value)); + } + return builder.toString(); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/ApiRequestLoggingFilter.java b/java-server/src/main/java/com/bigwo/javaserver/web/ApiRequestLoggingFilter.java new file mode 100644 index 0000000..5ebfc4f --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/ApiRequestLoggingFilter.java @@ -0,0 +1,31 @@ +package com.bigwo.javaserver.web; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +@Component +public class ApiRequestLoggingFilter extends OncePerRequestFilter { + + private static final Logger log = LoggerFactory.getLogger(ApiRequestLoggingFilter.class); + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) + throws ServletException, IOException { + long startTime = System.currentTimeMillis(); + try { + filterChain.doFilter(request, response); + } finally { + if (request.getRequestURI().startsWith("/api/")) { + long elapsed = System.currentTimeMillis() - startTime; + log.info("[{}] {} -> {} ({}ms)", request.getMethod(), request.getRequestURI(), response.getStatus(), elapsed); + } + } + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/request/AssistantProfileRefreshRequest.java b/java-server/src/main/java/com/bigwo/javaserver/web/request/AssistantProfileRefreshRequest.java new file mode 100644 index 0000000..f3ab4b3 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/request/AssistantProfileRefreshRequest.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.web.request; + +public record AssistantProfileRefreshRequest(String userId) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatSendRequest.java b/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatSendRequest.java new file mode 100644 index 0000000..4db2b00 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatSendRequest.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.web.request; + +public record ChatSendRequest(String sessionId, String message, String userId) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatStartRequest.java b/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatStartRequest.java new file mode 100644 index 0000000..025c95a --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/request/ChatStartRequest.java @@ -0,0 +1,7 @@ +package com.bigwo.javaserver.web.request; + +import com.bigwo.javaserver.model.ChatSubtitle; +import java.util.List; + +public record ChatStartRequest(String sessionId, List voiceSubtitles, String userId) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/request/SessionSwitchRequest.java b/java-server/src/main/java/com/bigwo/javaserver/web/request/SessionSwitchRequest.java new file mode 100644 index 0000000..597fc53 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/request/SessionSwitchRequest.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.web.request; + +public record SessionSwitchRequest(String targetMode) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/request/VideoAdminConfigRequest.java b/java-server/src/main/java/com/bigwo/javaserver/web/request/VideoAdminConfigRequest.java new file mode 100644 index 0000000..f95dc55 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/request/VideoAdminConfigRequest.java @@ -0,0 +1,4 @@ +package com.bigwo.javaserver.web.request; + +public record VideoAdminConfigRequest(String model) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/response/AssistantProfileResponseData.java b/java-server/src/main/java/com/bigwo/javaserver/web/response/AssistantProfileResponseData.java new file mode 100644 index 0000000..494a7bb --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/response/AssistantProfileResponseData.java @@ -0,0 +1,14 @@ +package com.bigwo.javaserver.web.response; + +import com.bigwo.javaserver.model.AssistantProfile; + +public record AssistantProfileResponseData( + String userId, + AssistantProfile profile, + String source, + boolean cached, + Long fetchedAt, + boolean configured, + String error +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthFeaturesResponse.java b/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthFeaturesResponse.java new file mode 100644 index 0000000..ad5ba5b --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthFeaturesResponse.java @@ -0,0 +1,15 @@ +package com.bigwo.javaserver.web.response; + +public record HealthFeaturesResponse( + boolean voiceChat, + boolean textChat, + String textChatProvider, + boolean webSearch, + boolean customSpeaker, + boolean arkKnowledgeBase, + boolean redis, + Object reranker, + String kbRetrievalMode, + boolean nativeVoiceGateway +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthResponse.java b/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthResponse.java new file mode 100644 index 0000000..9b4538e --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/web/response/HealthResponse.java @@ -0,0 +1,10 @@ +package com.bigwo.javaserver.web.response; + +public record HealthResponse( + String status, + String mode, + String apiVersion, + boolean configured, + HealthFeaturesResponse features +) { +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketConfig.java b/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketConfig.java new file mode 100644 index 0000000..481f192 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketConfig.java @@ -0,0 +1,22 @@ +package com.bigwo.javaserver.websocket; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + +@Configuration +@EnableWebSocket +public class VoiceWebSocketConfig implements WebSocketConfigurer { + + private final VoiceWebSocketHandler voiceWebSocketHandler; + + public VoiceWebSocketConfig(VoiceWebSocketHandler voiceWebSocketHandler) { + this.voiceWebSocketHandler = voiceWebSocketHandler; + } + + @Override + public void registerWebSocketHandlers(WebSocketHandlerRegistry registry) { + registry.addHandler(voiceWebSocketHandler, "/ws/realtime-dialog").setAllowedOriginPatterns("*"); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketHandler.java b/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketHandler.java new file mode 100644 index 0000000..971fb12 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/websocket/VoiceWebSocketHandler.java @@ -0,0 +1,47 @@ +package com.bigwo.javaserver.websocket; + +import com.bigwo.javaserver.service.VoiceGatewayService; +import java.net.URI; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.BinaryMessage; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.BinaryWebSocketHandler; +import org.springframework.web.util.UriComponentsBuilder; + +@Component +public class VoiceWebSocketHandler extends BinaryWebSocketHandler { + + private final VoiceGatewayService voiceGatewayService; + + public VoiceWebSocketHandler(VoiceGatewayService voiceGatewayService) { + this.voiceGatewayService = voiceGatewayService; + } + + @Override + public void afterConnectionEstablished(WebSocketSession session) { + URI uri = session.getUri(); + var queryParams = UriComponentsBuilder.fromUri(uri == null ? URI.create("/") : uri).build(true).getQueryParams(); + voiceGatewayService.afterConnectionEstablished(session, queryParams.getFirst("sessionId"), queryParams.getFirst("userId")); + } + + @Override + protected void handleTextMessage(WebSocketSession session, TextMessage message) { + voiceGatewayService.handleTextMessage(session, message == null ? "" : message.getPayload()); + } + + @Override + protected void handleBinaryMessage(WebSocketSession session, BinaryMessage message) { + voiceGatewayService.handleBinaryMessage(session, message == null ? new byte[0] : message.getPayload().array()); + } + + @Override + public void afterConnectionClosed(WebSocketSession session, org.springframework.web.socket.CloseStatus status) { + voiceGatewayService.afterConnectionClosed(session); + } + + @Override + public void handleTransportError(WebSocketSession session, Throwable exception) { + voiceGatewayService.handleTransportError(session, exception); + } +} diff --git a/java-server/src/main/java/com/bigwo/javaserver/websocket/VolcRealtimeProtocol.java b/java-server/src/main/java/com/bigwo/javaserver/websocket/VolcRealtimeProtocol.java new file mode 100644 index 0000000..2e0bc41 --- /dev/null +++ b/java-server/src/main/java/com/bigwo/javaserver/websocket/VolcRealtimeProtocol.java @@ -0,0 +1,169 @@ +package com.bigwo.javaserver.websocket; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public final class VolcRealtimeProtocol { + + private static final int HEADER_SIZE_4 = 0x1; + private static final int VERSION_1 = 0x10; + private static final int SERIALIZATION_JSON = 0x1 << 4; + private static final int SERIALIZATION_RAW = 0; + private static final int COMPRESSION_NONE = 0; + private static final int MSG_TYPE_FLAG_WITH_EVENT = 0b100; + + public static final int TYPE_FULL_CLIENT = 1; + public static final int TYPE_AUDIO_ONLY_CLIENT = 2; + public static final int TYPE_FULL_SERVER = 9; + public static final int TYPE_AUDIO_ONLY_SERVER = 11; + public static final int TYPE_ERROR = 15; + + private VolcRealtimeProtocol() { + } + + public static byte[] createStartConnectionMessage() { + return marshal(TYPE_FULL_CLIENT, MSG_TYPE_FLAG_WITH_EVENT, 1, "", "{}".getBytes(StandardCharsets.UTF_8), false); + } + + public static byte[] createStartSessionMessage(String sessionId, Map payload, ObjectMapper objectMapper) { + return marshal(TYPE_FULL_CLIENT, MSG_TYPE_FLAG_WITH_EVENT, 100, sessionId, writeJsonBytes(payload, objectMapper), false); + } + + public static byte[] createAudioMessage(String sessionId, byte[] audioBuffer) { + return marshal(TYPE_AUDIO_ONLY_CLIENT, MSG_TYPE_FLAG_WITH_EVENT, 200, sessionId, audioBuffer == null ? new byte[0] : audioBuffer, true); + } + + public static byte[] createSayHelloMessage(String sessionId, String content, ObjectMapper objectMapper) { + return marshal(TYPE_FULL_CLIENT, MSG_TYPE_FLAG_WITH_EVENT, 300, sessionId, writeJsonBytes(Map.of("content", content == null ? "" : content), objectMapper), false); + } + + public static byte[] createChatTTSTextMessage(String sessionId, boolean start, boolean end, String content, ObjectMapper objectMapper) { + Map payload = new java.util.LinkedHashMap<>(); + payload.put("session_id", sessionId); + payload.put("start", start); + payload.put("end", end); + payload.put("content", content == null ? "" : content); + return marshal(TYPE_FULL_CLIENT, MSG_TYPE_FLAG_WITH_EVENT, 500, sessionId, writeJsonBytes(payload, objectMapper), false); + } + + public static byte[] createChatRagTextMessage(String sessionId, String externalRag, ObjectMapper objectMapper) { + return marshal( + TYPE_FULL_CLIENT, + MSG_TYPE_FLAG_WITH_EVENT, + 502, + sessionId, + writeJsonBytes(Map.of("session_id", sessionId, "external_rag", externalRag == null ? "[]" : externalRag), objectMapper), + false + ); + } + + public static Frame unmarshal(byte[] data) { + byte[] buffer = data == null ? new byte[0] : data; + if (buffer.length < 4) { + throw new IllegalArgumentException("protocol message too short"); + } + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + byteBuffer.get(); + int typeAndFlag = Byte.toUnsignedInt(byteBuffer.get()); + byteBuffer.get(); + byteBuffer.get(); + int type = (typeAndFlag >> 4) & 0x0F; + int typeFlag = typeAndFlag & 0x0F; + if (type == TYPE_ERROR) { + byte[] payload = new byte[byteBuffer.remaining()]; + byteBuffer.get(payload); + return new Frame(type, typeFlag, 0, "", payload); + } + int event = 0; + String sessionId = ""; + if (containsEvent(typeFlag)) { + event = byteBuffer.getInt(); + } + if (containsEvent(typeFlag) && shouldHandleSessionId(event)) { + sessionId = readStringWithLength(byteBuffer); + } + byte[] payload = readPayload(byteBuffer); + return new Frame(type, typeFlag, event, sessionId, payload); + } + + private static byte[] marshal(int type, int typeFlag, int event, String sessionId, byte[] payload, boolean rawPayload) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + outputStream.write(new byte[] { + (byte) (VERSION_1 | HEADER_SIZE_4), + (byte) (((type & 0x0F) << 4) | (typeFlag & 0x0F)), + (byte) ((rawPayload ? SERIALIZATION_RAW : SERIALIZATION_JSON) | COMPRESSION_NONE), + 0 + }); + if (containsEvent(typeFlag)) { + writeInt(outputStream, event); + } + if (containsEvent(typeFlag) && shouldHandleSessionId(event)) { + writeStringWithLength(outputStream, sessionId); + } + writePayload(outputStream, payload == null ? new byte[0] : payload); + return outputStream.toByteArray(); + } catch (IOException exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + private static void writeInt(ByteArrayOutputStream outputStream, int value) throws IOException { + outputStream.write(ByteBuffer.allocate(4).putInt(value).array()); + } + + private static void writeStringWithLength(ByteArrayOutputStream outputStream, String value) throws IOException { + byte[] content = String.valueOf(value == null ? "" : value).getBytes(StandardCharsets.UTF_8); + writeInt(outputStream, content.length); + outputStream.write(content); + } + + private static void writePayload(ByteArrayOutputStream outputStream, byte[] payload) throws IOException { + writeInt(outputStream, payload.length); + outputStream.write(payload); + } + + private static String readStringWithLength(ByteBuffer byteBuffer) { + int size = byteBuffer.getInt(); + if (size <= 0) { + return ""; + } + byte[] value = new byte[size]; + byteBuffer.get(value); + return new String(value, StandardCharsets.UTF_8); + } + + private static byte[] readPayload(ByteBuffer byteBuffer) { + int size = byteBuffer.getInt(); + if (size <= 0) { + return new byte[0]; + } + byte[] payload = new byte[size]; + byteBuffer.get(payload); + return payload; + } + + private static boolean containsEvent(int typeFlag) { + return (typeFlag & MSG_TYPE_FLAG_WITH_EVENT) == MSG_TYPE_FLAG_WITH_EVENT; + } + + private static boolean shouldHandleSessionId(int event) { + return event != 1 && event != 2 && event != 50 && event != 51 && event != 52; + } + + private static byte[] writeJsonBytes(Object payload, ObjectMapper objectMapper) { + try { + return objectMapper.writeValueAsBytes(payload); + } catch (IOException exception) { + throw new IllegalStateException(exception.getMessage(), exception); + } + } + + public record Frame(int type, int typeFlag, int event, String sessionId, byte[] payload) { + } +} diff --git a/java-server/src/main/resources/application.yml b/java-server/src/main/resources/application.yml new file mode 100644 index 0000000..c2f6f92 --- /dev/null +++ b/java-server/src/main/resources/application.yml @@ -0,0 +1,89 @@ +server: + port: ${PORT:3012} + +spring: + application: + name: bigwo-java-server + mvc: + throw-exception-if-no-handler-found: true + servlet: + multipart: + max-file-size: 50MB + max-request-size: 50MB + web: + resources: + add-mappings: false + +logging: + level: + root: INFO + com.bigwo.javaserver: INFO + +bigwo: + mysql: + enabled: ${MYSQL_ENABLED:true} + host: ${MYSQL_HOST:localhost} + port: ${MYSQL_PORT:3306} + database: ${MYSQL_DATABASE:bigwo_chat} + username: ${MYSQL_USER:root} + password: ${MYSQL_PASSWORD:} + max-pool-size: ${MYSQL_MAX_POOL_SIZE:10} + connection-timeout-ms: ${MYSQL_CONNECTION_TIMEOUT_MS:5000} + redis: + enabled: ${ENABLE_REDIS_CONTEXT:true} + url: ${REDIS_URL:redis://127.0.0.1:6379} + password: ${REDIS_PASSWORD:} + database: ${REDIS_DB:0} + key-prefix: ${REDIS_KEY_PREFIX:bigwo:} + timeout-ms: ${REDIS_TIMEOUT_MS:5000} + voice: + enabled: ${ENABLE_NATIVE_VOICE_GATEWAY:true} + upstream-url: ${VOLC_S2S_UPSTREAM_URL:wss://openspeech.bytedance.com/api/v3/realtime/dialogue} + resource-id: ${VOLC_S2S_RESOURCE_ID:volc.speech.dialog} + app-id: ${VOLC_S2S_APP_ID:} + token: ${VOLC_S2S_TOKEN:} + app-key: ${VOLC_DIALOG_APP_KEY:PlgvMymc7f3tQnJ6} + default-speaker: ${VOLC_S2S_SPEAKER_ID:zh_female_vv_jupiter_bigtts} + idle-timeout-ms: ${VOICE_IDLE_TIMEOUT_MS:300000} + audio-keepalive-interval-ms: ${VOICE_AUDIO_KEEPALIVE_INTERVAL_MS:20000} + send-ready-early: ${VOICE_SEND_READY_EARLY:true} + assistant-profile: + api-url: ${ASSISTANT_PROFILE_API_URL:} + api-method: ${ASSISTANT_PROFILE_API_METHOD:GET} + api-token: ${ASSISTANT_PROFILE_API_TOKEN:} + api-headers: ${ASSISTANT_PROFILE_API_HEADERS:} + timeout-ms: ${ASSISTANT_PROFILE_API_TIMEOUT_MS:5000} + cache-ttl-ms: ${ASSISTANT_PROFILE_CACHE_TTL_MS:60000} + coze: + base-url: ${COZE_BASE_URL:https://api.coze.cn} + api-token: ${COZE_API_TOKEN:} + bot-id: ${COZE_BOT_ID:} + knowledge: + access-key-id: ${VOLC_ACCESS_KEY_ID:} + secret-access-key: ${VOLC_SECRET_ACCESS_KEY:} + endpoint-id: ${VOLC_ARK_KNOWLEDGE_ENDPOINT_ID:${VOLC_ARK_ENDPOINT_ID:}} + model: ${VOLC_ARK_KB_MODEL:${VOLC_ARK_KNOWLEDGE_ENDPOINT_ID:${VOLC_ARK_ENDPOINT_ID:}}} + dataset-ids: ${VOLC_ARK_KNOWLEDGE_BASE_IDS:} + retrieval-top-k: ${VOLC_ARK_KB_RETRIEVAL_TOP_K:25} + threshold: ${VOLC_ARK_KNOWLEDGE_THRESHOLD:0.1} + reranker-model: ${VOLC_ARK_RERANKER_MODEL:${VOLC_ARK_RERANKER_ENDPOINT_ID:doubao-seed-rerank}} + reranker-top-n: ${VOLC_ARK_RERANKER_TOP_N:3} + enable-reranker: ${ENABLE_RERANKER:true} + enable-redis-context: ${ENABLE_REDIS_CONTEXT:true} + collection-map-json: ${VIKINGDB_COLLECTION_MAP:} + video: + seedance-api-key: ${SEEDANCE_API_KEY:} + seedance-api-base: ${SEEDANCE_API_BASE:https://n.lconai.com} + seedance-model: ${SEEDANCE_MODEL:seedance-2.0} + gemini-model: ${GEMINI_MODEL:} + gemini-api-key: ${GEMINI_API_KEY:${SEEDANCE_API_KEY:}} + gemini-api-base: ${GEMINI_API_BASE:${SEEDANCE_API_BASE:https://n.lconai.com}} + poll-interval-ms: ${SEEDANCE_POLL_INTERVAL_MS:5000} + poll-timeout-ms: ${SEEDANCE_POLL_TIMEOUT_MS:900000} + request-timeout-ms: ${VIDEO_HTTP_TIMEOUT_MS:1800000} + work-dir: ${VIDEO_WORK_DIR:} + max-concurrent-tasks: ${VIDEO_MAX_CONCURRENT_TASKS:4} + logo-path: ${VIDEO_LOGO_PATH:} + videos-dir: ${VIDEO_VIDEOS_DIR:} + ffmpeg-threads: ${VIDEO_FFMPEG_THREADS:1} + post-process-timeout-ms: ${VIDEO_POST_PROCESS_TIMEOUT_MS:180000} diff --git a/java-server/src/main/resources/logback-spring.xml b/java-server/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..165ef1c --- /dev/null +++ b/java-server/src/main/resources/logback-spring.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/java-server/src/test/java/com/bigwo/javaserver/ApiContractSmokeTest.java b/java-server/src/test/java/com/bigwo/javaserver/ApiContractSmokeTest.java new file mode 100644 index 0000000..2215d35 --- /dev/null +++ b/java-server/src/test/java/com/bigwo/javaserver/ApiContractSmokeTest.java @@ -0,0 +1,64 @@ +package com.bigwo.javaserver; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest(properties = { + "MYSQL_ENABLED=false", + "ENABLE_REDIS_CONTEXT=false" +}) +@AutoConfigureMockMvc +class ApiContractSmokeTest { + + @Autowired + private MockMvc mockMvc; + + @Test + void healthEndpointShouldMatchExpectedContract() throws Exception { + mockMvc.perform(get("/api/health")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ok")) + .andExpect(jsonPath("$.mode").value("s2s-hybrid")) + .andExpect(jsonPath("$.apiVersion").value("2024-12-01")) + .andExpect(jsonPath("$.configured").value(true)) + .andExpect(jsonPath("$.features.voiceChat").value(true)) + .andExpect(jsonPath("$.features.redis").value(false)); + } + + @Test + void assistantProfileEndpointShouldReturnDefaultProfileWhenRemoteApiDisabled() throws Exception { + mockMvc.perform(get("/api/assistant-profile")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.message").value("success")) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(jsonPath("$.data.profile.agentName").value("大沃")) + .andExpect(jsonPath("$.data.configured").value(false)); + } + + @Test + void assistantProfileRefreshEndpointShouldReturnDefaultProfileWhenRemoteApiDisabled() throws Exception { + mockMvc.perform(post("/api/assistant-profile/refresh") + .contentType("application/json") + .content("{}")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.code").value(0)) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(jsonPath("$.data.profile.nickname").value("大沃")); + } + + @Test + void sessionListShouldReturnServerErrorWhenDatabaseIsUnavailable() throws Exception { + mockMvc.perform(get("/api/session/list")) + .andExpect(status().isInternalServerError()) + .andExpect(jsonPath("$.success").value(false)) + .andExpect(jsonPath("$.error").value("Database unavailable")); + } +} diff --git a/java-server/src/test/java/com/bigwo/javaserver/ChatApiContractSmokeTest.java b/java-server/src/test/java/com/bigwo/javaserver/ChatApiContractSmokeTest.java new file mode 100644 index 0000000..20ff7e2 --- /dev/null +++ b/java-server/src/test/java/com/bigwo/javaserver/ChatApiContractSmokeTest.java @@ -0,0 +1,61 @@ +package com.bigwo.javaserver; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest(properties = { + "MYSQL_ENABLED=false", + "ENABLE_REDIS_CONTEXT=false" +}) +@AutoConfigureMockMvc +class ChatApiContractSmokeTest { + + @Autowired + private MockMvc mockMvc; + + @Test + void chatStartShouldFailWhenCozeIsNotConfigured() throws Exception { + mockMvc.perform(post("/api/chat/start") + .contentType("application/json") + .content("{\"sessionId\":\"chat-test-1\"}")) + .andExpect(status().isInternalServerError()) + .andExpect(jsonPath("$.success").value(false)) + .andExpect(jsonPath("$.error").value("Coze 智能体未配置，请设置 COZE_API_TOKEN 和 COZE_BOT_ID")); + } + + @Test + void chatSendShouldReturnFastGreetingWithoutCoze() throws Exception { + mockMvc.perform(post("/api/chat/send") + .contentType("application/json") + .content("{\"sessionId\":\"chat-test-2\",\"message\":\"你好\"}")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(jsonPath("$.data.content").value("你好😊！我是大沃智能助手。你可以直接问我一成系统、德国PM产品、招商合作、营养科普等问题，我会尽量快速给你准确回复。")); + } + + @Test + void chatHistoryShouldReturnEmptyArrayForUnknownSession() throws Exception { + mockMvc.perform(get("/api/chat/history/unknown-session")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(jsonPath("$.data").isArray()); + } + + @Test + void chatDeleteShouldReturnSuccess() throws Exception { + mockMvc.perform(delete("/api/chat/chat-test-3")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(content().json("{\"success\":true}")); + } +} diff --git a/java-server/src/test/java/com/bigwo/javaserver/KnowledgeServicesTest.java b/java-server/src/test/java/com/bigwo/javaserver/KnowledgeServicesTest.java new file mode 100644 index 0000000..f41e5ca --- /dev/null +++ b/java-server/src/test/java/com/bigwo/javaserver/KnowledgeServicesTest.java @@ -0,0 +1,77 @@ +package com.bigwo.javaserver; + +import com.bigwo.javaserver.model.KnowledgeQueryInfo; +import com.bigwo.javaserver.model.LlmMessage; +import com.bigwo.javaserver.service.ContextKeywordTracker; +import com.bigwo.javaserver.service.FastAsrCorrector; +import com.bigwo.javaserver.service.KnowledgeKeywordCatalog; +import com.bigwo.javaserver.service.KnowledgeQueryResolver; +import com.bigwo.javaserver.service.KnowledgeRouteDecider; +import com.bigwo.javaserver.service.PinyinProductMatcher; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class KnowledgeServicesTest { + + private KnowledgeQueryResolver knowledgeQueryResolver; + private KnowledgeRouteDecider knowledgeRouteDecider; + private ContextKeywordTracker contextKeywordTracker; + + @BeforeEach + void setUp() { + KnowledgeKeywordCatalog keywordCatalog = new KnowledgeKeywordCatalog(); + PinyinProductMatcher pinyinProductMatcher = new PinyinProductMatcher(); + FastAsrCorrector fastAsrCorrector = new FastAsrCorrector(pinyinProductMatcher); + knowledgeQueryResolver = new KnowledgeQueryResolver(fastAsrCorrector, keywordCatalog); + knowledgeRouteDecider = new KnowledgeRouteDecider(keywordCatalog, knowledgeQueryResolver); + contextKeywordTracker = new ContextKeywordTracker(keywordCatalog, knowledgeQueryResolver); + } + + @Test + void rewriteKnowledgeQueryShouldNormalizeWarmFurnaceAlias() { + assertEquals("火炉原理", knowledgeQueryResolver.rewriteKnowledgeQuery("暖炉原理是什么", List.of())); + } + + @Test + void resolveKnowledgeQueryShouldApplySpecialEntityAlias() { + KnowledgeQueryInfo info = knowledgeQueryResolver.resolveKnowledgeQuery("太美怎么吃"); + + assertTrue(info.normalizedText().contains("肽美")); + assertEquals("肽美", info.primaryEntity()); + assertTrue(info.hasExplicitEntity()); + } + + @Test + void knowledgeRouteDeciderShouldUseContextForFollowUpQuestion() { + boolean forced = knowledgeRouteDecider.shouldForceKnowledgeRoute( + "怎么吃", + List.of(new LlmMessage("assistant", "小红产品 Activize Oxyplus 的功效和吃法")) + ); + + assertTrue(forced); + } + + @Test + void contextKeywordTrackerShouldReturnLatestKeywordForShortFollowUp() { + contextKeywordTracker.updateSession("session-1", "我想了解肽美的功效"); + + List contextTerms = contextKeywordTracker.suggestContextTerms("session-1", "怎么吃"); + + assertEquals(List.of("肽美"), contextTerms); + } + + @Test + void contextKeywordTrackerShouldNotInjectContextWhenQueryHasExplicitEntity() { + contextKeywordTracker.updateSession("session-2", "我想了解肽美的功效"); + + List contextTerms = contextKeywordTracker.suggestContextTerms("session-2", "Basics 怎么吃"); + + assertTrue(contextTerms.isEmpty()); + assertFalse(knowledgeRouteDecider.shouldForceKnowledgeRoute("你好", List.of())); + } +} diff --git a/java-server/src/test/java/com/bigwo/javaserver/VideoApiContractSmokeTest.java b/java-server/src/test/java/com/bigwo/javaserver/VideoApiContractSmokeTest.java new file mode 100644 index 0000000..9df7699 --- /dev/null +++ b/java-server/src/test/java/com/bigwo/javaserver/VideoApiContractSmokeTest.java @@ -0,0 +1,79 @@ +package com.bigwo.javaserver; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest(properties = { + "MYSQL_ENABLED=false", + "ENABLE_REDIS_CONTEXT=false", + "SEEDANCE_API_KEY=", + "GEMINI_MODEL=" +}) +@AutoConfigureMockMvc +class VideoApiContractSmokeTest { + + @Autowired + private MockMvc mockMvc; + + @Test + void videoAdminConfigShouldExposeDefaultModelContract() throws Exception { + mockMvc.perform(get("/api/video/admin/config")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.currentModel").value("seedance-2.0")) + .andExpect(jsonPath("$.source").value("default")) + .andExpect(jsonPath("$.validModels").isArray()) + .andExpect(jsonPath("$.validModels[0]").value("seedance-2.0")); + } + + @Test + void videoGenerateShouldReturnTaskIdImmediately() throws Exception { + MockMultipartFile emptyFile = new MockMultipartFile("image", "", "application/octet-stream", new byte[0]); + mockMvc.perform(multipart("/api/video/generate") + .file(emptyFile) + .param("prompt", "展示这个产品的高级质感") + .param("product", "PM 产品") + .param("username", "tester") + .param("template", "product") + .param("size", "720x1280") + .param("seconds", "15")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.taskId").exists()) + .andExpect(jsonPath("$.status").value("optimizing")) + .andExpect(jsonPath("$.username").value("tester")); + } + + @Test + void videoTaskShouldReturnNotFoundForUnknownId() throws Exception { + mockMvc.perform(get("/api/video/task/not-found-task")) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.error").value("任务不存在")); + } + + @Test + void videoHistoryShouldReturnRowsArray() throws Exception { + mockMvc.perform(get("/api/video/history")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.total").isNumber()) + .andExpect(jsonPath("$.rows").isArray()); + } + + @Test + void videoAdminConfigShouldRejectInvalidModel() throws Exception { + mockMvc.perform(post("/api/video/admin/config") + .contentType("application/json") + .content("{\"model\":\"invalid-model\"}")) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.error").value("无效的模型名: \"invalid-model\"")) + .andExpect(jsonPath("$.validModels").isArray()); + } +} diff --git a/java-server/src/test/java/com/bigwo/javaserver/VoiceGatewaySmokeTest.java b/java-server/src/test/java/com/bigwo/javaserver/VoiceGatewaySmokeTest.java new file mode 100644 index 0000000..6988e11 --- /dev/null +++ b/java-server/src/test/java/com/bigwo/javaserver/VoiceGatewaySmokeTest.java @@ -0,0 +1,183 @@ +package com.bigwo.javaserver; + +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import com.bigwo.javaserver.service.KnowledgeQueryResolver; +import com.bigwo.javaserver.service.KnowledgeRouteDecider; +import com.bigwo.javaserver.service.ProductLinkTrigger; +import com.bigwo.javaserver.service.VoiceAssistantProfileSupport; + +@SpringBootTest +class VoiceGatewaySmokeTest { + + @Autowired + private ProductLinkTrigger productLinkTrigger; + + @Autowired + private VoiceAssistantProfileSupport voiceAssistantProfileSupport; + + // ── ProductLinkTrigger ── + + @Test + void productLinkTrigger_matchesKnownProduct() { + ProductLinkTrigger.TriggerResult result = productLinkTrigger.check("看看基础三合一的详细介绍"); + assertThat(result.triggered()).isTrue(); + assertThat(result.productName()).isEqualTo("基础三合一"); + assertThat(result.link()).isNotBlank(); + } + + @Test + void productLinkTrigger_noIntentNoTrigger() { + ProductLinkTrigger.TriggerResult result = productLinkTrigger.check("基础三合一好不好"); + assertThat(result.triggered()).isFalse(); + } + + @Test + void productLinkTrigger_unknownProductNoTrigger() { + ProductLinkTrigger.TriggerResult result = productLinkTrigger.check("帮我查一下苹果手机的详细介绍"); + assertThat(result.triggered()).isFalse(); + } + + @Test + void productLinkTrigger_emptyInput() { + assertThat(productLinkTrigger.check("").triggered()).isFalse(); + assertThat(productLinkTrigger.check(null).triggered()).isFalse(); + } + + @Test + void productLinkTrigger_aliasMatch() { + ProductLinkTrigger.TriggerResult result = productLinkTrigger.check("我想了解一下三合一的详细信息"); + assertThat(result.triggered()).isTrue(); + assertThat(result.productName()).isEqualTo("基础三合一"); + } + + @Test + void productLinkTrigger_multiplePatternsMatch() { + assertThat(productLinkTrigger.check("给我看看活力健的详情").triggered()).isTrue(); + assertThat(productLinkTrigger.check("看一下肽美的详细介绍").triggered()).isTrue(); + assertThat(productLinkTrigger.check("给我发一下小白的链接").triggered()).isTrue(); + } + + // ── normalizeTextForSpeech ── + + @Test + void normalizeTextForSpeech_removesMarkdown() { + String result = voiceAssistantProfileSupport.normalizeTextForSpeech("## 标题\n**加粗文字**"); + assertThat(result).doesNotContain("#").doesNotContain("**"); + assertThat(result).contains("标题").contains("加粗文字"); + } + + @Test + void normalizeTextForSpeech_handlesNull() { + assertThat(voiceAssistantProfileSupport.normalizeTextForSpeech(null)).isEmpty(); + } + + // ── splitTextForSpeech (via reflection or indirect test) ── + // We test the normalizer which splitTextForSpeech depends on. + // splitTextForSpeech is private in VoiceGatewayService; direct test requires + // exposing it or using integration test. The compile+context start already + // validates the wiring. + + @Test + void normalizeTextForSpeech_preservesChinesePunctuation() { + String input = "你好，世界！这是一段测试。"; + String result = voiceAssistantProfileSupport.normalizeTextForSpeech(input); + assertThat(result).isEqualTo(input); + } + + // ── GAP-21: normalizeKnowledgeAlias ── + + @Autowired + private KnowledgeQueryResolver knowledgeQueryResolver; + + @Test + void normalizeKnowledgeAlias_dawo() { + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("大窝怎么样")).contains("大沃"); + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("大握是什么")).contains("大沃"); + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("大卧好不好")).contains("大沃"); + } + + @Test + void normalizeKnowledgeAlias_shengka() { + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("盛咖学院是什么")).contains("盛咖学愿"); + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("圣咖学院怎么样")).contains("盛咖学愿"); + } + + @Test + void normalizeKnowledgeAlias_aizhongxiang() { + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("爱众享是什么")).contains("Ai众享"); + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("艾众享怎么用")).contains("Ai众享"); + } + + @Test + void normalizeKnowledgeAlias_yicheng() { + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("一城系统介绍")).contains("一成系统"); + assertThat(knowledgeQueryResolver.normalizeKnowledgeQueryAlias("易成系统是什么")).contains("一成系统"); + } + + // ── GAP-19: isKnowledgeFollowUp ── + + @Autowired + private KnowledgeRouteDecider knowledgeRouteDecider; + + @Test + void isKnowledgeFollowUp_directQuestions() { + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("详细说说")).isTrue(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("怎么吃")).isTrue(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("多少钱")).isTrue(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("功效是什么")).isTrue(); + } + + @Test + void isKnowledgeFollowUp_corrections() { + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("你搞错了吧")).isTrue(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("不是的")).isTrue(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("帮我再查一下")).isTrue(); + } + + @Test + void isKnowledgeFollowUp_notFollowUp() { + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("你好")).isFalse(); + assertThat(knowledgeRouteDecider.isKnowledgeFollowUp("")).isFalse(); + } + + // ── GAP-20: Brand sensitive pattern ── + + private static final Pattern BRAND_SENSITIVE = Pattern.compile("传销|骗局|骗子|正规吗|合法吗|正不正规|合不合法|是不是传销|直销还是传销|层级分销|非法集资|拉人头|下线|发展下线|报单|人头费"); + + @Test + void brandSensitivePattern_matches() { + assertThat(BRAND_SENSITIVE.matcher("PM是不是传销").find()).isTrue(); + assertThat(BRAND_SENSITIVE.matcher("德国PM正规吗").find()).isTrue(); + assertThat(BRAND_SENSITIVE.matcher("这是骗局吧").find()).isTrue(); + } + + @Test + void brandSensitivePattern_noMatch() { + assertThat(BRAND_SENSITIVE.matcher("PM产品怎么样").find()).isFalse(); + assertThat(BRAND_SENSITIVE.matcher("基础三合一功效").find()).isFalse(); + } + + // ── GAP-15: Consultant referral pattern ── + + private static final Pattern CONSULTANT_REFERRAL = Pattern.compile("咨询(?:专业|你的)?顾问|健康管理顾问|联系顾问|一对一指导|咨询专业|咨询医生|咨询营养师|咨询专业人士|建议.*咨询|问问医生|问问.*营养师"); + + @Test + void consultantReferralPattern_matches() { + assertThat(CONSULTANT_REFERRAL.matcher("建议你咨询专业顾问").find()).isTrue(); + assertThat(CONSULTANT_REFERRAL.matcher("可以联系顾问了解").find()).isTrue(); + assertThat(CONSULTANT_REFERRAL.matcher("建议咨询医生").find()).isTrue(); + assertThat(CONSULTANT_REFERRAL.matcher("问问营养师").find()).isTrue(); + } + + @Test + void consultantReferralPattern_noMatch() { + assertThat(CONSULTANT_REFERRAL.matcher("产品功效很好").find()).isFalse(); + assertThat(CONSULTANT_REFERRAL.matcher("你好").find()).isFalse(); + } +} diff --git a/test2/client/src/services/nativeVoiceService.js b/test2/client/src/services/nativeVoiceService.js index f84cf95..bbb0597 100644 --- a/test2/client/src/services/nativeVoiceService.js +++ b/test2/client/src/services/nativeVoiceService.js @@ -10,6 +10,8 @@ class NativeVoiceService { this.playbackTime = 0; this.activeSources = new Set(); this.pendingSamples = []; + this.pendingAudioChunks = []; + this._resuming = false; this.readyResolver = null; this.readyRejector = null; this.callbacks = { @@ -19,6 +21,7 @@ class NativeVoiceService { onAssistantPending: null, onDiagnostic: null, onIdleTimeout: null, + onProductLink: null, }; } @@ -87,6 +90,15 @@ class NativeVoiceService { } this.playbackTime = this.playbackContext.currentTime; + // 安全上下文检查: getUserMedia 需要 HTTPS 或 localhost + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { + const errMsg = window.isSecureContext === false + ? '麦克风访问需要 HTTPS 连接，请使用 https:// 地址访问' + : '当前浏览器不支持麦克风访问'; + this.emitConnectionState('error', errMsg); + throw new Error(errMsg); + } + // 并行: 同时预获取麦克风和建立WS连接，节省500ms+ const micPromise = navigator.mediaDevices.getUserMedia({ audio: { @@ -97,7 +109,12 @@ class NativeVoiceService { }, video: false, }).catch((err) => { - console.warn('[NativeVoice] Pre-fetch getUserMedia failed:', err.message); + console.warn('[NativeVoice] Pre-fetch getUserMedia failed:', err.name, err.message); + if (err.name === 'NotAllowedError' || err.message?.includes('Permission denied')) { + const msg = '麦克风权限被拒绝，请在浏览器设置中允许本站访问麦克风后重试'; + this.emitConnectionState('error', msg); + throw new Error(msg); + } return null; }); @@ -206,6 +223,14 @@ class NativeVoiceService { this.callbacks.onIdleTimeout?.(msg.timeout || 300000); return; } + if (msg.type === 'product_link') { + this.callbacks.onProductLink?.({ + product: msg.product, + link: msg.link, + description: msg.description, + }); + return; + } if (msg.type === 'upstream_closed') { this.callbacks.onError?.(new Error('语音服务已断开，请重新开始通话')); return; @@ -224,35 +249,66 @@ class NativeVoiceService { if (!this.playbackContext) { return; } - const pcm16 = new Int16Array(arrayBuffer); - if (!pcm16.length) { + if (this.playbackContext.state === 'suspended') { + this.pendingAudioChunks.push(arrayBuffer); + this._tryResumePlayback(); return; } - const audioBuffer = this.playbackContext.createBuffer(1, pcm16.length, 24000); - const channel = audioBuffer.getChannelData(0); - for (let i = 0; i < pcm16.length; i += 1) { - channel[i] = pcm16[i] / 32768; + this._playPcm(arrayBuffer); + } + + _playPcm(arrayBuffer) { + try { + const pcm16 = new Int16Array(arrayBuffer); + if (!pcm16.length) { + return; + } + const audioBuffer = this.playbackContext.createBuffer(1, pcm16.length, 24000); + const channel = audioBuffer.getChannelData(0); + for (let i = 0; i < pcm16.length; i += 1) { + channel[i] = pcm16[i] / 32768; + } + const source = this.playbackContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.playbackContext.destination); + this.activeSources.add(source); + source.onended = () => { + this.activeSources.delete(source); + try { + source.disconnect(); + } catch (_) {} + }; + const now = this.playbackContext.currentTime; + if (this.playbackTime < now) { + this.playbackTime = now + 0.02; + } + source.start(this.playbackTime); + this.playbackTime += audioBuffer.duration; + this.emitDiagnostic('audio_chunk', { samples: pcm16.length, duration: audioBuffer.duration }); + } catch (err) { + console.warn('[NativeVoice] playPcm failed:', err.message); } - const source = this.playbackContext.createBufferSource(); - source.buffer = audioBuffer; - source.connect(this.playbackContext.destination); - this.activeSources.add(source); - source.onended = () => { - this.activeSources.delete(source); - try { - source.disconnect(); - } catch (_) {} - }; - const now = this.playbackContext.currentTime; - if (this.playbackTime < now) { - this.playbackTime = now + 0.02; + } + + async _tryResumePlayback() { + if (this._resuming) return; + this._resuming = true; + try { + await this.playbackContext.resume(); + while (this.pendingAudioChunks.length > 0) { + this._playPcm(this.pendingAudioChunks.shift()); + } + } catch (e) { + console.warn('[NativeVoice] resume failed:', e.message); + } finally { + this._resuming = false; } - source.start(this.playbackTime); - this.playbackTime += audioBuffer.duration; - this.emitDiagnostic('audio_chunk', { samples: pcm16.length, duration: audioBuffer.duration }); } async startCapture(preFetchedStream) { + if (!preFetchedStream && (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia)) { + throw new Error('麦克风不可用: 需要 HTTPS 安全连接'); + } this.mediaStream = preFetchedStream || await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, @@ -365,6 +421,8 @@ class NativeVoiceService { } this.playbackTime = 0; this.pendingSamples = []; + this.pendingAudioChunks = []; + this._resuming = false; this.emitConnectionState('disconnected'); }