知识库rerank设置

This commit is contained in:
2025-11-07 11:21:27 +08:00
parent d9947e273c
commit b98450df96
8 changed files with 360 additions and 34 deletions

View File

@@ -42,6 +42,7 @@ CREATE TABLE `tb_ai_knowledge` (
`embedding_model_provider` VARCHAR(100) DEFAULT NULL COMMENT '向量模型提供商',
`rerank_model` VARCHAR(100) DEFAULT NULL COMMENT 'Rerank模型名称',
`rerank_model_provider` VARCHAR(100) DEFAULT NULL COMMENT 'Rerank模型提供商',
`reranking_enable` TINYINT(1) DEFAULT 0 COMMENT '是否启用Rerank0否 1是',
`retrieval_top_k` INT(11) DEFAULT 2 COMMENT '检索Top K返回前K个结果',
`retrieval_score_threshold` DECIMAL(3,2) DEFAULT 0.00 COMMENT '检索分数阈值0.00-1.00',
`vector_id` VARCHAR(100) DEFAULT NULL COMMENT '向量ID用于向量检索',

View File

@@ -19,24 +19,24 @@ public class RetrievalModel {
@JSONField(name = "search_method")
private String searchMethod;
/**
* Rerank模型提供商
*/
@JSONField(name = "reranking_provider_name")
private String rerankingProviderName;
/**
* Rerank模型名称
*/
@JSONField(name = "reranking_model")
private String rerankingModel;
/**
* Rerank是否启用
*/
@JSONField(name = "reranking_enable")
private Boolean rerankingEnable;
/**
* Rerank模式字符串值为 "reranking_model"
*/
@JSONField(name = "reranking_mode")
private String rerankingMode;
/**
* Rerank模型配置当 reranking_enable=true 时必须设置)
*/
@JSONField(name = "reranking_model")
private RerankingModel rerankingModel;
/**
* Top K返回前K个结果
*/
@@ -54,5 +54,23 @@ public class RetrievalModel {
*/
@JSONField(name = "score_threshold_enabled")
private Boolean scoreThresholdEnabled;
/**
* Rerank模型配置嵌套对象
*/
@Data
public static class RerankingModel {
/**
* Rerank模型提供商
*/
@JSONField(name = "reranking_provider_name")
private String rerankingProviderName;
/**
* Rerank模型名称
*/
@JSONField(name = "reranking_model_name")
private String rerankingModelName;
}
}

View File

@@ -29,6 +29,8 @@ import org.xyzh.common.dto.user.TbSysUser;
import org.xyzh.common.vo.UserDeptRoleVO;
import org.xyzh.system.utils.LoginUtil;
import com.alibaba.fastjson2.JSON;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -118,7 +120,7 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
// 设置检索模型配置Rerank、Top K、Score 阈值)
RetrievalModel retrievalModel = new RetrievalModel();
retrievalModel.setSearchMethod("hybrid_search"); // 默认使用混合搜索
retrievalModel.setSearchMethod("hybrid_search"); // 必填字段
// Top K 配置
if (knowledge.getRetrievalTopK() != null && knowledge.getRetrievalTopK() > 0) {
@@ -136,20 +138,40 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
retrievalModel.setScoreThresholdEnabled(false);
}
// Rerank 模型配置
if (StringUtils.hasText(knowledge.getRerankModel())) {
retrievalModel.setRerankingEnable(true);
retrievalModel.setRerankingModel(knowledge.getRerankModel());
retrievalModel.setRerankingProviderName(knowledge.getRerankModelProvider());
log.info("创建知识库 - 启用Rerank: model={}, provider={}",
knowledge.getRerankModel(), knowledge.getRerankModelProvider());
// Rerank 模型配置(以前端传参为准)
Boolean rerankEnable = knowledge.getRerankingEnable() != null ?
knowledge.getRerankingEnable() : false;
retrievalModel.setRerankingEnable(rerankEnable);
if (rerankEnable) {
// 启用 Rerank 时model 和 provider 必须有值
if (!StringUtils.hasText(knowledge.getRerankModel())) {
throw new IllegalArgumentException("启用Rerank后必须指定rerankModel");
}
if (!StringUtils.hasText(knowledge.getRerankModelProvider())) {
throw new IllegalArgumentException("启用Rerank后必须指定rerankModelProvider");
}
// 设置 reranking_mode 为固定值 "reranking_model"
retrievalModel.setRerankingMode("reranking_model");
// 创建 RerankingModel 对象(嵌套在 reranking_model 字段中)
RetrievalModel.RerankingModel rerankingModel = new RetrievalModel.RerankingModel();
rerankingModel.setRerankingProviderName(knowledge.getRerankModelProvider());
rerankingModel.setRerankingModelName(knowledge.getRerankModel());
retrievalModel.setRerankingModel(rerankingModel);
log.info("创建知识库 - 启用Rerank: enable={}, mode=reranking_model, model={}, provider={}",
rerankEnable, knowledge.getRerankModel(), knowledge.getRerankModelProvider());
} else {
retrievalModel.setRerankingEnable(false);
// 禁用 Rerank不设置 rerankingMode 和 rerankingModel
log.info("创建知识库 - 禁用Rerank");
}
difyRequest.setRetrievalModel(retrievalModel);
// 调用Dify API创建知识库使用知识库API Key
log.info("创建知识库 - 请求参数: {}", JSON.toJSONString(difyRequest));
DatasetCreateResponse difyResponse = difyApiClient.createDataset(difyRequest);
difyDatasetId = difyResponse.getId();
@@ -300,6 +322,82 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
}
needUpdateDify = true;
}
// 检索配置变化Rerank、Top K、Score阈值
boolean retrievalConfigChanged = false;
// 检测 Rerank 开关状态变化
Boolean newRerankEnable = knowledge.getRerankingEnable();
Boolean existingRerankEnable = existing.getRerankingEnable();
boolean rerankEnableChanged = (newRerankEnable != null && !newRerankEnable.equals(existingRerankEnable));
// 检测 Rerank 模型变化
String newRerankModel = knowledge.getRerankModel();
String existingRerankModel = existing.getRerankModel();
boolean rerankModelChanged = (newRerankModel != null && !newRerankModel.equals(existingRerankModel));
if (rerankEnableChanged || rerankModelChanged ||
(knowledge.getRetrievalTopK() != null && !knowledge.getRetrievalTopK().equals(existing.getRetrievalTopK())) ||
(knowledge.getRetrievalScoreThreshold() != null && !knowledge.getRetrievalScoreThreshold().equals(existing.getRetrievalScoreThreshold()))) {
retrievalConfigChanged = true;
}
if (retrievalConfigChanged) {
RetrievalModel retrievalModel = new RetrievalModel();
retrievalModel.setSearchMethod("hybrid_search"); // 必填字段
// Top K
if (knowledge.getRetrievalTopK() != null) {
retrievalModel.setTopK(knowledge.getRetrievalTopK());
} else {
retrievalModel.setTopK(existing.getRetrievalTopK() != null ? existing.getRetrievalTopK() : 2);
}
// Score 阈值
Double scoreThreshold = knowledge.getRetrievalScoreThreshold() != null ?
knowledge.getRetrievalScoreThreshold() :
(existing.getRetrievalScoreThreshold() != null ? existing.getRetrievalScoreThreshold() : 0.0);
retrievalModel.setScoreThreshold(scoreThreshold);
retrievalModel.setScoreThresholdEnabled(scoreThreshold > 0);
// Rerank 配置(以前端传参为准)
Boolean finalRerankEnable = newRerankEnable != null ? newRerankEnable :
(existingRerankEnable != null ? existingRerankEnable : false);
String finalRerankModel = newRerankModel != null ? newRerankModel : existingRerankModel;
String finalRerankProvider = knowledge.getRerankModelProvider() != null ?
knowledge.getRerankModelProvider() : existing.getRerankModelProvider();
// 直接使用前端传入的开关状态
retrievalModel.setRerankingEnable(finalRerankEnable);
if (finalRerankEnable) {
// 启用 Rerank 时model 和 provider 必须有值
if (!StringUtils.hasText(finalRerankModel)) {
throw new IllegalArgumentException("启用Rerank后必须指定rerankModel");
}
if (!StringUtils.hasText(finalRerankProvider)) {
throw new IllegalArgumentException("启用Rerank后必须指定rerankModelProvider");
}
// 设置 reranking_mode 为固定值 "reranking_model"
retrievalModel.setRerankingMode("reranking_model");
// 创建 RerankingModel 对象(嵌套在 reranking_model 字段中)
RetrievalModel.RerankingModel rerankingModel = new RetrievalModel.RerankingModel();
rerankingModel.setRerankingProviderName(finalRerankProvider);
rerankingModel.setRerankingModelName(finalRerankModel);
retrievalModel.setRerankingModel(rerankingModel);
log.info("更新Rerank配置: 启用 - enable={}, mode=reranking_model, model={}, provider={}",
finalRerankEnable, finalRerankModel, finalRerankProvider);
} else {
// 禁用 Rerank不设置 rerankingMode 和 rerankingModel
log.info("更新Rerank配置: 禁用 - enable={}", finalRerankEnable);
}
updateRequest.setRetrievalModel(retrievalModel);
needUpdateDify = true;
}
// 同步到Dify
if (needUpdateDify && StringUtils.hasText(existing.getDifyDatasetId())) {

View File

@@ -21,6 +21,7 @@
<result column="embedding_model_provider" property="embeddingModelProvider" jdbcType="VARCHAR"/>
<result column="rerank_model" property="rerankModel" jdbcType="VARCHAR"/>
<result column="rerank_model_provider" property="rerankModelProvider" jdbcType="VARCHAR"/>
<result column="reranking_enable" property="rerankingEnable" jdbcType="BOOLEAN"/>
<result column="retrieval_top_k" property="retrievalTopK" jdbcType="INTEGER"/>
<result column="retrieval_score_threshold" property="retrievalScoreThreshold" jdbcType="DECIMAL"/>
<result column="vector_id" property="vectorID" jdbcType="VARCHAR"/>
@@ -40,7 +41,7 @@
<sql id="Base_Column_List">
id, title, avatar, description, content, source_type, source_id, file_name, file_path,
category, tags, dify_dataset_id, dify_indexing_technique, embedding_model, embedding_model_provider,
rerank_model, rerank_model_provider, retrieval_top_k, retrieval_score_threshold,
rerank_model, rerank_model_provider, reranking_enable, retrieval_top_k, retrieval_score_threshold,
vector_id, document_count, total_chunks, status, creator, creator_dept,
updater, create_time, update_time, delete_time, deleted
</sql>
@@ -170,13 +171,13 @@
INSERT INTO tb_ai_knowledge (
id, title, avatar, description, content, source_type, source_id, file_name, file_path,
category, tags, dify_dataset_id, dify_indexing_technique, embedding_model, embedding_model_provider,
rerank_model, rerank_model_provider, retrieval_top_k, retrieval_score_threshold,
rerank_model, rerank_model_provider, reranking_enable, retrieval_top_k, retrieval_score_threshold,
vector_id, document_count, total_chunks, status, creator, creator_dept,
updater, create_time, update_time, deleted
) VALUES (
#{ID}, #{title}, #{avatar}, #{description}, #{content}, #{sourceType}, #{sourceID}, #{fileName}, #{filePath},
#{category}, #{tags}, #{difyDatasetId}, #{difyIndexingTechnique}, #{embeddingModel}, #{embeddingModelProvider},
#{rerankModel}, #{rerankModelProvider}, #{retrievalTopK}, #{retrievalScoreThreshold},
#{rerankModel}, #{rerankModelProvider}, #{rerankingEnable}, #{retrievalTopK}, #{retrievalScoreThreshold},
#{vectorID}, #{documentCount}, #{totalChunks}, #{status}, #{creator}, #{creatorDept},
#{updater}, #{createTime}, #{updateTime}, #{deleted}
)
@@ -202,6 +203,7 @@
<if test="embeddingModelProvider != null">embedding_model_provider = #{embeddingModelProvider},</if>
<if test="rerankModel != null">rerank_model = #{rerankModel},</if>
<if test="rerankModelProvider != null">rerank_model_provider = #{rerankModelProvider},</if>
<if test="rerankingEnable != null">reranking_enable = #{rerankingEnable},</if>
<if test="retrievalTopK != null">retrieval_top_k = #{retrievalTopK},</if>
<if test="retrievalScoreThreshold != null">retrieval_score_threshold = #{retrievalScoreThreshold},</if>
<if test="vectorID != null">vector_id = #{vectorID},</if>

View File

@@ -93,6 +93,11 @@ public class TbAiKnowledge extends BaseDTO {
*/
private String rerankModelProvider;
/**
* @description 是否启用Rerank
*/
private Boolean rerankingEnable;
/**
* @description 检索Top K返回前K个结果
*/
@@ -322,6 +327,14 @@ public class TbAiKnowledge extends BaseDTO {
this.rerankModelProvider = rerankModelProvider;
}
public Boolean getRerankingEnable() {
return rerankingEnable;
}
public void setRerankingEnable(Boolean rerankingEnable) {
this.rerankingEnable = rerankingEnable;
}
public Integer getRetrievalTopK() {
return retrievalTopK;
}

View File

@@ -142,6 +142,15 @@ export const knowledgeApi = {
async getAvailableEmbeddingModels(): Promise<ResultDomain<any>> {
const response = await api.get<any>('/ai/knowledge/embedding-models');
return response.data;
},
/**
* 获取可用的Rerank模型列表
* @returns Promise<ResultDomain<any>>
*/
async getAvailableRerankModels(): Promise<ResultDomain<any>> {
const response = await api.get<any>('/ai/knowledge/rerank-models');
return response.data;
}
};

View File

@@ -75,6 +75,16 @@ export interface AiKnowledge extends BaseDTO {
embeddingModel?: string;
/** Embedding模型提供商 */
embeddingModelProvider?: string;
/** Rerank模型 */
rerankModel?: string;
/** Rerank模型提供商 */
rerankModelProvider?: string;
/** 是否启用Rerank */
rerankingEnable?: boolean;
/** 检索Top K返回前K个结果 */
retrievalTopK?: number;
/** 检索分数阈值0.00-1.00 */
retrievalScoreThreshold?: number;
/** 向量ID */
vectorID?: string;
/** 文档数量 */

View File

@@ -25,14 +25,32 @@
<span class="label">索引方式</span>
<span class="value">{{ getIndexingText(knowledge?.difyIndexingTechnique) }}</span>
</div>
<div class="meta-item" v-if="knowledge?.difyIndexingTechnique == 'high_quality'">
<span class="label">Embedding模型</span>
<span class="value">{{ knowledge?.embeddingModel || '-' }}</span>
</div>
<div class="meta-item">
<span class="label">文档数量</span>
<span class="value">{{ knowledge?.documentCount || 0 }}</span>
</div>
<div class="meta-item" v-if="knowledge?.difyIndexingTechnique == 'high_quality'">
<span class="label">Embedding模型</span>
<span class="value">{{ knowledge?.embeddingModel || '-' }}</span>
</div>
<div class="meta-item">
<span class="label">Rerank状态</span>
<el-tag :type="knowledge?.rerankingEnable ? 'success' : 'info'" size="small">
{{ knowledge?.rerankingEnable ? '已启用' : '未启用' }}
</el-tag>
</div>
<div class="meta-item" v-if="knowledge?.rerankingEnable && knowledge?.rerankModel">
<span class="label">Rerank模型</span>
<span class="value">{{ knowledge.rerankModel }}</span>
</div>
<div class="meta-item">
<span class="label">检索Top K</span>
<span class="value">{{ knowledge?.retrievalTopK || 2 }}</span>
</div>
<div class="meta-item" v-if="(knowledge?.retrievalScoreThreshold || 0) > 0">
<span class="label">检索阈值</span>
<span class="value">{{ knowledge?.retrievalScoreThreshold }}</span>
</div>
<div class="meta-item">
<span class="label">文档数量</span>
<span class="value">{{ knowledge?.documentCount || 0 }}</span>
</div>
<div class="meta-item">
<span class="label">创建时间</span>
<span class="value">{{ formatDate(knowledge?.createTime) }}</span>
@@ -138,6 +156,81 @@
</div>
</el-form-item>
<!-- Rerank模型配置 -->
<el-form-item label="启用Rerank" prop="rerankingEnable">
<el-switch
v-model="formData.rerankingEnable"
:active-value="true"
:inactive-value="false"
active-text="启用"
inactive-text="禁用"
/>
<div class="form-tip">
Rerank可以对检索结果进行重新排序提高精确度
</div>
</el-form-item>
<el-form-item v-if="formData.rerankingEnable" label="Rerank模型" prop="rerankModel">
<el-select
v-model="formData.rerankModel"
placeholder="请选择Rerank模型"
clearable
filterable
:loading="rerankModelsLoading"
@change="handleRerankModelChange"
>
<el-option-group
v-for="provider in rerankModels"
:key="provider.provider"
:label="provider.label || provider.provider"
>
<el-option
v-for="model in provider.models"
:key="model.model"
:label="getModelLabel(model)"
:value="model.model"
:data-provider="model.provider"
>
<span>{{ getModelLabel(model) }}</span>
<span v-if="model.contextSize" style="float: right; color: var(--el-text-color-secondary); font-size: 13px">
上下文: {{ model.contextSize }}
</span>
</el-option>
</el-option-group>
</el-select>
<div class="form-tip">
选择用于重新排序的Rerank模型
</div>
</el-form-item>
<!-- 检索参数配置 -->
<el-form-item label="检索Top K" prop="retrievalTopK">
<el-input-number
v-model="formData.retrievalTopK"
:min="1"
:max="20"
:step="1"
placeholder="返回前K个结果"
/>
<div class="form-tip">
返回前K个最相关的检索结果建议范围2-10
</div>
</el-form-item>
<el-form-item label="检索分数阈值" prop="retrievalScoreThreshold">
<el-input-number
v-model="formData.retrievalScoreThreshold"
:min="0"
:max="1"
:step="0.01"
:precision="2"
placeholder="分数阈值"
/>
<div class="form-tip">
只返回分数高于此阈值的结果0.00-1.000表示不启用
</div>
</el-form-item>
<el-form-item label="Dify数据集ID" prop="difyDatasetId">
<el-input
v-model="formData.difyDatasetId"
@@ -199,7 +292,9 @@ const emit = defineEmits<{
const formRef = ref<FormInstance>();
const submitting = ref(false);
const modelsLoading = ref(false);
const rerankModelsLoading = ref(false);
const embeddingModels = ref<any[]>([]);
const rerankModels = ref<any[]>([]);
// 表单数据
const formData = reactive<Partial<AiKnowledge>>({
@@ -209,6 +304,11 @@ const formData = reactive<Partial<AiKnowledge>>({
difyIndexingTechnique: 'high_quality',
embeddingModel: '',
embeddingModelProvider: '',
rerankModel: '',
rerankModelProvider: '',
rerankingEnable: false,
retrievalTopK: 2,
retrievalScoreThreshold: 0.0,
difyDatasetId: '',
status: 1
});
@@ -221,6 +321,18 @@ const rules: FormRules = {
],
difyIndexingTechnique: [
{ required: true, message: '请选择索引方式', trigger: 'change' }
],
rerankModel: [
{
validator: (rule: any, value: any, callback: any) => {
if (formData.rerankingEnable && !value) {
callback(new Error('启用Rerank后必须选择Rerank模型'));
} else {
callback();
}
},
trigger: 'change'
}
]
};
@@ -235,12 +347,25 @@ watch(() => props.knowledge, (newVal) => {
difyIndexingTechnique: newVal.difyIndexingTechnique || 'high_quality',
embeddingModel: newVal.embeddingModel,
embeddingModelProvider: newVal.embeddingModelProvider,
rerankModel: newVal.rerankModel,
rerankModelProvider: newVal.rerankModelProvider,
rerankingEnable: newVal.rerankingEnable ?? false,
retrievalTopK: newVal.retrievalTopK ?? 2,
retrievalScoreThreshold: newVal.retrievalScoreThreshold ?? 0.0,
difyDatasetId: newVal.difyDatasetId,
status: newVal.status ?? 1
});
}
}, { immediate: true });
// 监听 rerankingEnable 变化,触发表单验证
watch(() => formData.rerankingEnable, () => {
if (formRef.value) {
// 触发 rerankModel 字段的验证
formRef.value.validateField('rerankModel', () => {});
}
});
// 处理头像更新
function handleAvatarUpdate(val: string) {
formData.avatar = val;
@@ -295,12 +420,43 @@ async function loadEmbeddingModels() {
}
}
// 加载Rerank模型列表
async function loadRerankModels() {
try {
rerankModelsLoading.value = true;
const result = await knowledgeApi.getAvailableRerankModels();
if (result.success && result.data) {
// 按提供商分组
const providers = result.data.providers || [];
rerankModels.value = providers.map((provider: any) => ({
provider: provider.provider,
label: provider.label?.zh_Hans || provider.label?.en_US || provider.provider,
models: (provider.models || []).map((model: any) => ({
model: model.model,
provider: model.provider || provider.provider,
label: model.label?.zh_Hans || model.label?.en_US || model.model,
contextSize: model.model_properties?.context_size,
status: model.status
}))
}));
} else {
ElMessage.warning('获取Rerank模型列表失败');
rerankModels.value = [];
}
} catch (error: any) {
console.error('加载Rerank模型列表失败:', error);
rerankModels.value = [];
} finally {
rerankModelsLoading.value = false;
}
}
// 获取模型显示标签
function getModelLabel(model: any): string {
return model.label || model.model;
}
// 处理模型变化
// 处理Embedding模型变化
function handleModelChange(modelName: string) {
if (!modelName) {
formData.embeddingModelProvider = '';
@@ -312,7 +468,25 @@ function handleModelChange(modelName: string) {
const foundModel = providerGroup.models.find((m: any) => m.model === modelName);
if (foundModel) {
formData.embeddingModelProvider = foundModel.provider;
console.log('选择模型:', modelName, '提供商:', foundModel.provider);
console.log('选择Embedding模型:', modelName, '提供商:', foundModel.provider);
break;
}
}
}
// 处理Rerank模型变化
function handleRerankModelChange(modelName: string) {
if (!modelName) {
formData.rerankModelProvider = '';
return;
}
// 查找选中模型的提供商
for (const providerGroup of rerankModels.value) {
const foundModel = providerGroup.models.find((m: any) => m.model === modelName);
if (foundModel) {
formData.rerankModelProvider = foundModel.provider;
console.log('选择Rerank模型:', modelName, '提供商:', foundModel.provider);
break;
}
}
@@ -320,6 +494,7 @@ function handleModelChange(modelName: string) {
// 组件挂载时加载模型列表
loadEmbeddingModels();
loadRerankModels();
// 提交表单
async function handleSubmit() {