mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-06-04 13:30:32 +08:00
feat: 添加AI问题生成功能,支持配置生成数量并在前端展示生成的问题
This commit is contained in:
@@ -518,6 +518,34 @@ conversation:
|
||||
}
|
||||
]
|
||||
|
||||
generate_questions_prompt: |
|
||||
你是一个专业的问题生成助手。你的任务是根据给定的【主要内容】生成用户可能会问的相关问题。
|
||||
|
||||
{{.Context}}
|
||||
## 主要内容(请基于此内容生成问题)
|
||||
文档名称:{{.DocName}}
|
||||
文档内容:
|
||||
{{.Content}}
|
||||
|
||||
## 核心要求
|
||||
- 生成的问题必须与【主要内容】直接相关
|
||||
- 问题中禁止使用任何代词或指代词(如"它"、"这个"、"该文档"、"本文"、"文中"、"其"等),必须用具体名称替代
|
||||
- 问题必须是完整独立的,脱离上下文也能被理解
|
||||
- 问题应该是用户在实际场景中可能会提出的自然问题
|
||||
- 问题应该多样化,覆盖内容的不同方面
|
||||
- 每个问题应该简洁明了,长度控制在30字以内
|
||||
- 生成的问题数量为 {{.QuestionCount}} 个
|
||||
|
||||
## 问题类型建议
|
||||
- 定义类:什么是...?...是什么?
|
||||
- 原因类:为什么...?...的原因是什么?
|
||||
- 方法类:如何...?怎样...?
|
||||
- 比较类:...和...有什么区别?
|
||||
- 应用类:...可以用于什么场景?
|
||||
|
||||
## 输出格式
|
||||
直接输出问题列表,每行一个问题,不要有序号或其他前缀。
|
||||
|
||||
# 知识库配置
|
||||
knowledge_base:
|
||||
chunk_size: 512
|
||||
@@ -617,4 +645,4 @@ web_search:
|
||||
# 租户配置
|
||||
tenant:
|
||||
# 是否启用跨租户访问功能(内网环境可开启)
|
||||
enable_cross_tenant_access: true
|
||||
enable_cross_tenant_access: false
|
||||
|
||||
@@ -107,6 +107,10 @@ export interface KBModelConfigRequest {
|
||||
nodes: Node[]
|
||||
relations: Relation[]
|
||||
}
|
||||
questionGeneration?: {
|
||||
enabled: boolean
|
||||
questionCount: number
|
||||
}
|
||||
}
|
||||
|
||||
export function updateKBConfig(kbId: string, config: KBModelConfigRequest): Promise<any> {
|
||||
|
||||
@@ -185,6 +185,32 @@ const getChunkMeta = (item: any) => {
|
||||
return parts.join(' · ');
|
||||
};
|
||||
|
||||
// 解析生成的问题
|
||||
const getGeneratedQuestions = (item: any): string[] => {
|
||||
if (!item || !item.metadata) return [];
|
||||
try {
|
||||
const metadata = typeof item.metadata === 'string' ? JSON.parse(item.metadata) : item.metadata;
|
||||
return metadata.generated_questions || [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
// 展开状态管理
|
||||
const expandedChunks = ref<Set<number>>(new Set());
|
||||
|
||||
const toggleQuestions = (index: number) => {
|
||||
if (expandedChunks.value.has(index)) {
|
||||
expandedChunks.value.delete(index);
|
||||
} else {
|
||||
expandedChunks.value.add(index);
|
||||
}
|
||||
// 触发响应式更新
|
||||
expandedChunks.value = new Set(expandedChunks.value);
|
||||
};
|
||||
|
||||
const isExpanded = (index: number) => expandedChunks.value.has(index);
|
||||
|
||||
const downloadFile = () => {
|
||||
downKnowledgeDetails(props.details.id)
|
||||
.then((result) => {
|
||||
@@ -272,17 +298,44 @@ const handleDetailsScroll = () => {
|
||||
|
||||
<div v-if="details.md.length == 0" class="no_content">{{ $t('common.noData') }}</div>
|
||||
<div v-else class="chunk-list">
|
||||
<div
|
||||
class="chunk-item"
|
||||
<div class="chunk-item"
|
||||
v-for="(item, index) in details.md"
|
||||
:key="index"
|
||||
:class="getChunkClass(index)"
|
||||
>
|
||||
<div class="chunk-header">
|
||||
<span class="chunk-index">{{ $t('knowledgeBase.segment') || '片段' }} {{ index + 1 }}</span>
|
||||
<span class="chunk-meta">{{ getChunkMeta(item) }}</span>
|
||||
<div class="chunk-header-right">
|
||||
<t-tag
|
||||
v-if="getGeneratedQuestions(item).length > 0"
|
||||
size="small"
|
||||
theme="success"
|
||||
variant="light"
|
||||
>
|
||||
{{ $t('knowledgeBase.questions') || '问题' }} {{ getGeneratedQuestions(item).length }}
|
||||
</t-tag>
|
||||
<span class="chunk-meta">{{ getChunkMeta(item) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-content" v-html="processMarkdown(item.content)"></div>
|
||||
|
||||
<!-- 生成的问题展示 -->
|
||||
<div v-if="getGeneratedQuestions(item).length > 0" class="questions-section">
|
||||
<div class="questions-toggle" @click="toggleQuestions(index)">
|
||||
<t-icon :name="isExpanded(index) ? 'chevron-down' : 'chevron-right'" size="14px" />
|
||||
<span>{{ $t('knowledgeBase.generatedQuestions') || '生成的问题' }} ({{ getGeneratedQuestions(item).length }})</span>
|
||||
</div>
|
||||
<div v-show="isExpanded(index)" class="questions-list">
|
||||
<div
|
||||
v-for="(question, qIndex) in getGeneratedQuestions(item)"
|
||||
:key="qIndex"
|
||||
class="question-item"
|
||||
>
|
||||
<t-icon name="help-circle" size="14px" class="question-icon" />
|
||||
<span>{{ question }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -493,12 +546,65 @@ const handleDetailsScroll = () => {
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.chunk-header-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.chunk-meta {
|
||||
color: #00000066;
|
||||
font-size: 11px;
|
||||
}
|
||||
}
|
||||
|
||||
// 生成的问题样式
|
||||
.questions-section {
|
||||
margin-top: 12px;
|
||||
padding-top: 10px;
|
||||
border-top: 1px dashed #e0e0e0;
|
||||
}
|
||||
|
||||
.questions-toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
cursor: pointer;
|
||||
color: #059669;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
padding: 4px 0;
|
||||
transition: color 0.2s ease;
|
||||
|
||||
&:hover {
|
||||
color: #07c05f;
|
||||
}
|
||||
}
|
||||
|
||||
.questions-list {
|
||||
margin-top: 8px;
|
||||
padding-left: 4px;
|
||||
}
|
||||
|
||||
.question-item {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
padding: 6px 8px;
|
||||
margin-bottom: 4px;
|
||||
background: #f0fdf4;
|
||||
border-radius: 4px;
|
||||
font-size: 13px;
|
||||
color: #1d2129;
|
||||
line-height: 1.5;
|
||||
|
||||
.question-icon {
|
||||
color: #059669;
|
||||
flex-shrink: 0;
|
||||
margin-top: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
.md-content {
|
||||
word-break: break-word;
|
||||
line-height: 1.6;
|
||||
|
||||
@@ -92,6 +92,8 @@ export default {
|
||||
createTime: 'Create Time',
|
||||
characters: 'chars',
|
||||
segment: 'Segment',
|
||||
questions: 'Questions',
|
||||
generatedQuestions: 'Generated Questions',
|
||||
notInitialized: 'Knowledge base is not initialized. Please configure models in settings before uploading files',
|
||||
getInfoFailed: 'Failed to get knowledge base information, file upload is not possible',
|
||||
missingId: 'Knowledge base ID is missing',
|
||||
@@ -1001,7 +1003,13 @@ export default {
|
||||
},
|
||||
advanced: {
|
||||
title: 'Advanced Settings',
|
||||
description: 'Configure multimodal features',
|
||||
description: 'Configure question generation, multimodal features',
|
||||
questionGeneration: {
|
||||
label: 'AI Question Generation',
|
||||
description: 'Generate related questions for each chunk using LLM during document parsing to improve retrieval recall. Enabling this will increase document parsing time.',
|
||||
countLabel: 'Question Count',
|
||||
countDescription: 'Number of questions to generate per document chunk (1-10)',
|
||||
},
|
||||
multimodal: {
|
||||
label: 'Multimodal Feature',
|
||||
description: 'Enable understanding of multimodal content such as images and videos',
|
||||
|
||||
@@ -92,6 +92,8 @@ export default {
|
||||
createTime: 'Время создания',
|
||||
characters: 'символов',
|
||||
segment: 'Фрагмент',
|
||||
questions: 'Вопросы',
|
||||
generatedQuestions: 'Сгенерированные вопросы',
|
||||
notInitialized: 'База знаний не инициализирована. Пожалуйста, настройте модели в разделе настроек перед загрузкой файлов',
|
||||
getInfoFailed: 'Не удалось получить информацию о базе знаний, загрузка файла невозможна',
|
||||
missingId: 'Отсутствует ID базы знаний',
|
||||
@@ -1095,7 +1097,13 @@ export default {
|
||||
},
|
||||
advanced: {
|
||||
title: 'Расширенные настройки',
|
||||
description: 'Настройте мультимодальные возможности',
|
||||
description: 'Настройте генерацию вопросов и мультимодальные возможности',
|
||||
questionGeneration: {
|
||||
label: 'AI генерация вопросов',
|
||||
description: 'Генерация связанных вопросов для каждого фрагмента с помощью LLM при парсинге документа для улучшения полноты поиска. Включение увеличит время парсинга документа.',
|
||||
countLabel: 'Количество вопросов',
|
||||
countDescription: 'Количество вопросов для генерации на фрагмент документа (1-10)',
|
||||
},
|
||||
multimodal: {
|
||||
label: 'Мультимодальная функция',
|
||||
description: 'Включите понимание мультимедийного контента, такого как изображения и видео',
|
||||
|
||||
@@ -91,6 +91,8 @@ export default {
|
||||
createTime: "创建时间",
|
||||
characters: "字符",
|
||||
segment: "片段",
|
||||
questions: "问题",
|
||||
generatedQuestions: "生成的问题",
|
||||
docActionUnsupported: "当前知识库类型不支持该操作",
|
||||
notInitialized:
|
||||
"该知识库尚未完成初始化配置,请先前往设置页面配置模型信息后再上传文件",
|
||||
@@ -1334,7 +1336,13 @@ export default {
|
||||
},
|
||||
advanced: {
|
||||
title: "高级设置",
|
||||
description: "配置多模态等高级功能",
|
||||
description: "配置问题生成、多模态等高级功能",
|
||||
questionGeneration: {
|
||||
label: "AI 问题生成",
|
||||
description: "解析文档时调用大模型为每个分块生成相关问题,提高检索召回率。启用后会增加文档解析耗时。",
|
||||
countLabel: "生成问题数量",
|
||||
countDescription: "每个文档分块生成的问题数量(1-10)",
|
||||
},
|
||||
multimodal: {
|
||||
label: "多模态功能",
|
||||
description: "启用图片、视频等多模态内容的理解能力",
|
||||
|
||||
@@ -144,8 +144,10 @@
|
||||
ref="advancedSettingsRef"
|
||||
v-if="formData"
|
||||
:multimodal="formData.multimodalConfig"
|
||||
:question-generation="formData.questionGenerationConfig"
|
||||
:all-models="allModels"
|
||||
@update:multimodal="handleMultimodalUpdate"
|
||||
@update:question-generation="handleQuestionGenerationUpdate"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -296,6 +298,10 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
|
||||
type: string
|
||||
}>
|
||||
},
|
||||
questionGenerationConfig: {
|
||||
enabled: false,
|
||||
questionCount: 3
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -377,6 +383,10 @@ const loadKBData = async () => {
|
||||
})),
|
||||
relations: kb.extract_config?.relations || []
|
||||
},
|
||||
questionGenerationConfig: {
|
||||
enabled: kb.question_generation_config?.enabled || false,
|
||||
questionCount: kb.question_generation_config?.question_count || 3
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load knowledge base data:', error)
|
||||
@@ -406,6 +416,12 @@ const handleMultimodalUpdate = (config: any) => {
|
||||
}
|
||||
}
|
||||
|
||||
const handleQuestionGenerationUpdate = (config: any) => {
|
||||
if (formData.value) {
|
||||
formData.value.questionGenerationConfig = { ...config }
|
||||
}
|
||||
}
|
||||
|
||||
const handleNodeExtractUpdate = (config: any) => {
|
||||
if (formData.value) {
|
||||
formData.value.nodeExtractConfig = { ...config }
|
||||
@@ -514,6 +530,14 @@ const buildSubmitData = () => {
|
||||
}
|
||||
}
|
||||
|
||||
// 添加问题生成配置
|
||||
if (formData.value.questionGenerationConfig?.enabled) {
|
||||
data.question_generation_config = {
|
||||
enabled: true,
|
||||
question_count: formData.value.questionGenerationConfig.questionCount || 3
|
||||
}
|
||||
}
|
||||
|
||||
if (formData.value.type === 'faq') {
|
||||
data.faq_config = {
|
||||
index_mode: formData.value.faqConfig?.indexMode || 'question_only',
|
||||
@@ -598,6 +622,10 @@ const handleSubmit = async () => {
|
||||
tags: data.extract_config?.tags || [],
|
||||
nodes: data.extract_config?.nodes || [],
|
||||
relations: data.extract_config?.relations || []
|
||||
},
|
||||
questionGeneration: {
|
||||
enabled: data.question_generation_config?.enabled || false,
|
||||
questionCount: data.question_generation_config?.question_count || 3
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,42 @@
|
||||
</div>
|
||||
|
||||
<div class="settings-group">
|
||||
<!-- Question Generation feature -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.questionGeneration.label') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.questionGeneration.description') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-switch
|
||||
v-model="localQuestionGeneration.enabled"
|
||||
@change="handleQuestionGenerationToggle"
|
||||
size="large"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Question Generation configuration -->
|
||||
<div v-if="localQuestionGeneration.enabled" class="subsection">
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
<label>{{ $t('knowledgeEditor.advanced.questionGeneration.countLabel') }}</label>
|
||||
<p class="desc">{{ $t('knowledgeEditor.advanced.questionGeneration.countDescription') }}</p>
|
||||
</div>
|
||||
<div class="setting-control">
|
||||
<t-input-number
|
||||
v-model="localQuestionGeneration.questionCount"
|
||||
:min="1"
|
||||
:max="10"
|
||||
:step="1"
|
||||
theme="normal"
|
||||
@change="handleQuestionGenerationChange"
|
||||
style="width: 120px;"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Multimodal feature -->
|
||||
<div class="setting-row">
|
||||
<div class="setting-info">
|
||||
@@ -241,8 +277,14 @@ interface MultimodalConfig {
|
||||
}
|
||||
}
|
||||
|
||||
interface QuestionGenerationConfig {
|
||||
enabled: boolean
|
||||
questionCount: number
|
||||
}
|
||||
|
||||
interface Props {
|
||||
multimodal: MultimodalConfig
|
||||
questionGeneration?: QuestionGenerationConfig
|
||||
allModels?: any[]
|
||||
}
|
||||
|
||||
@@ -250,9 +292,13 @@ const props = defineProps<Props>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:multimodal': [value: MultimodalConfig]
|
||||
'update:questionGeneration': [value: QuestionGenerationConfig]
|
||||
}>()
|
||||
|
||||
const localMultimodal = ref<MultimodalConfig>({ ...props.multimodal })
|
||||
const localQuestionGeneration = ref<QuestionGenerationConfig>(
|
||||
props.questionGeneration || { enabled: false, questionCount: 3 }
|
||||
)
|
||||
|
||||
const vllmSelectorRef = ref()
|
||||
const isMinioEnabled = ref(false)
|
||||
@@ -287,6 +333,25 @@ watch(() => props.multimodal, (newVal) => {
|
||||
localMultimodal.value = { ...newVal }
|
||||
}, { deep: true })
|
||||
|
||||
watch(() => props.questionGeneration, (newVal) => {
|
||||
if (newVal) {
|
||||
localQuestionGeneration.value = { ...newVal }
|
||||
}
|
||||
}, { deep: true })
|
||||
|
||||
// Handle question generation toggle
|
||||
const handleQuestionGenerationToggle = () => {
|
||||
if (!localQuestionGeneration.value.enabled) {
|
||||
localQuestionGeneration.value.questionCount = 3
|
||||
}
|
||||
emit('update:questionGeneration', localQuestionGeneration.value)
|
||||
}
|
||||
|
||||
// Handle question generation config change
|
||||
const handleQuestionGenerationChange = () => {
|
||||
emit('update:questionGeneration', localQuestionGeneration.value)
|
||||
}
|
||||
|
||||
// Handle multimodal toggle
|
||||
const handleMultimodalToggle = () => {
|
||||
// Reset related configuration when multimodal is disabled
|
||||
|
||||
@@ -60,7 +60,6 @@ func (r *chunkRepository) ListChunksByKnowledgeID(
|
||||
) ([]*types.Chunk, error) {
|
||||
var chunks []*types.Chunk
|
||||
if err := r.db.WithContext(ctx).
|
||||
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info").
|
||||
Where("tenant_id = ? AND knowledge_id = ? and chunk_type = ?", tenantID, knowledgeID, "text").
|
||||
Order("chunk_index ASC").
|
||||
Find(&chunks).Error; err != nil {
|
||||
@@ -85,7 +84,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
|
||||
|
||||
baseFilter := func(db *gorm.DB) *gorm.DB {
|
||||
db = db.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
|
||||
tenantID, knowledgeID, chunkType, []types.ChunkStatus{types.ChunkStatusIndexed, types.ChunkStatusDefault})
|
||||
tenantID, knowledgeID, chunkType, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)})
|
||||
if tagID != "" {
|
||||
db = db.Where("tag_id = ?", tagID)
|
||||
}
|
||||
@@ -106,7 +105,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
|
||||
// Then query the paginated data
|
||||
dataQuery := baseFilter(
|
||||
r.db.WithContext(ctx).
|
||||
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id"),
|
||||
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id, status"),
|
||||
)
|
||||
|
||||
if err := dataQuery.
|
||||
|
||||
@@ -291,7 +291,7 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
ORDER BY embedding::halfvec(%d) <=> $1::halfvec
|
||||
LIMIT $%d
|
||||
) AS candidates
|
||||
WHERE distance < $%d
|
||||
WHERE distance <= $%d
|
||||
ORDER BY distance ASC
|
||||
LIMIT $%d
|
||||
`, dimension, whereClause, dimension, subqueryLimitParam, thresholdParam, finalLimitParam)
|
||||
|
||||
@@ -5,34 +5,24 @@ import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/yanyiwu/gojieba"
|
||||
)
|
||||
|
||||
// PluginPreprocess Query preprocessing plugin
|
||||
type PluginPreprocess struct {
|
||||
config *config.Config
|
||||
jieba *gojieba.Jieba
|
||||
stopwords map[string]struct{}
|
||||
modelService interfaces.ModelService
|
||||
}
|
||||
|
||||
// Regular expressions for text cleaning
|
||||
var (
|
||||
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
|
||||
urlRegex = regexp.MustCompile(`https?://\S+`) // URLs
|
||||
emailRegex = regexp.MustCompile(`\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,}\b`) // Email addresses
|
||||
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
|
||||
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
|
||||
)
|
||||
|
||||
const maxProcessedTokens = 12
|
||||
|
||||
// NewPluginPreprocess Creates a new query preprocessing plugin
|
||||
func NewPluginPreprocess(
|
||||
eventManager *EventManager,
|
||||
@@ -40,51 +30,15 @@ func NewPluginPreprocess(
|
||||
cleaner interfaces.ResourceCleaner,
|
||||
modelService interfaces.ModelService,
|
||||
) *PluginPreprocess {
|
||||
// Use default dictionary for Jieba tokenizer
|
||||
jieba := gojieba.NewJieba()
|
||||
|
||||
// Load stopwords from built-in stopword library
|
||||
stopwords := loadStopwords()
|
||||
|
||||
res := &PluginPreprocess{
|
||||
config: config,
|
||||
jieba: jieba,
|
||||
stopwords: stopwords,
|
||||
modelService: modelService,
|
||||
}
|
||||
|
||||
// Register resource cleanup function
|
||||
if cleaner != nil {
|
||||
cleaner.RegisterWithName("JiebaPreprocessor", func() error {
|
||||
res.Close()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
}
|
||||
|
||||
// Load stopwords
|
||||
func loadStopwords() map[string]struct{} {
|
||||
// Directly use some common stopwords built into Jieba
|
||||
commonStopwords := []string{
|
||||
"的", "了", "和", "是", "在", "我", "你", "他", "她", "它",
|
||||
"这", "那", "什么", "怎么", "如何", "为什么", "哪里", "什么时候",
|
||||
"the", "is", "are", "am", "I", "you", "he", "she", "it", "this",
|
||||
"that", "what", "how", "a", "an", "and", "or", "but", "if", "of",
|
||||
"to", "in", "on", "at", "by", "for", "with", "about", "from",
|
||||
"有", "无", "好", "来", "去", "说", "看", "想", "会", "可以",
|
||||
"吗", "呢", "啊", "吧", "的话", "就是", "只是", "因为", "所以",
|
||||
}
|
||||
|
||||
result := make(map[string]struct{}, len(commonStopwords))
|
||||
for _, word := range commonStopwords {
|
||||
result[word] = struct{}{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ActivationEvents Register activation events
|
||||
func (p *PluginPreprocess) ActivationEvents() []types.EventType {
|
||||
return []types.EventType{types.PREPROCESS_QUERY}
|
||||
@@ -107,171 +61,22 @@ func (p *PluginPreprocess) OnEvent(
|
||||
"rewrite_query": rawQuery,
|
||||
})
|
||||
|
||||
normalized := normalizeWhitespace(rawQuery)
|
||||
sanitized := strings.TrimSpace(p.cleanText(normalized))
|
||||
if sanitized == "" {
|
||||
sanitized = normalized
|
||||
}
|
||||
|
||||
var (
|
||||
processed = sanitized
|
||||
strategy = "original"
|
||||
tokenPreview string
|
||||
tokenCount int
|
||||
)
|
||||
|
||||
switch {
|
||||
case containsChineseCharacters(sanitized):
|
||||
segments := p.segmentText(sanitized)
|
||||
tokens := p.selectMeaningfulTokens(segments)
|
||||
tokenCount = len(tokens)
|
||||
if len(tokens) >= 2 {
|
||||
processed = strings.Join(tokens, " ")
|
||||
strategy = "zh_tokens"
|
||||
tokenPreview = strings.Join(tokens, ",")
|
||||
} else {
|
||||
strategy = "fallback_original"
|
||||
}
|
||||
case containsLatinLetters(sanitized):
|
||||
processed = normalizeLatinQuery(sanitized)
|
||||
if processed != sanitized {
|
||||
strategy = "latin_normalize"
|
||||
}
|
||||
default:
|
||||
strategy = "original"
|
||||
}
|
||||
|
||||
if strings.TrimSpace(processed) == "" {
|
||||
processed = rawQuery
|
||||
strategy = "fallback_original"
|
||||
}
|
||||
// Lightweight normalization: just collapse multiple spaces
|
||||
processed := multiSpaceRegex.ReplaceAllString(rawQuery, " ")
|
||||
processed = strings.TrimSpace(processed)
|
||||
|
||||
chatManage.ProcessedQuery = processed
|
||||
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
|
||||
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, processed)
|
||||
|
||||
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"processed_query": processed,
|
||||
"strategy": strategy,
|
||||
"token_count": tokenCount,
|
||||
"token_preview": tokenPreview,
|
||||
"query_intent": chatManage.QueryIntent,
|
||||
})
|
||||
|
||||
return next()
|
||||
}
|
||||
|
||||
// cleanText Basic text cleaning
|
||||
func (p *PluginPreprocess) cleanText(text string) string {
|
||||
// Remove URLs
|
||||
text = urlRegex.ReplaceAllString(text, " ")
|
||||
|
||||
// Remove email addresses
|
||||
text = emailRegex.ReplaceAllString(text, " ")
|
||||
|
||||
// Remove excessive spaces
|
||||
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
|
||||
// Remove punctuation marks
|
||||
text = punctRegex.ReplaceAllString(text, " ")
|
||||
|
||||
// Trim leading and trailing spaces
|
||||
text = strings.TrimSpace(text)
|
||||
|
||||
return text
|
||||
}
|
||||
|
||||
// segmentText Text tokenization
|
||||
func (p *PluginPreprocess) segmentText(text string) []string {
|
||||
// Use Jieba tokenizer for tokenization, using search engine mode
|
||||
segments := p.jieba.CutForSearch(text, true)
|
||||
return segments
|
||||
}
|
||||
|
||||
// filterStopwords Filter stopwords
|
||||
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
|
||||
var tokens []string
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for _, word := range segments {
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
if _, stop := p.stopwords[word]; stop {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[word]; exists {
|
||||
continue
|
||||
}
|
||||
if !isInformativeToken(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[word] = struct{}{}
|
||||
tokens = append(tokens, word)
|
||||
if len(tokens) >= maxProcessedTokens {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
|
||||
// isBlank Check if a string is blank
|
||||
func isInformativeToken(token string) bool {
|
||||
if token == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
runeCount := utf8.RuneCountInString(token)
|
||||
if runeCount == 1 {
|
||||
r, _ := utf8.DecodeRuneInString(token)
|
||||
if unicode.IsDigit(r) {
|
||||
return true
|
||||
}
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// containsChineseCharacters checks if a string contains Chinese characters
|
||||
func containsChineseCharacters(text string) bool {
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// containsLatinLetters checks if a string contains Latin letters
|
||||
func containsLatinLetters(text string) bool {
|
||||
for _, r := range text {
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// normalizeWhitespace normalizes whitespace in a string
|
||||
func normalizeWhitespace(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
return multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
}
|
||||
|
||||
// normalizeLatinQuery normalizes a Latin query
|
||||
func normalizeLatinQuery(text string) string {
|
||||
text = strings.ToLower(text)
|
||||
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
// intentResp is a response for intent detection
|
||||
type intentResp struct {
|
||||
Intent string `json:"intent"`
|
||||
@@ -349,12 +154,8 @@ func extractJSONBody(text string) string {
|
||||
return "{}"
|
||||
}
|
||||
|
||||
// Ensure resources are properly released
|
||||
// Close Releases resources
|
||||
func (p *PluginPreprocess) Close() {
|
||||
if p.jieba != nil {
|
||||
p.jieba.Free()
|
||||
p.jieba = nil
|
||||
}
|
||||
}
|
||||
|
||||
// ShutdownHandler Returns shutdown function
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/utils"
|
||||
"github.com/Tencent/WeKnora/internal/tracing"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -283,14 +282,26 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
enableMultimodelValue = kb.VLMConfig.Enabled
|
||||
}
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
}
|
||||
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
FilePath: filePath,
|
||||
FileName: safeFilename,
|
||||
FileType: getFileType(safeFilename),
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
FilePath: filePath,
|
||||
FileName: safeFilename,
|
||||
FileType: getFileType(safeFilename),
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -405,12 +416,24 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
enableMultimodelValue = kb.VLMConfig.Enabled
|
||||
}
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
}
|
||||
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
URL: url,
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
URL: url,
|
||||
EnableMultimodel: enableMultimodelValue,
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -595,12 +618,25 @@ func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Contex
|
||||
// Enqueue passage processing task to Asynq
|
||||
logger.Info(ctx, "Enqueuing passage processing task to Asynq")
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
// Check question generation config
|
||||
enableQuestionGeneration := false
|
||||
questionCount := 3 // default
|
||||
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
|
||||
enableQuestionGeneration = true
|
||||
if kb.QuestionGenerationConfig.QuestionCount > 0 {
|
||||
questionCount = kb.QuestionGenerationConfig.QuestionCount
|
||||
}
|
||||
}
|
||||
|
||||
taskPayload := types.DocumentProcessPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Passages: safePassages,
|
||||
EnableMultimodel: false, // 文本段落不支持多模态
|
||||
TenantID: tenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Passages: safePassages,
|
||||
EnableMultimodel: false, // 文本段落不支持多模态
|
||||
EnableQuestionGeneration: enableQuestionGeneration,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -917,10 +953,23 @@ func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
|
||||
s.processChunks(ctx, kb, knowledge, chunks)
|
||||
}
|
||||
|
||||
// ProcessChunksOptions contains options for processing chunks
|
||||
type ProcessChunksOptions struct {
|
||||
EnableQuestionGeneration bool
|
||||
QuestionCount int
|
||||
}
|
||||
|
||||
// processChunks processes chunks and creates embeddings for knowledge content
|
||||
func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*proto.Chunk,
|
||||
opts ...ProcessChunksOptions,
|
||||
) {
|
||||
// Get options
|
||||
var options ProcessChunksOptions
|
||||
if len(opts) > 0 {
|
||||
options = opts[0]
|
||||
}
|
||||
|
||||
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processChunks")
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
@@ -969,14 +1018,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
|
||||
logger.Infof(ctx, "Cleanup completed, starting to process new chunks")
|
||||
|
||||
// Generate document summary - 只使用文本类型的 Chunk
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get summary model failed")
|
||||
span.RecordError(err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create chunk objects from proto chunks
|
||||
maxSeq := 0
|
||||
|
||||
@@ -1118,52 +1159,19 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
span.AddEvent("extract summary")
|
||||
summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
|
||||
WithField("error", err).Errorf("processChunks get summary failed, use first chunk as description")
|
||||
if len(textChunks) > 0 {
|
||||
knowledge.Description = textChunks[0].Content
|
||||
}
|
||||
} else {
|
||||
knowledge.Description = summary
|
||||
}
|
||||
span.SetAttributes(attribute.String("summary", knowledge.Description))
|
||||
|
||||
// 批量索引
|
||||
if strings.TrimSpace(knowledge.Description) != "" && len(textChunks) > 0 {
|
||||
sChunk := &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, knowledge.Description),
|
||||
ChunkIndex: maxSeq + 3, // 使用不冲突的索引方式
|
||||
IsEnabled: true,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
StartAt: 0,
|
||||
EndAt: 0,
|
||||
ChunkType: types.ChunkTypeSummary,
|
||||
ParentChunkID: textChunks[0].ID,
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("Created summary chunk for %s with index %d",
|
||||
sChunk.ParentChunkID, sChunk.ChunkIndex)
|
||||
insertChunks = append(insertChunks, sChunk)
|
||||
}
|
||||
|
||||
// Create index information for each chunk
|
||||
indexInfoList := utils.MapSlice(insertChunks, func(chunk *types.Chunk) *types.IndexInfo {
|
||||
return &types.IndexInfo{
|
||||
// Create index information for each chunk (without generated questions for now)
|
||||
indexInfoList := make([]*types.IndexInfo, 0, len(insertChunks))
|
||||
for _, chunk := range insertChunks {
|
||||
// Add original chunk content to index
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: chunk.Content,
|
||||
SourceID: chunk.ID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Initialize retrieval engine
|
||||
|
||||
@@ -1249,6 +1257,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed")
|
||||
}
|
||||
|
||||
// Enqueue question generation task if enabled (async, non-blocking)
|
||||
if options.EnableQuestionGeneration && len(textChunks) > 0 {
|
||||
questionCount := options.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
if questionCount > 10 {
|
||||
questionCount = 10
|
||||
}
|
||||
s.enqueueQuestionGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID, questionCount)
|
||||
}
|
||||
|
||||
// Enqueue summary generation task (async, non-blocking)
|
||||
if len(textChunks) > 0 {
|
||||
s.enqueueSummaryGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
|
||||
}
|
||||
|
||||
// Update tenant's storage usage
|
||||
tenantInfo.StorageUsed += totalStorageSize
|
||||
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
|
||||
@@ -1354,6 +1379,471 @@ func (s *knowledgeService) getSummary(ctx context.Context,
|
||||
return summary.Content, nil
|
||||
}
|
||||
|
||||
// enqueueQuestionGenerationTask enqueues an async task for question generation
|
||||
func (s *knowledgeService) enqueueQuestionGenerationTask(ctx context.Context,
|
||||
kbID, knowledgeID string, questionCount int,
|
||||
) {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
payload := types.QuestionGenerationPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal question generation payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue question generation task: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued question generation task: %s for knowledge: %s", info.ID, knowledgeID)
|
||||
}
|
||||
|
||||
// enqueueSummaryGenerationTask enqueues an async task for summary generation
|
||||
func (s *knowledgeService) enqueueSummaryGenerationTask(ctx context.Context,
|
||||
kbID, knowledgeID string,
|
||||
) {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
payload := types.SummaryGenerationPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal summary generation payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue summary generation task: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued summary generation task: %s for knowledge: %s", info.ID, knowledgeID)
|
||||
}
|
||||
|
||||
// ProcessSummaryGeneration handles async summary generation task
|
||||
func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.SummaryGenerationPayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal summary generation payload: %v", err)
|
||||
return nil // Don't retry on unmarshal error
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing summary generation for knowledge: %s", payload.KnowledgeID)
|
||||
|
||||
// Set tenant context
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
// Get knowledge base
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get knowledge
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get text chunks for this knowledge
|
||||
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chunks: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Filter text chunks only
|
||||
textChunks := make([]*types.Chunk, 0)
|
||||
for _, chunk := range chunks {
|
||||
if chunk.ChunkType == types.ChunkTypeText {
|
||||
textChunks = append(textChunks, chunk)
|
||||
}
|
||||
}
|
||||
|
||||
if len(textChunks) == 0 {
|
||||
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort chunks by ChunkIndex for proper ordering
|
||||
sort.Slice(textChunks, func(i, j int) bool {
|
||||
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
|
||||
})
|
||||
|
||||
// Initialize chat model for summary
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chat model: %v", err)
|
||||
return fmt.Errorf("failed to get chat model: %w", err)
|
||||
}
|
||||
|
||||
// Generate summary
|
||||
summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to generate summary for knowledge %s: %v", payload.KnowledgeID, err)
|
||||
// Use first chunk content as fallback
|
||||
if len(textChunks) > 0 {
|
||||
summary = textChunks[0].Content
|
||||
if len(summary) > 500 {
|
||||
summary = summary[:500]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update knowledge description
|
||||
knowledge.Description = summary
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update knowledge description: %v", err)
|
||||
return fmt.Errorf("failed to update knowledge: %w", err)
|
||||
}
|
||||
|
||||
// Create summary chunk and index it
|
||||
if strings.TrimSpace(summary) != "" {
|
||||
// Get max chunk index
|
||||
maxChunkIndex := 0
|
||||
for _, chunk := range chunks {
|
||||
if chunk.ChunkIndex > maxChunkIndex {
|
||||
maxChunkIndex = chunk.ChunkIndex
|
||||
}
|
||||
}
|
||||
|
||||
summaryChunk := &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, summary),
|
||||
ChunkIndex: maxChunkIndex + 1,
|
||||
IsEnabled: true,
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
StartAt: 0,
|
||||
EndAt: 0,
|
||||
ChunkType: types.ChunkTypeSummary,
|
||||
ParentChunkID: textChunks[0].ID,
|
||||
}
|
||||
|
||||
// Save summary chunk
|
||||
if err := s.chunkService.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create summary chunk: %v", err)
|
||||
return fmt.Errorf("failed to create summary chunk: %w", err)
|
||||
}
|
||||
|
||||
// Index summary chunk
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
|
||||
return fmt.Errorf("failed to get tenant info: %w", err)
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
||||
}
|
||||
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
|
||||
return fmt.Errorf("failed to get embedding model: %w", err)
|
||||
}
|
||||
|
||||
indexInfo := []*types.IndexInfo{{
|
||||
Content: summaryChunk.Content,
|
||||
SourceID: summaryChunk.ID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: summaryChunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
}}
|
||||
|
||||
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
|
||||
logger.Errorf(ctx, "Failed to index summary chunk: %v", err)
|
||||
return fmt.Errorf("failed to index summary chunk: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Successfully created and indexed summary chunk for knowledge: %s", payload.KnowledgeID)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Successfully generated summary for knowledge: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProcessQuestionGeneration handles async question generation task
|
||||
func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error {
|
||||
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.ProcessQuestionGeneration")
|
||||
defer span.End()
|
||||
|
||||
var payload types.QuestionGenerationPayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal question generation payload: %v", err)
|
||||
return nil // Don't retry on unmarshal error
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing question generation for knowledge: %s", payload.KnowledgeID)
|
||||
|
||||
// Set tenant context
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
|
||||
// Get knowledge base
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get knowledge
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get text chunks for this knowledge
|
||||
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chunks: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Filter text chunks only
|
||||
textChunks := make([]*types.Chunk, 0)
|
||||
for _, chunk := range chunks {
|
||||
if chunk.ChunkType == types.ChunkTypeText {
|
||||
textChunks = append(textChunks, chunk)
|
||||
}
|
||||
}
|
||||
|
||||
if len(textChunks) == 0 {
|
||||
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort chunks by StartAt for context building
|
||||
sort.Slice(textChunks, func(i, j int) bool {
|
||||
return textChunks[i].StartAt < textChunks[j].StartAt
|
||||
})
|
||||
|
||||
// Initialize chat model
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chat model: %v", err)
|
||||
return fmt.Errorf("failed to get chat model: %w", err)
|
||||
}
|
||||
|
||||
// Initialize embedding model and retrieval engine
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
|
||||
return fmt.Errorf("failed to get embedding model: %w", err)
|
||||
}
|
||||
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
|
||||
return fmt.Errorf("failed to get tenant info: %w", err)
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
||||
}
|
||||
|
||||
questionCount := payload.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
if questionCount > 10 {
|
||||
questionCount = 10
|
||||
}
|
||||
|
||||
// Generate questions for each chunk with context
|
||||
var indexInfoList []*types.IndexInfo
|
||||
for i, chunk := range textChunks {
|
||||
// Build context from adjacent chunks
|
||||
var prevContent, nextContent string
|
||||
if i > 0 {
|
||||
prevContent = textChunks[i-1].Content
|
||||
// Limit context size
|
||||
if len(prevContent) > 500 {
|
||||
prevContent = prevContent[len(prevContent)-500:]
|
||||
}
|
||||
}
|
||||
if i < len(textChunks)-1 {
|
||||
nextContent = textChunks[i+1].Content
|
||||
// Limit context size
|
||||
if len(nextContent) > 500 {
|
||||
nextContent = nextContent[:500]
|
||||
}
|
||||
}
|
||||
|
||||
questions, err := s.generateQuestionsWithContext(ctx, chatModel, chunk.Content, prevContent, nextContent, knowledge.Title, questionCount)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(questions) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Update chunk metadata
|
||||
meta := &types.DocumentChunkMetadata{
|
||||
GeneratedQuestions: questions,
|
||||
}
|
||||
if err := chunk.SetDocumentMetadata(meta); err != nil {
|
||||
logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update chunk in database
|
||||
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create index entries for generated questions
|
||||
for j, question := range questions {
|
||||
sourceID := fmt.Sprintf("%s-q%d", chunk.ID, j)
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: question,
|
||||
SourceID: sourceID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
logger.Debugf(ctx, "Generated %d questions for chunk %s", len(questions), chunk.ID)
|
||||
}
|
||||
|
||||
// Index generated questions
|
||||
if len(indexInfoList) > 0 {
|
||||
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
|
||||
logger.Errorf(ctx, "Failed to index generated questions: %v", err)
|
||||
return fmt.Errorf("failed to index questions: %w", err)
|
||||
}
|
||||
logger.Infof(ctx, "Successfully indexed %d generated questions for knowledge: %s", len(indexInfoList), payload.KnowledgeID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateQuestionsWithContext generates questions for a chunk with surrounding context
|
||||
func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
|
||||
chatModel chat.Chat, content, prevContent, nextContent, docName string, questionCount int,
|
||||
) ([]string, error) {
|
||||
if content == "" || questionCount <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build prompt with context
|
||||
prompt := s.config.Conversation.GenerateQuestionsPrompt
|
||||
if prompt == "" {
|
||||
prompt = defaultQuestionGenerationPrompt
|
||||
}
|
||||
|
||||
// Build context section
|
||||
var contextSection string
|
||||
if prevContent != "" || nextContent != "" {
|
||||
contextSection = "## 上下文信息(仅供参考,帮助理解主要内容)\n"
|
||||
if prevContent != "" {
|
||||
contextSection += fmt.Sprintf("【前文】%s\n", prevContent)
|
||||
}
|
||||
if nextContent != "" {
|
||||
contextSection += fmt.Sprintf("【后文】%s\n", nextContent)
|
||||
}
|
||||
contextSection += "\n"
|
||||
}
|
||||
|
||||
// Replace placeholders
|
||||
prompt = strings.ReplaceAll(prompt, "{{.QuestionCount}}", fmt.Sprintf("%d", questionCount))
|
||||
prompt = strings.ReplaceAll(prompt, "{{.Content}}", content)
|
||||
prompt = strings.ReplaceAll(prompt, "{{.Context}}", contextSection)
|
||||
prompt = strings.ReplaceAll(prompt, "{{.DocName}}", docName)
|
||||
|
||||
thinking := false
|
||||
response, err := chatModel.Chat(ctx, []chat.Message{
|
||||
{
|
||||
Role: "user",
|
||||
Content: prompt,
|
||||
},
|
||||
}, &chat.ChatOptions{
|
||||
Temperature: 0.7,
|
||||
MaxTokens: 512,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate questions: %w", err)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
lines := strings.Split(response.Content, "\n")
|
||||
questions := make([]string, 0, questionCount)
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
line = strings.TrimLeft(line, "0123456789.-*) ")
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" && len(line) > 5 {
|
||||
questions = append(questions, line)
|
||||
if len(questions) >= questionCount {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return questions, nil
|
||||
}
|
||||
|
||||
// Default prompt for question generation with context support
|
||||
const defaultQuestionGenerationPrompt = `你是一个专业的问题生成助手。你的任务是根据给定的【主要内容】生成用户可能会问的相关问题。
|
||||
|
||||
{{.Context}}
|
||||
## 主要内容(请基于此内容生成问题)
|
||||
文档名称:{{.DocName}}
|
||||
文档内容:
|
||||
{{.Content}}
|
||||
|
||||
## 核心要求
|
||||
- 生成的问题必须与【主要内容】直接相关
|
||||
- 问题中禁止使用任何代词或指代词(如"它"、"这个"、"该文档"、"本文"、"文中"、"其"等),必须用具体名称替代
|
||||
- 问题必须是完整独立的,脱离上下文也能被理解
|
||||
- 问题应该是用户在实际场景中可能会提出的自然问题
|
||||
- 问题应该多样化,覆盖内容的不同方面
|
||||
- 每个问题应该简洁明了,长度控制在30字以内
|
||||
- 生成的问题数量为 {{.QuestionCount}} 个
|
||||
|
||||
## 问题类型建议
|
||||
- 定义类:什么是...?...是什么?
|
||||
- 原因类:为什么...?...的原因是什么?
|
||||
- 方法类:如何...?怎样...?
|
||||
- 比较类:...和...有什么区别?
|
||||
- 应用类:...可以用于什么场景?
|
||||
|
||||
## 输出格式
|
||||
直接输出问题列表,每行一个问题,不要有序号或其他前缀。`
|
||||
|
||||
// GetKnowledgeFile retrieves the physical file associated with a knowledge entry
|
||||
func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) {
|
||||
// Get knowledge record
|
||||
@@ -4051,7 +4541,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
}
|
||||
|
||||
// 处理chunks(这会更新状态为completed)
|
||||
s.processChunks(ctx, kb, knowledge, chunks)
|
||||
s.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{
|
||||
EnableQuestionGeneration: payload.EnableQuestionGeneration,
|
||||
QuestionCount: payload.QuestionCount,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ type ConversationConfig struct {
|
||||
SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"`
|
||||
ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"`
|
||||
ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"`
|
||||
// GenerateQuestionsPrompt is used to generate questions for document chunks to improve recall
|
||||
GenerateQuestionsPrompt string `yaml:"generate_questions_prompt" json:"generate_questions_prompt"`
|
||||
}
|
||||
|
||||
// SummaryConfig 摘要配置
|
||||
|
||||
@@ -123,6 +123,12 @@ type KBModelConfigRequest struct {
|
||||
Nodes []types.GraphNode `json:"nodes"`
|
||||
Relations []types.GraphRelation `json:"relations"`
|
||||
} `json:"nodeExtract"`
|
||||
|
||||
// 问题生成配置
|
||||
QuestionGeneration struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
QuestionCount int `json:"questionCount"`
|
||||
} `json:"questionGeneration"`
|
||||
}
|
||||
|
||||
// InitializationRequest 初始化请求结构
|
||||
@@ -192,6 +198,11 @@ type InitializationRequest struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"relations"`
|
||||
} `json:"nodeExtract"`
|
||||
|
||||
QuestionGeneration struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
QuestionCount int `json:"questionCount"`
|
||||
} `json:"questionGeneration"`
|
||||
}
|
||||
|
||||
// UpdateKBConfig 根据知识库ID和模型ID更新配置(简化版)
|
||||
@@ -333,6 +344,23 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// 更新问题生成配置
|
||||
if req.QuestionGeneration.Enabled {
|
||||
questionCount := req.QuestionGeneration.QuestionCount
|
||||
if questionCount <= 0 {
|
||||
questionCount = 3
|
||||
}
|
||||
if questionCount > 10 {
|
||||
questionCount = 10
|
||||
}
|
||||
kb.QuestionGenerationConfig = &types.QuestionGenerationConfig{
|
||||
Enabled: true,
|
||||
QuestionCount: questionCount,
|
||||
}
|
||||
} else {
|
||||
kb.QuestionGenerationConfig = &types.QuestionGenerationConfig{Enabled: false}
|
||||
}
|
||||
|
||||
// 保存更新后的知识库
|
||||
if err := h.kbRepository.UpdateKnowledgeBase(ctx, kb); err != nil {
|
||||
logger.Error(ctx, "Failed to update knowledge base", err)
|
||||
|
||||
@@ -67,6 +67,12 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register FAQ import handler
|
||||
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
|
||||
|
||||
// Register question generation handler
|
||||
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
|
||||
|
||||
// Register summary generation handler
|
||||
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
|
||||
|
||||
go func() {
|
||||
// Start the server
|
||||
if err := params.Server.Run(mux); err != nil {
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package types
|
||||
|
||||
const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -15,16 +17,18 @@ type ExtractChunkPayload struct {
|
||||
|
||||
// DocumentProcessPayload represents the document process task payload
|
||||
type DocumentProcessPayload struct {
|
||||
RequestId string `json:"request_id"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
|
||||
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
|
||||
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
|
||||
URL string `json:"url,omitempty"` // URL(URL导入时使用)
|
||||
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
|
||||
EnableMultimodel bool `json:"enable_multimodel"`
|
||||
RequestId string `json:"request_id"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
|
||||
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
|
||||
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
|
||||
URL string `json:"url,omitempty"` // URL(URL导入时使用)
|
||||
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
|
||||
EnableMultimodel bool `json:"enable_multimodel"`
|
||||
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
|
||||
QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量
|
||||
}
|
||||
|
||||
// FAQImportPayload represents the FAQ import task payload
|
||||
@@ -37,6 +41,29 @@ type FAQImportPayload struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
// QuestionGenerationPayload represents the question generation task payload
|
||||
type QuestionGenerationPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
QuestionCount int `json:"question_count"`
|
||||
}
|
||||
|
||||
// SummaryGenerationPayload represents the summary generation task payload
|
||||
type SummaryGenerationPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
}
|
||||
|
||||
// ChunkContext represents chunk content with surrounding context
|
||||
type ChunkContext struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
Content string `json:"content"`
|
||||
PrevContent string `json:"prev_content,omitempty"` // Previous chunk content for context
|
||||
NextContent string `json:"next_content,omitempty"` // Next chunk content for context
|
||||
}
|
||||
|
||||
// PromptTemplateStructured represents the prompt template structured
|
||||
type PromptTemplateStructured struct {
|
||||
Description string `json:"description"`
|
||||
|
||||
@@ -19,6 +19,43 @@ type FAQChunkMetadata struct {
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
// DocumentChunkMetadata 定义文档 Chunk 的元数据结构
|
||||
// 用于存储AI生成的问题等增强信息
|
||||
type DocumentChunkMetadata struct {
|
||||
// GeneratedQuestions 存储AI为该Chunk生成的相关问题
|
||||
// 这些问题会被独立索引以提高召回率
|
||||
GeneratedQuestions []string `json:"generated_questions,omitempty"`
|
||||
}
|
||||
|
||||
// DocumentMetadata 解析 Chunk 中的文档元数据
|
||||
func (c *Chunk) DocumentMetadata() (*DocumentChunkMetadata, error) {
|
||||
if c == nil || len(c.Metadata) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var meta DocumentChunkMetadata
|
||||
if err := json.Unmarshal(c.Metadata, &meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &meta, nil
|
||||
}
|
||||
|
||||
// SetDocumentMetadata 设置 Chunk 的文档元数据
|
||||
func (c *Chunk) SetDocumentMetadata(meta *DocumentChunkMetadata) error {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
if meta == nil {
|
||||
c.Metadata = nil
|
||||
return nil
|
||||
}
|
||||
bytes, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Metadata = JSON(bytes)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Normalize 清理空白与重复项
|
||||
func (m *FAQChunkMetadata) Normalize() {
|
||||
if m == nil {
|
||||
|
||||
@@ -99,6 +99,10 @@ type KnowledgeService interface {
|
||||
ProcessDocument(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessFAQImport handles Asynq FAQ import tasks
|
||||
ProcessFAQImport(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessQuestionGeneration handles Asynq question generation tasks
|
||||
ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error
|
||||
// ProcessSummaryGeneration handles Asynq summary generation tasks
|
||||
ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error
|
||||
}
|
||||
|
||||
// KnowledgeRepository defines the interface for knowledge repositories.
|
||||
|
||||
@@ -65,6 +65,8 @@ type KnowledgeBase struct {
|
||||
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
|
||||
// FAQConfig stores FAQ specific configuration such as indexing strategy
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config" gorm:"column:faq_config;type:json"`
|
||||
// QuestionGenerationConfig stores question generation configuration for document knowledge bases
|
||||
QuestionGenerationConfig *QuestionGenerationConfig `yaml:"question_generation_config" json:"question_generation_config" gorm:"column:question_generation_config;type:json"`
|
||||
// Creation time of the knowledge base
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time of the knowledge base
|
||||
@@ -180,6 +182,32 @@ type VLMConfig struct {
|
||||
ModelID string `yaml:"model_id" json:"model_id"`
|
||||
}
|
||||
|
||||
// QuestionGenerationConfig represents the question generation configuration for document knowledge bases
|
||||
// When enabled, the system will use LLM to generate questions for each chunk during document parsing
|
||||
// These generated questions will be indexed separately to improve recall
|
||||
type QuestionGenerationConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
// Number of questions to generate per chunk (default: 3, max: 10)
|
||||
QuestionCount int `yaml:"question_count" json:"question_count"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface
|
||||
func (c QuestionGenerationConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface
|
||||
func (c *QuestionGenerationConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert VLMConfig to database value
|
||||
func (c VLMConfig) Value() (driver.Value, error) {
|
||||
return json.Marshal(c)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Remove question_generation_config column from knowledge_bases table
|
||||
|
||||
ALTER TABLE knowledge_bases
|
||||
DROP COLUMN IF EXISTS question_generation_config;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Add question_generation_config column to knowledge_bases table
|
||||
-- This column stores configuration for AI question generation feature
|
||||
-- When enabled, the system generates questions for document chunks to improve recall
|
||||
|
||||
ALTER TABLE knowledge_bases
|
||||
ADD COLUMN IF NOT EXISTS question_generation_config JSON NULL
|
||||
COMMENT 'Question generation configuration for document knowledge bases';
|
||||
Reference in New Issue
Block a user