feat: 添加AI问题生成功能,支持配置生成数量并在前端展示生成的问题

This commit is contained in:
wizardchen
2025-12-03 01:42:31 +08:00
parent 12497a2fea
commit 16562eebbb
21 changed files with 987 additions and 296 deletions

View File

@@ -518,6 +518,34 @@ conversation:
}
]
generate_questions_prompt: |
你是一个专业的问题生成助手。你的任务是根据给定的【主要内容】生成用户可能会问的相关问题。
{{.Context}}
## 主要内容(请基于此内容生成问题)
文档名称:{{.DocName}}
文档内容:
{{.Content}}
## 核心要求
- 生成的问题必须与【主要内容】直接相关
- 问题中禁止使用任何代词或指代词(如"它"、"这个"、"该文档"、"本文"、"文中"、"其"等),必须用具体名称替代
- 问题必须是完整独立的,脱离上下文也能被理解
- 问题应该是用户在实际场景中可能会提出的自然问题
- 问题应该多样化,覆盖内容的不同方面
- 每个问题应该简洁明了长度控制在30字以内
- 生成的问题数量为 {{.QuestionCount}} 个
## 问题类型建议
- 定义类:什么是......是什么?
- 原因类:为什么......的原因是什么?
- 方法类:如何...?怎样...
- 比较类:...和...有什么区别?
- 应用类:...可以用于什么场景?
## 输出格式
直接输出问题列表,每行一个问题,不要有序号或其他前缀。
# 知识库配置
knowledge_base:
chunk_size: 512
@@ -617,4 +645,4 @@ web_search:
# 租户配置
tenant:
# 是否启用跨租户访问功能(内网环境可开启)
enable_cross_tenant_access: true
enable_cross_tenant_access: false

View File

@@ -107,6 +107,10 @@ export interface KBModelConfigRequest {
nodes: Node[]
relations: Relation[]
}
questionGeneration?: {
enabled: boolean
questionCount: number
}
}
export function updateKBConfig(kbId: string, config: KBModelConfigRequest): Promise<any> {

View File

@@ -185,6 +185,32 @@ const getChunkMeta = (item: any) => {
return parts.join(' · ');
};
// 解析生成的问题
const getGeneratedQuestions = (item: any): string[] => {
if (!item || !item.metadata) return [];
try {
const metadata = typeof item.metadata === 'string' ? JSON.parse(item.metadata) : item.metadata;
return metadata.generated_questions || [];
} catch {
return [];
}
};
// 展开状态管理
const expandedChunks = ref<Set<number>>(new Set());
const toggleQuestions = (index: number) => {
if (expandedChunks.value.has(index)) {
expandedChunks.value.delete(index);
} else {
expandedChunks.value.add(index);
}
// 触发响应式更新
expandedChunks.value = new Set(expandedChunks.value);
};
const isExpanded = (index: number) => expandedChunks.value.has(index);
const downloadFile = () => {
downKnowledgeDetails(props.details.id)
.then((result) => {
@@ -272,17 +298,44 @@ const handleDetailsScroll = () => {
<div v-if="details.md.length == 0" class="no_content">{{ $t('common.noData') }}</div>
<div v-else class="chunk-list">
<div
class="chunk-item"
<div class="chunk-item"
v-for="(item, index) in details.md"
:key="index"
:class="getChunkClass(index)"
>
<div class="chunk-header">
<span class="chunk-index">{{ $t('knowledgeBase.segment') || '片段' }} {{ index + 1 }}</span>
<span class="chunk-meta">{{ getChunkMeta(item) }}</span>
<div class="chunk-header-right">
<t-tag
v-if="getGeneratedQuestions(item).length > 0"
size="small"
theme="success"
variant="light"
>
{{ $t('knowledgeBase.questions') || '问题' }} {{ getGeneratedQuestions(item).length }}
</t-tag>
<span class="chunk-meta">{{ getChunkMeta(item) }}</span>
</div>
</div>
<div class="md-content" v-html="processMarkdown(item.content)"></div>
<!-- 生成的问题展示 -->
<div v-if="getGeneratedQuestions(item).length > 0" class="questions-section">
<div class="questions-toggle" @click="toggleQuestions(index)">
<t-icon :name="isExpanded(index) ? 'chevron-down' : 'chevron-right'" size="14px" />
<span>{{ $t('knowledgeBase.generatedQuestions') || '生成的问题' }} ({{ getGeneratedQuestions(item).length }})</span>
</div>
<div v-show="isExpanded(index)" class="questions-list">
<div
v-for="(question, qIndex) in getGeneratedQuestions(item)"
:key="qIndex"
class="question-item"
>
<t-icon name="help-circle" size="14px" class="question-icon" />
<span>{{ question }}</span>
</div>
</div>
</div>
</div>
</div>
@@ -493,12 +546,65 @@ const handleDetailsScroll = () => {
letter-spacing: 0.5px;
}
.chunk-header-right {
display: flex;
align-items: center;
gap: 8px;
}
.chunk-meta {
color: #00000066;
font-size: 11px;
}
}
// 生成的问题样式
.questions-section {
margin-top: 12px;
padding-top: 10px;
border-top: 1px dashed #e0e0e0;
}
.questions-toggle {
display: flex;
align-items: center;
gap: 6px;
cursor: pointer;
color: #059669;
font-size: 12px;
font-weight: 500;
padding: 4px 0;
transition: color 0.2s ease;
&:hover {
color: #07c05f;
}
}
.questions-list {
margin-top: 8px;
padding-left: 4px;
}
.question-item {
display: flex;
align-items: flex-start;
gap: 8px;
padding: 6px 8px;
margin-bottom: 4px;
background: #f0fdf4;
border-radius: 4px;
font-size: 13px;
color: #1d2129;
line-height: 1.5;
.question-icon {
color: #059669;
flex-shrink: 0;
margin-top: 2px;
}
}
.md-content {
word-break: break-word;
line-height: 1.6;

View File

@@ -92,6 +92,8 @@ export default {
createTime: 'Create Time',
characters: 'chars',
segment: 'Segment',
questions: 'Questions',
generatedQuestions: 'Generated Questions',
notInitialized: 'Knowledge base is not initialized. Please configure models in settings before uploading files',
getInfoFailed: 'Failed to get knowledge base information, file upload is not possible',
missingId: 'Knowledge base ID is missing',
@@ -1001,7 +1003,13 @@ export default {
},
advanced: {
title: 'Advanced Settings',
description: 'Configure multimodal features',
description: 'Configure question generation, multimodal features',
questionGeneration: {
label: 'AI Question Generation',
description: 'Generate related questions for each chunk using LLM during document parsing to improve retrieval recall. Enabling this will increase document parsing time.',
countLabel: 'Question Count',
countDescription: 'Number of questions to generate per document chunk (1-10)',
},
multimodal: {
label: 'Multimodal Feature',
description: 'Enable understanding of multimodal content such as images and videos',

View File

@@ -92,6 +92,8 @@ export default {
createTime: 'Время создания',
characters: 'символов',
segment: 'Фрагмент',
questions: 'Вопросы',
generatedQuestions: 'Сгенерированные вопросы',
notInitialized: 'База знаний не инициализирована. Пожалуйста, настройте модели в разделе настроек перед загрузкой файлов',
getInfoFailed: 'Не удалось получить информацию о базе знаний, загрузка файла невозможна',
missingId: 'Отсутствует ID базы знаний',
@@ -1095,7 +1097,13 @@ export default {
},
advanced: {
title: 'Расширенные настройки',
description: 'Настройте мультимодальные возможности',
description: 'Настройте генерацию вопросов и мультимодальные возможности',
questionGeneration: {
label: 'AI генерация вопросов',
description: 'Генерация связанных вопросов для каждого фрагмента с помощью LLM при парсинге документа для улучшения полноты поиска. Включение увеличит время парсинга документа.',
countLabel: 'Количество вопросов',
countDescription: 'Количество вопросов для генерации на фрагмент документа (1-10)',
},
multimodal: {
label: 'Мультимодальная функция',
description: 'Включите понимание мультимедийного контента, такого как изображения и видео',

View File

@@ -91,6 +91,8 @@ export default {
createTime: "创建时间",
characters: "字符",
segment: "片段",
questions: "问题",
generatedQuestions: "生成的问题",
docActionUnsupported: "当前知识库类型不支持该操作",
notInitialized:
"该知识库尚未完成初始化配置,请先前往设置页面配置模型信息后再上传文件",
@@ -1334,7 +1336,13 @@ export default {
},
advanced: {
title: "高级设置",
description: "配置多模态等高级功能",
description: "配置问题生成、多模态等高级功能",
questionGeneration: {
label: "AI 问题生成",
description: "解析文档时调用大模型为每个分块生成相关问题,提高检索召回率。启用后会增加文档解析耗时。",
countLabel: "生成问题数量",
countDescription: "每个文档分块生成的问题数量1-10",
},
multimodal: {
label: "多模态功能",
description: "启用图片、视频等多模态内容的理解能力",

View File

@@ -144,8 +144,10 @@
ref="advancedSettingsRef"
v-if="formData"
:multimodal="formData.multimodalConfig"
:question-generation="formData.questionGenerationConfig"
:all-models="allModels"
@update:multimodal="handleMultimodalUpdate"
@update:question-generation="handleQuestionGenerationUpdate"
/>
</div>
</div>
@@ -296,6 +298,10 @@ const initFormData = (type: 'document' | 'faq' = 'document') => {
type: string
}>
},
questionGenerationConfig: {
enabled: false,
questionCount: 3
},
}
}
@@ -377,6 +383,10 @@ const loadKBData = async () => {
})),
relations: kb.extract_config?.relations || []
},
questionGenerationConfig: {
enabled: kb.question_generation_config?.enabled || false,
questionCount: kb.question_generation_config?.question_count || 3
},
}
} catch (error) {
console.error('Failed to load knowledge base data:', error)
@@ -406,6 +416,12 @@ const handleMultimodalUpdate = (config: any) => {
}
}
const handleQuestionGenerationUpdate = (config: any) => {
if (formData.value) {
formData.value.questionGenerationConfig = { ...config }
}
}
const handleNodeExtractUpdate = (config: any) => {
if (formData.value) {
formData.value.nodeExtractConfig = { ...config }
@@ -514,6 +530,14 @@ const buildSubmitData = () => {
}
}
// 添加问题生成配置
if (formData.value.questionGenerationConfig?.enabled) {
data.question_generation_config = {
enabled: true,
question_count: formData.value.questionGenerationConfig.questionCount || 3
}
}
if (formData.value.type === 'faq') {
data.faq_config = {
index_mode: formData.value.faqConfig?.indexMode || 'question_only',
@@ -598,6 +622,10 @@ const handleSubmit = async () => {
tags: data.extract_config?.tags || [],
nodes: data.extract_config?.nodes || [],
relations: data.extract_config?.relations || []
},
questionGeneration: {
enabled: data.question_generation_config?.enabled || false,
questionCount: data.question_generation_config?.question_count || 3
}
}

View File

@@ -6,6 +6,42 @@
</div>
<div class="settings-group">
<!-- Question Generation feature -->
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.questionGeneration.label') }}</label>
<p class="desc">{{ $t('knowledgeEditor.advanced.questionGeneration.description') }}</p>
</div>
<div class="setting-control">
<t-switch
v-model="localQuestionGeneration.enabled"
@change="handleQuestionGenerationToggle"
size="large"
/>
</div>
</div>
<!-- Question Generation configuration -->
<div v-if="localQuestionGeneration.enabled" class="subsection">
<div class="setting-row">
<div class="setting-info">
<label>{{ $t('knowledgeEditor.advanced.questionGeneration.countLabel') }}</label>
<p class="desc">{{ $t('knowledgeEditor.advanced.questionGeneration.countDescription') }}</p>
</div>
<div class="setting-control">
<t-input-number
v-model="localQuestionGeneration.questionCount"
:min="1"
:max="10"
:step="1"
theme="normal"
@change="handleQuestionGenerationChange"
style="width: 120px;"
/>
</div>
</div>
</div>
<!-- Multimodal feature -->
<div class="setting-row">
<div class="setting-info">
@@ -241,8 +277,14 @@ interface MultimodalConfig {
}
}
interface QuestionGenerationConfig {
enabled: boolean
questionCount: number
}
interface Props {
multimodal: MultimodalConfig
questionGeneration?: QuestionGenerationConfig
allModels?: any[]
}
@@ -250,9 +292,13 @@ const props = defineProps<Props>()
const emit = defineEmits<{
'update:multimodal': [value: MultimodalConfig]
'update:questionGeneration': [value: QuestionGenerationConfig]
}>()
const localMultimodal = ref<MultimodalConfig>({ ...props.multimodal })
const localQuestionGeneration = ref<QuestionGenerationConfig>(
props.questionGeneration || { enabled: false, questionCount: 3 }
)
const vllmSelectorRef = ref()
const isMinioEnabled = ref(false)
@@ -287,6 +333,25 @@ watch(() => props.multimodal, (newVal) => {
localMultimodal.value = { ...newVal }
}, { deep: true })
watch(() => props.questionGeneration, (newVal) => {
if (newVal) {
localQuestionGeneration.value = { ...newVal }
}
}, { deep: true })
// Handle question generation toggle
const handleQuestionGenerationToggle = () => {
if (!localQuestionGeneration.value.enabled) {
localQuestionGeneration.value.questionCount = 3
}
emit('update:questionGeneration', localQuestionGeneration.value)
}
// Handle question generation config change
const handleQuestionGenerationChange = () => {
emit('update:questionGeneration', localQuestionGeneration.value)
}
// Handle multimodal toggle
const handleMultimodalToggle = () => {
// Reset related configuration when multimodal is disabled

View File

@@ -60,7 +60,6 @@ func (r *chunkRepository) ListChunksByKnowledgeID(
) ([]*types.Chunk, error) {
var chunks []*types.Chunk
if err := r.db.WithContext(ctx).
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info").
Where("tenant_id = ? AND knowledge_id = ? and chunk_type = ?", tenantID, knowledgeID, "text").
Order("chunk_index ASC").
Find(&chunks).Error; err != nil {
@@ -85,7 +84,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
baseFilter := func(db *gorm.DB) *gorm.DB {
db = db.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
tenantID, knowledgeID, chunkType, []types.ChunkStatus{types.ChunkStatusIndexed, types.ChunkStatusDefault})
tenantID, knowledgeID, chunkType, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)})
if tagID != "" {
db = db.Where("tag_id = ?", tagID)
}
@@ -106,7 +105,7 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
// Then query the paginated data
dataQuery := baseFilter(
r.db.WithContext(ctx).
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id"),
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id, status"),
)
if err := dataQuery.

View File

@@ -291,7 +291,7 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
ORDER BY embedding::halfvec(%d) <=> $1::halfvec
LIMIT $%d
) AS candidates
WHERE distance < $%d
WHERE distance <= $%d
ORDER BY distance ASC
LIMIT $%d
`, dimension, whereClause, dimension, subqueryLimitParam, thresholdParam, finalLimitParam)

View File

@@ -5,34 +5,24 @@ import (
"encoding/json"
"regexp"
"strings"
"unicode"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/yanyiwu/gojieba"
)
// PluginPreprocess Query preprocessing plugin
type PluginPreprocess struct {
config *config.Config
jieba *gojieba.Jieba
stopwords map[string]struct{}
modelService interfaces.ModelService
}
// Regular expressions for text cleaning
var (
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
urlRegex = regexp.MustCompile(`https?://\S+`) // URLs
emailRegex = regexp.MustCompile(`\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,}\b`) // Email addresses
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces
)
const maxProcessedTokens = 12
// NewPluginPreprocess Creates a new query preprocessing plugin
func NewPluginPreprocess(
eventManager *EventManager,
@@ -40,51 +30,15 @@ func NewPluginPreprocess(
cleaner interfaces.ResourceCleaner,
modelService interfaces.ModelService,
) *PluginPreprocess {
// Use default dictionary for Jieba tokenizer
jieba := gojieba.NewJieba()
// Load stopwords from built-in stopword library
stopwords := loadStopwords()
res := &PluginPreprocess{
config: config,
jieba: jieba,
stopwords: stopwords,
modelService: modelService,
}
// Register resource cleanup function
if cleaner != nil {
cleaner.RegisterWithName("JiebaPreprocessor", func() error {
res.Close()
return nil
})
}
eventManager.Register(res)
return res
}
// Load stopwords
func loadStopwords() map[string]struct{} {
// Directly use some common stopwords built into Jieba
commonStopwords := []string{
"的", "了", "和", "是", "在", "我", "你", "他", "她", "它",
"这", "那", "什么", "怎么", "如何", "为什么", "哪里", "什么时候",
"the", "is", "are", "am", "I", "you", "he", "she", "it", "this",
"that", "what", "how", "a", "an", "and", "or", "but", "if", "of",
"to", "in", "on", "at", "by", "for", "with", "about", "from",
"有", "无", "好", "来", "去", "说", "看", "想", "会", "可以",
"吗", "呢", "啊", "吧", "的话", "就是", "只是", "因为", "所以",
}
result := make(map[string]struct{}, len(commonStopwords))
for _, word := range commonStopwords {
result[word] = struct{}{}
}
return result
}
// ActivationEvents Register activation events
func (p *PluginPreprocess) ActivationEvents() []types.EventType {
return []types.EventType{types.PREPROCESS_QUERY}
@@ -107,171 +61,22 @@ func (p *PluginPreprocess) OnEvent(
"rewrite_query": rawQuery,
})
normalized := normalizeWhitespace(rawQuery)
sanitized := strings.TrimSpace(p.cleanText(normalized))
if sanitized == "" {
sanitized = normalized
}
var (
processed = sanitized
strategy = "original"
tokenPreview string
tokenCount int
)
switch {
case containsChineseCharacters(sanitized):
segments := p.segmentText(sanitized)
tokens := p.selectMeaningfulTokens(segments)
tokenCount = len(tokens)
if len(tokens) >= 2 {
processed = strings.Join(tokens, " ")
strategy = "zh_tokens"
tokenPreview = strings.Join(tokens, ",")
} else {
strategy = "fallback_original"
}
case containsLatinLetters(sanitized):
processed = normalizeLatinQuery(sanitized)
if processed != sanitized {
strategy = "latin_normalize"
}
default:
strategy = "original"
}
if strings.TrimSpace(processed) == "" {
processed = rawQuery
strategy = "fallback_original"
}
// Lightweight normalization: just collapse multiple spaces
processed := multiSpaceRegex.ReplaceAllString(rawQuery, " ")
processed = strings.TrimSpace(processed)
chatManage.ProcessedQuery = processed
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, processed)
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"processed_query": processed,
"strategy": strategy,
"token_count": tokenCount,
"token_preview": tokenPreview,
"query_intent": chatManage.QueryIntent,
})
return next()
}
// cleanText Basic text cleaning
func (p *PluginPreprocess) cleanText(text string) string {
// Remove URLs
text = urlRegex.ReplaceAllString(text, " ")
// Remove email addresses
text = emailRegex.ReplaceAllString(text, " ")
// Remove excessive spaces
text = multiSpaceRegex.ReplaceAllString(text, " ")
// Remove punctuation marks
text = punctRegex.ReplaceAllString(text, " ")
// Trim leading and trailing spaces
text = strings.TrimSpace(text)
return text
}
// segmentText Text tokenization
func (p *PluginPreprocess) segmentText(text string) []string {
// Use Jieba tokenizer for tokenization, using search engine mode
segments := p.jieba.CutForSearch(text, true)
return segments
}
// filterStopwords Filter stopwords
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
var tokens []string
seen := make(map[string]struct{})
for _, word := range segments {
word = strings.TrimSpace(word)
if word == "" {
continue
}
if _, stop := p.stopwords[word]; stop {
continue
}
if _, exists := seen[word]; exists {
continue
}
if !isInformativeToken(word) {
continue
}
seen[word] = struct{}{}
tokens = append(tokens, word)
if len(tokens) >= maxProcessedTokens {
break
}
}
return tokens
}
// isBlank Check if a string is blank
func isInformativeToken(token string) bool {
if token == "" {
return false
}
runeCount := utf8.RuneCountInString(token)
if runeCount == 1 {
r, _ := utf8.DecodeRuneInString(token)
if unicode.IsDigit(r) {
return true
}
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
return false
}
return true
}
// containsChineseCharacters checks if a string contains Chinese characters
func containsChineseCharacters(text string) bool {
for _, r := range text {
if unicode.Is(unicode.Han, r) {
return true
}
}
return false
}
// containsLatinLetters checks if a string contains Latin letters
func containsLatinLetters(text string) bool {
for _, r := range text {
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
}
return false
}
// normalizeWhitespace normalizes whitespace in a string
func normalizeWhitespace(text string) string {
text = strings.TrimSpace(text)
return multiSpaceRegex.ReplaceAllString(text, " ")
}
// normalizeLatinQuery normalizes a Latin query
func normalizeLatinQuery(text string) string {
text = strings.ToLower(text)
text = multiSpaceRegex.ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}
// intentResp is a response for intent detection
type intentResp struct {
Intent string `json:"intent"`
@@ -349,12 +154,8 @@ func extractJSONBody(text string) string {
return "{}"
}
// Ensure resources are properly released
// Close Releases resources
func (p *PluginPreprocess) Close() {
if p.jieba != nil {
p.jieba.Free()
p.jieba = nil
}
}
// ShutdownHandler Returns shutdown function

View File

@@ -24,7 +24,6 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/Tencent/WeKnora/internal/tracing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -283,14 +282,26 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
enableMultimodelValue = kb.VLMConfig.Enabled
}
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
FilePath: filePath,
FileName: safeFilename,
FileType: getFileType(safeFilename),
EnableMultimodel: enableMultimodelValue,
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
FilePath: filePath,
FileName: safeFilename,
FileType: getFileType(safeFilename),
EnableMultimodel: enableMultimodelValue,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
@@ -405,12 +416,24 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
enableMultimodelValue = kb.VLMConfig.Enabled
}
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
URL: url,
EnableMultimodel: enableMultimodelValue,
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
URL: url,
EnableMultimodel: enableMultimodelValue,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
@@ -595,12 +618,25 @@ func (s *knowledgeService) createKnowledgeFromPassageInternal(ctx context.Contex
// Enqueue passage processing task to Asynq
logger.Info(ctx, "Enqueuing passage processing task to Asynq")
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// Check question generation config
enableQuestionGeneration := false
questionCount := 3 // default
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
Passages: safePassages,
EnableMultimodel: false, // 文本段落不支持多模态
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
Passages: safePassages,
EnableMultimodel: false, // 文本段落不支持多模态
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
@@ -917,10 +953,23 @@ func (s *knowledgeService) processDocumentFromPassage(ctx context.Context,
s.processChunks(ctx, kb, knowledge, chunks)
}
// ProcessChunksOptions contains options for processing chunks
type ProcessChunksOptions struct {
EnableQuestionGeneration bool
QuestionCount int
}
// processChunks processes chunks and creates embeddings for knowledge content
func (s *knowledgeService) processChunks(ctx context.Context,
kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*proto.Chunk,
opts ...ProcessChunksOptions,
) {
// Get options
var options ProcessChunksOptions
if len(opts) > 0 {
options = opts[0]
}
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processChunks")
defer span.End()
span.SetAttributes(
@@ -969,14 +1018,6 @@ func (s *knowledgeService) processChunks(ctx context.Context,
logger.Infof(ctx, "Cleanup completed, starting to process new chunks")
// Generate document summary - 只使用文本类型的 Chunk
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get summary model failed")
span.RecordError(err)
return
}
// Create chunk objects from proto chunks
maxSeq := 0
@@ -1118,52 +1159,19 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
}
span.AddEvent("extract summary")
summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
if err != nil {
logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID).
WithField("error", err).Errorf("processChunks get summary failed, use first chunk as description")
if len(textChunks) > 0 {
knowledge.Description = textChunks[0].Content
}
} else {
knowledge.Description = summary
}
span.SetAttributes(attribute.String("summary", knowledge.Description))
// 批量索引
if strings.TrimSpace(knowledge.Description) != "" && len(textChunks) > 0 {
sChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, knowledge.Description),
ChunkIndex: maxSeq + 3, // 使用不冲突的索引方式
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: 0,
EndAt: 0,
ChunkType: types.ChunkTypeSummary,
ParentChunkID: textChunks[0].ID,
}
logger.GetLogger(ctx).Infof("Created summary chunk for %s with index %d",
sChunk.ParentChunkID, sChunk.ChunkIndex)
insertChunks = append(insertChunks, sChunk)
}
// Create index information for each chunk
indexInfoList := utils.MapSlice(insertChunks, func(chunk *types.Chunk) *types.IndexInfo {
return &types.IndexInfo{
// Create index information for each chunk (without generated questions for now)
indexInfoList := make([]*types.IndexInfo, 0, len(insertChunks))
for _, chunk := range insertChunks {
// Add original chunk content to index
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: chunk.Content,
SourceID: chunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
}
})
})
}
// Initialize retrieval engine
@@ -1249,6 +1257,23 @@ func (s *knowledgeService) processChunks(ctx context.Context,
logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed")
}
// Enqueue question generation task if enabled (async, non-blocking)
if options.EnableQuestionGeneration && len(textChunks) > 0 {
questionCount := options.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
s.enqueueQuestionGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID, questionCount)
}
// Enqueue summary generation task (async, non-blocking)
if len(textChunks) > 0 {
s.enqueueSummaryGenerationTask(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
}
// Update tenant's storage usage
tenantInfo.StorageUsed += totalStorageSize
if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil {
@@ -1354,6 +1379,471 @@ func (s *knowledgeService) getSummary(ctx context.Context,
return summary.Content, nil
}
// enqueueQuestionGenerationTask enqueues an async task for question generation
func (s *knowledgeService) enqueueQuestionGenerationTask(ctx context.Context,
kbID, knowledgeID string, questionCount int,
) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
payload := types.QuestionGenerationPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue question generation task: %v", err)
return
}
logger.Infof(ctx, "Enqueued question generation task: %s for knowledge: %s", info.ID, knowledgeID)
}
// enqueueSummaryGenerationTask enqueues an async task for summary generation
func (s *knowledgeService) enqueueSummaryGenerationTask(ctx context.Context,
kbID, knowledgeID string,
) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
payload := types.SummaryGenerationPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal summary generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue summary generation task: %v", err)
return
}
logger.Infof(ctx, "Enqueued summary generation task: %s for knowledge: %s", info.ID, knowledgeID)
}
// ProcessSummaryGeneration handles async summary generation task
func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error {
var payload types.SummaryGenerationPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal summary generation payload: %v", err)
return nil // Don't retry on unmarshal error
}
logger.Infof(ctx, "Processing summary generation for knowledge: %s", payload.KnowledgeID)
// Set tenant context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
// Get knowledge base
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return nil
}
// Get knowledge
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return nil
}
// Get text chunks for this knowledge
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get chunks: %v", err)
return nil
}
// Filter text chunks only
textChunks := make([]*types.Chunk, 0)
for _, chunk := range chunks {
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
if len(textChunks) == 0 {
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
return nil
}
// Sort chunks by ChunkIndex for proper ordering
sort.Slice(textChunks, func(i, j int) bool {
return textChunks[i].ChunkIndex < textChunks[j].ChunkIndex
})
// Initialize chat model for summary
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
// Generate summary
summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks)
if err != nil {
logger.Errorf(ctx, "Failed to generate summary for knowledge %s: %v", payload.KnowledgeID, err)
// Use first chunk content as fallback
if len(textChunks) > 0 {
summary = textChunks[0].Content
if len(summary) > 500 {
summary = summary[:500]
}
}
}
// Update knowledge description
knowledge.Description = summary
knowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to update knowledge description: %v", err)
return fmt.Errorf("failed to update knowledge: %w", err)
}
// Create summary chunk and index it
if strings.TrimSpace(summary) != "" {
// Get max chunk index
maxChunkIndex := 0
for _, chunk := range chunks {
if chunk.ChunkIndex > maxChunkIndex {
maxChunkIndex = chunk.ChunkIndex
}
}
summaryChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, summary),
ChunkIndex: maxChunkIndex + 1,
IsEnabled: true,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
StartAt: 0,
EndAt: 0,
ChunkType: types.ChunkTypeSummary,
ParentChunkID: textChunks[0].ID,
}
// Save summary chunk
if err := s.chunkService.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
logger.Errorf(ctx, "Failed to create summary chunk: %v", err)
return fmt.Errorf("failed to create summary chunk: %w", err)
}
// Index summary chunk
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
return fmt.Errorf("failed to get embedding model: %w", err)
}
indexInfo := []*types.IndexInfo{{
Content: summaryChunk.Content,
SourceID: summaryChunk.ID,
SourceType: types.ChunkSourceType,
ChunkID: summaryChunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
}}
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
logger.Errorf(ctx, "Failed to index summary chunk: %v", err)
return fmt.Errorf("failed to index summary chunk: %w", err)
}
logger.Infof(ctx, "Successfully created and indexed summary chunk for knowledge: %s", payload.KnowledgeID)
}
logger.Infof(ctx, "Successfully generated summary for knowledge: %s", payload.KnowledgeID)
return nil
}
// ProcessQuestionGeneration handles async question generation task
func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error {
ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.ProcessQuestionGeneration")
defer span.End()
var payload types.QuestionGenerationPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal question generation payload: %v", err)
return nil // Don't retry on unmarshal error
}
logger.Infof(ctx, "Processing question generation for knowledge: %s", payload.KnowledgeID)
// Set tenant context
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
// Get knowledge base
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return nil
}
// Get knowledge
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
return nil
}
// Get text chunks for this knowledge
chunks, err := s.chunkService.ListChunksByKnowledgeID(ctx, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to get chunks: %v", err)
return nil
}
// Filter text chunks only
textChunks := make([]*types.Chunk, 0)
for _, chunk := range chunks {
if chunk.ChunkType == types.ChunkTypeText {
textChunks = append(textChunks, chunk)
}
}
if len(textChunks) == 0 {
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
return nil
}
// Sort chunks by StartAt for context building
sort.Slice(textChunks, func(i, j int) bool {
return textChunks[i].StartAt < textChunks[j].StartAt
})
// Initialize chat model
chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
// Initialize embedding model and retrieval engine
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
return fmt.Errorf("failed to get embedding model: %w", err)
}
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
return fmt.Errorf("failed to get tenant info: %w", err)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
questionCount := payload.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
// Generate questions for each chunk with context
var indexInfoList []*types.IndexInfo
for i, chunk := range textChunks {
// Build context from adjacent chunks
var prevContent, nextContent string
if i > 0 {
prevContent = textChunks[i-1].Content
// Limit context size
if len(prevContent) > 500 {
prevContent = prevContent[len(prevContent)-500:]
}
}
if i < len(textChunks)-1 {
nextContent = textChunks[i+1].Content
// Limit context size
if len(nextContent) > 500 {
nextContent = nextContent[:500]
}
}
questions, err := s.generateQuestionsWithContext(ctx, chatModel, chunk.Content, prevContent, nextContent, knowledge.Title, questionCount)
if err != nil {
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
continue
}
if len(questions) == 0 {
continue
}
// Update chunk metadata
meta := &types.DocumentChunkMetadata{
GeneratedQuestions: questions,
}
if err := chunk.SetDocumentMetadata(meta); err != nil {
logger.Warnf(ctx, "Failed to set document metadata for chunk %s: %v", chunk.ID, err)
continue
}
// Update chunk in database
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
logger.Warnf(ctx, "Failed to update chunk %s: %v", chunk.ID, err)
continue
}
// Create index entries for generated questions
for j, question := range questions {
sourceID := fmt.Sprintf("%s-q%d", chunk.ID, j)
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: question,
SourceID: sourceID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
})
}
logger.Debugf(ctx, "Generated %d questions for chunk %s", len(questions), chunk.ID)
}
// Index generated questions
if len(indexInfoList) > 0 {
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList); err != nil {
logger.Errorf(ctx, "Failed to index generated questions: %v", err)
return fmt.Errorf("failed to index questions: %w", err)
}
logger.Infof(ctx, "Successfully indexed %d generated questions for knowledge: %s", len(indexInfoList), payload.KnowledgeID)
}
return nil
}
// generateQuestionsWithContext generates questions for a chunk with surrounding context
func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
chatModel chat.Chat, content, prevContent, nextContent, docName string, questionCount int,
) ([]string, error) {
if content == "" || questionCount <= 0 {
return nil, nil
}
// Build prompt with context
prompt := s.config.Conversation.GenerateQuestionsPrompt
if prompt == "" {
prompt = defaultQuestionGenerationPrompt
}
// Build context section
var contextSection string
if prevContent != "" || nextContent != "" {
contextSection = "## 上下文信息(仅供参考,帮助理解主要内容)\n"
if prevContent != "" {
contextSection += fmt.Sprintf("【前文】%s\n", prevContent)
}
if nextContent != "" {
contextSection += fmt.Sprintf("【后文】%s\n", nextContent)
}
contextSection += "\n"
}
// Replace placeholders
prompt = strings.ReplaceAll(prompt, "{{.QuestionCount}}", fmt.Sprintf("%d", questionCount))
prompt = strings.ReplaceAll(prompt, "{{.Content}}", content)
prompt = strings.ReplaceAll(prompt, "{{.Context}}", contextSection)
prompt = strings.ReplaceAll(prompt, "{{.DocName}}", docName)
thinking := false
response, err := chatModel.Chat(ctx, []chat.Message{
{
Role: "user",
Content: prompt,
},
}, &chat.ChatOptions{
Temperature: 0.7,
MaxTokens: 512,
Thinking: &thinking,
})
if err != nil {
return nil, fmt.Errorf("failed to generate questions: %w", err)
}
// Parse response
lines := strings.Split(response.Content, "\n")
questions := make([]string, 0, questionCount)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
line = strings.TrimLeft(line, "0123456789.-*) ")
line = strings.TrimSpace(line)
if line != "" && len(line) > 5 {
questions = append(questions, line)
if len(questions) >= questionCount {
break
}
}
}
return questions, nil
}
// Default prompt for question generation with context support
const defaultQuestionGenerationPrompt = `你是一个专业的问题生成助手。你的任务是根据给定的【主要内容】生成用户可能会问的相关问题。
{{.Context}}
## 主要内容(请基于此内容生成问题)
文档名称:{{.DocName}}
文档内容:
{{.Content}}
## 核心要求
- 生成的问题必须与【主要内容】直接相关
- 问题中禁止使用任何代词或指代词(如"它"、"这个"、"该文档"、"本文"、"文中"、"其"等),必须用具体名称替代
- 问题必须是完整独立的,脱离上下文也能被理解
- 问题应该是用户在实际场景中可能会提出的自然问题
- 问题应该多样化,覆盖内容的不同方面
- 每个问题应该简洁明了长度控制在30字以内
- 生成的问题数量为 {{.QuestionCount}}
## 问题类型建议
- 定义类:什么是......是什么?
- 原因类:为什么......的原因是什么?
- 方法类:如何...?怎样...
- 比较类:...和...有什么区别?
- 应用类:...可以用于什么场景?
## 输出格式
直接输出问题列表,每行一个问题,不要有序号或其他前缀。`
// GetKnowledgeFile retrieves the physical file associated with a knowledge entry
func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) {
// Get knowledge record
@@ -4051,7 +4541,10 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
}
// 处理chunks这会更新状态为completed
s.processChunks(ctx, kb, knowledge, chunks)
s.processChunks(ctx, kb, knowledge, chunks, ProcessChunksOptions{
EnableQuestionGeneration: payload.EnableQuestionGeneration,
QuestionCount: payload.QuestionCount,
})
return nil
}

View File

@@ -57,6 +57,8 @@ type ConversationConfig struct {
SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"`
ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"`
ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"`
// GenerateQuestionsPrompt is used to generate questions for document chunks to improve recall
GenerateQuestionsPrompt string `yaml:"generate_questions_prompt" json:"generate_questions_prompt"`
}
// SummaryConfig 摘要配置

View File

@@ -123,6 +123,12 @@ type KBModelConfigRequest struct {
Nodes []types.GraphNode `json:"nodes"`
Relations []types.GraphRelation `json:"relations"`
} `json:"nodeExtract"`
// 问题生成配置
QuestionGeneration struct {
Enabled bool `json:"enabled"`
QuestionCount int `json:"questionCount"`
} `json:"questionGeneration"`
}
// InitializationRequest 初始化请求结构
@@ -192,6 +198,11 @@ type InitializationRequest struct {
Type string `json:"type"`
} `json:"relations"`
} `json:"nodeExtract"`
QuestionGeneration struct {
Enabled bool `json:"enabled"`
QuestionCount int `json:"questionCount"`
} `json:"questionGeneration"`
}
// UpdateKBConfig 根据知识库ID和模型ID更新配置简化版
@@ -333,6 +344,23 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
return
}
// 更新问题生成配置
if req.QuestionGeneration.Enabled {
questionCount := req.QuestionGeneration.QuestionCount
if questionCount <= 0 {
questionCount = 3
}
if questionCount > 10 {
questionCount = 10
}
kb.QuestionGenerationConfig = &types.QuestionGenerationConfig{
Enabled: true,
QuestionCount: questionCount,
}
} else {
kb.QuestionGenerationConfig = &types.QuestionGenerationConfig{Enabled: false}
}
// 保存更新后的知识库
if err := h.kbRepository.UpdateKnowledgeBase(ctx, kb); err != nil {
logger.Error(ctx, "Failed to update knowledge base", err)

View File

@@ -67,6 +67,12 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register FAQ import handler
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
// Register question generation handler
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
// Register summary generation handler
mux.HandleFunc(types.TypeSummaryGeneration, params.KnowledgeService.ProcessSummaryGeneration)
go func() {
// Start the server
if err := params.Server.Run(mux); err != nil {

View File

@@ -1,9 +1,11 @@
package types
const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -15,16 +17,18 @@ type ExtractChunkPayload struct {
// DocumentProcessPayload represents the document process task payload
type DocumentProcessPayload struct {
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
URL string `json:"url,omitempty"` // URLURL导入时使用
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
EnableMultimodel bool `json:"enable_multimodel"`
RequestId string `json:"request_id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用)
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
URL string `json:"url,omitempty"` // URLURL导入时使用
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
EnableMultimodel bool `json:"enable_multimodel"`
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量
}
// FAQImportPayload represents the FAQ import task payload
@@ -37,6 +41,29 @@ type FAQImportPayload struct {
Mode string `json:"mode"`
}
// QuestionGenerationPayload represents the question generation task payload
type QuestionGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
QuestionCount int `json:"question_count"`
}
// SummaryGenerationPayload represents the summary generation task payload
type SummaryGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
}
// ChunkContext represents chunk content with surrounding context
type ChunkContext struct {
ChunkID string `json:"chunk_id"`
Content string `json:"content"`
PrevContent string `json:"prev_content,omitempty"` // Previous chunk content for context
NextContent string `json:"next_content,omitempty"` // Next chunk content for context
}
// PromptTemplateStructured represents the prompt template structured
type PromptTemplateStructured struct {
Description string `json:"description"`

View File

@@ -19,6 +19,43 @@ type FAQChunkMetadata struct {
Source string `json:"source,omitempty"`
}
// DocumentChunkMetadata 定义文档 Chunk 的元数据结构
// 用于存储AI生成的问题等增强信息
type DocumentChunkMetadata struct {
// GeneratedQuestions 存储AI为该Chunk生成的相关问题
// 这些问题会被独立索引以提高召回率
GeneratedQuestions []string `json:"generated_questions,omitempty"`
}
// DocumentMetadata 解析 Chunk 中的文档元数据
func (c *Chunk) DocumentMetadata() (*DocumentChunkMetadata, error) {
if c == nil || len(c.Metadata) == 0 {
return nil, nil
}
var meta DocumentChunkMetadata
if err := json.Unmarshal(c.Metadata, &meta); err != nil {
return nil, err
}
return &meta, nil
}
// SetDocumentMetadata 设置 Chunk 的文档元数据
func (c *Chunk) SetDocumentMetadata(meta *DocumentChunkMetadata) error {
if c == nil {
return nil
}
if meta == nil {
c.Metadata = nil
return nil
}
bytes, err := json.Marshal(meta)
if err != nil {
return err
}
c.Metadata = JSON(bytes)
return nil
}
// Normalize 清理空白与重复项
func (m *FAQChunkMetadata) Normalize() {
if m == nil {

View File

@@ -99,6 +99,10 @@ type KnowledgeService interface {
ProcessDocument(ctx context.Context, t *asynq.Task) error
// ProcessFAQImport handles Asynq FAQ import tasks
ProcessFAQImport(ctx context.Context, t *asynq.Task) error
// ProcessQuestionGeneration handles Asynq question generation tasks
ProcessQuestionGeneration(ctx context.Context, t *asynq.Task) error
// ProcessSummaryGeneration handles Asynq summary generation tasks
ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error
}
// KnowledgeRepository defines the interface for knowledge repositories.

View File

@@ -65,6 +65,8 @@ type KnowledgeBase struct {
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
// FAQConfig stores FAQ specific configuration such as indexing strategy
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config" gorm:"column:faq_config;type:json"`
// QuestionGenerationConfig stores question generation configuration for document knowledge bases
QuestionGenerationConfig *QuestionGenerationConfig `yaml:"question_generation_config" json:"question_generation_config" gorm:"column:question_generation_config;type:json"`
// Creation time of the knowledge base
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
// Last updated time of the knowledge base
@@ -180,6 +182,32 @@ type VLMConfig struct {
ModelID string `yaml:"model_id" json:"model_id"`
}
// QuestionGenerationConfig represents the question generation configuration for document knowledge bases
// When enabled, the system will use LLM to generate questions for each chunk during document parsing
// These generated questions will be indexed separately to improve recall
type QuestionGenerationConfig struct {
Enabled bool `yaml:"enabled" json:"enabled"`
// Number of questions to generate per chunk (default: 3, max: 10)
QuestionCount int `yaml:"question_count" json:"question_count"`
}
// Value implements the driver.Valuer interface
func (c QuestionGenerationConfig) Value() (driver.Value, error) {
return json.Marshal(c)
}
// Scan implements the sql.Scanner interface
func (c *QuestionGenerationConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(b, c)
}
// Value implements the driver.Valuer interface, used to convert VLMConfig to database value
func (c VLMConfig) Value() (driver.Value, error) {
return json.Marshal(c)

View File

@@ -0,0 +1,4 @@
-- Remove question_generation_config column from knowledge_bases table
ALTER TABLE knowledge_bases
DROP COLUMN IF EXISTS question_generation_config;

View File

@@ -0,0 +1,7 @@
-- Add question_generation_config column to knowledge_bases table
-- This column stores configuration for AI question generation feature
-- When enabled, the system generates questions for document chunks to improve recall
ALTER TABLE knowledge_bases
ADD COLUMN IF NOT EXISTS question_generation_config JSON NULL
COMMENT 'Question generation configuration for document knowledge bases';