mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-06-04 13:30:32 +08:00
refactor: Update function signatures across multiple files to improve readability by adding context parameters and enhancing code structure
This commit is contained in:
@@ -150,7 +150,11 @@ func (as *AgentSession) Ask(ctx context.Context, query string, callback AgentEve
|
||||
}
|
||||
|
||||
// AskWithRequest sends a customized agent request for this session.
|
||||
func (as *AgentSession) AskWithRequest(ctx context.Context, request *AgentQARequest, callback AgentEventCallback) error {
|
||||
func (as *AgentSession) AskWithRequest(
|
||||
ctx context.Context,
|
||||
request *AgentQARequest,
|
||||
callback AgentEventCallback,
|
||||
) error {
|
||||
return as.client.AgentQAStreamWithRequest(ctx, as.sessionID, request, callback)
|
||||
}
|
||||
|
||||
|
||||
@@ -250,7 +250,12 @@ func (cli *CLI) askQuestion(query string) {
|
||||
case client.AgentResponseTypeReferences:
|
||||
if resp.KnowledgeReferences != nil {
|
||||
references = append(references, resp.KnowledgeReferences...)
|
||||
fmt.Printf("%s📚 Knowledge References:%s Found %d reference(s)\n", ColorCyan, ColorReset, len(resp.KnowledgeReferences))
|
||||
fmt.Printf(
|
||||
"%s📚 Knowledge References:%s Found %d reference(s)\n",
|
||||
ColorCyan,
|
||||
ColorReset,
|
||||
len(resp.KnowledgeReferences),
|
||||
)
|
||||
for i, ref := range resp.KnowledgeReferences {
|
||||
fmt.Printf(" %d. [Score: %.3f] %s\n", i+1, ref.Score, truncateString(ref.Content, 80))
|
||||
fmt.Printf(" Knowledge: %s (Chunk: %d)\n", ref.KnowledgeTitle, ref.ChunkIndex)
|
||||
|
||||
@@ -172,7 +172,12 @@ func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
|
||||
}
|
||||
|
||||
// CreateKnowledgeFromURL creates a knowledge entry from a web URL
|
||||
func (c *Client) CreateKnowledgeFromURL(ctx context.Context, knowledgeBaseID string, url string, enableMultimodel *bool) (*Knowledge, error) {
|
||||
func (c *Client) CreateKnowledgeFromURL(
|
||||
ctx context.Context,
|
||||
knowledgeBaseID string,
|
||||
url string,
|
||||
enableMultimodel *bool,
|
||||
) (*Knowledge, error) {
|
||||
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID)
|
||||
|
||||
reqBody := struct {
|
||||
|
||||
@@ -19,7 +19,7 @@ type Message struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Content string `json:"content"`
|
||||
Role string `json:"role"`
|
||||
KnowledgeReferences []*SearchResult `json:"knowledge_references" `
|
||||
KnowledgeReferences []*SearchResult `json:"knowledge_references"`
|
||||
IsCompleted bool `json:"is_completed"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
@@ -32,7 +32,12 @@ type MessageListResponse struct {
|
||||
}
|
||||
|
||||
// LoadMessages loads session messages, supports pagination and time filtering
|
||||
func (c *Client) LoadMessages(ctx context.Context, sessionID string, limit int, beforeTime *time.Time) ([]Message, error) {
|
||||
func (c *Client) LoadMessages(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
limit int,
|
||||
beforeTime *time.Time,
|
||||
) ([]Message, error) {
|
||||
path := fmt.Sprintf("/api/v1/messages/%s/load", sessionID)
|
||||
|
||||
queryParams := url.Values{}
|
||||
@@ -61,7 +66,12 @@ func (c *Client) GetRecentMessages(ctx context.Context, sessionID string, limit
|
||||
}
|
||||
|
||||
// GetMessagesBefore gets messages before a specified time
|
||||
func (c *Client) GetMessagesBefore(ctx context.Context, sessionID string, beforeTime time.Time, limit int) ([]Message, error) {
|
||||
func (c *Client) GetMessagesBefore(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
beforeTime time.Time,
|
||||
limit int,
|
||||
) ([]Message, error) {
|
||||
return c.LoadMessages(ctx, sessionID, limit, &beforeTime)
|
||||
}
|
||||
|
||||
|
||||
@@ -236,7 +236,12 @@ type StreamResponse struct {
|
||||
}
|
||||
|
||||
// KnowledgeQAStream knowledge Q&A streaming API
|
||||
func (c *Client) KnowledgeQAStream(ctx context.Context, sessionID string, query string, callback func(*StreamResponse) error) error {
|
||||
func (c *Client) KnowledgeQAStream(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
query string,
|
||||
callback func(*StreamResponse) error,
|
||||
) error {
|
||||
path := fmt.Sprintf("/api/v1/knowledge-chat/%s", sessionID)
|
||||
fmt.Printf("Starting KnowledgeQAStream request, session ID: %s, query: %s\n", sessionID, query)
|
||||
|
||||
@@ -315,7 +320,12 @@ func (c *Client) KnowledgeQAStream(ctx context.Context, sessionID string, query
|
||||
}
|
||||
|
||||
// ContinueStream continues to receive an active stream for a session
|
||||
func (c *Client) ContinueStream(ctx context.Context, sessionID string, messageID string, callback func(*StreamResponse) error) error {
|
||||
func (c *Client) ContinueStream(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
messageID string,
|
||||
callback func(*StreamResponse) error,
|
||||
) error {
|
||||
path := fmt.Sprintf("/api/v1/sessions/continue-stream/%s", sessionID)
|
||||
|
||||
queryParams := url.Values{}
|
||||
|
||||
@@ -24,23 +24,23 @@ type RetrieverEngineParams struct {
|
||||
|
||||
// Tenant represents tenant information in the system
|
||||
type Tenant struct {
|
||||
ID uint `yaml:"id" json:"id" gorm:"primaryKey"`
|
||||
ID uint `yaml:"id" json:"id" gorm:"primaryKey"`
|
||||
// Tenant name
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
// Tenant description
|
||||
Description string `yaml:"description" json:"description"`
|
||||
Description string `yaml:"description" json:"description"`
|
||||
// API key for authentication
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
// Tenant status (active, inactive)
|
||||
Status string `yaml:"status" json:"status" gorm:"default:'active'"`
|
||||
Status string `yaml:"status" json:"status" gorm:"default:'active'"`
|
||||
// Configured retrieval engines
|
||||
RetrieverEngines RetrieverEngines `yaml:"retriever_engines" json:"retriever_engines" gorm:"type:json"`
|
||||
// Business/department information
|
||||
Business string `yaml:"business" json:"business"`
|
||||
Business string `yaml:"business" json:"business"`
|
||||
// Creation timestamp
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last update timestamp
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
// TenantResponse represents the API response structure for tenant operations
|
||||
|
||||
@@ -71,7 +71,11 @@ func NewAgentEngine(
|
||||
|
||||
// Execute executes the agent with conversation history and streaming output
|
||||
// All events are emitted to EventBus and handled by subscribers (like Handler layer)
|
||||
func (e *AgentEngine) Execute(ctx context.Context, sessionID, messageID, query string, llmContext []chat.Message) (*types.AgentState, error) {
|
||||
func (e *AgentEngine) Execute(
|
||||
ctx context.Context,
|
||||
sessionID, messageID, query string,
|
||||
llmContext []chat.Message,
|
||||
) (*types.AgentState, error) {
|
||||
logger.Infof(ctx, "========== Agent Execution Started ==========")
|
||||
logger.Infof(ctx, "[Agent] SessionID: %s, MessageID: %s", sessionID, messageID)
|
||||
logger.Infof(ctx, "[Agent] User Query: %s", query)
|
||||
@@ -92,7 +96,11 @@ func (e *AgentEngine) Execute(ctx context.Context, sessionID, messageID, query s
|
||||
}
|
||||
|
||||
// Build system prompt using progressive RAG prompt
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(e.knowledgeBasesInfo, e.config.WebSearchEnabled, e.systemPromptTemplate)
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(
|
||||
e.knowledgeBasesInfo,
|
||||
e.config.WebSearchEnabled,
|
||||
e.systemPromptTemplate,
|
||||
)
|
||||
logger.Debugf(ctx, "[Agent] SystemPrompt Length: %d characters", len(systemPrompt))
|
||||
logger.Debugf(ctx, "[Agent] SystemPrompt (stream)\n----\n%s\n----", systemPrompt)
|
||||
|
||||
@@ -193,7 +201,12 @@ func (e *AgentEngine) executeLoop(
|
||||
// Debug: log finish reason and tool call count from LLM
|
||||
logger.Infof(ctx, "[Agent][Round-%d] LLM response received: finish_reason=%s, tool_calls=%d, content_length=%d",
|
||||
state.CurrentRound+1, response.FinishReason, len(response.ToolCalls), len(response.Content))
|
||||
logger.Debugf(ctx, "[Agent] LLM response finish=%s, toolCalls=%d", response.FinishReason, len(response.ToolCalls))
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"[Agent] LLM response finish=%s, toolCalls=%d",
|
||||
response.FinishReason,
|
||||
len(response.ToolCalls),
|
||||
)
|
||||
if response.Content != "" {
|
||||
logger.Debugf(ctx, "[Agent][Round-%d] LLM thought content:\n%s", state.CurrentRound+1, response.Content)
|
||||
}
|
||||
@@ -229,13 +242,23 @@ func (e *AgentEngine) executeLoop(
|
||||
Done: true,
|
||||
},
|
||||
})
|
||||
logger.Infof(ctx, "[Agent][Round-%d] Duration: %dms", state.CurrentRound+1, time.Since(roundStart).Milliseconds())
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Agent][Round-%d] Duration: %dms",
|
||||
state.CurrentRound+1,
|
||||
time.Since(roundStart).Milliseconds(),
|
||||
)
|
||||
break
|
||||
}
|
||||
|
||||
// 3. Act: Execute tool calls if any
|
||||
if len(response.ToolCalls) > 0 {
|
||||
logger.Infof(ctx, "[Agent][Round-%d] Executing %d tool calls...", state.CurrentRound+1, len(response.ToolCalls))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Agent][Round-%d] Executing %d tool calls...",
|
||||
state.CurrentRound+1,
|
||||
len(response.ToolCalls),
|
||||
)
|
||||
|
||||
for i, tc := range response.ToolCalls {
|
||||
logger.Infof(ctx, "[Agent][Round-%d][Tool-%d/%d] Tool: %s, ID: %s",
|
||||
@@ -483,7 +506,11 @@ func (e *AgentEngine) buildToolsForLLM() []chat.Tool {
|
||||
|
||||
// appendToolResults adds tool results to the message history following OpenAI's tool calling format
|
||||
// Also writes these messages to the context manager for persistence
|
||||
func (e *AgentEngine) appendToolResults(ctx context.Context, messages []chat.Message, step types.AgentStep) []chat.Message {
|
||||
func (e *AgentEngine) appendToolResults(
|
||||
ctx context.Context,
|
||||
messages []chat.Message,
|
||||
step types.AgentStep,
|
||||
) []chat.Message {
|
||||
// Add assistant message with tool calls (if any)
|
||||
if step.Thought != "" || len(step.ToolCalls) > 0 {
|
||||
assistantMsg := chat.Message{
|
||||
@@ -744,7 +771,11 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
||||
})
|
||||
|
||||
// Build messages with all context
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(e.knowledgeBasesInfo, e.config.WebSearchEnabled, e.systemPromptTemplate)
|
||||
systemPrompt := BuildProgressiveRAGSystemPrompt(
|
||||
e.knowledgeBasesInfo,
|
||||
e.config.WebSearchEnabled,
|
||||
e.systemPromptTemplate,
|
||||
)
|
||||
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
@@ -837,7 +868,10 @@ func countTotalToolCalls(steps []types.AgentStep) int {
|
||||
}
|
||||
|
||||
// buildMessagesWithLLMContext builds the message array with LLM context
|
||||
func (e *AgentEngine) buildMessagesWithLLMContext(systemPrompt, currentQuery string, llmContext []chat.Message) []chat.Message {
|
||||
func (e *AgentEngine) buildMessagesWithLLMContext(
|
||||
systemPrompt, currentQuery string,
|
||||
llmContext []chat.Message,
|
||||
) []chat.Message {
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: systemPrompt},
|
||||
}
|
||||
|
||||
@@ -189,7 +189,12 @@ func renderPromptPlaceholders(template string, knowledgeBases []*KnowledgeBaseIn
|
||||
// - {{knowledge_bases}}
|
||||
// - {{web_search_status}} -> "Enabled" or "Disabled"
|
||||
// - {{current_time}} -> current time string
|
||||
func renderPromptPlaceholdersWithStatus(template string, knowledgeBases []*KnowledgeBaseInfo, webSearchEnabled bool, currentTime string) string {
|
||||
func renderPromptPlaceholdersWithStatus(
|
||||
template string,
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
webSearchEnabled bool,
|
||||
currentTime string,
|
||||
) string {
|
||||
result := renderPromptPlaceholders(template, knowledgeBases)
|
||||
status := "Disabled"
|
||||
if webSearchEnabled {
|
||||
@@ -205,7 +210,10 @@ func renderPromptPlaceholdersWithStatus(template string, knowledgeBases []*Knowl
|
||||
}
|
||||
|
||||
// BuildProgressiveRAGSystemPromptWithWeb builds the progressive RAG system prompt with web search enabled
|
||||
func BuildProgressiveRAGSystemPromptWithWeb(knowledgeBases []*KnowledgeBaseInfo, systemPromptTemplate ...string) string {
|
||||
func BuildProgressiveRAGSystemPromptWithWeb(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
var template string
|
||||
if len(systemPromptTemplate) > 0 && systemPromptTemplate[0] != "" {
|
||||
template = systemPromptTemplate[0]
|
||||
@@ -217,7 +225,10 @@ func BuildProgressiveRAGSystemPromptWithWeb(knowledgeBases []*KnowledgeBaseInfo,
|
||||
}
|
||||
|
||||
// BuildProgressiveRAGSystemPromptWithoutWeb builds the progressive RAG system prompt without web search
|
||||
func BuildProgressiveRAGSystemPromptWithoutWeb(knowledgeBases []*KnowledgeBaseInfo, systemPromptTemplate ...string) string {
|
||||
func BuildProgressiveRAGSystemPromptWithoutWeb(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
var template string
|
||||
if len(systemPromptTemplate) > 0 && systemPromptTemplate[0] != "" {
|
||||
template = systemPromptTemplate[0]
|
||||
@@ -230,7 +241,11 @@ func BuildProgressiveRAGSystemPromptWithoutWeb(knowledgeBases []*KnowledgeBaseIn
|
||||
|
||||
// BuildProgressiveRAGSystemPrompt builds the progressive RAG system prompt based on web search status
|
||||
// This is the main function to use - it automatically selects the appropriate version
|
||||
func BuildProgressiveRAGSystemPrompt(knowledgeBases []*KnowledgeBaseInfo, webSearchEnabled bool, systemPromptTemplate ...string) string {
|
||||
func BuildProgressiveRAGSystemPrompt(
|
||||
knowledgeBases []*KnowledgeBaseInfo,
|
||||
webSearchEnabled bool,
|
||||
systemPromptTemplate ...string,
|
||||
) string {
|
||||
if webSearchEnabled {
|
||||
return BuildProgressiveRAGSystemPromptWithWeb(knowledgeBases, systemPromptTemplate...)
|
||||
}
|
||||
|
||||
@@ -373,7 +373,11 @@ func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string) (string, error
|
||||
}
|
||||
|
||||
// formatQueryResults formats query results into readable text
|
||||
func (t *DatabaseQueryTool) formatQueryResults(columns []string, results []map[string]interface{}, query string) string {
|
||||
func (t *DatabaseQueryTool) formatQueryResults(
|
||||
columns []string,
|
||||
results []map[string]interface{},
|
||||
query string,
|
||||
) string {
|
||||
output := "=== 查询结果 ===\n\n"
|
||||
output += fmt.Sprintf("执行的SQL: %s\n\n", query)
|
||||
output += fmt.Sprintf("返回 %d 行数据\n\n", len(results))
|
||||
|
||||
@@ -133,10 +133,11 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter
|
||||
}
|
||||
|
||||
// Get chunk count
|
||||
_, total, err := t.chunkService.GetRepository().ListPagedChunksByKnowledgeID(ctx, t.tenantID, id, &types.Pagination{
|
||||
Page: 1,
|
||||
PageSize: 1000,
|
||||
}, []types.ChunkType{"text"}, "")
|
||||
_, total, err := t.chunkService.GetRepository().
|
||||
ListPagedChunksByKnowledgeID(ctx, t.tenantID, id, &types.Pagination{
|
||||
Page: 1,
|
||||
PageSize: 1000,
|
||||
}, []types.ChunkType{"text"}, "")
|
||||
if err != nil {
|
||||
mu.Lock()
|
||||
results[id] = &docInfo{
|
||||
|
||||
@@ -203,7 +203,12 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{
|
||||
if maxResults > 0 && mmrK > maxResults {
|
||||
mmrK = maxResults
|
||||
}
|
||||
logger.Debugf(ctx, "[Tool][GrepChunks] Applying MMR: k=%d, lambda=0.7, input=%d results", mmrK, len(scoredResults))
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"[Tool][GrepChunks] Applying MMR: k=%d, lambda=0.7, input=%d results",
|
||||
mmrK,
|
||||
len(scoredResults),
|
||||
)
|
||||
mmrResults := t.applyMMR(ctx, scoredResults, patterns, mmrK, 0.7)
|
||||
if len(mmrResults) > 0 {
|
||||
finalResults = mmrResults
|
||||
@@ -252,9 +257,9 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{
|
||||
|
||||
type chunkWithTitle struct {
|
||||
types.Chunk
|
||||
KnowledgeTitle string `json:"knowledge_title" gorm:"column:knowledge_title"`
|
||||
MatchScore float64 `json:"match_score" gorm:"column:match_score"` // Score based on match count and position
|
||||
MatchedPatterns int `json:"matched_patterns"` // Number of unique patterns matched
|
||||
KnowledgeTitle string `json:"knowledge_title" gorm:"column:knowledge_title"`
|
||||
MatchScore float64 `json:"match_score" gorm:"column:match_score"` // Score based on match count and position
|
||||
MatchedPatterns int `json:"matched_patterns"` // Number of unique patterns matched
|
||||
TotalChunkCount int `json:"total_chunk_count" gorm:"column:total_chunk_count"`
|
||||
}
|
||||
|
||||
@@ -344,14 +349,16 @@ func (t *GrepChunksTool) formatOutput(
|
||||
patternSummaries = append(patternSummaries, fmt.Sprintf("%s=%d", pattern, count))
|
||||
}
|
||||
|
||||
output.WriteString(fmt.Sprintf("%d) knowledge_id=%s | title=%s | chunk_hits=%d | chunk_total=%d | pattern_hits=[%s]\n",
|
||||
idx+1,
|
||||
result.KnowledgeID,
|
||||
result.KnowledgeTitle,
|
||||
result.ChunkHitCount,
|
||||
result.TotalChunkCount,
|
||||
strings.Join(patternSummaries, ", "),
|
||||
))
|
||||
output.WriteString(
|
||||
fmt.Sprintf("%d) knowledge_id=%s | title=%s | chunk_hits=%d | chunk_total=%d | pattern_hits=[%s]\n",
|
||||
idx+1,
|
||||
result.KnowledgeID,
|
||||
result.KnowledgeTitle,
|
||||
result.ChunkHitCount,
|
||||
result.TotalChunkCount,
|
||||
strings.Join(patternSummaries, ", "),
|
||||
),
|
||||
)
|
||||
}
|
||||
return output.String()
|
||||
}
|
||||
@@ -553,7 +560,11 @@ func (t *GrepChunksTool) buildContentSignature(content string) string {
|
||||
}
|
||||
|
||||
// scoreChunks calculates match scores for chunks based on pattern matches
|
||||
func (t *GrepChunksTool) scoreChunks(ctx context.Context, results []chunkWithTitle, patterns []string) []chunkWithTitle {
|
||||
func (t *GrepChunksTool) scoreChunks(
|
||||
ctx context.Context,
|
||||
results []chunkWithTitle,
|
||||
patterns []string,
|
||||
) []chunkWithTitle {
|
||||
scored := make([]chunkWithTitle, len(results))
|
||||
for i := range results {
|
||||
scored[i] = results[i]
|
||||
|
||||
@@ -246,8 +246,14 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
|
||||
minScore = 0.3
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Search params: top_k=%d, vector_threshold=%.2f, keyword_threshold=%.2f, min_score=%.2f",
|
||||
topK, vectorThreshold, keywordThreshold, minScore)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Search params: top_k=%d, vector_threshold=%.2f, keyword_threshold=%.2f, min_score=%.2f",
|
||||
topK,
|
||||
vectorThreshold,
|
||||
keywordThreshold,
|
||||
minScore,
|
||||
)
|
||||
|
||||
// Execute concurrent search (hybrid search handles both vector and keyword)
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Starting concurrent search across %d KBs", len(kbIDs))
|
||||
@@ -280,8 +286,13 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
|
||||
}
|
||||
|
||||
if t.chatModel != nil && len(deduplicatedBeforeRerank) > 0 && rerankQuery != "" {
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying LLM-based rerank with model: %s, input: %d results, queries: %v",
|
||||
t.chatModel.GetModelName(), len(deduplicatedBeforeRerank), queries)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Applying LLM-based rerank with model: %s, input: %d results, queries: %v",
|
||||
t.chatModel.GetModelName(),
|
||||
len(deduplicatedBeforeRerank),
|
||||
queries,
|
||||
)
|
||||
rerankedResults, err := t.rerankResults(ctx, rerankQuery, deduplicatedBeforeRerank)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] LLM rerank failed, using original results: %v", err)
|
||||
@@ -320,7 +331,12 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter
|
||||
mmrK = 1
|
||||
}
|
||||
// Apply MMR with lambda=0.7 (balance between relevance and diversity)
|
||||
logger.Debugf(ctx, "[Tool][KnowledgeSearch] Applying MMR: k=%d, lambda=0.7, input=%d results", mmrK, len(filteredResults))
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Applying MMR: k=%d, lambda=0.7, input=%d results",
|
||||
mmrK,
|
||||
len(filteredResults),
|
||||
)
|
||||
mmrResults := t.applyMMR(ctx, filteredResults, mmrK, 0.7)
|
||||
if len(mmrResults) > 0 {
|
||||
filteredResults = mmrResults
|
||||
@@ -659,7 +675,8 @@ func (t *KnowledgeSearchTool) rerankWithLLM(
|
||||
}
|
||||
|
||||
// Optimized prompt focused on retrieval matching and reranking
|
||||
prompt := fmt.Sprintf(`You are a search result reranking expert. Your task is to evaluate how well each retrieved passage matches the user's search query and information need.
|
||||
prompt := fmt.Sprintf(
|
||||
`You are a search result reranking expert. Your task is to evaluate how well each retrieved passage matches the user's search query and information need.
|
||||
|
||||
User Query: %s
|
||||
|
||||
@@ -690,7 +707,12 @@ Passage 3: X.XX
|
||||
...
|
||||
Passage %d: X.XX
|
||||
|
||||
Output only the scores, no explanations or additional text.`, query, passagesBuilder.String(), len(batch), len(batch))
|
||||
Output only the scores, no explanations or additional text.`,
|
||||
query,
|
||||
passagesBuilder.String(),
|
||||
len(batch),
|
||||
len(batch),
|
||||
)
|
||||
|
||||
messages := []chat.Message{
|
||||
{
|
||||
@@ -727,8 +749,13 @@ Output only the scores, no explanations or additional text.`, query, passagesBui
|
||||
// Parse scores from response
|
||||
batchScores, err := t.parseScoresFromResponse(response.Content, len(batch))
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to parse LLM scores for batch %d-%d: %v, using original scores",
|
||||
batchStart+1, batchEnd, err)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Failed to parse LLM scores for batch %d-%d: %v, using original scores",
|
||||
batchStart+1,
|
||||
batchEnd,
|
||||
err,
|
||||
)
|
||||
// Use original scores for this batch on parsing error
|
||||
for i := batchStart; i < batchEnd; i++ {
|
||||
allScores[i] = results[i].Score
|
||||
@@ -862,7 +889,12 @@ func (t *KnowledgeSearchTool) rerankWithModel(
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Reranked %d results from %d original results", len(reranked), len(results))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Reranked %d results from %d original results",
|
||||
len(reranked),
|
||||
len(results),
|
||||
)
|
||||
return reranked, nil
|
||||
}
|
||||
|
||||
@@ -1027,7 +1059,12 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
if result.KnowledgeBaseType == types.KnowledgeBaseTypeFAQ {
|
||||
meta, err := t.getFAQMetadata(ctx, result.ID, faqMetadataCache)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to load FAQ metadata for chunk %s: %v", result.ID, err)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Failed to load FAQ metadata for chunk %s: %v",
|
||||
result.ID,
|
||||
err,
|
||||
)
|
||||
} else {
|
||||
faqMeta = meta
|
||||
}
|
||||
@@ -1055,7 +1092,12 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
&types.Pagination{Page: 1, PageSize: 1},
|
||||
[]types.ChunkType{types.ChunkTypeText}, "")
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Failed to get total chunks for knowledge %s: %v", result.KnowledgeID, err)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Failed to get total chunks for knowledge %s: %v",
|
||||
result.KnowledgeID,
|
||||
err,
|
||||
)
|
||||
knowledgeTotalMap[result.KnowledgeID] = 0
|
||||
} else {
|
||||
knowledgeTotalMap[result.KnowledgeID] = total
|
||||
@@ -1065,7 +1107,12 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
|
||||
// relevanceLevel := GetRelevanceLevel(result.Score)
|
||||
output += fmt.Sprintf("\nResult #%d:\n", i+1)
|
||||
output += fmt.Sprintf(" [chunk_id: %s][chunk_index: %d]\nContent: %s\n", result.ID, result.ChunkIndex, result.Content)
|
||||
output += fmt.Sprintf(
|
||||
" [chunk_id: %s][chunk_index: %d]\nContent: %s\n",
|
||||
result.ID,
|
||||
result.ChunkIndex,
|
||||
result.Content,
|
||||
)
|
||||
|
||||
if faqMeta != nil {
|
||||
if faqMeta.StandardQuestion != "" {
|
||||
@@ -1134,7 +1181,11 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
|
||||
if len(missingRanges) == 0 {
|
||||
// No gaps found (shouldn't happen if remaining > 0, but handle it)
|
||||
output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks)
|
||||
output += fmt.Sprintf(
|
||||
" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n",
|
||||
knowledgeID,
|
||||
totalChunks,
|
||||
)
|
||||
} else if len(missingRanges) == 1 && missingRanges[0].start == 0 && missingRanges[0].end == int(totalChunks)-1 {
|
||||
// All chunks are missing (shouldn't happen, but handle it)
|
||||
output += fmt.Sprintf(" - 获取全部内容: list_knowledge_chunks(knowledge_id=\"%s\", offset=0, limit=%d)\n", knowledgeID, totalChunks)
|
||||
@@ -1284,8 +1335,12 @@ func (t *KnowledgeSearchTool) normalizeKeywordSearchResults(ctx context.Context,
|
||||
for _, r := range keywordResults {
|
||||
r.Score = 1.0
|
||||
}
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Keyword scores have no variance, all set to 1.0: count=%d, score=%.3f",
|
||||
len(keywordResults), minS)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Keyword scores have no variance, all set to 1.0: count=%d, score=%.3f",
|
||||
len(keywordResults),
|
||||
minS,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1336,8 +1391,15 @@ func (t *KnowledgeSearchTool) normalizeKeywordSearchResults(ctx context.Context,
|
||||
r.Score = ns
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Tool][KnowledgeSearch] Normalized keyword scores: count=%d, raw_min=%.3f, raw_max=%.3f, normalize_min=%.3f, normalize_max=%.3f",
|
||||
len(keywordResults), minS, maxS, normalizeMin, normalizeMax)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][KnowledgeSearch] Normalized keyword scores: count=%d, raw_min=%.3f, raw_max=%.3f, normalize_min=%.3f, normalize_max=%.3f",
|
||||
len(keywordResults),
|
||||
minS,
|
||||
maxS,
|
||||
normalizeMin,
|
||||
normalizeMax,
|
||||
)
|
||||
} else {
|
||||
// Fallback: all scores are the same after percentile filtering
|
||||
for _, r := range keywordResults {
|
||||
|
||||
@@ -203,7 +203,13 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
fmt.Fprintf(builder, "本次拉取: %d 条, 检索范围: %d - %d\n\n", fetched, chunks[0].ChunkIndex, chunks[len(chunks)-1].ChunkIndex)
|
||||
fmt.Fprintf(
|
||||
builder,
|
||||
"本次拉取: %d 条, 检索范围: %d - %d\n\n",
|
||||
fetched,
|
||||
chunks[0].ChunkIndex,
|
||||
chunks[len(chunks)-1].ChunkIndex,
|
||||
)
|
||||
|
||||
builder.WriteString("=== 分块内容预览 ===\n\n")
|
||||
for idx, c := range chunks {
|
||||
|
||||
@@ -177,7 +177,12 @@ func sanitizeName(name string) string {
|
||||
}
|
||||
|
||||
// RegisterMCPTools registers MCP tools from given services
|
||||
func RegisterMCPTools(ctx context.Context, registry *ToolRegistry, services []*types.MCPService, mcpManager *mcp.MCPManager) error {
|
||||
func RegisterMCPTools(
|
||||
ctx context.Context,
|
||||
registry *ToolRegistry,
|
||||
services []*types.MCPService,
|
||||
mcpManager *mcp.MCPManager,
|
||||
) error {
|
||||
if len(services) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -239,7 +244,11 @@ func RegisterMCPTools(ctx context.Context, registry *ToolRegistry, services []*t
|
||||
}
|
||||
|
||||
// GetMCPToolsInfo returns information about available MCP tools
|
||||
func GetMCPToolsInfo(ctx context.Context, services []*types.MCPService, mcpManager *mcp.MCPManager) (map[string][]string, error) {
|
||||
func GetMCPToolsInfo(
|
||||
ctx context.Context,
|
||||
services []*types.MCPService,
|
||||
mcpManager *mcp.MCPManager,
|
||||
) (map[string][]string, error) {
|
||||
result := make(map[string][]string)
|
||||
|
||||
// Use provided context with timeout
|
||||
|
||||
@@ -369,7 +369,10 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i
|
||||
}
|
||||
|
||||
// buildGraphVisualizationData builds structured data for graph visualization
|
||||
func buildGraphVisualizationData(results []*types.SearchResult, graphConfigs map[string]map[string]interface{}) map[string]interface{} {
|
||||
func buildGraphVisualizationData(
|
||||
results []*types.SearchResult,
|
||||
graphConfigs map[string]map[string]interface{},
|
||||
) map[string]interface{} {
|
||||
// Build a simple graph structure for frontend visualization
|
||||
nodes := make([]map[string]interface{}, 0)
|
||||
edges := make([]map[string]interface{}, 0)
|
||||
|
||||
@@ -69,7 +69,11 @@ func (r *ToolRegistry) GetFunctionDefinitions() []types.FunctionDefinition {
|
||||
}
|
||||
|
||||
// ExecuteTool executes a tool by name with the given arguments
|
||||
func (r *ToolRegistry) ExecuteTool(ctx context.Context, name string, args map[string]interface{}) (*types.ToolResult, error) {
|
||||
func (r *ToolRegistry) ExecuteTool(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
args map[string]interface{},
|
||||
) (*types.ToolResult, error) {
|
||||
common.PipelineInfo(ctx, "AgentTool", "execute_start", map[string]interface{}{
|
||||
"tool": name,
|
||||
"args": args,
|
||||
|
||||
@@ -196,7 +196,8 @@ func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]in
|
||||
branchKeys = append(branchKeys, k)
|
||||
}
|
||||
|
||||
incomplete := thoughtData.NextThoughtNeeded || thoughtData.NeedsMoreThoughts || thoughtData.ThoughtNumber < thoughtData.TotalThoughts
|
||||
incomplete := thoughtData.NextThoughtNeeded || thoughtData.NeedsMoreThoughts ||
|
||||
thoughtData.ThoughtNumber < thoughtData.TotalThoughts
|
||||
|
||||
responseData := map[string]interface{}{
|
||||
"thought_number": thoughtData.ThoughtNumber,
|
||||
@@ -209,7 +210,12 @@ func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]in
|
||||
"incomplete_steps": incomplete,
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Tool][SequentialThinking] Execute completed - Thought %d/%d", thoughtData.ThoughtNumber, thoughtData.TotalThoughts)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][SequentialThinking] Execute completed - Thought %d/%d",
|
||||
thoughtData.ThoughtNumber,
|
||||
thoughtData.TotalThoughts,
|
||||
)
|
||||
|
||||
outputMsg := "Thought process recorded"
|
||||
if incomplete {
|
||||
|
||||
@@ -252,7 +252,10 @@ func (t *WebFetchTool) validateParams(p webFetchParams) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *WebFetchTool) executeFetch(ctx context.Context, params webFetchParams) (string, map[string]interface{}, error) {
|
||||
func (t *WebFetchTool) executeFetch(
|
||||
ctx context.Context,
|
||||
params webFetchParams,
|
||||
) (string, map[string]interface{}, error) {
|
||||
logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", params.URL)
|
||||
|
||||
finalURL := t.normalizeGitHubURL(params.URL)
|
||||
@@ -375,14 +378,17 @@ func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (
|
||||
func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) {
|
||||
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL)
|
||||
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
opts := append(
|
||||
chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.Flag("headless", true),
|
||||
chromedp.Flag("disable-setuid-sandbox", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
chromedp.Flag("disable-gpu", true),
|
||||
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
||||
chromedp.Flag("disable-features", "VizDisplayCompositor"),
|
||||
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
||||
chromedp.UserAgent(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
),
|
||||
)
|
||||
|
||||
allocCtx, cancel := chromedp.NewExecAllocator(ctx, opts...)
|
||||
@@ -435,7 +441,10 @@ func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WebFetchTool/1.0)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
|
||||
req.Header.Set(
|
||||
"Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
)
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
|
||||
|
||||
@@ -57,7 +57,10 @@ func NewWebSearchTool(
|
||||
**Parameters**:
|
||||
- query (required): Search query string
|
||||
|
||||
**Returns**: Web search results with title, URL, snippet, and content (up to ` + fmt.Sprintf("%d", maxResults) + ` results)
|
||||
**Returns**: Web search results with title, URL, snippet, and content (up to ` + fmt.Sprintf(
|
||||
"%d",
|
||||
maxResults,
|
||||
) + ` results)
|
||||
|
||||
## Examples
|
||||
|
||||
@@ -80,7 +83,10 @@ func NewWebSearchTool(
|
||||
- Use this tool when knowledge bases don't have the information you need
|
||||
- Results include URL, title, snippet, and content snippet (may be truncated)
|
||||
- **CRITICAL**: If content is truncated or you need full details, use **web_fetch** to fetch complete page content
|
||||
- Maximum ` + fmt.Sprintf("%d", maxResults) + ` results will be returned per search`
|
||||
- Maximum ` + fmt.Sprintf(
|
||||
"%d",
|
||||
maxResults,
|
||||
) + ` results will be returned per search`
|
||||
|
||||
return &WebSearchTool{
|
||||
BaseTool: NewBaseTool("web_search", description),
|
||||
@@ -152,7 +158,12 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
searchConfig.MaxResults = t.maxResults
|
||||
|
||||
// Perform web search
|
||||
logger.Infof(ctx, "[Tool][WebSearch] Performing web search with provider: %s, maxResults: %d", searchConfig.Provider, searchConfig.MaxResults)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[Tool][WebSearch] Performing web search with provider: %s, maxResults: %d",
|
||||
searchConfig.Provider,
|
||||
searchConfig.MaxResults,
|
||||
)
|
||||
webResults, err := t.webSearchService.Search(ctx, &searchConfig, query)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[Tool][WebSearch] Web search failed: %v", err)
|
||||
@@ -165,7 +176,8 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{}
|
||||
logger.Infof(ctx, "[Tool][WebSearch] Web search returned %d results", len(webResults))
|
||||
|
||||
// Apply RAG compression if configured
|
||||
if len(webResults) > 0 && tenant.WebSearchConfig.CompressionMethod != "none" && tenant.WebSearchConfig.CompressionMethod != "" {
|
||||
if len(webResults) > 0 && tenant.WebSearchConfig.CompressionMethod != "none" &&
|
||||
tenant.WebSearchConfig.CompressionMethod != "" {
|
||||
// Load session-scoped temp KB state from Redis using SessionService
|
||||
tempKBID, seen, ids := t.sessionService.GetWebSearchTempKBState(ctx, t.sessionID)
|
||||
|
||||
|
||||
@@ -112,7 +112,11 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
|
||||
return chunks, total, nil
|
||||
}
|
||||
|
||||
func (r *chunkRepository) ListChunkByParentID(ctx context.Context, tenantID uint64, parentID string) ([]*types.Chunk, error) {
|
||||
func (r *chunkRepository) ListChunkByParentID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
parentID string,
|
||||
) ([]*types.Chunk, error) {
|
||||
var chunks []*types.Chunk
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND parent_chunk_id = ?", tenantID, parentID).
|
||||
@@ -166,7 +170,11 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui
|
||||
}
|
||||
|
||||
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
|
||||
func (r *chunkRepository) CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error) {
|
||||
func (r *chunkRepository) CountChunksByKnowledgeBaseID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) (int64, error) {
|
||||
var count int64
|
||||
err := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
||||
@@ -175,7 +183,11 @@ func (r *chunkRepository) CountChunksByKnowledgeBaseID(ctx context.Context, tena
|
||||
}
|
||||
|
||||
// DeleteUnindexedChunks by knowledge id and chunk index range
|
||||
func (r *chunkRepository) DeleteUnindexedChunks(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) {
|
||||
func (r *chunkRepository) DeleteUnindexedChunks(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
) ([]*types.Chunk, error) {
|
||||
var chunks []*types.Chunk
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND knowledge_id = ? AND status = ?", tenantID, knowledgeID, types.ChunkStatusStored).
|
||||
@@ -194,7 +206,11 @@ func (r *chunkRepository) DeleteUnindexedChunks(ctx context.Context, tenantID ui
|
||||
|
||||
// ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID (only essential fields for efficiency)
|
||||
// Uses batch query to handle large datasets
|
||||
func (r *chunkRepository) ListAllFAQChunksByKnowledgeID(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) {
|
||||
func (r *chunkRepository) ListAllFAQChunksByKnowledgeID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
knowledgeID string,
|
||||
) ([]*types.Chunk, error) {
|
||||
const batchSize = 1000 // 每批查询1000条
|
||||
var allChunks []*types.Chunk
|
||||
offset := 0
|
||||
|
||||
@@ -28,7 +28,11 @@ func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *ty
|
||||
}
|
||||
|
||||
// GetKnowledgeByID gets knowledge
|
||||
func (r *knowledgeRepository) GetKnowledgeByID(ctx context.Context, tenantID uint64, id string) (*types.Knowledge, error) {
|
||||
func (r *knowledgeRepository) GetKnowledgeByID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
id string,
|
||||
) (*types.Knowledge, error) {
|
||||
var knowledge types.Knowledge
|
||||
if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&knowledge).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
@@ -217,13 +221,22 @@ func (r *knowledgeRepository) AminusB(
|
||||
return knowledgeIDs, err
|
||||
}
|
||||
|
||||
func (r *knowledgeRepository) UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error {
|
||||
func (r *knowledgeRepository) UpdateKnowledgeColumn(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
column string,
|
||||
value interface{},
|
||||
) error {
|
||||
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error
|
||||
return err
|
||||
}
|
||||
|
||||
// CountKnowledgeByKnowledgeBaseID counts the number of knowledge items in a knowledge base
|
||||
func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error) {
|
||||
func (r *knowledgeRepository) CountKnowledgeByKnowledgeBaseID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) (int64, error) {
|
||||
var count int64
|
||||
err := r.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
||||
|
||||
@@ -69,7 +69,11 @@ func (r *mcpServiceRepository) ListEnabled(ctx context.Context, tenantID uint64)
|
||||
}
|
||||
|
||||
// ListByIDs retrieves MCP services by multiple IDs for a tenant
|
||||
func (r *mcpServiceRepository) ListByIDs(ctx context.Context, tenantID uint64, ids []string) ([]*types.MCPService, error) {
|
||||
func (r *mcpServiceRepository) ListByIDs(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
ids []string,
|
||||
) ([]*types.MCPService, error) {
|
||||
if len(ids) == 0 {
|
||||
return []*types.MCPService{}, nil
|
||||
}
|
||||
|
||||
@@ -79,7 +79,12 @@ func (r *modelRepository) Delete(ctx context.Context, tenantID uint64, id string
|
||||
|
||||
// ClearDefaultByType clears the default flag for all models of a specific type
|
||||
// This is a batch operation that updates all matching records in one query
|
||||
func (r *modelRepository) ClearDefaultByType(ctx context.Context, tenantID uint, modelType types.ModelType, excludeID string) error {
|
||||
func (r *modelRepository) ClearDefaultByType(
|
||||
ctx context.Context,
|
||||
tenantID uint,
|
||||
modelType types.ModelType,
|
||||
excludeID string,
|
||||
) error {
|
||||
query := r.db.WithContext(ctx).Model(&types.Model{}).Where(
|
||||
"tenant_id = ? AND type = ? AND is_default = ?", tenantID, modelType, true,
|
||||
)
|
||||
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
|
||||
// VectorEmbedding defines the Elasticsearch document structure for vector embeddings
|
||||
type VectorEmbedding struct {
|
||||
Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"` // Type of the source document
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` // Unique ID of the text chunk
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` // ID of the knowledge item
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` // ID of the knowledge base
|
||||
Embedding []float32 `json:"embedding" gorm:"column:embedding;not null"` // Vector embedding of the content
|
||||
IsEnabled bool `json:"is_enabled"` // Whether the chunk is enabled
|
||||
Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"` // Type of the source document
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` // Unique ID of the text chunk
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` // ID of the knowledge item
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` // ID of the knowledge base
|
||||
Embedding []float32 `json:"embedding" gorm:"column:embedding;not null"` // Vector embedding of the content
|
||||
IsEnabled bool `json:"is_enabled"` // Whether the chunk is enabled
|
||||
}
|
||||
|
||||
// VectorEmbeddingWithScore extends VectorEmbedding with similarity score
|
||||
|
||||
@@ -1094,7 +1094,10 @@ func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexIn
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkStatusMap) == 0 {
|
||||
log.Warnf("[ElasticsearchV7] Chunk status map is empty, skipping update")
|
||||
|
||||
@@ -547,7 +547,10 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
func (e *elasticsearchRepository) BatchUpdateChunkEnabledStatus(
|
||||
ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkStatusMap) == 0 {
|
||||
log.Warnf("[Elasticsearch] Chunk status map is empty, skipping update")
|
||||
|
||||
@@ -152,7 +152,11 @@ func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameS
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Neo4jRepository) SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) {
|
||||
func (n *Neo4jRepository) SearchNode(
|
||||
ctx context.Context,
|
||||
namespace types.NameSpace,
|
||||
nodes []string,
|
||||
) (*types.GraphData, error) {
|
||||
if n.driver == nil {
|
||||
logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
||||
return nil, nil
|
||||
|
||||
@@ -481,7 +481,8 @@ func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkS
|
||||
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update enabled chunks: %v", result.Error)
|
||||
return result.Error
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Updated %d chunks to enabled, rows affected: %d", len(enabledChunkIDs), result.RowsAffected)
|
||||
logger.GetLogger(ctx).
|
||||
Infof("[Postgres] Updated %d chunks to enabled, rows affected: %d", len(enabledChunkIDs), result.RowsAffected)
|
||||
}
|
||||
|
||||
// Batch update disabled chunks
|
||||
@@ -493,7 +494,8 @@ func (g *pgRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkS
|
||||
logger.GetLogger(ctx).Errorf("[Postgres] Failed to update disabled chunks: %v", result.Error)
|
||||
return result.Error
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Updated %d chunks to disabled, rows affected: %d", len(disabledChunkIDs), result.RowsAffected)
|
||||
logger.GetLogger(ctx).
|
||||
Infof("[Postgres] Updated %d chunks to disabled, rows affected: %d", len(disabledChunkIDs), result.RowsAffected)
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Successfully batch updated chunk enabled status")
|
||||
|
||||
@@ -13,35 +13,35 @@ import (
|
||||
|
||||
// pgVector defines the database model for vector embeddings storage
|
||||
type pgVector struct {
|
||||
ID uint `json:"id" gorm:"primarykey"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"`
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"`
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
ID uint `json:"id" gorm:"primarykey"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"`
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"`
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
IsEnabled bool `json:"is_enabled" gorm:"column:is_enabled;default:true;index"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
IsEnabled bool `json:"is_enabled" gorm:"column:is_enabled;default:true;index"`
|
||||
}
|
||||
|
||||
// pgVectorWithScore extends pgVector with similarity score field
|
||||
type pgVectorWithScore struct {
|
||||
ID uint `json:"id" gorm:"primarykey"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"`
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"`
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
ID uint `json:"id" gorm:"primarykey"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
|
||||
SourceID string `json:"source_id" gorm:"column:source_id;not null"`
|
||||
SourceType int `json:"source_type" gorm:"column:source_type;not null"`
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
IsEnabled bool `json:"is_enabled" gorm:"column:is_enabled;default:true;index"`
|
||||
Score float64 `json:"score" gorm:"column:score"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
IsEnabled bool `json:"is_enabled" gorm:"column:is_enabled;default:true;index"`
|
||||
Score float64 `json:"score" gorm:"column:score"`
|
||||
}
|
||||
|
||||
// TableName specifies the database table name for pgVectorWithScore
|
||||
|
||||
@@ -35,7 +35,11 @@ func (r *knowledgeTagRepository) GetByID(ctx context.Context, tenantID uint64, i
|
||||
return &tag, nil
|
||||
}
|
||||
|
||||
func (r *knowledgeTagRepository) ListByKB(ctx context.Context, tenantID uint64, kbID string) ([]*types.KnowledgeTag, error) {
|
||||
func (r *knowledgeTagRepository) ListByKB(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) ([]*types.KnowledgeTag, error) {
|
||||
var tags []*types.KnowledgeTag
|
||||
if err := r.db.WithContext(ctx).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID).
|
||||
|
||||
@@ -184,7 +184,13 @@ func (s *agentService) registerTools(
|
||||
if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok {
|
||||
tenantID = tid
|
||||
}
|
||||
logger.Infof(ctx, "Registering tools: %v, tenant ID: %d, webSearchEnabled: %v", allowedTools, tenantID, config.WebSearchEnabled)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Registering tools: %v, tenant ID: %d, webSearchEnabled: %v",
|
||||
allowedTools,
|
||||
tenantID,
|
||||
config.WebSearchEnabled,
|
||||
)
|
||||
|
||||
// Register each allowed tool
|
||||
for _, toolName := range allowedTools {
|
||||
@@ -224,7 +230,12 @@ func (s *agentService) registerTools(
|
||||
sessionID,
|
||||
config.WebSearchMaxResults,
|
||||
))
|
||||
logger.Infof(ctx, "Registered web_search tool for session: %s, maxResults: %d", sessionID, config.WebSearchMaxResults)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Registered web_search tool for session: %s, maxResults: %d",
|
||||
sessionID,
|
||||
config.WebSearchMaxResults,
|
||||
)
|
||||
|
||||
case "web_fetch":
|
||||
registry.RegisterTool(tools.NewWebFetchTool(chatModel))
|
||||
|
||||
@@ -212,7 +212,11 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PluginMerge) populateFAQAnswers(ctx context.Context, chatManage *types.ChatManage, results []*types.SearchResult) []*types.SearchResult {
|
||||
func (p *PluginMerge) populateFAQAnswers(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
results []*types.SearchResult,
|
||||
) []*types.SearchResult {
|
||||
if len(results) == 0 || p.chunkRepo == nil {
|
||||
return results
|
||||
}
|
||||
@@ -332,7 +336,11 @@ func buildFAQAnswerContent(meta *types.FAQChunkMetadata) string {
|
||||
return strings.TrimSpace(builder.String())
|
||||
}
|
||||
|
||||
func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatManage *types.ChatManage, results []*types.SearchResult) []*types.SearchResult {
|
||||
func (p *PluginMerge) expandShortContextWithNeighbors(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
results []*types.SearchResult,
|
||||
) []*types.SearchResult {
|
||||
const (
|
||||
minLen = 350
|
||||
maxLen = 850
|
||||
@@ -495,7 +503,8 @@ func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatM
|
||||
expanded := false
|
||||
if prevCursor != "" {
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, prevCursor)
|
||||
if prevChunk := chunkMap[prevCursor]; prevChunk != nil && prevChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
if prevChunk := chunkMap[prevCursor]; prevChunk != nil &&
|
||||
prevChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
prevContent = concatNoOverlap(prevChunk.Content, prevContent)
|
||||
prevIDs = append([]string{prevChunk.ID}, prevIDs...)
|
||||
prevCursor = prevChunk.PreChunkID
|
||||
@@ -512,7 +521,8 @@ func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatM
|
||||
|
||||
if nextCursor != "" {
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, nextCursor)
|
||||
if nextChunk := chunkMap[nextCursor]; nextChunk != nil && nextChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
if nextChunk := chunkMap[nextCursor]; nextChunk != nil &&
|
||||
nextChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
nextContent = concatNoOverlap(nextContent, nextChunk.Content)
|
||||
nextIDs = append(nextIDs, nextChunk.ID)
|
||||
nextCursor = nextChunk.NextChunkID
|
||||
@@ -625,7 +635,12 @@ func containsID(ids []string, target string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *PluginMerge) fetchChunksIfMissing(ctx context.Context, tenantID uint64, chunkMap map[string]*types.Chunk, chunkIDs ...string) {
|
||||
func (p *PluginMerge) fetchChunksIfMissing(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
chunkMap map[string]*types.Chunk,
|
||||
chunkIDs ...string,
|
||||
) {
|
||||
missing := make([]string, 0, len(chunkIDs))
|
||||
for _, id := range chunkIDs {
|
||||
if id == "" {
|
||||
|
||||
@@ -91,7 +91,12 @@ func (p *PluginPreprocess) ActivationEvents() []types.EventType {
|
||||
}
|
||||
|
||||
// OnEvent Process events
|
||||
func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError {
|
||||
func (p *PluginPreprocess) OnEvent(
|
||||
ctx context.Context,
|
||||
eventType types.EventType,
|
||||
chatManage *types.ChatManage,
|
||||
next func() *PluginError,
|
||||
) *PluginError {
|
||||
rawQuery := strings.TrimSpace(chatManage.RewriteQuery)
|
||||
if rawQuery == "" {
|
||||
return next()
|
||||
@@ -270,15 +275,30 @@ type intentResp struct {
|
||||
|
||||
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
|
||||
if p.modelService == nil || chatManage.ChatModelID == "" {
|
||||
pipelineWarn(ctx, "IntentDetect", "skip", map[string]interface{}{"reason": "no_model", "session_id": chatManage.SessionID})
|
||||
pipelineWarn(
|
||||
ctx,
|
||||
"IntentDetect",
|
||||
"skip",
|
||||
map[string]interface{}{"reason": "no_model", "session_id": chatManage.SessionID},
|
||||
)
|
||||
return "general"
|
||||
}
|
||||
chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "IntentDetect", "get_model_failed", map[string]interface{}{"error": err.Error(), "model_id": chatManage.ChatModelID})
|
||||
pipelineWarn(
|
||||
ctx,
|
||||
"IntentDetect",
|
||||
"get_model_failed",
|
||||
map[string]interface{}{"error": err.Error(), "model_id": chatManage.ChatModelID},
|
||||
)
|
||||
return "general"
|
||||
}
|
||||
pipelineInfo(ctx, "IntentDetect", "start", map[string]interface{}{"session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID})
|
||||
pipelineInfo(
|
||||
ctx,
|
||||
"IntentDetect",
|
||||
"start",
|
||||
map[string]interface{}{"session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID},
|
||||
)
|
||||
sys := "You are a query intent classifier. Classify the user's query into one of: definition, howto, compare, qa, general. Respond ONLY with a JSON object {\"intent\": \"...\", \"confidence\": 0.0 } inside a markdown fenced block."
|
||||
usr := text
|
||||
think := false
|
||||
@@ -296,7 +316,12 @@ func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *type
|
||||
pipelineWarn(ctx, "IntentDetect", "parse_failed", map[string]interface{}{"body": body, "error": err.Error()})
|
||||
return "general"
|
||||
}
|
||||
pipelineInfo(ctx, "IntentDetect", "result", map[string]interface{}{"intent": ir.Intent, "confidence": ir.Confidence})
|
||||
pipelineInfo(
|
||||
ctx,
|
||||
"IntentDetect",
|
||||
"result",
|
||||
map[string]interface{}{"intent": ir.Intent, "confidence": ir.Confidence},
|
||||
)
|
||||
switch strings.ToLower(strings.TrimSpace(ir.Intent)) {
|
||||
case "definition", "howto", "compare", "qa", "general":
|
||||
return strings.ToLower(ir.Intent)
|
||||
|
||||
@@ -222,7 +222,13 @@ func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatM
|
||||
return composite
|
||||
}
|
||||
|
||||
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
|
||||
func applyMMR(
|
||||
ctx context.Context,
|
||||
results []*types.SearchResult,
|
||||
chatManage *types.ChatManage,
|
||||
k int,
|
||||
lambda float64,
|
||||
) []*types.SearchResult {
|
||||
if k <= 0 || len(results) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -299,7 +299,11 @@ func buildContentSignature(content string) string {
|
||||
}
|
||||
|
||||
// searchKnowledgeBases performs KB searches for rewrite and processed queries across KB IDs
|
||||
func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseIDs []string, chatManage *types.ChatManage) []*types.SearchResult {
|
||||
func (p *PluginSearch) searchKnowledgeBases(
|
||||
ctx context.Context,
|
||||
knowledgeBaseIDs []string,
|
||||
chatManage *types.ChatManage,
|
||||
) []*types.SearchResult {
|
||||
// Build base params for rewrite query
|
||||
baseParams := types.SearchParams{
|
||||
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
||||
|
||||
@@ -77,7 +77,13 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "KB %s entity search result count: %d nodes, %d relations", knowledgeBaseID, len(graph.Node), len(graph.Relation))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"KB %s entity search result count: %d nodes, %d relations",
|
||||
knowledgeBaseID,
|
||||
len(graph.Node),
|
||||
len(graph.Relation),
|
||||
)
|
||||
|
||||
mu.Lock()
|
||||
allNodes = append(allNodes, graph.Node...)
|
||||
@@ -109,7 +115,11 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
for _, chunk := range chunks {
|
||||
knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID)
|
||||
}
|
||||
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, ctx.Value(types.TenantIDContextKey).(uint64), knowledgeIDs)
|
||||
knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(
|
||||
ctx,
|
||||
ctx.Value(types.TenantIDContextKey).(uint64),
|
||||
knowledgeIDs,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
return next()
|
||||
@@ -129,7 +139,12 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID)
|
||||
return ErrSearchNothing
|
||||
}
|
||||
logger.Infof(ctx, "search entity result count: %d, session_id: %s", len(chatManage.SearchResult), chatManage.SessionID)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"search entity result count: %d, session_id: %s",
|
||||
len(chatManage.SearchResult),
|
||||
chatManage.SessionID,
|
||||
)
|
||||
return next()
|
||||
}
|
||||
|
||||
|
||||
@@ -138,7 +138,14 @@ func (s *chunkService) ListPagedChunksByKnowledgeID(ctx context.Context,
|
||||
knowledgeID string, page *types.Pagination, chunkType []types.ChunkType,
|
||||
) (*types.PageResult, error) {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
chunks, total, err := s.chunkRepository.ListPagedChunksByKnowledgeID(ctx, tenantID, knowledgeID, page, chunkType, "")
|
||||
chunks, total, err := s.chunkRepository.ListPagedChunksByKnowledgeID(
|
||||
ctx,
|
||||
tenantID,
|
||||
knowledgeID,
|
||||
page,
|
||||
chunkType,
|
||||
"",
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
@@ -298,7 +305,11 @@ func (s *chunkService) DeleteByKnowledgeList(ctx context.Context, ids []string)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *chunkService) ListChunkByParentID(ctx context.Context, tenantID uint64, parentID string) ([]*types.Chunk, error) {
|
||||
func (s *chunkService) ListChunkByParentID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
parentID string,
|
||||
) ([]*types.Chunk, error) {
|
||||
logger.Info(ctx, "Start listing chunk by parent ID")
|
||||
logger.Infof(ctx, "Parent ID: %s", parentID)
|
||||
|
||||
|
||||
@@ -16,7 +16,13 @@ import (
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
func NewChunkExtractTask(ctx context.Context, client *asynq.Client, tenantID uint64, chunkID string, modelID string) error {
|
||||
func NewChunkExtractTask(
|
||||
ctx context.Context,
|
||||
client *asynq.Client,
|
||||
tenantID uint64,
|
||||
chunkID string,
|
||||
modelID string,
|
||||
) error {
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Warn(ctx, "NEO4J is not enabled, skip chunk extract task")
|
||||
return nil
|
||||
|
||||
@@ -307,7 +307,13 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
// 即使入队失败,也返回knowledge,因为文件已保存
|
||||
return knowledge, nil
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued document process task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, knowledge.ID)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Enqueued document process task: id=%s queue=%s knowledge_id=%s",
|
||||
info.ID,
|
||||
info.Queue,
|
||||
knowledge.ID,
|
||||
)
|
||||
|
||||
logger.Infof(ctx, "Knowledge from file created successfully, ID: %s", knowledge.ID)
|
||||
return knowledge, nil
|
||||
@@ -664,7 +670,10 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
// Delete knowledge embeddings from vector store
|
||||
wg.Go(func() error {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
@@ -738,7 +747,10 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
// 2. Delete knowledge embeddings from vector store
|
||||
wg.Go(func() error {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
@@ -754,7 +766,9 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
return err
|
||||
}
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
logger.GetLogger(ctx).
|
||||
WithField("error", err).
|
||||
Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -792,7 +806,10 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
wg.Go(func() error {
|
||||
namespaces := []types.NameSpace{}
|
||||
for _, knowledge := range knowledgeList {
|
||||
namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID})
|
||||
namespaces = append(
|
||||
namespaces,
|
||||
types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID},
|
||||
)
|
||||
}
|
||||
if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed")
|
||||
@@ -808,7 +825,11 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
return s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids)
|
||||
}
|
||||
|
||||
func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowledge, targetKB *types.KnowledgeBase) (err error) {
|
||||
func (s *knowledgeService) cloneKnowledge(
|
||||
ctx context.Context,
|
||||
src *types.Knowledge,
|
||||
targetKB *types.KnowledgeBase,
|
||||
) (err error) {
|
||||
if src.ParseStatus != "completed" {
|
||||
logger.GetLogger(ctx).WithField("knowledge_id", src.ID).Errorf("MoveKnowledge parse status is not completed")
|
||||
return nil
|
||||
@@ -1657,7 +1678,12 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID string, chunkID string, imageInfo string) error {
|
||||
func (s *knowledgeService) UpdateImageInfo(
|
||||
ctx context.Context,
|
||||
knowledgeID string,
|
||||
chunkID string,
|
||||
imageInfo string,
|
||||
) error {
|
||||
var images []*types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(imageInfo), &images); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal image info: %v", err)
|
||||
@@ -2165,7 +2191,12 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
|
||||
if payload.Mode == types.FAQBatchModeReplace {
|
||||
// Replace模式:计算需要删除、创建、更新的条目
|
||||
entriesToProcess, chunksToDelete, skippedCount, err = s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
|
||||
entriesToProcess, chunksToDelete, skippedCount, err = s.calculateReplaceOperations(
|
||||
ctx,
|
||||
tenantID,
|
||||
faqKnowledge.ID,
|
||||
payload.Entries,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to calculate replace operations: %w", err)
|
||||
}
|
||||
@@ -2193,7 +2224,14 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ import task %s: total entries: %d, to process: %d, skipped: %d", taskID, len(payload.Entries), len(entriesToProcess), skippedCount)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: total entries: %d, to process: %d, skipped: %d",
|
||||
taskID,
|
||||
len(payload.Entries),
|
||||
len(entriesToProcess),
|
||||
skippedCount,
|
||||
)
|
||||
|
||||
// 如果没有需要处理的条目,直接返回
|
||||
if len(entriesToProcess) == 0 {
|
||||
@@ -2206,7 +2244,14 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
totalStartTime := time.Now()
|
||||
actualProcessed := skippedCount + processedCount
|
||||
|
||||
logger.Infof(ctx, "FAQ import task %s: starting batch processing, remaining entries: %d, total entries: %d, batch size: %d", taskID, remainingEntries, totalEntries, faqImportBatchSize)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: starting batch processing, remaining entries: %d, total entries: %d, batch size: %d",
|
||||
taskID,
|
||||
remainingEntries,
|
||||
totalEntries,
|
||||
faqImportBatchSize,
|
||||
)
|
||||
|
||||
for i := 0; i < remainingEntries; i += faqImportBatchSize {
|
||||
batchStartTime := time.Now()
|
||||
@@ -2263,7 +2308,15 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
return fmt.Errorf("failed to create chunks: %w", err)
|
||||
}
|
||||
createDuration := time.Since(createStartTime)
|
||||
logger.Infof(ctx, "FAQ import task %s: batch %d-%d created %d chunks in %v", taskID, i+1, end, len(chunks), createDuration)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: batch %d-%d created %d chunks in %v",
|
||||
taskID,
|
||||
i+1,
|
||||
end,
|
||||
len(chunks),
|
||||
createDuration,
|
||||
)
|
||||
|
||||
// 索引chunks
|
||||
indexStartTime := time.Now()
|
||||
@@ -2272,7 +2325,15 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
return fmt.Errorf("failed to index chunks: %w", err)
|
||||
}
|
||||
indexDuration := time.Since(indexStartTime)
|
||||
logger.Infof(ctx, "FAQ import task %s: batch %d-%d indexed %d chunks in %v", taskID, i+1, end, len(chunks), indexDuration)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: batch %d-%d indexed %d chunks in %v",
|
||||
taskID,
|
||||
i+1,
|
||||
end,
|
||||
len(chunks),
|
||||
indexDuration,
|
||||
)
|
||||
|
||||
// 更新chunks的Status为已索引
|
||||
chunksToUpdate := make([]*types.Chunk, 0, len(chunks))
|
||||
@@ -2292,13 +2353,32 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
}
|
||||
|
||||
batchDuration := time.Since(batchStartTime)
|
||||
logger.Infof(ctx, "FAQ import task %s: batch %d-%d completed in %v (build: %v, create: %v, index: %v), total progress: %d/%d (%d%%)",
|
||||
taskID, i+1, end, batchDuration, buildDuration, createDuration, indexDuration, actualProcessed, totalEntries, progress)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: batch %d-%d completed in %v (build: %v, create: %v, index: %v), total progress: %d/%d (%d%%)",
|
||||
taskID,
|
||||
i+1,
|
||||
end,
|
||||
batchDuration,
|
||||
buildDuration,
|
||||
createDuration,
|
||||
indexDuration,
|
||||
actualProcessed,
|
||||
totalEntries,
|
||||
progress,
|
||||
)
|
||||
}
|
||||
|
||||
totalDuration := time.Since(totalStartTime)
|
||||
logger.Infof(ctx, "FAQ import task %s: all batches completed, processed: %d entries (skipped: %d) in %v, avg: %v per entry",
|
||||
taskID, actualProcessed, skippedCount, totalDuration, totalDuration/time.Duration(actualProcessed))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"FAQ import task %s: all batches completed, processed: %d entries (skipped: %d) in %v, avg: %v per entry",
|
||||
taskID,
|
||||
actualProcessed,
|
||||
skippedCount,
|
||||
totalDuration,
|
||||
totalDuration/time.Duration(actualProcessed),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -2358,7 +2438,10 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
|
||||
if isEnabledUpdated {
|
||||
chunkStatusMap := map[string]bool{chunk.ID: chunk.IsEnabled}
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2464,7 +2547,10 @@ func (s *knowledgeService) UpdateFAQEntryStatusBatch(ctx context.Context,
|
||||
// Sync update to retriever engines
|
||||
if len(chunkStatusMap) > 0 {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2839,7 +2925,11 @@ func (s *knowledgeService) validateFAQKnowledgeBase(ctx context.Context, kbID st
|
||||
return kb, nil
|
||||
}
|
||||
|
||||
func (s *knowledgeService) findFAQKnowledge(ctx context.Context, tenantID uint64, kbID string) (*types.Knowledge, error) {
|
||||
func (s *knowledgeService) findFAQKnowledge(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) (*types.Knowledge, error) {
|
||||
knowledges, err := s.repo.ListKnowledgeByKnowledgeBaseID(ctx, tenantID, kbID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2852,7 +2942,11 @@ func (s *knowledgeService) findFAQKnowledge(ctx context.Context, tenantID uint64
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint64, kb *types.KnowledgeBase) (*types.Knowledge, error) {
|
||||
func (s *knowledgeService) ensureFAQKnowledge(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kb *types.KnowledgeBase,
|
||||
) (*types.Knowledge, error) {
|
||||
existing, err := s.findFAQKnowledge(ctx, tenantID, kb.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2879,15 +2973,23 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint
|
||||
return knowledge, nil
|
||||
}
|
||||
|
||||
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
||||
progress, total, processed int, errorMsg string,
|
||||
func (s *knowledgeService) updateFAQImportStatus(
|
||||
ctx context.Context,
|
||||
knowledgeID string,
|
||||
status types.FAQImportTaskStatus,
|
||||
progress, total, processed int,
|
||||
errorMsg string,
|
||||
) error {
|
||||
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
|
||||
}
|
||||
|
||||
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
|
||||
func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
|
||||
progress, total, processed int, errorMsg string,
|
||||
func (s *knowledgeService) updateFAQImportStatusWithRanges(
|
||||
ctx context.Context,
|
||||
knowledgeID string,
|
||||
status types.FAQImportTaskStatus,
|
||||
progress, total, processed int,
|
||||
errorMsg string,
|
||||
) error {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
|
||||
@@ -2925,7 +3027,11 @@ func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context,
|
||||
}
|
||||
|
||||
// getRunningFAQImportTask 获取指定知识库的进行中导入任务
|
||||
func (s *knowledgeService) getRunningFAQImportTask(ctx context.Context, kbID string, tenantID uint64) (*types.Knowledge, error) {
|
||||
func (s *knowledgeService) getRunningFAQImportTask(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
tenantID uint64,
|
||||
) (*types.Knowledge, error) {
|
||||
faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kbID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -3023,7 +3129,11 @@ func buildFAQIndexContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode)
|
||||
}
|
||||
|
||||
// buildFAQIndexInfoList 构建FAQ索引信息列表,支持分别索引模式
|
||||
func (s *knowledgeService) buildFAQIndexInfoList(ctx context.Context, kb *types.KnowledgeBase, chunk *types.Chunk) ([]*types.IndexInfo, error) {
|
||||
func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
chunk *types.Chunk,
|
||||
) ([]*types.IndexInfo, error) {
|
||||
indexMode := types.FAQIndexModeQuestionAnswer
|
||||
questionIndexMode := types.FAQQuestionIndexModeCombined
|
||||
if kb.FAQConfig != nil {
|
||||
@@ -3137,7 +3247,13 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
|
||||
chunkIDs = append(chunkIDs, chunk.ID)
|
||||
}
|
||||
buildIndexInfoDuration := time.Since(buildIndexInfoStartTime)
|
||||
logger.Debugf(ctx, "indexFAQChunks: built %d index info entries for %d chunks in %v", len(indexInfo), len(chunks), buildIndexInfoDuration)
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"indexFAQChunks: built %d index info entries for %d chunks in %v",
|
||||
len(indexInfo),
|
||||
len(chunks),
|
||||
buildIndexInfoDuration,
|
||||
)
|
||||
|
||||
var size int64
|
||||
if adjustStorage {
|
||||
@@ -3195,8 +3311,16 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
|
||||
}
|
||||
|
||||
totalDuration := time.Since(indexStartTime)
|
||||
logger.Debugf(ctx, "indexFAQChunks: completed indexing %d chunks in %v (build: %v, delete: %v, batchIndex: %v, update: %v)",
|
||||
len(chunks), totalDuration, buildIndexInfoDuration, deleteDuration, batchIndexDuration, updateDuration)
|
||||
logger.Debugf(
|
||||
ctx,
|
||||
"indexFAQChunks: completed indexing %d chunks in %v (build: %v, delete: %v, batchIndex: %v, update: %v)",
|
||||
len(chunks),
|
||||
totalDuration,
|
||||
buildIndexInfoDuration,
|
||||
deleteDuration,
|
||||
batchIndexDuration,
|
||||
updateDuration,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -3311,7 +3435,9 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
Separators: kb.ChunkingConfig.Separators,
|
||||
EnableMultimodal: enableMultimodel,
|
||||
StorageConfig: &proto.StorageConfig{
|
||||
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
|
||||
Provider: proto.StorageProvider(
|
||||
proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)],
|
||||
),
|
||||
Region: kb.StorageConfig.Region,
|
||||
BucketName: kb.StorageConfig.BucketName,
|
||||
AccessKeyId: kb.StorageConfig.SecretID,
|
||||
@@ -3354,7 +3480,10 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if knowledge.EmbeddingModelID != "" {
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup")
|
||||
cleanupErr = errors.Join(cleanupErr, err)
|
||||
@@ -3470,12 +3599,18 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
if knowledge.ParseStatus == "failed" {
|
||||
// 检查是否可恢复(例如:超时、临时错误等)
|
||||
// 对于不可恢复的错误,直接返回
|
||||
logger.Warnf(ctx, "Document processing previously failed: %s, error: %s", payload.KnowledgeID, knowledge.ErrorMessage)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"Document processing previously failed: %s, error: %s",
|
||||
payload.KnowledgeID,
|
||||
knowledge.ErrorMessage,
|
||||
)
|
||||
// 这里可以根据错误类型判断是否可恢复,暂时允许重试
|
||||
}
|
||||
|
||||
// 检查是否有部分处理(有chunks但状态不是completed)
|
||||
if knowledge.ParseStatus != "completed" && knowledge.ParseStatus != "pending" && knowledge.ParseStatus != "processing" {
|
||||
if knowledge.ParseStatus != "completed" && knowledge.ParseStatus != "pending" &&
|
||||
knowledge.ParseStatus != "processing" {
|
||||
// 状态异常,记录日志但继续处理
|
||||
logger.Warnf(ctx, "Unexpected parse status: %s for knowledge: %s", knowledge.ParseStatus, payload.KnowledgeID)
|
||||
}
|
||||
@@ -3546,7 +3681,9 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
Separators: kb.ChunkingConfig.Separators,
|
||||
EnableMultimodal: payload.EnableMultimodel,
|
||||
StorageConfig: &proto.StorageConfig{
|
||||
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
|
||||
Provider: proto.StorageProvider(
|
||||
proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)],
|
||||
),
|
||||
Region: kb.StorageConfig.Region,
|
||||
BucketName: kb.StorageConfig.BucketName,
|
||||
AccessKeyId: kb.StorageConfig.SecretID,
|
||||
@@ -3721,7 +3858,10 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
// 删除索引数据
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err == nil {
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err == nil {
|
||||
chunkIDs := make([]string, 0, len(chunksDeleted))
|
||||
for _, chunk := range chunksDeleted {
|
||||
@@ -3740,7 +3880,12 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
payload.Entries = payload.Entries[processedCount:]
|
||||
}
|
||||
// Replace 模式使用hash去重,不截断payload.Entries
|
||||
logger.Infof(ctx, "Continuing FAQ import from entry %d, remaining: %d entries", processedCount, len(payload.Entries))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Continuing FAQ import from entry %d, remaining: %d entries",
|
||||
processedCount,
|
||||
len(payload.Entries),
|
||||
)
|
||||
}
|
||||
|
||||
// 更新任务状态为运行中
|
||||
|
||||
@@ -248,7 +248,10 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
// Delete embeddings from vector store
|
||||
logger.Infof(ctx, "Deleting embeddings from vector store")
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.RetrieverEngines.Engines,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to create retrieve engine: %v", err)
|
||||
} else {
|
||||
@@ -489,7 +492,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
}
|
||||
|
||||
// Add keyword retrieval params if supported and not FAQ
|
||||
if retrieveEngine.SupportRetriever(types.KeywordsRetrieverType) && !params.DisableKeywordsMatch && kb.Type != types.KnowledgeBaseTypeFAQ {
|
||||
if retrieveEngine.SupportRetriever(types.KeywordsRetrieverType) && !params.DisableKeywordsMatch &&
|
||||
kb.Type != types.KnowledgeBaseTypeFAQ {
|
||||
logger.Info(ctx, "Keyword retrieval supported, preparing keyword retrieval parameters")
|
||||
retrieveParams = append(retrieveParams, types.RetrieveParams{
|
||||
Query: params.QueryText,
|
||||
@@ -537,7 +541,9 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
logger.Infof(ctx, "Result count before deduplication: %d", len(matchResults))
|
||||
|
||||
// First, try standard deduplication
|
||||
deduplicatedChunks := common.DeduplicateWithScore(func(r *types.IndexWithScore) string { return r.ChunkID }, matchResults...)
|
||||
deduplicatedChunks := common.DeduplicateWithScore(
|
||||
func(r *types.IndexWithScore) string { return r.ChunkID },
|
||||
matchResults...)
|
||||
logger.Infof(ctx, "Result count after deduplication: %d", len(deduplicatedChunks))
|
||||
|
||||
kb.EnsureDefaults()
|
||||
@@ -549,7 +555,13 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
if needsIterativeRetrieval {
|
||||
logger.Info(ctx, "Not enough unique chunks, using iterative retrieval for FAQ")
|
||||
// Use iterative retrieval to get more unique chunks (with negative question filtering inside)
|
||||
deduplicatedChunks = s.iterativeRetrieveWithDeduplication(ctx, retrieveEngine, retrieveParams, params.MatchCount, params.QueryText)
|
||||
deduplicatedChunks = s.iterativeRetrieveWithDeduplication(
|
||||
ctx,
|
||||
retrieveEngine,
|
||||
retrieveParams,
|
||||
params.MatchCount,
|
||||
params.QueryText,
|
||||
)
|
||||
} else if kb.Type == types.KnowledgeBaseTypeFAQ {
|
||||
// Filter by negative questions if not using iterative retrieval
|
||||
deduplicatedChunks = s.filterByNegativeQuestions(ctx, deduplicatedChunks, params.QueryText)
|
||||
@@ -606,7 +618,12 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
// Check if we got fewer results than requested - means no more results available
|
||||
totalRetrieved := len(iterationResults)
|
||||
if totalRetrieved < currentTopK {
|
||||
logger.Infof(ctx, "Retrieved %d results (less than TopK %d), no more results available", totalRetrieved, currentTopK)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Retrieved %d results (less than TopK %d), no more results available",
|
||||
totalRetrieved,
|
||||
currentTopK,
|
||||
)
|
||||
}
|
||||
|
||||
// Deduplicate and merge (keep highest score for each chunk)
|
||||
@@ -631,8 +648,14 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
uniqueChunks[chunk.ChunkID] = chunk
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "After iteration %d: retrieved %d results, found %d unique chunks after filtering (target: %d)",
|
||||
i+1, totalRetrieved, len(uniqueChunks), matchCount)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"After iteration %d: retrieved %d results, found %d unique chunks after filtering (target: %d)",
|
||||
i+1,
|
||||
totalRetrieved,
|
||||
len(uniqueChunks),
|
||||
matchCount,
|
||||
)
|
||||
|
||||
// Early stop: Check if we have enough unique chunks after deduplication and filtering
|
||||
if len(uniqueChunks) >= matchCount {
|
||||
|
||||
@@ -24,7 +24,11 @@ func NewSlidingWindowStrategy(recentMessageCount int) interfaces.CompressionStra
|
||||
|
||||
// Compress implements the sliding window compression
|
||||
// Keeps system messages and the most recent N messages
|
||||
func (s *slidingWindowStrategy) Compress(ctx context.Context, messages []chat.Message, maxTokens int) ([]chat.Message, error) {
|
||||
func (s *slidingWindowStrategy) Compress(
|
||||
ctx context.Context,
|
||||
messages []chat.Message,
|
||||
maxTokens int,
|
||||
) ([]chat.Message, error) {
|
||||
if len(messages) <= s.recentMessageCount {
|
||||
return messages, nil
|
||||
}
|
||||
@@ -83,7 +87,11 @@ type smartCompressionStrategy struct {
|
||||
}
|
||||
|
||||
// NewSmartCompressionStrategy creates a new smart compression strategy
|
||||
func NewSmartCompressionStrategy(recentMessageCount int, chatModel chat.Chat, summarizeThreshold int) interfaces.CompressionStrategy {
|
||||
func NewSmartCompressionStrategy(
|
||||
recentMessageCount int,
|
||||
chatModel chat.Chat,
|
||||
summarizeThreshold int,
|
||||
) interfaces.CompressionStrategy {
|
||||
return &smartCompressionStrategy{
|
||||
recentMessageCount: recentMessageCount,
|
||||
chatModel: chatModel,
|
||||
@@ -93,7 +101,11 @@ func NewSmartCompressionStrategy(recentMessageCount int, chatModel chat.Chat, su
|
||||
|
||||
// Compress implements smart compression with LLM summarization
|
||||
// Summarizes old messages and keeps recent messages intact
|
||||
func (s *smartCompressionStrategy) Compress(ctx context.Context, messages []chat.Message, maxTokens int) ([]chat.Message, error) {
|
||||
func (s *smartCompressionStrategy) Compress(
|
||||
ctx context.Context,
|
||||
messages []chat.Message,
|
||||
maxTokens int,
|
||||
) ([]chat.Message, error) {
|
||||
if len(messages) <= s.recentMessageCount {
|
||||
return messages, nil
|
||||
}
|
||||
@@ -156,8 +168,15 @@ func (s *smartCompressionStrategy) Compress(ctx context.Context, messages []chat
|
||||
})
|
||||
result = append(result, recentMessages...)
|
||||
|
||||
logger.Infof(ctx, "[SmartCompression] Compressed %d messages to %d messages (summarized %d old + kept %d recent + %d system)",
|
||||
len(messages), len(result), len(oldMessages), len(recentMessages), len(systemMessages))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[SmartCompression] Compressed %d messages to %d messages (summarized %d old + kept %d recent + %d system)",
|
||||
len(messages),
|
||||
len(result),
|
||||
len(oldMessages),
|
||||
len(recentMessages),
|
||||
len(systemMessages),
|
||||
)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -18,7 +18,11 @@ type contextManager struct {
|
||||
}
|
||||
|
||||
// NewContextManager creates a new context manager with the specified storage and compression strategy
|
||||
func NewContextManager(storage ContextStorage, compressionStrategy interfaces.CompressionStrategy, maxTokens int) interfaces.ContextManager {
|
||||
func NewContextManager(
|
||||
storage ContextStorage,
|
||||
compressionStrategy interfaces.CompressionStrategy,
|
||||
maxTokens int,
|
||||
) interfaces.ContextManager {
|
||||
return &contextManager{
|
||||
storage: storage,
|
||||
compressionStrategy: compressionStrategy,
|
||||
@@ -27,7 +31,10 @@ func NewContextManager(storage ContextStorage, compressionStrategy interfaces.Co
|
||||
}
|
||||
|
||||
// NewContextManagerWithMemory creates a context manager with in-memory storage (for backward compatibility)
|
||||
func NewContextManagerWithMemory(compressionStrategy interfaces.CompressionStrategy, maxTokens int) interfaces.ContextManager {
|
||||
func NewContextManagerWithMemory(
|
||||
compressionStrategy interfaces.CompressionStrategy,
|
||||
maxTokens int,
|
||||
) interfaces.ContextManager {
|
||||
return &contextManager{
|
||||
storage: NewMemoryStorage(),
|
||||
compressionStrategy: compressionStrategy,
|
||||
@@ -86,7 +93,12 @@ func (cm *contextManager) AddMessage(ctx context.Context, sessionID string, mess
|
||||
return fmt.Errorf("failed to save context: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[ContextManager][Session-%s] Successfully added message (total: %d messages)", sessionID, len(messages))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"[ContextManager][Session-%s] Successfully added message (total: %d messages)",
|
||||
sessionID,
|
||||
len(messages),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,11 @@ const (
|
||||
)
|
||||
|
||||
// NewContextManagerFromConfig creates a ContextManager based on configuration
|
||||
func NewContextManagerFromConfig(contextCfg *types.ContextConfig, storage ContextStorage, chatModel chat.Chat) interfaces.ContextManager {
|
||||
func NewContextManagerFromConfig(
|
||||
contextCfg *types.ContextConfig,
|
||||
storage ContextStorage,
|
||||
chatModel chat.Chat,
|
||||
) interfaces.ContextManager {
|
||||
// Use default values if config is nil
|
||||
if contextCfg == nil {
|
||||
logger.Info(context.TODO(), "ContextManager config not found, using default memory-based context manager")
|
||||
|
||||
@@ -49,7 +49,11 @@ func (s *mcpServiceService) CreateMCPService(ctx context.Context, service *types
|
||||
}
|
||||
|
||||
// GetMCPServiceByID retrieves an MCP service by ID
|
||||
func (s *mcpServiceService) GetMCPServiceByID(ctx context.Context, tenantID uint64, id string) (*types.MCPService, error) {
|
||||
func (s *mcpServiceService) GetMCPServiceByID(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
id string,
|
||||
) (*types.MCPService, error) {
|
||||
service, err := s.mcpServiceRepo.GetByID(ctx, tenantID, id)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Failed to get MCP service: %v", err)
|
||||
@@ -80,7 +84,11 @@ func (s *mcpServiceService) ListMCPServices(ctx context.Context, tenantID uint64
|
||||
}
|
||||
|
||||
// ListMCPServicesByIDs retrieves multiple MCP services by IDs
|
||||
func (s *mcpServiceService) ListMCPServicesByIDs(ctx context.Context, tenantID uint64, ids []string) ([]*types.MCPService, error) {
|
||||
func (s *mcpServiceService) ListMCPServicesByIDs(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
ids []string,
|
||||
) ([]*types.MCPService, error) {
|
||||
if len(ids) == 0 {
|
||||
return []*types.MCPService{}, nil
|
||||
}
|
||||
@@ -221,7 +229,11 @@ func (s *mcpServiceService) DeleteMCPService(ctx context.Context, tenantID uint6
|
||||
}
|
||||
|
||||
// TestMCPService tests the connection to an MCP service and returns available tools/resources
|
||||
func (s *mcpServiceService) TestMCPService(ctx context.Context, tenantID uint64, id string) (*types.MCPTestResult, error) {
|
||||
func (s *mcpServiceService) TestMCPService(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
id string,
|
||||
) (*types.MCPTestResult, error) {
|
||||
// Get service
|
||||
service, err := s.mcpServiceRepo.GetByID(ctx, tenantID, id)
|
||||
if err != nil {
|
||||
@@ -280,15 +292,23 @@ func (s *mcpServiceService) TestMCPService(ctx context.Context, tenantID uint64,
|
||||
}
|
||||
|
||||
return &types.MCPTestResult{
|
||||
Success: true,
|
||||
Message: fmt.Sprintf("Connected successfully to %s v%s", initResult.ServerInfo.Name, initResult.ServerInfo.Version),
|
||||
Success: true,
|
||||
Message: fmt.Sprintf(
|
||||
"Connected successfully to %s v%s",
|
||||
initResult.ServerInfo.Name,
|
||||
initResult.ServerInfo.Version,
|
||||
),
|
||||
Tools: tools,
|
||||
Resources: resources,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetMCPServiceTools retrieves the list of tools from an MCP service
|
||||
func (s *mcpServiceService) GetMCPServiceTools(ctx context.Context, tenantID uint64, id string) ([]*types.MCPTool, error) {
|
||||
func (s *mcpServiceService) GetMCPServiceTools(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
id string,
|
||||
) ([]*types.MCPTool, error) {
|
||||
// Get service
|
||||
service, err := s.mcpServiceRepo.GetByID(ctx, tenantID, id)
|
||||
if err != nil {
|
||||
@@ -314,7 +334,11 @@ func (s *mcpServiceService) GetMCPServiceTools(ctx context.Context, tenantID uin
|
||||
}
|
||||
|
||||
// GetMCPServiceResources retrieves the list of resources from an MCP service
|
||||
func (s *mcpServiceService) GetMCPServiceResources(ctx context.Context, tenantID uint64, id string) ([]*types.MCPResource, error) {
|
||||
func (s *mcpServiceService) GetMCPServiceResources(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
id string,
|
||||
) ([]*types.MCPResource, error) {
|
||||
// Get service
|
||||
service, err := s.mcpServiceRepo.GetByID(ctx, tenantID, id)
|
||||
if err != nil {
|
||||
|
||||
@@ -198,7 +198,10 @@ func unionLcs(evaluatedSentences []string, referenceSentence string, prevUnion *
|
||||
return newLcsCount, lcsUnion
|
||||
}
|
||||
|
||||
func rougeLSummaryLevel(evaluatedSentences, referenceSentences []string, rawResults, exclusive bool) map[string]float64 {
|
||||
func rougeLSummaryLevel(
|
||||
evaluatedSentences, referenceSentences []string,
|
||||
rawResults, exclusive bool,
|
||||
) map[string]float64 {
|
||||
referenceNgrams := NewNgrams(exclusive)
|
||||
referenceNgrams.BatchAdd(splitIntoWords(referenceSentences))
|
||||
m := referenceNgrams.Len()
|
||||
|
||||
@@ -62,7 +62,10 @@ func (c *CompositeRetrieveEngine) Retrieve(ctx context.Context,
|
||||
}
|
||||
|
||||
// NewCompositeRetrieveEngine creates a new composite retrieve engine with the given parameters
|
||||
func NewCompositeRetrieveEngine(registry interfaces.RetrieveEngineRegistry, engineParams []types.RetrieverEngineParams) (*CompositeRetrieveEngine, error) {
|
||||
func NewCompositeRetrieveEngine(
|
||||
registry interfaces.RetrieveEngineRegistry,
|
||||
engineParams []types.RetrieverEngineParams,
|
||||
) (*CompositeRetrieveEngine, error) {
|
||||
engineInfos := make(map[types.RetrieverEngineType]*engineInfo)
|
||||
for _, engineParam := range engineParams {
|
||||
repo, err := registry.GetRetrieveEngineService(engineParam.RetrieverEngineType)
|
||||
@@ -100,7 +103,10 @@ func (c *CompositeRetrieveEngine) SupportRetriever(r types.RetrieverType) bool {
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
func (c *CompositeRetrieveEngine) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
func (c *CompositeRetrieveEngine) BatchUpdateChunkEnabledStatus(
|
||||
ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap); err != nil {
|
||||
return err
|
||||
|
||||
@@ -165,6 +165,9 @@ func (v *KeywordsVectorHybridRetrieveEngineService) CopyIndices(
|
||||
}
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) BatchUpdateChunkEnabledStatus(
|
||||
ctx context.Context,
|
||||
chunkStatusMap map[string]bool,
|
||||
) error {
|
||||
return v.indexRepository.BatchUpdateChunkEnabledStatus(ctx, chunkStatusMap)
|
||||
}
|
||||
|
||||
@@ -326,7 +326,12 @@ func (s *sessionService) GenerateTitle(ctx context.Context,
|
||||
// GenerateTitleAsync generates a title for the session asynchronously
|
||||
// This method clones the session and generates the title in a goroutine
|
||||
// It emits an event when the title is generated
|
||||
func (s *sessionService) GenerateTitleAsync(ctx context.Context, session *types.Session, userQuery string, eventBus *event.EventBus) {
|
||||
func (s *sessionService) GenerateTitleAsync(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
userQuery string,
|
||||
eventBus *event.EventBus,
|
||||
) {
|
||||
// Extract values from context before cloning
|
||||
tenantID := ctx.Value(types.TenantIDContextKey)
|
||||
requestID := ctx.Value(types.RequestIDContextKey)
|
||||
@@ -384,8 +389,23 @@ func (s *sessionService) GenerateTitleAsync(ctx context.Context, session *types.
|
||||
|
||||
// KnowledgeQA performs knowledge base question answering with LLM summarization
|
||||
// Events are emitted through eventBus (references, answer chunks, completion)
|
||||
func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus) error {
|
||||
logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s, webSearchEnabled: %v", session.ID, query, webSearchEnabled)
|
||||
func (s *sessionService) KnowledgeQA(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
query string,
|
||||
knowledgeBaseIDs []string,
|
||||
assistantMessageID string,
|
||||
summaryModelID string,
|
||||
webSearchEnabled bool,
|
||||
eventBus *event.EventBus,
|
||||
) error {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge base question answering parameters, session ID: %s, query: %s, webSearchEnabled: %v",
|
||||
session.ID,
|
||||
query,
|
||||
webSearchEnabled,
|
||||
)
|
||||
|
||||
// If no knowledge base IDs provided, fall back to session's default
|
||||
if len(knowledgeBaseIDs) == 0 {
|
||||
@@ -492,7 +512,12 @@ func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session
|
||||
}
|
||||
|
||||
// Create chat management object with session settings
|
||||
logger.Infof(ctx, "Creating chat manage object, knowledge base IDs: %v, chat model ID: %s", knowledgeBaseIDs, chatModelID)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Creating chat manage object, knowledge base IDs: %v, chat model ID: %s",
|
||||
knowledgeBaseIDs,
|
||||
chatModelID,
|
||||
)
|
||||
chatManage := &types.ChatManage{
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
@@ -563,13 +588,23 @@ func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session
|
||||
// 3. First knowledge base with a Remote model
|
||||
// 4. Session's SummaryModelID (if not Remote)
|
||||
// 5. First knowledge base's SummaryModelID
|
||||
func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, session *types.Session, knowledgeBaseIDs []string, summaryModelID string) (string, error) {
|
||||
func (s *sessionService) selectChatModelIDWithOverride(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
knowledgeBaseIDs []string,
|
||||
summaryModelID string,
|
||||
) (string, error) {
|
||||
// First, check if request has summaryModelID override
|
||||
if summaryModelID != "" {
|
||||
// Validate that the model exists
|
||||
model, err := s.modelService.GetModelByID(ctx, summaryModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Request provided invalid summary model ID %s: %v, falling back to default selection", summaryModelID, err)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"Request provided invalid summary model ID %s: %v, falling back to default selection",
|
||||
summaryModelID,
|
||||
err,
|
||||
)
|
||||
} else if model != nil {
|
||||
logger.Infof(ctx, "Using request's summary model override: %s", summaryModelID)
|
||||
return summaryModelID, nil
|
||||
@@ -586,7 +621,11 @@ func (s *sessionService) selectChatModelIDWithOverride(ctx context.Context, sess
|
||||
// 2. First knowledge base with a Remote model
|
||||
// 3. Session's SummaryModelID (if not Remote)
|
||||
// 4. First knowledge base's SummaryModelID
|
||||
func (s *sessionService) selectChatModelID(ctx context.Context, session *types.Session, knowledgeBaseIDs []string) (string, error) {
|
||||
func (s *sessionService) selectChatModelID(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
knowledgeBaseIDs []string,
|
||||
) (string, error) {
|
||||
// First, check if session has a SummaryModelID and if it's a Remote model
|
||||
if session.SummaryModelID != "" {
|
||||
model, err := s.modelService.GetModelByID(ctx, session.SummaryModelID)
|
||||
@@ -630,7 +669,12 @@ func (s *sessionService) selectChatModelID(ctx context.Context, session *types.S
|
||||
return "", fmt.Errorf("failed to get knowledge base %s: %w", knowledgeBaseIDs[0], err)
|
||||
}
|
||||
if kb != nil && kb.SummaryModelID != "" {
|
||||
logger.Infof(ctx, "Using summary model from first knowledge base %s: %s", knowledgeBaseIDs[0], kb.SummaryModelID)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Using summary model from first knowledge base %s: %s",
|
||||
knowledgeBaseIDs[0],
|
||||
kb.SummaryModelID,
|
||||
)
|
||||
return kb.SummaryModelID, nil
|
||||
} else {
|
||||
logger.Errorf(ctx, "Knowledge base %s has no summary model ID", knowledgeBaseIDs[0])
|
||||
@@ -639,7 +683,9 @@ func (s *sessionService) selectChatModelID(ctx context.Context, session *types.S
|
||||
}
|
||||
|
||||
logger.Error(ctx, "No chat model ID available")
|
||||
return "", errors.New("no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID")
|
||||
return "", errors.New(
|
||||
"no chat model ID available: session has no SummaryModelID and knowledge bases have no SummaryModelID",
|
||||
)
|
||||
}
|
||||
|
||||
// KnowledgeQAByEvent processes knowledge QA through a series of events in the pipeline
|
||||
@@ -674,7 +720,12 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
|
||||
|
||||
// Handle case where search returns no results
|
||||
if err == chatpipline.ErrSearchNothing {
|
||||
logger.Warnf(ctx, "Event %v triggered, search result is empty, using fallback response, strategy: %v", eventType, chatManage.FallbackStrategy)
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"Event %v triggered, search result is empty, using fallback response, strategy: %v",
|
||||
eventType,
|
||||
chatManage.FallbackStrategy,
|
||||
)
|
||||
s.handleFallbackResponse(ctx, chatManage)
|
||||
return nil
|
||||
}
|
||||
@@ -800,7 +851,13 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
|
||||
}
|
||||
|
||||
// AgentQA performs agent-based question answering with conversation history and streaming support
|
||||
func (s *sessionService) AgentQA(ctx context.Context, session *types.Session, query string, assistantMessageID string, eventBus *event.EventBus) error {
|
||||
func (s *sessionService) AgentQA(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
query string,
|
||||
assistantMessageID string,
|
||||
eventBus *event.EventBus,
|
||||
) error {
|
||||
sessionID := session.ID
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
sessionJSON, err := json.Marshal(session)
|
||||
@@ -940,7 +997,16 @@ func (s *sessionService) AgentQA(ctx context.Context, session *types.Session, qu
|
||||
|
||||
// Create agent engine with EventBus and ContextManager
|
||||
logger.Info(ctx, "Creating agent engine")
|
||||
engine, err := s.agentService.CreateAgentEngine(ctx, agentConfig, summaryModel, rerankModel, eventBus, contextManager, session.ID, s)
|
||||
engine, err := s.agentService.CreateAgentEngine(
|
||||
ctx,
|
||||
agentConfig,
|
||||
summaryModel,
|
||||
rerankModel,
|
||||
eventBus,
|
||||
contextManager,
|
||||
session.ID,
|
||||
s,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create agent engine: %v", err)
|
||||
return err
|
||||
@@ -968,7 +1034,11 @@ func (s *sessionService) AgentQA(ctx context.Context, session *types.Session, qu
|
||||
|
||||
// getContextManagerForSession creates a context manager for the session based on configuration
|
||||
// Returns the configured context manager (tenant-level or session-level) or default
|
||||
func (s *sessionService) getContextManagerForSession(ctx context.Context, session *types.Session, chatModel chat.Chat) interfaces.ContextManager {
|
||||
func (s *sessionService) getContextManagerForSession(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
chatModel chat.Chat,
|
||||
) interfaces.ContextManager {
|
||||
// Get tenant to access global context configuration
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
// Determine which context config to use: session-specific or tenant-level
|
||||
@@ -995,7 +1065,11 @@ func (s *sessionService) getContextManagerForSession(ctx context.Context, sessio
|
||||
}
|
||||
|
||||
// getContextForSession retrieves LLM context for a session
|
||||
func (s *sessionService) getContextForSession(ctx context.Context, contextManager interfaces.ContextManager, sessionID string) ([]chat.Message, error) {
|
||||
func (s *sessionService) getContextForSession(
|
||||
ctx context.Context,
|
||||
contextManager interfaces.ContextManager,
|
||||
sessionID string,
|
||||
) ([]chat.Message, error) {
|
||||
history, err := contextManager.GetContext(ctx, sessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get context: %w", err)
|
||||
@@ -1019,7 +1093,10 @@ func (s *sessionService) ClearContext(ctx context.Context, sessionID string) err
|
||||
}
|
||||
|
||||
// GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis
|
||||
func (s *sessionService) GetWebSearchTempKBState(ctx context.Context, sessionID string) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) {
|
||||
func (s *sessionService) GetWebSearchTempKBState(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) {
|
||||
stateKey := fmt.Sprintf("tempkb:%s", sessionID)
|
||||
if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 {
|
||||
var state struct {
|
||||
@@ -1042,7 +1119,13 @@ func (s *sessionService) GetWebSearchTempKBState(ctx context.Context, sessionID
|
||||
}
|
||||
|
||||
// SaveWebSearchTempKBState saves the temporary KB state for web search to Redis
|
||||
func (s *sessionService) SaveWebSearchTempKBState(ctx context.Context, sessionID string, tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) {
|
||||
func (s *sessionService) SaveWebSearchTempKBState(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
tempKBID string,
|
||||
seenURLs map[string]bool,
|
||||
knowledgeIDs []string,
|
||||
) {
|
||||
stateKey := fmt.Sprintf("tempkb:%s", sessionID)
|
||||
state := struct {
|
||||
KBID string `json:"kbID"`
|
||||
@@ -1207,7 +1290,11 @@ func (s *sessionService) renderFallbackPrompt(ctx context.Context, chatManage *t
|
||||
}
|
||||
|
||||
// consumeFallbackStream consumes the streaming response and emits events
|
||||
func (s *sessionService) consumeFallbackStream(ctx context.Context, chatManage *types.ChatManage, responseChan <-chan types.StreamResponse) {
|
||||
func (s *sessionService) consumeFallbackStream(
|
||||
ctx context.Context,
|
||||
chatManage *types.ChatManage,
|
||||
responseChan <-chan types.StreamResponse,
|
||||
) {
|
||||
fallbackID := generateEventID("fallback")
|
||||
eventBus := chatManage.EventBus
|
||||
var finalContent string
|
||||
|
||||
@@ -69,7 +69,13 @@ func (s *knowledgeTagService) ListTags(ctx context.Context, kbID string) ([]*typ
|
||||
}
|
||||
|
||||
// CreateTag creates a new tag under a KB.
|
||||
func (s *knowledgeTagService) CreateTag(ctx context.Context, kbID string, name string, color string, sortOrder int) (*types.KnowledgeTag, error) {
|
||||
func (s *knowledgeTagService) CreateTag(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
name string,
|
||||
color string,
|
||||
sortOrder int,
|
||||
) (*types.KnowledgeTag, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if kbID == "" || name == "" {
|
||||
return nil, werrors.NewBadRequestError("知识库ID和标签名称不能为空")
|
||||
@@ -96,7 +102,13 @@ func (s *knowledgeTagService) CreateTag(ctx context.Context, kbID string, name s
|
||||
}
|
||||
|
||||
// UpdateTag updates tag basic information.
|
||||
func (s *knowledgeTagService) UpdateTag(ctx context.Context, id string, name *string, color *string, sortOrder *int) (*types.KnowledgeTag, error) {
|
||||
func (s *knowledgeTagService) UpdateTag(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
name *string,
|
||||
color *string,
|
||||
sortOrder *int,
|
||||
) (*types.KnowledgeTag, error) {
|
||||
if id == "" {
|
||||
return nil, werrors.NewBadRequestError("标签ID不能为空")
|
||||
}
|
||||
|
||||
@@ -52,7 +52,11 @@ type userService struct {
|
||||
}
|
||||
|
||||
// NewUserService creates a new user service instance
|
||||
func NewUserService(userRepo interfaces.UserRepository, tokenRepo interfaces.AuthTokenRepository, tenantService interfaces.TenantService) interfaces.UserService {
|
||||
func NewUserService(
|
||||
userRepo interfaces.UserRepository,
|
||||
tokenRepo interfaces.AuthTokenRepository,
|
||||
tenantService interfaces.TenantService,
|
||||
) interfaces.UserService {
|
||||
return &userService{
|
||||
userRepo: userRepo,
|
||||
tokenRepo: tokenRepo,
|
||||
@@ -294,7 +298,10 @@ func (s *userService) ValidatePassword(ctx context.Context, userID string, passw
|
||||
}
|
||||
|
||||
// GenerateTokens generates access and refresh tokens for user
|
||||
func (s *userService) GenerateTokens(ctx context.Context, user *types.User) (accessToken, refreshToken string, err error) {
|
||||
func (s *userService) GenerateTokens(
|
||||
ctx context.Context,
|
||||
user *types.User,
|
||||
) (accessToken, refreshToken string, err error) {
|
||||
// Generate access token (expires in 24 hours)
|
||||
accessClaims := jwt.MapClaims{
|
||||
"user_id": user.ID,
|
||||
@@ -385,7 +392,10 @@ func (s *userService) ValidateToken(ctx context.Context, tokenString string) (*t
|
||||
}
|
||||
|
||||
// RefreshToken refreshes access token using refresh token
|
||||
func (s *userService) RefreshToken(ctx context.Context, refreshTokenString string) (accessToken, newRefreshToken string, err error) {
|
||||
func (s *userService) RefreshToken(
|
||||
ctx context.Context,
|
||||
refreshTokenString string,
|
||||
) (accessToken, newRefreshToken string, err error) {
|
||||
token, err := jwt.Parse(refreshTokenString, func(token *jwt.Token) (interface{}, error) {
|
||||
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
|
||||
|
||||
@@ -55,7 +55,10 @@ func (s *WebSearchService) CompressWithRAG(
|
||||
}
|
||||
createdKB, err = kbSvc.CreateKnowledgeBase(ctx, kb)
|
||||
if err != nil {
|
||||
return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf("failed to create temporary knowledge base: %w", err)
|
||||
return nil, tempKBID, seenURLs, knowledgeIDs, fmt.Errorf(
|
||||
"failed to create temporary knowledge base: %w",
|
||||
err,
|
||||
)
|
||||
}
|
||||
tempKBID = createdKB.ID
|
||||
}
|
||||
@@ -240,7 +243,11 @@ func stripMarker(content string) string {
|
||||
|
||||
// Search performs web search using the specified provider
|
||||
// This method implements the interface expected by PluginSearch
|
||||
func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) {
|
||||
func (s *WebSearchService) Search(
|
||||
ctx context.Context,
|
||||
config *types.WebSearchConfig,
|
||||
query string,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("web search config is required")
|
||||
}
|
||||
@@ -310,7 +317,10 @@ func NewWebSearchService(cfg *config.Config) (interfaces.WebSearchService, error
|
||||
}
|
||||
|
||||
// filterBlacklist filters results based on blacklist rules
|
||||
func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult {
|
||||
func (s *WebSearchService) filterBlacklist(
|
||||
results []*types.WebSearchResult,
|
||||
blacklist []string,
|
||||
) []*types.WebSearchResult {
|
||||
if len(blacklist) == 0 {
|
||||
return results
|
||||
}
|
||||
|
||||
@@ -38,7 +38,12 @@ func (p *DuckDuckGoProvider) Name() string {
|
||||
}
|
||||
|
||||
// Search performs a web search using DuckDuckGo HTML endpoint with API fallback
|
||||
func (p *DuckDuckGoProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) {
|
||||
func (p *DuckDuckGoProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
maxResults int,
|
||||
includeDate bool,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
if maxResults <= 0 {
|
||||
maxResults = 5
|
||||
}
|
||||
@@ -58,7 +63,11 @@ func (p *DuckDuckGoProvider) Search(ctx context.Context, query string, maxResult
|
||||
return nil, fmt.Errorf("duckduckgo API search failed: %w", apiErr)
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoProvider) searchHTML(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) {
|
||||
func (p *DuckDuckGoProvider) searchHTML(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
maxResults int,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
baseURL := "https://html.duckduckgo.com/html/"
|
||||
params := url.Values{}
|
||||
params.Set("q", query)
|
||||
@@ -71,10 +80,16 @@ func (p *DuckDuckGoProvider) searchHTML(ctx context.Context, query string, maxRe
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
// Use a realistic UA to avoid blocks
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set(
|
||||
"User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
)
|
||||
|
||||
// print curl of request
|
||||
curlCommand := fmt.Sprintf("curl -X GET '%s' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'", req.URL.String())
|
||||
curlCommand := fmt.Sprintf(
|
||||
"curl -X GET '%s' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'",
|
||||
req.URL.String(),
|
||||
)
|
||||
logger.Infof(ctx, "Curl of request: %s", secutils.SanitizeForLog(curlCommand))
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
@@ -119,7 +134,11 @@ func (p *DuckDuckGoProvider) searchHTML(ctx context.Context, query string, maxRe
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoProvider) searchAPI(ctx context.Context, query string, maxResults int) ([]*types.WebSearchResult, error) {
|
||||
func (p *DuckDuckGoProvider) searchAPI(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
maxResults int,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
baseURL := "https://api.duckduckgo.com/"
|
||||
params := url.Values{}
|
||||
params.Set("q", query)
|
||||
|
||||
@@ -95,10 +95,12 @@ func TestDuckDuckGoProvider_Search_HTMLSuccess(t *testing.T) {
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("expected 2 results, got %d", len(results))
|
||||
}
|
||||
if results[0].Title != "Example One" || !strings.HasPrefix(results[0].URL, "https://example.com/") || results[0].Snippet != "Snippet one" {
|
||||
if results[0].Title != "Example One" || !strings.HasPrefix(results[0].URL, "https://example.com/") ||
|
||||
results[0].Snippet != "Snippet one" {
|
||||
t.Fatalf("unexpected first result: %+v", results[0])
|
||||
}
|
||||
if results[1].Title != "Example Two" || !strings.HasPrefix(results[1].URL, "https://example.org/") || results[1].Snippet != "Snippet two" {
|
||||
if results[1].Title != "Example Two" || !strings.HasPrefix(results[1].URL, "https://example.org/") ||
|
||||
results[1].Snippet != "Snippet two" {
|
||||
t.Fatalf("unexpected second result: %+v", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,16 +14,16 @@ import (
|
||||
|
||||
// Config 应用程序总配置
|
||||
type Config struct {
|
||||
Conversation *ConversationConfig `yaml:"conversation" json:"conversation"`
|
||||
Server *ServerConfig `yaml:"server" json:"server"`
|
||||
KnowledgeBase *KnowledgeBaseConfig `yaml:"knowledge_base" json:"knowledge_base"`
|
||||
Tenant *TenantConfig `yaml:"tenant" json:"tenant"`
|
||||
Models []ModelConfig `yaml:"models" json:"models"`
|
||||
Conversation *ConversationConfig `yaml:"conversation" json:"conversation"`
|
||||
Server *ServerConfig `yaml:"server" json:"server"`
|
||||
KnowledgeBase *KnowledgeBaseConfig `yaml:"knowledge_base" json:"knowledge_base"`
|
||||
Tenant *TenantConfig `yaml:"tenant" json:"tenant"`
|
||||
Models []ModelConfig `yaml:"models" json:"models"`
|
||||
VectorDatabase *VectorDatabaseConfig `yaml:"vector_database" json:"vector_database"`
|
||||
DocReader *DocReaderConfig `yaml:"docreader" json:"docreader"`
|
||||
StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"`
|
||||
ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"`
|
||||
WebSearch *WebSearchConfig `yaml:"web_search" json:"web_search"`
|
||||
DocReader *DocReaderConfig `yaml:"docreader" json:"docreader"`
|
||||
StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"`
|
||||
ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"`
|
||||
WebSearch *WebSearchConfig `yaml:"web_search" json:"web_search"`
|
||||
}
|
||||
|
||||
type DocReaderConfig struct {
|
||||
@@ -36,59 +36,59 @@ type VectorDatabaseConfig struct {
|
||||
|
||||
// ConversationConfig 对话服务配置
|
||||
type ConversationConfig struct {
|
||||
MaxRounds int `yaml:"max_rounds" json:"max_rounds"`
|
||||
KeywordThreshold float64 `yaml:"keyword_threshold" json:"keyword_threshold"`
|
||||
EmbeddingTopK int `yaml:"embedding_top_k" json:"embedding_top_k"`
|
||||
VectorThreshold float64 `yaml:"vector_threshold" json:"vector_threshold"`
|
||||
RerankTopK int `yaml:"rerank_top_k" json:"rerank_top_k"`
|
||||
RerankThreshold float64 `yaml:"rerank_threshold" json:"rerank_threshold"`
|
||||
FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"`
|
||||
FallbackResponse string `yaml:"fallback_response" json:"fallback_response"`
|
||||
FallbackPrompt string `yaml:"fallback_prompt" json:"fallback_prompt"`
|
||||
EnableRewrite bool `yaml:"enable_rewrite" json:"enable_rewrite"`
|
||||
EnableQueryExpansion bool `yaml:"enable_query_expansion" json:"enable_query_expansion"`
|
||||
EnableRerank bool `yaml:"enable_rerank" json:"enable_rerank"`
|
||||
Summary *SummaryConfig `yaml:"summary" json:"summary"`
|
||||
MaxRounds int `yaml:"max_rounds" json:"max_rounds"`
|
||||
KeywordThreshold float64 `yaml:"keyword_threshold" json:"keyword_threshold"`
|
||||
EmbeddingTopK int `yaml:"embedding_top_k" json:"embedding_top_k"`
|
||||
VectorThreshold float64 `yaml:"vector_threshold" json:"vector_threshold"`
|
||||
RerankTopK int `yaml:"rerank_top_k" json:"rerank_top_k"`
|
||||
RerankThreshold float64 `yaml:"rerank_threshold" json:"rerank_threshold"`
|
||||
FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"`
|
||||
FallbackResponse string `yaml:"fallback_response" json:"fallback_response"`
|
||||
FallbackPrompt string `yaml:"fallback_prompt" json:"fallback_prompt"`
|
||||
EnableRewrite bool `yaml:"enable_rewrite" json:"enable_rewrite"`
|
||||
EnableQueryExpansion bool `yaml:"enable_query_expansion" json:"enable_query_expansion"`
|
||||
EnableRerank bool `yaml:"enable_rerank" json:"enable_rerank"`
|
||||
Summary *SummaryConfig `yaml:"summary" json:"summary"`
|
||||
GenerateSessionTitlePrompt string `yaml:"generate_session_title_prompt" json:"generate_session_title_prompt"`
|
||||
GenerateSummaryPrompt string `yaml:"generate_summary_prompt" json:"generate_summary_prompt"`
|
||||
RewritePromptSystem string `yaml:"rewrite_prompt_system" json:"rewrite_prompt_system"`
|
||||
RewritePromptUser string `yaml:"rewrite_prompt_user" json:"rewrite_prompt_user"`
|
||||
SimplifyQueryPrompt string `yaml:"simplify_query_prompt" json:"simplify_query_prompt"`
|
||||
SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"`
|
||||
ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"`
|
||||
ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"`
|
||||
GenerateSummaryPrompt string `yaml:"generate_summary_prompt" json:"generate_summary_prompt"`
|
||||
RewritePromptSystem string `yaml:"rewrite_prompt_system" json:"rewrite_prompt_system"`
|
||||
RewritePromptUser string `yaml:"rewrite_prompt_user" json:"rewrite_prompt_user"`
|
||||
SimplifyQueryPrompt string `yaml:"simplify_query_prompt" json:"simplify_query_prompt"`
|
||||
SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"`
|
||||
ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"`
|
||||
ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"`
|
||||
}
|
||||
|
||||
// SummaryConfig 摘要配置
|
||||
type SummaryConfig struct {
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||
RepeatPenalty float64 `yaml:"repeat_penalty" json:"repeat_penalty"`
|
||||
TopK int `yaml:"top_k" json:"top_k"`
|
||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||
FrequencyPenalty float64 `yaml:"frequency_penalty" json:"frequency_penalty"`
|
||||
PresencePenalty float64 `yaml:"presence_penalty" json:"presence_penalty"`
|
||||
Prompt string `yaml:"prompt" json:"prompt"`
|
||||
ContextTemplate string `yaml:"context_template" json:"context_template"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
Seed int `yaml:"seed" json:"seed"`
|
||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
||||
RepeatPenalty float64 `yaml:"repeat_penalty" json:"repeat_penalty"`
|
||||
TopK int `yaml:"top_k" json:"top_k"`
|
||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
||||
FrequencyPenalty float64 `yaml:"frequency_penalty" json:"frequency_penalty"`
|
||||
PresencePenalty float64 `yaml:"presence_penalty" json:"presence_penalty"`
|
||||
Prompt string `yaml:"prompt" json:"prompt"`
|
||||
ContextTemplate string `yaml:"context_template" json:"context_template"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
Seed int `yaml:"seed" json:"seed"`
|
||||
MaxCompletionTokens int `yaml:"max_completion_tokens" json:"max_completion_tokens"`
|
||||
NoMatchPrefix string `yaml:"no_match_prefix" json:"no_match_prefix"`
|
||||
NoMatchPrefix string `yaml:"no_match_prefix" json:"no_match_prefix"`
|
||||
}
|
||||
|
||||
// ServerConfig 服务器配置
|
||||
type ServerConfig struct {
|
||||
Port int `yaml:"port" json:"port"`
|
||||
Host string `yaml:"host" json:"host"`
|
||||
LogPath string `yaml:"log_path" json:"log_path"`
|
||||
Port int `yaml:"port" json:"port"`
|
||||
Host string `yaml:"host" json:"host"`
|
||||
LogPath string `yaml:"log_path" json:"log_path"`
|
||||
ShutdownTimeout time.Duration `yaml:"shutdown_timeout" json:"shutdown_timeout" default:"30s"`
|
||||
}
|
||||
|
||||
// KnowledgeBaseConfig 知识库配置
|
||||
type KnowledgeBaseConfig struct {
|
||||
ChunkSize int `yaml:"chunk_size" json:"chunk_size"`
|
||||
ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"`
|
||||
SplitMarkers []string `yaml:"split_markers" json:"split_markers"`
|
||||
KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"`
|
||||
ChunkSize int `yaml:"chunk_size" json:"chunk_size"`
|
||||
ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"`
|
||||
SplitMarkers []string `yaml:"split_markers" json:"split_markers"`
|
||||
KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"`
|
||||
ImageProcessing *ImageProcessingConfig `yaml:"image_processing" json:"image_processing"`
|
||||
}
|
||||
|
||||
@@ -99,44 +99,44 @@ type ImageProcessingConfig struct {
|
||||
|
||||
// TenantConfig 租户配置
|
||||
type TenantConfig struct {
|
||||
DefaultSessionName string `yaml:"default_session_name" json:"default_session_name"`
|
||||
DefaultSessionTitle string `yaml:"default_session_title" json:"default_session_title"`
|
||||
DefaultSessionName string `yaml:"default_session_name" json:"default_session_name"`
|
||||
DefaultSessionTitle string `yaml:"default_session_title" json:"default_session_title"`
|
||||
DefaultSessionDescription string `yaml:"default_session_description" json:"default_session_description"`
|
||||
}
|
||||
|
||||
// ModelConfig 模型配置
|
||||
type ModelConfig struct {
|
||||
Type string `yaml:"type" json:"type"`
|
||||
Source string `yaml:"source" json:"source"`
|
||||
Type string `yaml:"type" json:"type"`
|
||||
Source string `yaml:"source" json:"source"`
|
||||
ModelName string `yaml:"model_name" json:"model_name"`
|
||||
Parameters map[string]interface{} `yaml:"parameters" json:"parameters"`
|
||||
}
|
||||
|
||||
// StreamManagerConfig 流管理器配置
|
||||
type StreamManagerConfig struct {
|
||||
Type string `yaml:"type" json:"type"` // 类型: "memory" 或 "redis"
|
||||
Redis RedisConfig `yaml:"redis" json:"redis"` // Redis配置
|
||||
Type string `yaml:"type" json:"type"` // 类型: "memory" 或 "redis"
|
||||
Redis RedisConfig `yaml:"redis" json:"redis"` // Redis配置
|
||||
CleanupTimeout time.Duration `yaml:"cleanup_timeout" json:"cleanup_timeout"` // 清理超时,单位秒
|
||||
}
|
||||
|
||||
// RedisConfig Redis配置
|
||||
type RedisConfig struct {
|
||||
Address string `yaml:"address" json:"address"` // Redis地址
|
||||
Address string `yaml:"address" json:"address"` // Redis地址
|
||||
Password string `yaml:"password" json:"password"` // Redis密码
|
||||
DB int `yaml:"db" json:"db"` // Redis数据库
|
||||
Prefix string `yaml:"prefix" json:"prefix"` // 键前缀
|
||||
TTL time.Duration `yaml:"ttl" json:"ttl"` // 过期时间(小时)
|
||||
DB int `yaml:"db" json:"db"` // Redis数据库
|
||||
Prefix string `yaml:"prefix" json:"prefix"` // 键前缀
|
||||
TTL time.Duration `yaml:"ttl" json:"ttl"` // 过期时间(小时)
|
||||
}
|
||||
|
||||
// ExtractManagerConfig 抽取管理器配置
|
||||
type ExtractManagerConfig struct {
|
||||
ExtractGraph *types.PromptTemplateStructured `yaml:"extract_graph" json:"extract_graph"`
|
||||
ExtractGraph *types.PromptTemplateStructured `yaml:"extract_graph" json:"extract_graph"`
|
||||
ExtractEntity *types.PromptTemplateStructured `yaml:"extract_entity" json:"extract_entity"`
|
||||
FabriText *FebriText `yaml:"fabri_text" json:"fabri_text"`
|
||||
FabriText *FebriText `yaml:"fabri_text" json:"fabri_text"`
|
||||
}
|
||||
|
||||
type FebriText struct {
|
||||
WithTag string `yaml:"with_tag" json:"with_tag"`
|
||||
WithTag string `yaml:"with_tag" json:"with_tag"`
|
||||
WithNoTag string `yaml:"with_no_tag" json:"with_no_tag"`
|
||||
}
|
||||
|
||||
@@ -194,25 +194,25 @@ func LoadConfig() (*Config, error) {
|
||||
// WebSearchConfig represents the web search configuration
|
||||
type WebSearchConfig struct {
|
||||
Providers []WebSearchProviderConfig `yaml:"providers" json:"providers"`
|
||||
Default WebSearchDefaultConfig `yaml:"default" json:"default"`
|
||||
Timeout int `yaml:"timeout" json:"timeout"` // 超时时间(秒)
|
||||
Default WebSearchDefaultConfig `yaml:"default" json:"default"`
|
||||
Timeout int `yaml:"timeout" json:"timeout"` // 超时时间(秒)
|
||||
}
|
||||
|
||||
// WebSearchProviderConfig represents configuration for a web search provider
|
||||
type WebSearchProviderConfig struct {
|
||||
ID string `yaml:"id" json:"id"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Free bool `yaml:"free" json:"free"`
|
||||
RequiresAPIKey bool `yaml:"requires_api_key" json:"requires_api_key"`
|
||||
ID string `yaml:"id" json:"id"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Free bool `yaml:"free" json:"free"`
|
||||
RequiresAPIKey bool `yaml:"requires_api_key" json:"requires_api_key"`
|
||||
Description string `yaml:"description,omitempty" json:"description,omitempty"`
|
||||
APIURL string `yaml:"api_url,omitempty" json:"api_url,omitempty"`
|
||||
APIURL string `yaml:"api_url,omitempty" json:"api_url,omitempty"`
|
||||
}
|
||||
|
||||
// WebSearchDefaultConfig represents the default web search configuration
|
||||
type WebSearchDefaultConfig struct {
|
||||
Provider string `yaml:"provider" json:"provider"`
|
||||
MaxResults int `yaml:"max_results" json:"max_results"`
|
||||
IncludeDate bool `yaml:"include_date" json:"include_date"`
|
||||
Provider string `yaml:"provider" json:"provider"`
|
||||
MaxResults int `yaml:"max_results" json:"max_results"`
|
||||
IncludeDate bool `yaml:"include_date" json:"include_date"`
|
||||
CompressionMethod string `yaml:"compression_method" json:"compression_method"`
|
||||
Blacklist []string `yaml:"blacklist" json:"blacklist"`
|
||||
Blacklist []string `yaml:"blacklist" json:"blacklist"`
|
||||
}
|
||||
|
||||
@@ -282,7 +282,10 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
||||
if err := database.RunMigrations(migrateDSN); err != nil {
|
||||
// Log warning but don't fail startup - migrations might be handled externally
|
||||
logger.Warnf(context.Background(), "Database migration failed: %v", err)
|
||||
logger.Warnf(context.Background(), "Continuing with application startup. Please run migrations manually if needed.")
|
||||
logger.Warnf(
|
||||
context.Background(),
|
||||
"Continuing with application startup. Please run migrations manually if needed.",
|
||||
)
|
||||
}
|
||||
} else {
|
||||
logger.Infof(context.Background(), "Auto-migration is disabled (AUTO_MIGRATE=false)")
|
||||
|
||||
@@ -189,7 +189,10 @@ func (h *ChunkHandler) validateAndGetChunk(c *gin.Context) (*types.Chunk, string
|
||||
logger.Warnf(
|
||||
ctx,
|
||||
"Tenant has no permission to access chunk, knowledge ID: %s, chunk ID: %s, req tenant: %d, chunk tenant: %d",
|
||||
knowledgeID, id, tenantID, chunk.TenantID,
|
||||
knowledgeID,
|
||||
id,
|
||||
tenantID,
|
||||
chunk.TenantID,
|
||||
)
|
||||
return nil, knowledgeID, errors.NewForbiddenError("No permission to access this chunk")
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ func NewInitializationHandler(
|
||||
|
||||
// KBModelConfigRequest 知识库模型配置请求(简化版,只传模型ID)
|
||||
type KBModelConfigRequest struct {
|
||||
LLMModelID string `json:"llmModelId" binding:"required"`
|
||||
LLMModelID string `json:"llmModelId" binding:"required"`
|
||||
EmbeddingModelID string `json:"embeddingModelId" binding:"required"`
|
||||
VLMConfig *types.VLMConfig `json:"vlm_config"`
|
||||
|
||||
@@ -357,7 +357,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
|
||||
c.Error(errors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", utils.SanitizeForLog(kbIdStr), utils.SanitizeForLog(utils.ToJSON(req)))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Starting knowledge base configuration update, kbId: %s, request: %s",
|
||||
utils.SanitizeForLog(kbIdStr),
|
||||
utils.SanitizeForLog(utils.ToJSON(req)),
|
||||
)
|
||||
|
||||
// 获取指定知识库信息
|
||||
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
|
||||
@@ -1253,7 +1258,7 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models
|
||||
// RemoteModelCheckRequest 远程模型检查请求结构
|
||||
type RemoteModelCheckRequest struct {
|
||||
ModelName string `json:"modelName" binding:"required"`
|
||||
BaseURL string `json:"baseUrl" binding:"required"`
|
||||
BaseURL string `json:"baseUrl" binding:"required"`
|
||||
APIKey string `json:"apiKey"`
|
||||
}
|
||||
|
||||
@@ -1764,8 +1769,8 @@ func (h *InitializationHandler) testMultimodalWithDocReader(
|
||||
|
||||
// TextRelationExtractionRequest 文本关系提取请求结构
|
||||
type TextRelationExtractionRequest struct {
|
||||
Text string `json:"text" binding:"required"`
|
||||
Tags []string `json:"tags" binding:"required"`
|
||||
Text string `json:"text" binding:"required"`
|
||||
Tags []string `json:"tags" binding:"required"`
|
||||
LLMConfig LLMConfig `json:"llmConfig"`
|
||||
}
|
||||
|
||||
@@ -1824,7 +1829,12 @@ func (h *InitializationHandler) ExtractTextRelations(c *gin.Context) {
|
||||
}
|
||||
|
||||
// extractRelationsFromText 从文本中提取关系
|
||||
func (h *InitializationHandler) extractRelationsFromText(ctx context.Context, text string, tags []string, llm LLMConfig) (*TextRelationExtractionResponse, error) {
|
||||
func (h *InitializationHandler) extractRelationsFromText(
|
||||
ctx context.Context,
|
||||
text string,
|
||||
tags []string,
|
||||
llm LLMConfig,
|
||||
) (*TextRelationExtractionResponse, error) {
|
||||
chatModel, err := chat.NewChat(&chat.ChatConfig{
|
||||
ModelID: "initialization",
|
||||
APIKey: llm.ApiKey,
|
||||
|
||||
@@ -155,7 +155,12 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge created successfully, ID: %s, title: %s", secutils.SanitizeForLog(knowledge.ID), secutils.SanitizeForLog(knowledge.Title))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge created successfully, ID: %s, title: %s",
|
||||
secutils.SanitizeForLog(knowledge.ID),
|
||||
secutils.SanitizeForLog(knowledge.Title),
|
||||
)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": knowledge,
|
||||
@@ -186,7 +191,12 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Received URL request: %s", secutils.SanitizeForLog(req.URL))
|
||||
logger.Infof(ctx, "Creating knowledge from URL, knowledge base ID: %s, URL: %s", secutils.SanitizeForLog(kbID), secutils.SanitizeForLog(req.URL))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Creating knowledge from URL, knowledge base ID: %s, URL: %s",
|
||||
secutils.SanitizeForLog(kbID),
|
||||
secutils.SanitizeForLog(req.URL),
|
||||
)
|
||||
|
||||
// Create knowledge entry from the URL
|
||||
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel)
|
||||
@@ -200,7 +210,12 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge created successfully from URL, ID: %s, title: %s", secutils.SanitizeForLog(knowledge.ID), secutils.SanitizeForLog(knowledge.Title))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge created successfully from URL, ID: %s, title: %s",
|
||||
secutils.SanitizeForLog(knowledge.ID),
|
||||
secutils.SanitizeForLog(knowledge.Title),
|
||||
)
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
"success": true,
|
||||
"data": knowledge,
|
||||
@@ -268,7 +283,12 @@ func (h *KnowledgeHandler) GetKnowledge(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, title: %s", secutils.SanitizeForLog(knowledge.ID), secutils.SanitizeForLog(knowledge.Title))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge retrieved successfully, ID: %s, title: %s",
|
||||
secutils.SanitizeForLog(knowledge.ID),
|
||||
secutils.SanitizeForLog(knowledge.Title),
|
||||
)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": knowledge,
|
||||
@@ -299,8 +319,14 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
|
||||
|
||||
tagID := c.Query("tag_id")
|
||||
|
||||
logger.Infof(ctx, "Retrieving knowledge list under knowledge base, knowledge base ID: %s, tag_id: %s, page: %d, page size: %d",
|
||||
secutils.SanitizeForLog(kbID), secutils.SanitizeForLog(tagID), pagination.Page, pagination.PageSize)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Retrieving knowledge list under knowledge base, knowledge base ID: %s, tag_id: %s, page: %d, page size: %d",
|
||||
secutils.SanitizeForLog(kbID),
|
||||
secutils.SanitizeForLog(tagID),
|
||||
pagination.Page,
|
||||
pagination.PageSize,
|
||||
)
|
||||
|
||||
// Retrieve paginated knowledge entries
|
||||
result, err := h.kgService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbID, &pagination, tagID)
|
||||
@@ -310,7 +336,12 @@ func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge list retrieved successfully, knowledge base ID: %s, total: %d", secutils.SanitizeForLog(kbID), result.Total)
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge list retrieved successfully, knowledge base ID: %s, total: %d",
|
||||
secutils.SanitizeForLog(kbID),
|
||||
result.Total,
|
||||
)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": result.Data,
|
||||
@@ -374,7 +405,12 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) {
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
logger.Infof(ctx, "Knowledge file retrieved successfully, ID: %s, filename: %s", secutils.SanitizeForLog(id), secutils.SanitizeForLog(filename))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge file retrieved successfully, ID: %s, filename: %s",
|
||||
secutils.SanitizeForLog(id),
|
||||
secutils.SanitizeForLog(filename),
|
||||
)
|
||||
|
||||
// Set response headers for file download
|
||||
c.Header("Content-Description", "File Transfer")
|
||||
|
||||
@@ -178,9 +178,9 @@ func (h *KnowledgeBaseHandler) ListKnowledgeBases(c *gin.Context) {
|
||||
|
||||
// UpdateKnowledgeBaseRequest defines the request body structure for updating a knowledge base
|
||||
type UpdateKnowledgeBaseRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
Description string `json:"description"`
|
||||
Config *types.KnowledgeBaseConfig `json:"config" binding:"required"`
|
||||
Config *types.KnowledgeBaseConfig `json:"config" binding:"required"`
|
||||
}
|
||||
|
||||
// UpdateKnowledgeBase handles requests to update an existing knowledge base
|
||||
|
||||
@@ -61,11 +61,11 @@ func hideSensitiveInfo(model *types.Model) *types.Model {
|
||||
// CreateModelRequest defines the structure for model creation requests
|
||||
// Contains all fields required to create a new model in the system
|
||||
type CreateModelRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Type types.ModelType `json:"type" binding:"required"`
|
||||
Source types.ModelSource `json:"source" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
Type types.ModelType `json:"type" binding:"required"`
|
||||
Source types.ModelSource `json:"source" binding:"required"`
|
||||
Description string `json:"description"`
|
||||
Parameters types.ModelParameters `json:"parameters" binding:"required"`
|
||||
Parameters types.ModelParameters `json:"parameters" binding:"required"`
|
||||
}
|
||||
|
||||
// CreateModel handles the HTTP request to create a new model
|
||||
@@ -109,7 +109,12 @@ func (h *ModelHandler) CreateModel(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Model created successfully, ID: %s, Name: %s", secutils.SanitizeForLog(model.ID), secutils.SanitizeForLog(model.Name))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Model created successfully, ID: %s, Name: %s",
|
||||
secutils.SanitizeForLog(model.ID),
|
||||
secutils.SanitizeForLog(model.Name),
|
||||
)
|
||||
|
||||
// Hide sensitive information for builtin models (though newly created models are unlikely to be builtin)
|
||||
responseModel := hideSensitiveInfo(model)
|
||||
|
||||
@@ -104,7 +104,12 @@ func (h *Handler) KnowledgeQA(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Knowledge QA request, session ID: %s, query: %s", sessionID, secutils.SanitizeForLog(request.Query))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Knowledge QA request, session ID: %s, query: %s",
|
||||
sessionID,
|
||||
secutils.SanitizeForLog(request.Query),
|
||||
)
|
||||
|
||||
// Get session to prepare knowledge base IDs
|
||||
session, err := h.sessionService.GetSession(ctx, sessionID)
|
||||
@@ -118,7 +123,11 @@ func (h *Handler) KnowledgeQA(c *gin.Context) {
|
||||
knowledgeBaseIDs := request.KnowledgeBaseIDs
|
||||
if len(knowledgeBaseIDs) == 0 && session.KnowledgeBaseID != "" {
|
||||
knowledgeBaseIDs = []string{session.KnowledgeBaseID}
|
||||
logger.Infof(ctx, "No knowledge base IDs in request, using session default: %s", secutils.SanitizeForLog(session.KnowledgeBaseID))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"No knowledge base IDs in request, using session default: %s",
|
||||
secutils.SanitizeForLog(session.KnowledgeBaseID),
|
||||
)
|
||||
}
|
||||
|
||||
// Use shared function to handle KnowledgeQA request
|
||||
@@ -284,21 +293,42 @@ func (h *Handler) AgentQA(c *gin.Context) {
|
||||
// If still empty, use session default knowledge base
|
||||
if len(knowledgeBaseIDs) == 0 && session.KnowledgeBaseID != "" {
|
||||
knowledgeBaseIDs = []string{session.KnowledgeBaseID}
|
||||
logger.Infof(ctx, "Using session default knowledge base: %s", secutils.SanitizeForLog(session.KnowledgeBaseID))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Using session default knowledge base: %s",
|
||||
secutils.SanitizeForLog(session.KnowledgeBaseID),
|
||||
)
|
||||
}
|
||||
|
||||
// Validate at least one knowledge base is available
|
||||
if len(knowledgeBaseIDs) == 0 {
|
||||
logger.Error(ctx, "No knowledge base available for delegation")
|
||||
c.Error(errors.NewBadRequestError("No knowledge base available. Please configure at least one knowledge base."))
|
||||
c.Error(
|
||||
errors.NewBadRequestError("No knowledge base available. Please configure at least one knowledge base."),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Delegating to KnowledgeQA with knowledge bases: %s", secutils.SanitizeForLog(fmt.Sprintf("%v", knowledgeBaseIDs)))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Delegating to KnowledgeQA with knowledge bases: %s",
|
||||
secutils.SanitizeForLog(fmt.Sprintf("%v", knowledgeBaseIDs)),
|
||||
)
|
||||
|
||||
// Use shared function to handle KnowledgeQA request (no title generation for AgentQA fallback)
|
||||
h.handleKnowledgeQARequest(ctx, c, session, secutils.SanitizeForLog(request.Query),
|
||||
secutils.SanitizeForLogArray(knowledgeBaseIDs), assistantMessage, false, secutils.SanitizeForLog(request.SummaryModelID), request.WebSearchEnabled)
|
||||
h.handleKnowledgeQARequest(
|
||||
ctx,
|
||||
c,
|
||||
session,
|
||||
secutils.SanitizeForLog(request.Query),
|
||||
secutils.SanitizeForLogArray(
|
||||
knowledgeBaseIDs,
|
||||
),
|
||||
assistantMessage,
|
||||
false,
|
||||
secutils.SanitizeForLog(request.SummaryModelID),
|
||||
request.WebSearchEnabled,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -369,7 +399,13 @@ func (h *Handler) AgentQA(c *gin.Context) {
|
||||
h.completeAssistantMessage(asyncCtx, assistantMessage)
|
||||
logger.Infof(asyncCtx, "Agent QA service completed for session: %s", sessionID)
|
||||
}()
|
||||
err := h.sessionService.AgentQA(asyncCtx, session, secutils.SanitizeForLog(request.Query), assistantMessage.ID, eventBus)
|
||||
err := h.sessionService.AgentQA(
|
||||
asyncCtx,
|
||||
session,
|
||||
secutils.SanitizeForLog(request.Query),
|
||||
assistantMessage.ID,
|
||||
eventBus,
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(asyncCtx, err, nil)
|
||||
// Emit error event to dedicated EventBus
|
||||
@@ -479,10 +515,23 @@ func (h *Handler) handleKnowledgeQARequest(
|
||||
if r := recover(); r != nil {
|
||||
buf := make([]byte, 10240)
|
||||
runtime.Stack(buf, true)
|
||||
logger.ErrorWithFields(asyncCtx, errors.NewInternalServerError(fmt.Sprintf("Knowledge QA service panicked: %v\n%s", r, string(buf))), nil)
|
||||
logger.ErrorWithFields(
|
||||
asyncCtx,
|
||||
errors.NewInternalServerError(fmt.Sprintf("Knowledge QA service panicked: %v\n%s", r, string(buf))),
|
||||
nil,
|
||||
)
|
||||
}
|
||||
}()
|
||||
err := h.sessionService.KnowledgeQA(asyncCtx, session, query, knowledgeBaseIDs, assistantMessage.ID, summaryModelID, webSearchEnabled, eventBus)
|
||||
err := h.sessionService.KnowledgeQA(
|
||||
asyncCtx,
|
||||
session,
|
||||
query,
|
||||
knowledgeBaseIDs,
|
||||
assistantMessage.ID,
|
||||
summaryModelID,
|
||||
webSearchEnabled,
|
||||
eventBus,
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(asyncCtx, err, nil)
|
||||
// Emit error event to dedicated EventBus
|
||||
|
||||
@@ -290,7 +290,11 @@ func (h *Handler) handleAgentEventsForSSE(
|
||||
select {
|
||||
case <-c.Request.Context().Done():
|
||||
// Connection closed, exit gracefully without panic
|
||||
log.Infof("Client disconnected, stopping SSE streaming for session=%s, message=%s", sessionID, assistantMessageID)
|
||||
log.Infof(
|
||||
"Client disconnected, stopping SSE streaming for session=%s, message=%s",
|
||||
sessionID,
|
||||
assistantMessageID,
|
||||
)
|
||||
return
|
||||
|
||||
case <-ticker.C:
|
||||
|
||||
@@ -53,16 +53,16 @@ type GenerateTitleRequest struct {
|
||||
|
||||
// CreateKnowledgeQARequest defines the request structure for knowledge QA
|
||||
type CreateKnowledgeQARequest struct {
|
||||
Query string `json:"query" binding:"required"` // Query text for knowledge base search
|
||||
KnowledgeBaseIDs []string `json:"knowledge_base_ids"` // Selected knowledge base ID for this request
|
||||
AgentEnabled bool `json:"agent_enabled"` // Whether agent mode is enabled for this request
|
||||
WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this request
|
||||
SummaryModelID string `json:"summary_model_id"` // Optional summary model ID for this request (overrides session default)
|
||||
Query string `json:"query" binding:"required"` // Query text for knowledge base search
|
||||
KnowledgeBaseIDs []string `json:"knowledge_base_ids"` // Selected knowledge base ID for this request
|
||||
AgentEnabled bool `json:"agent_enabled"` // Whether agent mode is enabled for this request
|
||||
WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this request
|
||||
SummaryModelID string `json:"summary_model_id"` // Optional summary model ID for this request (overrides session default)
|
||||
}
|
||||
|
||||
// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization
|
||||
type SearchKnowledgeRequest struct {
|
||||
Query string `json:"query" binding:"required"` // Query text to search for
|
||||
Query string `json:"query" binding:"required"` // Query text to search for
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" binding:"required"` // ID of the knowledge base to search
|
||||
}
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ func (h *TagHandler) ListTags(c *gin.Context) {
|
||||
}
|
||||
|
||||
type createTagRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
Color string `json:"color"`
|
||||
SortOrder int `json:"sort_order"`
|
||||
}
|
||||
|
||||
@@ -70,7 +70,12 @@ func (h *TenantHandler) CreateTenant(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Tenant created successfully, ID: %d, name: %s", createdTenant.ID, secutils.SanitizeForLog(createdTenant.Name))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Tenant created successfully, ID: %d, name: %s",
|
||||
createdTenant.ID,
|
||||
secutils.SanitizeForLog(createdTenant.Name),
|
||||
)
|
||||
c.JSON(http.StatusCreated, gin.H{
|
||||
"success": true,
|
||||
"data": createdTenant,
|
||||
@@ -151,7 +156,12 @@ func (h *TenantHandler) UpdateTenant(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Tenant updated successfully, ID: %d, Name: %s", updatedTenant.ID, secutils.SanitizeForLog(updatedTenant.Name))
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Tenant updated successfully, ID: %d, Name: %s",
|
||||
updatedTenant.ID,
|
||||
secutils.SanitizeForLog(updatedTenant.Name),
|
||||
)
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": updatedTenant,
|
||||
|
||||
@@ -38,7 +38,11 @@ func isNoAuthAPI(path string, method string) bool {
|
||||
}
|
||||
|
||||
// Auth 认证中间件
|
||||
func Auth(tenantService interfaces.TenantService, userService interfaces.UserService, cfg *config.Config) gin.HandlerFunc {
|
||||
func Auth(
|
||||
tenantService interfaces.TenantService,
|
||||
userService interfaces.UserService,
|
||||
cfg *config.Config,
|
||||
) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// ignore OPTIONS request
|
||||
if c.Request.Method == "OPTIONS" {
|
||||
|
||||
@@ -263,7 +263,11 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
}
|
||||
|
||||
// chatWithQwen 使用自定义请求处理 qwen 模型
|
||||
func (c *RemoteAPIChat) chatWithQwen(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||
func (c *RemoteAPIChat) chatWithQwen(
|
||||
ctx context.Context,
|
||||
messages []Message,
|
||||
opts *ChatOptions,
|
||||
) (*types.ChatResponse, error) {
|
||||
// 构建 qwen 请求参数
|
||||
req := c.buildQwenChatCompletionRequest(messages, opts, false)
|
||||
|
||||
|
||||
@@ -103,7 +103,8 @@ func (e *OpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte
|
||||
if backoffTime > 10*time.Second {
|
||||
backoffTime = 10 * time.Second
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("OpenAIEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
|
||||
logger.GetLogger(ctx).
|
||||
Infof("OpenAIEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime)
|
||||
|
||||
select {
|
||||
case <-time.After(backoffTime):
|
||||
|
||||
@@ -88,7 +88,10 @@ type RerankerConfig struct {
|
||||
// NewReranker creates a reranker
|
||||
func NewReranker(config *RerankerConfig) (Reranker, error) {
|
||||
// 根据URL判断模型来源,而不是依赖Source字段
|
||||
if strings.Contains(config.BaseURL, "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank") {
|
||||
if strings.Contains(
|
||||
config.BaseURL,
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank",
|
||||
) {
|
||||
return NewAliyunReranker(config)
|
||||
} else {
|
||||
return NewOpenAIReranker(config)
|
||||
|
||||
@@ -186,7 +186,8 @@ func (s *OllamaService) EnsureModelAvailable(ctx context.Context, modelName stri
|
||||
available, err := s.IsModelAvailable(ctx, modelName)
|
||||
if err != nil {
|
||||
if s.isOptional {
|
||||
logger.GetLogger(ctx).Warnf("Failed to check model %s availability, but Ollama is set as optional", modelName)
|
||||
logger.GetLogger(ctx).
|
||||
Warnf("Failed to check model %s availability, but Ollama is set as optional", modelName)
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
|
||||
@@ -60,7 +60,11 @@ func (m *MemoryStreamManager) getStream(sessionID, messageID string) *memoryStre
|
||||
}
|
||||
|
||||
// AppendEvent appends a single event to the stream
|
||||
func (m *MemoryStreamManager) AppendEvent(ctx context.Context, sessionID, messageID string, event interfaces.StreamEvent) error {
|
||||
func (m *MemoryStreamManager) AppendEvent(
|
||||
ctx context.Context,
|
||||
sessionID, messageID string,
|
||||
event interfaces.StreamEvent,
|
||||
) error {
|
||||
stream := m.getOrCreateStream(sessionID, messageID)
|
||||
|
||||
stream.mu.Lock()
|
||||
@@ -80,7 +84,11 @@ func (m *MemoryStreamManager) AppendEvent(ctx context.Context, sessionID, messag
|
||||
|
||||
// GetEvents gets events starting from offset
|
||||
// Returns: events slice, next offset, error
|
||||
func (m *MemoryStreamManager) GetEvents(ctx context.Context, sessionID, messageID string, fromOffset int) ([]interfaces.StreamEvent, int, error) {
|
||||
func (m *MemoryStreamManager) GetEvents(
|
||||
ctx context.Context,
|
||||
sessionID, messageID string,
|
||||
fromOffset int,
|
||||
) ([]interfaces.StreamEvent, int, error) {
|
||||
stream := m.getStream(sessionID, messageID)
|
||||
if stream == nil {
|
||||
// Stream doesn't exist yet
|
||||
|
||||
@@ -54,7 +54,11 @@ func (r *RedisStreamManager) buildKey(sessionID, messageID string) string {
|
||||
}
|
||||
|
||||
// AppendEvent appends a single event to the stream using Redis RPush
|
||||
func (r *RedisStreamManager) AppendEvent(ctx context.Context, sessionID, messageID string, event interfaces.StreamEvent) error {
|
||||
func (r *RedisStreamManager) AppendEvent(
|
||||
ctx context.Context,
|
||||
sessionID, messageID string,
|
||||
event interfaces.StreamEvent,
|
||||
) error {
|
||||
key := r.buildKey(sessionID, messageID)
|
||||
|
||||
// Set timestamp if not already set
|
||||
@@ -83,7 +87,11 @@ func (r *RedisStreamManager) AppendEvent(ctx context.Context, sessionID, message
|
||||
|
||||
// GetEvents gets events starting from offset using Redis LRange
|
||||
// Returns: events slice, next offset, error
|
||||
func (r *RedisStreamManager) GetEvents(ctx context.Context, sessionID, messageID string, fromOffset int) ([]interfaces.StreamEvent, int, error) {
|
||||
func (r *RedisStreamManager) GetEvents(
|
||||
ctx context.Context,
|
||||
sessionID, messageID string,
|
||||
fromOffset int,
|
||||
) ([]interfaces.StreamEvent, int, error) {
|
||||
key := r.buildKey(sessionID, messageID)
|
||||
|
||||
// Get all events from offset to end using LRange
|
||||
|
||||
@@ -44,7 +44,7 @@ const (
|
||||
// ImageInfo 表示与 Chunk 关联的图片信息
|
||||
type ImageInfo struct {
|
||||
// 图片URL(COS)
|
||||
URL string `json:"url" gorm:"type:text"`
|
||||
URL string `json:"url" gorm:"type:text"`
|
||||
// 原始图片URL
|
||||
OriginalURL string `json:"original_url" gorm:"type:text"`
|
||||
// 图片在文本中的开始位置
|
||||
@@ -65,7 +65,7 @@ type ImageInfo struct {
|
||||
// Chunks can be independently embedded as vectors and retrieved, supporting precise content localization
|
||||
type Chunk struct {
|
||||
// Unique identifier of the chunk, using UUID format
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Tenant ID, used for multi-tenant isolation
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
// ID of the parent knowledge, associated with the Knowledge model
|
||||
@@ -73,17 +73,17 @@ type Chunk struct {
|
||||
// ID of the knowledge base, for quick location
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
// Optional tag ID for categorization within a knowledge base (used for FAQ)
|
||||
TagID string `json:"tag_id" gorm:"type:varchar(36);index"`
|
||||
TagID string `json:"tag_id" gorm:"type:varchar(36);index"`
|
||||
// Actual text content of the chunk
|
||||
Content string `json:"content"`
|
||||
// Index position of the chunk in the original document
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
// Whether the chunk is enabled, can be used to temporarily disable certain chunks
|
||||
IsEnabled bool `json:"is_enabled" gorm:"default:true"`
|
||||
IsEnabled bool `json:"is_enabled" gorm:"default:true"`
|
||||
// Status of the chunk
|
||||
Status int `json:"status" gorm:"default:0"`
|
||||
Status int `json:"status" gorm:"default:0"`
|
||||
// Starting character position in the original text
|
||||
StartAt int `json:"start_at" `
|
||||
StartAt int `json:"start_at"`
|
||||
// Ending character position in the original text
|
||||
EndAt int `json:"end_at"`
|
||||
// Previous chunk ID
|
||||
@@ -91,23 +91,23 @@ type Chunk struct {
|
||||
// Next chunk ID
|
||||
NextChunkID string `json:"next_chunk_id"`
|
||||
// Chunk 类型,用于区分不同类型的 Chunk
|
||||
ChunkType ChunkType `json:"chunk_type" gorm:"type:varchar(20);default:'text'"`
|
||||
ChunkType ChunkType `json:"chunk_type" gorm:"type:varchar(20);default:'text'"`
|
||||
// 父 Chunk ID,用于关联图片 Chunk 和原始文本 Chunk
|
||||
ParentChunkID string `json:"parent_chunk_id" gorm:"type:varchar(36);index"`
|
||||
ParentChunkID string `json:"parent_chunk_id" gorm:"type:varchar(36);index"`
|
||||
// 关系 Chunk ID,用于关联关系 Chunk 和原始文本 Chunk
|
||||
RelationChunks JSON `json:"relation_chunks" gorm:"type:json"`
|
||||
RelationChunks JSON `json:"relation_chunks" gorm:"type:json"`
|
||||
// 间接关系 Chunk ID,用于关联间接关系 Chunk 和原始文本 Chunk
|
||||
IndirectRelationChunks JSON `json:"indirect_relation_chunks" gorm:"type:json"`
|
||||
// Metadata 存储 chunk 级别的扩展信息,例如 FAQ 元数据
|
||||
Metadata JSON `json:"metadata" gorm:"type:json"`
|
||||
Metadata JSON `json:"metadata" gorm:"type:json"`
|
||||
// ContentHash 存储内容的 hash 值,用于快速匹配(主要用于 FAQ)
|
||||
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
|
||||
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
|
||||
// 图片信息,存储为 JSON
|
||||
ImageInfo string `json:"image_info" gorm:"type:text"`
|
||||
ImageInfo string `json:"image_info" gorm:"type:text"`
|
||||
// Chunk creation time
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Chunk last update time
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
// Soft delete marker, supports data recovery
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
@@ -129,10 +129,10 @@ type FAQEntry struct {
|
||||
|
||||
// FAQEntryPayload 用于创建/更新 FAQ 条目的 payload
|
||||
type FAQEntryPayload struct {
|
||||
StandardQuestion string `json:"standard_question" binding:"required"`
|
||||
StandardQuestion string `json:"standard_question" binding:"required"`
|
||||
SimilarQuestions []string `json:"similar_questions"`
|
||||
NegativeQuestions []string `json:"negative_questions"`
|
||||
Answers []string `json:"answers" binding:"required"`
|
||||
Answers []string `json:"answers" binding:"required"`
|
||||
TagID string `json:"tag_id"`
|
||||
IsEnabled *bool `json:"is_enabled,omitempty"`
|
||||
}
|
||||
@@ -144,14 +144,14 @@ const (
|
||||
|
||||
// FAQBatchUpsertPayload 批量导入 FAQ 条目
|
||||
type FAQBatchUpsertPayload struct {
|
||||
Entries []FAQEntryPayload `json:"entries" binding:"required"`
|
||||
Mode string `json:"mode" binding:"oneof=append replace"`
|
||||
Entries []FAQEntryPayload `json:"entries" binding:"required"`
|
||||
Mode string `json:"mode" binding:"oneof=append replace"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
}
|
||||
|
||||
// FAQSearchRequest FAQ检索请求参数
|
||||
type FAQSearchRequest struct {
|
||||
QueryText string `json:"query_text" binding:"required"`
|
||||
QueryText string `json:"query_text" binding:"required"`
|
||||
VectorThreshold float64 `json:"vector_threshold"`
|
||||
MatchCount int `json:"match_count"`
|
||||
}
|
||||
|
||||
@@ -21,13 +21,26 @@ type AgentStreamEvent struct {
|
||||
// AgentEngine defines the interface for agent execution engine
|
||||
type AgentEngine interface {
|
||||
// Execute executes the agent with conversation history and returns a stream of events
|
||||
Execute(ctx context.Context, sessionID, messageID, query string, llmContext []chat.Message) (*types.AgentState, error)
|
||||
Execute(
|
||||
ctx context.Context,
|
||||
sessionID, messageID, query string,
|
||||
llmContext []chat.Message,
|
||||
) (*types.AgentState, error)
|
||||
}
|
||||
|
||||
// AgentService defines the interface for agent-related operations
|
||||
type AgentService interface {
|
||||
// CreateAgentEngine creates an agent engine with the given configuration, EventBus, and ContextManager
|
||||
CreateAgentEngine(ctx context.Context, config *types.AgentConfig, chatModel chat.Chat, rerankModel rerank.Reranker, eventBus *event.EventBus, contextManager ContextManager, sessionID string, sessionService SessionService) (AgentEngine, error)
|
||||
CreateAgentEngine(
|
||||
ctx context.Context,
|
||||
config *types.AgentConfig,
|
||||
chatModel chat.Chat,
|
||||
rerankModel rerank.Reranker,
|
||||
eventBus *event.EventBus,
|
||||
contextManager ContextManager,
|
||||
sessionID string,
|
||||
sessionService SessionService,
|
||||
) (AgentEngine, error)
|
||||
|
||||
// ValidateConfig validates an agent configuration
|
||||
ValidateConfig(config *types.AgentConfig) error
|
||||
|
||||
@@ -21,13 +21,22 @@ type KnowledgeService interface {
|
||||
customFileName string,
|
||||
) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromURL creates knowledge from a URL.
|
||||
CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool) (*types.Knowledge, error)
|
||||
CreateKnowledgeFromURL(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
url string,
|
||||
enableMultimodel *bool,
|
||||
) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromPassage creates knowledge from text passages.
|
||||
CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromPassageSync creates knowledge from text passages and waits until chunks are indexed.
|
||||
CreateKnowledgeFromPassageSync(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
|
||||
// CreateKnowledgeFromManual creates or saves manual Markdown knowledge content.
|
||||
CreateKnowledgeFromManual(ctx context.Context, kbID string, payload *types.ManualKnowledgePayload) (*types.Knowledge, error)
|
||||
CreateKnowledgeFromManual(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
payload *types.ManualKnowledgePayload,
|
||||
) (*types.Knowledge, error)
|
||||
// GetKnowledgeByID retrieves knowledge by ID.
|
||||
GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error)
|
||||
// GetKnowledgeBatch retrieves a batch of knowledge by IDs.
|
||||
@@ -49,7 +58,11 @@ type KnowledgeService interface {
|
||||
// UpdateKnowledge updates knowledge information.
|
||||
UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error
|
||||
// UpdateManualKnowledge updates manual Markdown knowledge content.
|
||||
UpdateManualKnowledge(ctx context.Context, knowledgeID string, payload *types.ManualKnowledgePayload) (*types.Knowledge, error)
|
||||
UpdateManualKnowledge(
|
||||
ctx context.Context,
|
||||
knowledgeID string,
|
||||
payload *types.ManualKnowledgePayload,
|
||||
) (*types.Knowledge, error)
|
||||
// CloneKnowledgeBase clones knowledge to another knowledge base.
|
||||
CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error
|
||||
// UpdateImageInfo updates image information for a knowledge chunk.
|
||||
|
||||
@@ -40,13 +40,28 @@ type SessionService interface {
|
||||
SearchKnowledge(ctx context.Context, knowledgeBaseID, query string) ([]*types.SearchResult, error)
|
||||
// AgentQA performs agent-based question answering with conversation history and streaming support
|
||||
// eventBus is optional - if nil, uses service's default EventBus
|
||||
AgentQA(ctx context.Context, session *types.Session, query string, assistantMessageID string, eventBus *event.EventBus) error
|
||||
AgentQA(
|
||||
ctx context.Context,
|
||||
session *types.Session,
|
||||
query string,
|
||||
assistantMessageID string,
|
||||
eventBus *event.EventBus,
|
||||
) error
|
||||
// ClearContext clears the LLM context for a session
|
||||
ClearContext(ctx context.Context, sessionID string) error
|
||||
// GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis
|
||||
GetWebSearchTempKBState(ctx context.Context, sessionID string) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string)
|
||||
GetWebSearchTempKBState(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string)
|
||||
// SaveWebSearchTempKBState saves the temporary KB state for web search to Redis
|
||||
SaveWebSearchTempKBState(ctx context.Context, sessionID string, tempKBID string, seenURLs map[string]bool, knowledgeIDs []string)
|
||||
SaveWebSearchTempKBState(
|
||||
ctx context.Context,
|
||||
sessionID string,
|
||||
tempKBID string,
|
||||
seenURLs map[string]bool,
|
||||
knowledgeIDs []string,
|
||||
)
|
||||
// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis and cleans up associated knowledge base and knowledge items
|
||||
DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error
|
||||
}
|
||||
|
||||
@@ -26,5 +26,10 @@ type KnowledgeTagRepository interface {
|
||||
ListByKB(ctx context.Context, tenantID uint64, kbID string) ([]*types.KnowledgeTag, error)
|
||||
Delete(ctx context.Context, tenantID uint64, id string) error
|
||||
// CountReferences returns number of knowledges and chunks that reference the tag.
|
||||
CountReferences(ctx context.Context, tenantID uint64, kbID string, tagID string) (knowledgeCount int64, chunkCount int64, err error)
|
||||
CountReferences(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
tagID string,
|
||||
) (knowledgeCount int64, chunkCount int64, err error)
|
||||
}
|
||||
|
||||
@@ -28,13 +28,13 @@ const (
|
||||
// and references to the physical file if applicable.
|
||||
type Knowledge struct {
|
||||
// Unique identifier of the knowledge
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Tenant ID
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
// ID of the knowledge base
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
// Optional tag ID for categorization within a knowledge base
|
||||
TagID string `json:"tag_id" gorm:"type:varchar(36);index"`
|
||||
TagID string `json:"tag_id" gorm:"type:varchar(36);index"`
|
||||
// Type of the knowledge
|
||||
Type string `json:"type"`
|
||||
// Title of the knowledge
|
||||
@@ -62,7 +62,7 @@ type Knowledge struct {
|
||||
// Storage size of the knowledge
|
||||
StorageSize int64 `json:"storage_size"`
|
||||
// Metadata of the knowledge
|
||||
Metadata JSON `json:"metadata" gorm:"type:json"`
|
||||
Metadata JSON `json:"metadata" gorm:"type:json"`
|
||||
// Creation time of the knowledge
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Last updated time of the knowledge
|
||||
@@ -72,7 +72,7 @@ type Knowledge struct {
|
||||
// Error message of the knowledge
|
||||
ErrorMessage string `json:"error_message"`
|
||||
// Deletion time of the knowledge
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// GetMetadata returns the metadata as a map[string]string.
|
||||
|
||||
@@ -38,85 +38,85 @@ const (
|
||||
// KnowledgeBase represents a knowledge base entity
|
||||
type KnowledgeBase struct {
|
||||
// Unique identifier of the knowledge base
|
||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Name of the knowledge base
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
// Type of the knowledge base (document, faq, etc.)
|
||||
Type string `yaml:"type" json:"type" gorm:"type:varchar(32);default:'document'"`
|
||||
Type string `yaml:"type" json:"type" gorm:"type:varchar(32);default:'document'"`
|
||||
// Whether this knowledge base is temporary (ephemeral) and should be hidden from UI
|
||||
IsTemporary bool `yaml:"is_temporary" json:"is_temporary" gorm:"default:false"`
|
||||
IsTemporary bool `yaml:"is_temporary" json:"is_temporary" gorm:"default:false"`
|
||||
// Description of the knowledge base
|
||||
Description string `yaml:"description" json:"description"`
|
||||
Description string `yaml:"description" json:"description"`
|
||||
// Tenant ID
|
||||
TenantID uint64 `yaml:"tenant_id" json:"tenant_id"`
|
||||
TenantID uint64 `yaml:"tenant_id" json:"tenant_id"`
|
||||
// Chunking configuration
|
||||
ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config" gorm:"type:json"`
|
||||
ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config" gorm:"type:json"`
|
||||
// Image processing configuration
|
||||
ImageProcessingConfig ImageProcessingConfig `yaml:"image_processing_config" json:"image_processing_config" gorm:"type:json"`
|
||||
// ID of the embedding model
|
||||
EmbeddingModelID string `yaml:"embedding_model_id" json:"embedding_model_id"`
|
||||
EmbeddingModelID string `yaml:"embedding_model_id" json:"embedding_model_id"`
|
||||
// Summary model ID
|
||||
SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"`
|
||||
SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"`
|
||||
// VLM config
|
||||
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
|
||||
VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"`
|
||||
// Storage config
|
||||
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
|
||||
StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"`
|
||||
// Extract config
|
||||
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
|
||||
ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"`
|
||||
// FAQConfig stores FAQ specific configuration such as indexing strategy
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config" gorm:"column:faq_config;type:json"`
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config" gorm:"column:faq_config;type:json"`
|
||||
// Creation time of the knowledge base
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time of the knowledge base
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
// Deletion time of the knowledge base
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
// Knowledge count (not stored in database, calculated on query)
|
||||
KnowledgeCount int64 `yaml:"knowledge_count" json:"knowledge_count" gorm:"-"`
|
||||
KnowledgeCount int64 `yaml:"knowledge_count" json:"knowledge_count" gorm:"-"`
|
||||
// Chunk count (not stored in database, calculated on query)
|
||||
ChunkCount int64 `yaml:"chunk_count" json:"chunk_count" gorm:"-"`
|
||||
ChunkCount int64 `yaml:"chunk_count" json:"chunk_count" gorm:"-"`
|
||||
// IsProcessing indicates if there is a processing import task (for FAQ type knowledge bases)
|
||||
IsProcessing bool `yaml:"is_processing" json:"is_processing" gorm:"-"`
|
||||
IsProcessing bool `yaml:"is_processing" json:"is_processing" gorm:"-"`
|
||||
// ProcessingCount indicates the number of knowledge items being processed (for document type knowledge bases)
|
||||
ProcessingCount int64 `yaml:"processing_count" json:"processing_count" gorm:"-"`
|
||||
ProcessingCount int64 `yaml:"processing_count" json:"processing_count" gorm:"-"`
|
||||
}
|
||||
|
||||
// KnowledgeBaseConfig represents the knowledge base configuration
|
||||
type KnowledgeBaseConfig struct {
|
||||
// Chunking configuration
|
||||
ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config"`
|
||||
ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config"`
|
||||
// Image processing configuration
|
||||
ImageProcessingConfig ImageProcessingConfig `yaml:"image_processing_config" json:"image_processing_config"`
|
||||
// FAQ configuration (only for FAQ type knowledge bases)
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config"`
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config"`
|
||||
}
|
||||
|
||||
// ChunkingConfig represents the document splitting configuration
|
||||
type ChunkingConfig struct {
|
||||
// Chunk size
|
||||
ChunkSize int `yaml:"chunk_size" json:"chunk_size"`
|
||||
ChunkSize int `yaml:"chunk_size" json:"chunk_size"`
|
||||
// Chunk overlap
|
||||
ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"`
|
||||
// Separators
|
||||
Separators []string `yaml:"separators" json:"separators"`
|
||||
Separators []string `yaml:"separators" json:"separators"`
|
||||
}
|
||||
|
||||
// COSConfig represents the COS configuration
|
||||
type StorageConfig struct {
|
||||
// Secret ID
|
||||
SecretID string `yaml:"secret_id" json:"secret_id"`
|
||||
SecretID string `yaml:"secret_id" json:"secret_id"`
|
||||
// Secret Key
|
||||
SecretKey string `yaml:"secret_key" json:"secret_key"`
|
||||
SecretKey string `yaml:"secret_key" json:"secret_key"`
|
||||
// Region
|
||||
Region string `yaml:"region" json:"region"`
|
||||
Region string `yaml:"region" json:"region"`
|
||||
// Bucket Name
|
||||
BucketName string `yaml:"bucket_name" json:"bucket_name"`
|
||||
// App ID
|
||||
AppID string `yaml:"app_id" json:"app_id"`
|
||||
AppID string `yaml:"app_id" json:"app_id"`
|
||||
// Path Prefix
|
||||
PathPrefix string `yaml:"path_prefix" json:"path_prefix"`
|
||||
// Provider
|
||||
Provider string `yaml:"provider" json:"provider"`
|
||||
Provider string `yaml:"provider" json:"provider"`
|
||||
}
|
||||
|
||||
func (c StorageConfig) Value() (driver.Value, error) {
|
||||
@@ -176,7 +176,7 @@ func (c *ImageProcessingConfig) Scan(value interface{}) error {
|
||||
|
||||
// VLMConfig represents the VLM configuration
|
||||
type VLMConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
ModelID string `yaml:"model_id" json:"model_id"`
|
||||
}
|
||||
|
||||
@@ -199,10 +199,10 @@ func (c *VLMConfig) Scan(value interface{}) error {
|
||||
|
||||
// ExtractConfig represents the extract configuration for a knowledge base
|
||||
type ExtractConfig struct {
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Text string `yaml:"text" json:"text,omitempty"`
|
||||
Tags []string `yaml:"tags" json:"tags,omitempty"`
|
||||
Nodes []*GraphNode `yaml:"nodes" json:"nodes,omitempty"`
|
||||
Enabled bool `yaml:"enabled" json:"enabled"`
|
||||
Text string `yaml:"text" json:"text,omitempty"`
|
||||
Tags []string `yaml:"tags" json:"tags,omitempty"`
|
||||
Nodes []*GraphNode `yaml:"nodes" json:"nodes,omitempty"`
|
||||
Relations []*GraphRelation `yaml:"relations" json:"relations,omitempty"`
|
||||
}
|
||||
|
||||
@@ -225,7 +225,7 @@ func (e *ExtractConfig) Scan(value interface{}) error {
|
||||
|
||||
// FAQConfig 存储 FAQ 知识库的特有配置
|
||||
type FAQConfig struct {
|
||||
IndexMode FAQIndexMode `yaml:"index_mode" json:"index_mode"`
|
||||
IndexMode FAQIndexMode `yaml:"index_mode" json:"index_mode"`
|
||||
QuestionIndexMode FAQQuestionIndexMode `yaml:"question_index_mode" json:"question_index_mode"`
|
||||
}
|
||||
|
||||
|
||||
@@ -20,21 +20,21 @@ const (
|
||||
|
||||
// MCPService represents an MCP (Model Context Protocol) service configuration
|
||||
type MCPService struct {
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null"`
|
||||
Description string `json:"description" gorm:"type:text"`
|
||||
Enabled bool `json:"enabled" gorm:"default:true;index"`
|
||||
TransportType MCPTransportType `json:"transport_type" gorm:"type:varchar(50);not null"`
|
||||
URL *string `json:"url,omitempty" gorm:"type:varchar(512)"` // Optional: required for SSE/HTTP Streamable
|
||||
Headers MCPHeaders `json:"headers" gorm:"type:json"`
|
||||
AuthConfig *MCPAuthConfig `json:"auth_config" gorm:"type:json"`
|
||||
AdvancedConfig *MCPAdvancedConfig `json:"advanced_config" gorm:"type:json"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null"`
|
||||
Description string `json:"description" gorm:"type:text"`
|
||||
Enabled bool `json:"enabled" gorm:"default:true;index"`
|
||||
TransportType MCPTransportType `json:"transport_type" gorm:"type:varchar(50);not null"`
|
||||
URL *string `json:"url,omitempty" gorm:"type:varchar(512)"` // Optional: required for SSE/HTTP Streamable
|
||||
Headers MCPHeaders `json:"headers" gorm:"type:json"`
|
||||
AuthConfig *MCPAuthConfig `json:"auth_config" gorm:"type:json"`
|
||||
AdvancedConfig *MCPAdvancedConfig `json:"advanced_config" gorm:"type:json"`
|
||||
StdioConfig *MCPStdioConfig `json:"stdio_config,omitempty" gorm:"type:json"` // Required for stdio transport
|
||||
EnvVars MCPEnvVars `json:"env_vars,omitempty" gorm:"type:json"` // Environment variables for stdio
|
||||
EnvVars MCPEnvVars `json:"env_vars,omitempty" gorm:"type:json"` // Environment variables for stdio
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// MCPHeaders represents HTTP headers as a map
|
||||
|
||||
@@ -25,7 +25,7 @@ type History struct {
|
||||
// Messages can contain references to knowledge chunks used to generate responses
|
||||
type Message struct {
|
||||
// Unique identifier for the message
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// ID of the session this message belongs to
|
||||
SessionID string `json:"session_id"`
|
||||
// Request identifier for tracking API requests
|
||||
@@ -35,7 +35,7 @@ type Message struct {
|
||||
// Message role: "user", "assistant", "system"
|
||||
Role string `json:"role"`
|
||||
// References to knowledge chunks used in the response
|
||||
KnowledgeReferences References `json:"knowledge_references" gorm:"type:json,column:knowledge_references"`
|
||||
KnowledgeReferences References `json:"knowledge_references" gorm:"type:json,column:knowledge_references"`
|
||||
// Agent execution steps (only for assistant messages generated by agent)
|
||||
// This contains the detailed reasoning process and tool calls made by the agent
|
||||
// Stored for user history display, but NOT included in LLM context to avoid redundancy
|
||||
@@ -47,7 +47,7 @@ type Message struct {
|
||||
// Last update timestamp
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
// Soft delete timestamp
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// AgentSteps represents a collection of agent execution steps
|
||||
|
||||
@@ -39,46 +39,46 @@ const (
|
||||
|
||||
// EmbeddingParameters represents the embedding parameters for a model
|
||||
type EmbeddingParameters struct {
|
||||
Dimension int `yaml:"dimension" json:"dimension"`
|
||||
Dimension int `yaml:"dimension" json:"dimension"`
|
||||
TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"`
|
||||
}
|
||||
|
||||
type ModelParameters struct {
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
InterfaceType string `yaml:"interface_type" json:"interface_type"`
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
InterfaceType string `yaml:"interface_type" json:"interface_type"`
|
||||
EmbeddingParameters EmbeddingParameters `yaml:"embedding_parameters" json:"embedding_parameters"`
|
||||
ParameterSize string `yaml:"parameter_size" json:"parameter_size"` // Ollama model parameter size (e.g., "7B", "13B", "70B")
|
||||
ParameterSize string `yaml:"parameter_size" json:"parameter_size"` // Ollama model parameter size (e.g., "7B", "13B", "70B")
|
||||
}
|
||||
|
||||
// Model represents the AI model
|
||||
type Model struct {
|
||||
// Unique identifier of the model
|
||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Tenant ID
|
||||
TenantID uint64 `yaml:"tenant_id" json:"tenant_id"`
|
||||
TenantID uint64 `yaml:"tenant_id" json:"tenant_id"`
|
||||
// Name of the model
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
// Type of the model
|
||||
Type ModelType `yaml:"type" json:"type"`
|
||||
Type ModelType `yaml:"type" json:"type"`
|
||||
// Source of the model
|
||||
Source ModelSource `yaml:"source" json:"source"`
|
||||
Source ModelSource `yaml:"source" json:"source"`
|
||||
// Description of the model
|
||||
Description string `yaml:"description" json:"description"`
|
||||
// Model parameters in JSON format
|
||||
Parameters ModelParameters `yaml:"parameters" json:"parameters" gorm:"type:json"`
|
||||
Parameters ModelParameters `yaml:"parameters" json:"parameters" gorm:"type:json"`
|
||||
// Whether the model is the default model
|
||||
IsDefault bool `yaml:"is_default" json:"is_default"`
|
||||
IsDefault bool `yaml:"is_default" json:"is_default"`
|
||||
// Whether the model is a builtin model (visible to all tenants)
|
||||
IsBuiltin bool `yaml:"is_builtin" json:"is_builtin" gorm:"default:false"`
|
||||
IsBuiltin bool `yaml:"is_builtin" json:"is_builtin" gorm:"default:false"`
|
||||
// Model status, default: active, possible: downloading, download_failed
|
||||
Status ModelStatus `yaml:"status" json:"status"`
|
||||
Status ModelStatus `yaml:"status" json:"status"`
|
||||
// Creation time of the model
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time of the model
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
// Deletion time of the model
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert ModelParameters to database value
|
||||
|
||||
@@ -48,7 +48,7 @@ type RetrieverEngineParams struct {
|
||||
// Retriever engine type
|
||||
RetrieverEngineType RetrieverEngineType `yaml:"retriever_engine_type" json:"retriever_engine_type"`
|
||||
// Retriever type
|
||||
RetrieverType RetrieverType `yaml:"retriever_type" json:"retriever_type"`
|
||||
RetrieverType RetrieverType `yaml:"retriever_type" json:"retriever_type"`
|
||||
}
|
||||
|
||||
// IndexWithScore represents the index with score
|
||||
|
||||
@@ -8,29 +8,29 @@ import (
|
||||
// SearchResult represents the search result
|
||||
type SearchResult struct {
|
||||
// ID
|
||||
ID string `gorm:"column:id" json:"id"`
|
||||
ID string `gorm:"column:id" json:"id"`
|
||||
// Content
|
||||
Content string `gorm:"column:content" json:"content"`
|
||||
Content string `gorm:"column:content" json:"content"`
|
||||
// Knowledge ID
|
||||
KnowledgeID string `gorm:"column:knowledge_id" json:"knowledge_id"`
|
||||
KnowledgeID string `gorm:"column:knowledge_id" json:"knowledge_id"`
|
||||
// Chunk index
|
||||
ChunkIndex int `gorm:"column:chunk_index" json:"chunk_index"`
|
||||
ChunkIndex int `gorm:"column:chunk_index" json:"chunk_index"`
|
||||
// Knowledge title
|
||||
KnowledgeTitle string `gorm:"column:knowledge_title" json:"knowledge_title"`
|
||||
// Start at
|
||||
StartAt int `gorm:"column:start_at" json:"start_at"`
|
||||
StartAt int `gorm:"column:start_at" json:"start_at"`
|
||||
// End at
|
||||
EndAt int `gorm:"column:end_at" json:"end_at"`
|
||||
EndAt int `gorm:"column:end_at" json:"end_at"`
|
||||
// Seq
|
||||
Seq int `gorm:"column:seq" json:"seq"`
|
||||
Seq int `gorm:"column:seq" json:"seq"`
|
||||
// Score
|
||||
Score float64 `json:"score"`
|
||||
Score float64 ` json:"score"`
|
||||
// Match type
|
||||
MatchType MatchType `json:"match_type"`
|
||||
MatchType MatchType ` json:"match_type"`
|
||||
// SubChunkIndex
|
||||
SubChunkID []string `json:"sub_chunk_id"`
|
||||
SubChunkID []string ` json:"sub_chunk_id"`
|
||||
// Metadata
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
Metadata map[string]string ` json:"metadata"`
|
||||
|
||||
// Chunk 类型
|
||||
ChunkType string `json:"chunk_type"`
|
||||
@@ -78,7 +78,7 @@ func (c *SearchResult) Scan(value interface{}) error {
|
||||
// Pagination represents the pagination parameters
|
||||
type Pagination struct {
|
||||
// Page
|
||||
Page int `form:"page" json:"page" binding:"omitempty,min=1"`
|
||||
Page int `form:"page" json:"page" binding:"omitempty,min=1"`
|
||||
// Page size
|
||||
PageSize int `form:"page_size" json:"page_size" binding:"omitempty,min=1,max=100"`
|
||||
}
|
||||
|
||||
@@ -72,30 +72,30 @@ type ContextConfig struct {
|
||||
// Session represents the session
|
||||
type Session struct {
|
||||
// ID
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Title
|
||||
Title string `json:"title"`
|
||||
// Description
|
||||
Description string `json:"description"`
|
||||
// Tenant ID
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
|
||||
// Strategy configuration
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"` // 关联的知识库ID
|
||||
MaxRounds int `json:"max_rounds"` // 多轮保持轮数
|
||||
EnableRewrite bool `json:"enable_rewrite"` // 多轮改写开关
|
||||
FallbackStrategy FallbackStrategy `json:"fallback_strategy"` // 兜底策略
|
||||
FallbackResponse string `json:"fallback_response"` // 固定回复内容
|
||||
EmbeddingTopK int `json:"embedding_top_k"` // 向量召回TopK
|
||||
KeywordThreshold float64 `json:"keyword_threshold"` // 关键词召回阈值
|
||||
VectorThreshold float64 `json:"vector_threshold"` // 向量召回阈值
|
||||
RerankModelID string `json:"rerank_model_id"` // 排序模型ID
|
||||
RerankTopK int `json:"rerank_top_k"` // 排序TopK
|
||||
RerankThreshold float64 `json:"rerank_threshold"` // 排序阈值
|
||||
SummaryModelID string `json:"summary_model_id"` // 总结模型ID
|
||||
SummaryParameters *SummaryConfig `json:"summary_parameters" gorm:"type:json"` // 总结模型参数
|
||||
AgentConfig *SessionAgentConfig `json:"agent_config" gorm:"type:jsonb"` // Agent 配置(会话级别,仅存储enabled和knowledge_bases)
|
||||
ContextConfig *ContextConfig `json:"context_config" gorm:"type:jsonb"` // 上下文管理配置(可选)
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"` // 关联的知识库ID
|
||||
MaxRounds int `json:"max_rounds"` // 多轮保持轮数
|
||||
EnableRewrite bool `json:"enable_rewrite"` // 多轮改写开关
|
||||
FallbackStrategy FallbackStrategy `json:"fallback_strategy"` // 兜底策略
|
||||
FallbackResponse string `json:"fallback_response"` // 固定回复内容
|
||||
EmbeddingTopK int `json:"embedding_top_k"` // 向量召回TopK
|
||||
KeywordThreshold float64 `json:"keyword_threshold"` // 关键词召回阈值
|
||||
VectorThreshold float64 `json:"vector_threshold"` // 向量召回阈值
|
||||
RerankModelID string `json:"rerank_model_id"` // 排序模型ID
|
||||
RerankTopK int `json:"rerank_top_k"` // 排序TopK
|
||||
RerankThreshold float64 `json:"rerank_threshold"` // 排序阈值
|
||||
SummaryModelID string `json:"summary_model_id"` // 总结模型ID
|
||||
SummaryParameters *SummaryConfig `json:"summary_parameters" gorm:"type:json"` // 总结模型参数
|
||||
AgentConfig *SessionAgentConfig `json:"agent_config" gorm:"type:jsonb"` // Agent 配置(会话级别,仅存储enabled和knowledge_bases)
|
||||
ContextConfig *ContextConfig `json:"context_config" gorm:"type:jsonb"` // 上下文管理配置(可选)
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
|
||||
@@ -7,17 +7,17 @@ import "time"
|
||||
// Knowledge (documents) and FAQ Chunks.
|
||||
type KnowledgeTag struct {
|
||||
// Unique identifier of the tag (UUID)
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Tenant ID
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
// Knowledge base ID that this tag belongs to
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);index"`
|
||||
// Tag name, unique within the same knowledge base
|
||||
Name string `json:"name" gorm:"type:varchar(128);not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(128);not null"`
|
||||
// Optional display color
|
||||
Color string `json:"color" gorm:"type:varchar(32)"`
|
||||
Color string `json:"color" gorm:"type:varchar(32)"`
|
||||
// Sort order within the same knowledge base
|
||||
SortOrder int `json:"sort_order" gorm:"default:0"`
|
||||
SortOrder int `json:"sort_order" gorm:"default:0"`
|
||||
// Creation time
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Last updated time
|
||||
|
||||
@@ -11,37 +11,37 @@ import (
|
||||
// Tenant represents the tenant
|
||||
type Tenant struct {
|
||||
// ID
|
||||
ID uint64 `yaml:"id" json:"id" gorm:"primaryKey"`
|
||||
ID uint64 `yaml:"id" json:"id" gorm:"primaryKey"`
|
||||
// Name
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
// Description
|
||||
Description string `yaml:"description" json:"description"`
|
||||
Description string `yaml:"description" json:"description"`
|
||||
// API key
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
// Status
|
||||
Status string `yaml:"status" json:"status" gorm:"default:'active'"`
|
||||
Status string `yaml:"status" json:"status" gorm:"default:'active'"`
|
||||
// Retriever engines
|
||||
RetrieverEngines RetrieverEngines `yaml:"retriever_engines" json:"retriever_engines" gorm:"type:json"`
|
||||
RetrieverEngines RetrieverEngines `yaml:"retriever_engines" json:"retriever_engines" gorm:"type:json"`
|
||||
// Business
|
||||
Business string `yaml:"business" json:"business"`
|
||||
Business string `yaml:"business" json:"business"`
|
||||
// Storage quota (Bytes), default is 10GB, including vector, original file, text, index, etc.
|
||||
StorageQuota int64 `yaml:"storage_quota" json:"storage_quota" gorm:"default:10737418240"`
|
||||
StorageQuota int64 `yaml:"storage_quota" json:"storage_quota" gorm:"default:10737418240"`
|
||||
// Storage used (Bytes)
|
||||
StorageUsed int64 `yaml:"storage_used" json:"storage_used" gorm:"default:0"`
|
||||
StorageUsed int64 `yaml:"storage_used" json:"storage_used" gorm:"default:0"`
|
||||
// Global Agent configuration for this tenant (default for all sessions)
|
||||
AgentConfig *AgentConfig `yaml:"agent_config" json:"agent_config" gorm:"type:jsonb"`
|
||||
AgentConfig *AgentConfig `yaml:"agent_config" json:"agent_config" gorm:"type:jsonb"`
|
||||
// Global Context configuration for this tenant (default for all sessions)
|
||||
ContextConfig *ContextConfig `yaml:"context_config" json:"context_config" gorm:"type:jsonb"`
|
||||
ContextConfig *ContextConfig `yaml:"context_config" json:"context_config" gorm:"type:jsonb"`
|
||||
// Global WebSearch configuration for this tenant
|
||||
WebSearchConfig *WebSearchConfig `yaml:"web_search_config" json:"web_search_config" gorm:"type:jsonb"`
|
||||
WebSearchConfig *WebSearchConfig `yaml:"web_search_config" json:"web_search_config" gorm:"type:jsonb"`
|
||||
// Global Conversation configuration for this tenant (default for normal mode sessions)
|
||||
ConversationConfig *ConversationConfig `yaml:"conversation_config" json:"conversation_config" gorm:"type:jsonb"`
|
||||
// Creation time
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"`
|
||||
// Deletion time
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
// RetrieverEngines represents the retriever engines for a tenant
|
||||
|
||||
@@ -9,19 +9,19 @@ import (
|
||||
// User represents a user in the system
|
||||
type User struct {
|
||||
// Unique identifier of the user
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// Username of the user
|
||||
Username string `json:"username" gorm:"type:varchar(100);uniqueIndex;not null"`
|
||||
Username string `json:"username" gorm:"type:varchar(100);uniqueIndex;not null"`
|
||||
// Email address of the user
|
||||
Email string `json:"email" gorm:"type:varchar(255);uniqueIndex;not null"`
|
||||
Email string `json:"email" gorm:"type:varchar(255);uniqueIndex;not null"`
|
||||
// Hashed password of the user
|
||||
PasswordHash string `json:"-" gorm:"type:varchar(255);not null"`
|
||||
PasswordHash string `json:"-" gorm:"type:varchar(255);not null"`
|
||||
// Avatar URL of the user
|
||||
Avatar string `json:"avatar" gorm:"type:varchar(500)"`
|
||||
Avatar string `json:"avatar" gorm:"type:varchar(500)"`
|
||||
// Tenant ID that the user belongs to
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"index"`
|
||||
// Whether the user is active
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
// Creation time of the user
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Last updated time of the user
|
||||
@@ -36,11 +36,11 @@ type User struct {
|
||||
// AuthToken represents an authentication token
|
||||
type AuthToken struct {
|
||||
// Unique identifier of the token
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey"`
|
||||
// User ID that owns this token
|
||||
UserID string `json:"user_id" gorm:"type:varchar(36);index;not null"`
|
||||
UserID string `json:"user_id" gorm:"type:varchar(36);index;not null"`
|
||||
// Token value (JWT or other format)
|
||||
Token string `json:"token" gorm:"type:text;not null"`
|
||||
Token string `json:"token" gorm:"type:text;not null"`
|
||||
// Token type (access_token, refresh_token)
|
||||
TokenType string `json:"token_type" gorm:"type:varchar(50);not null"`
|
||||
// Token expiration time
|
||||
@@ -58,14 +58,14 @@ type AuthToken struct {
|
||||
|
||||
// LoginRequest represents a login request
|
||||
type LoginRequest struct {
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Password string `json:"password" binding:"required,min=6"`
|
||||
}
|
||||
|
||||
// RegisterRequest represents a registration request
|
||||
type RegisterRequest struct {
|
||||
Username string `json:"username" binding:"required,min=3,max=50"`
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Email string `json:"email" binding:"required,email"`
|
||||
Password string `json:"password" binding:"required,min=6"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user