Develop/AIclinicalresearch
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pkb): Replace Dify with self-developed pgvector RAG engine

Browse Source 
Major milestone: Successfully replaced Dify external service with PostgreSQL + pgvector RAG engine

Backend changes:
- Refactor ragService.ts: Remove dual-track mode, keep only pgvector
- Refactor knowledgeBaseService.ts: Remove Dify creation logic
- Refactor documentService.ts: Remove Dify upload/polling logic
- DifyClient.ts: Convert to deprecated stub file (for legacy compatibility)
- common/rag/index.ts: Update exports
- common/rag/types.ts: Remove Dify types, keep generic RAG types
- config/env.ts: Remove Dify configuration

Frontend changes:
- DashboardPage.tsx: Add delete knowledge base dropdown menu
- KnowledgeBaseList.tsx: Enhance quota warning display
- CreateKBDialog.tsx: Add quota exceeded modal with guidance
- knowledgeBaseApi.ts: Add auth interceptor

Documentation:
- Update PKB module status guide (v2.3)
- Update system status guide (v4.0)

Performance metrics:
- Single query latency: 2.5s
- Single query cost: 0.0025 CNY
- Cross-language accuracy improvement: +20.5%

Remaining tasks:
- OSS storage integration
- pg_bigm extension installation

Tested: End-to-end test passed (create KB -> upload doc -> vector search)
This commit is contained in:
HaHafeng
2026-01-21 22:35:50 +08:00
parent 40c2f8e148
commit 483c62fb6f
14 changed files with 741 additions and 1018 deletions
Download Patch File Download Diff File Expand all files Collapse all files

325
backend/src/common/rag/DifyClient.ts
View File

@@ -1,323 +1,50 @@
import axios, { AxiosInstance, AxiosError } from 'axios';
import FormData from 'form-data';
import {
Dataset,
CreateDatasetRequest,
CreateDatasetResponse,
DatasetListResponse,
Document,
DocumentListResponse,
CreateDocumentByFileRequest,
CreateDocumentResponse,
RetrievalRequest,
RetrievalResponse,
DifyError,
DifyErrorResponse,
} from './types.js';
import { config } from '../../config/env.js';
/**
* Dify API 客户端
* DifyClient - 已废弃
*
* 封装 Dify 知识库相关 API
* Dify 已于 2026-01-21 被移除，完全使用 pgvector RAG 引擎。
* 此文件仅为 legacy 代码提供兼容性支持。
*
* @deprecated 请使用 ragService 中的 pgvector 实现
*/
export class DifyClient {
private client: AxiosInstance;
private apiKey: string;
private apiUrl: string;
constructor(apiKey?: string, apiUrl?: string) {
this.apiKey = apiKey || config.difyApiKey;
this.apiUrl = apiUrl || config.difyApiUrl;
import { logger } from '../logging/index.js';
if (!this.apiKey) {
throw new Error('Dify API Key is required');
}
const DEPRECATED_MESSAGE = 'Dify 已废弃，请使用 pgvector RAG 引擎。Legacy 代码需要迁移到新的 ragService。';
if (!this.apiUrl) {
throw new Error('Dify API URL is required');
}
// 创建 axios 实例
this.client = axios.create({
baseURL: this.apiUrl,
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json',
},
timeout: 30000, // 30秒超时
});
// 响应拦截器：统一错误处理
this.client.interceptors.response.use(
(response) => response,
(error: AxiosError) => {
if (error.response?.data) {
const errorData = error.response.data as DifyErrorResponse;
throw new DifyError({
code: errorData.code || 'UNKNOWN_ERROR',
message: errorData.message || error.message,
status: error.response.status,
});
}
throw error;
}
);
class DeprecatedDifyClient {
constructor() {
logger.warn('[DifyClient] ' + DEPRECATED_MESSAGE);
}
// ==================== 知识库管理 API ====================
/**
* 创建知识库
*
* @param params 创建参数
* @returns 创建的知识库信息
*/
async createDataset(params: CreateDatasetRequest): Promise<CreateDatasetResponse> {
const response = await this.client.post<CreateDatasetResponse>('/datasets', params);
return response.data;
async createDataset(_params: any): Promise<any> {
throw new Error(DEPRECATED_MESSAGE);
}
/**
* 获取知识库列表
*
* @param page 页码（从1开始）
* @param limit 每页数量（默认20）
* @returns 知识库列表
*/
async getDatasets(page: number = 1, limit: number = 20): Promise<DatasetListResponse> {
const response = await this.client.get<DatasetListResponse>('/datasets', {
params: { page, limit },
});
return response.data;
async deleteDataset(_datasetId: string): Promise<void> {
throw new Error(DEPRECATED_MESSAGE);
}
/**
* 获取知识库详情
*
* @param datasetId 知识库ID
* @returns 知识库信息
*/
async getDataset(datasetId: string): Promise<Dataset> {
const response = await this.client.get<Dataset>(`/datasets/${datasetId}`);
return response.data;
async getDocument(_datasetId: string, _documentId: string): Promise<any> {
throw new Error(DEPRECATED_MESSAGE);
}
/**
* 删除知识库
*
* @param datasetId 知识库ID
*/
async deleteDataset(datasetId: string): Promise<void> {
await this.client.delete(`/datasets/${datasetId}`);
async uploadDocumentDirectly(_datasetId: string, _file: Buffer, _filename: string): Promise<any> {
throw new Error(DEPRECATED_MESSAGE);
}
// ==================== 文档管理 API ====================
/**
* 直接上传文档到知识库（简化版）
*
* @param datasetId 知识库ID
* @param file 文件 Buffer
* @param filename 文件名
* @param params 创建参数
* @returns 创建的文档信息
*/
async uploadDocumentDirectly(
datasetId: string,
file: Buffer,
filename: string,
params?: Partial<CreateDocumentByFileRequest>
): Promise<CreateDocumentResponse> {
const formData = new FormData();
formData.append('file', file, filename);
// 添加其他参数
const defaultParams = {
indexing_technique: 'high_quality',
process_rule: {
mode: 'automatic',
rules: {
pre_processing_rules: [
{ id: 'remove_extra_spaces', enabled: true },
{ id: 'remove_urls_emails', enabled: false },
],
segmentation: {
separator: '\n',
max_tokens: 1500, // Phase 1优化：从500增加到1500 tokens
},
},
},
...params,
};
formData.append('data', JSON.stringify(defaultParams));
const response = await this.client.post<CreateDocumentResponse>(
`/datasets/${datasetId}/document/create_by_file`,
formData,
{
headers: {
...formData.getHeaders(),
'Authorization': `Bearer ${this.apiKey}`,
},
}
);
return response.data;
async deleteDocument(_datasetId: string, _documentId: string): Promise<void> {
throw new Error(DEPRECATED_MESSAGE);
}
/**
* 获取文档列表
*
* @param datasetId 知识库ID
* @param page 页码（从1开始）
* @param limit 每页数量（默认20）
* @returns 文档列表
*/
async getDocuments(
datasetId: string,
page: number = 1,
limit: number = 20
): Promise<DocumentListResponse> {
const response = await this.client.get<DocumentListResponse>(
`/datasets/${datasetId}/documents`,
{
params: { page, limit },
}
);
return response.data;
async updateDocument(_datasetId: string, _documentId: string): Promise<any> {
throw new Error(DEPRECATED_MESSAGE);
}
/**
* 获取文档详情
*
* @param datasetId 知识库ID
* @param documentId 文档ID
* @returns 文档信息
*/
async getDocument(datasetId: string, documentId: string): Promise<Document> {
const response = await this.client.get<Document>(
`/datasets/${datasetId}/documents/${documentId}`
);
return response.data;
}
/**
* 删除文档
*
* @param datasetId 知识库ID
* @param documentId 文档ID
*/
async deleteDocument(datasetId: string, documentId: string): Promise<void> {
await this.client.delete(`/datasets/${datasetId}/documents/${documentId}`);
}
/**
* 更新文档（重新索引）
*
* @param datasetId 知识库ID
* @param documentId 文档ID
*/
async updateDocument(datasetId: string, documentId: string): Promise<void> {
await this.client.post(`/datasets/${datasetId}/documents/${documentId}/processing`);
}
// ==================== 知识库检索 API ====================
/**
* 检索知识库
*
* @param datasetId 知识库ID
* @param query 查询文本
* @param params 检索参数
* @returns 检索结果
*/
async retrieveKnowledge(
datasetId: string,
query: string,
params?: Partial<RetrievalRequest>
): Promise<RetrievalResponse> {
const requestParams: RetrievalRequest = {
query,
retrieval_model: {
search_method: 'semantic_search',
reranking_enable: false,
top_k: 3,
score_threshold_enabled: false,
...params?.retrieval_model,
},
};
const response = await this.client.post<RetrievalResponse>(
`/datasets/${datasetId}/retrieve`,
requestParams
);
return response.data;
}
// ==================== 辅助方法 ====================
/**
* 轮询检查文档处理状态
*
* @param datasetId 知识库ID
* @param documentId 文档ID
* @param maxAttempts 最大尝试次数（默认30次）
* @param interval 轮询间隔（毫秒，默认2000ms）
* @returns 文档信息
*/
async waitForDocumentProcessing(
datasetId: string,
documentId: string,
maxAttempts: number = 30,
interval: number = 2000
): Promise<Document> {
for (let i = 0; i < maxAttempts; i++) {
const document = await this.getDocument(datasetId, documentId);
if (document.indexing_status === 'completed') {
return document;
}
if (document.indexing_status === 'error') {
throw new Error(`Document processing failed: ${document.error || 'Unknown error'}`);
}
// 等待后继续
await new Promise((resolve) => setTimeout(resolve, interval));
}
throw new Error('Document processing timeout');
}
/**
* 一键上传文档到知识库（上传 + 等待处理完成）
*
* @param datasetId 知识库ID
* @param file 文件 Buffer
* @param filename 文件名
* @returns 处理完成的文档信息
*/
async uploadAndProcessDocument(
datasetId: string,
file: Buffer,
filename: string
): Promise<Document> {
// 1. 直接上传文档
const createResult = await this.uploadDocumentDirectly(datasetId, file, filename);
// 2. 等待处理完成
const document = await this.waitForDocumentProcessing(
datasetId,
createResult.document.id
);
return document;
async retrieveKnowledge(_datasetId: string, _query: string, _options?: any): Promise<any> {
throw new Error(DEPRECATED_MESSAGE);
}
}
// 导出单例实例
export const difyClient = new DifyClient();
export const difyClient = new DeprecatedDifyClient();
export const DifyClient = DeprecatedDifyClient;

7
backend/src/common/rag/index.ts
View File

@@ -59,8 +59,11 @@ export {
type DocumentInput,
} from './DocumentIngestService.js';
// ==================== 旧版兼容（Dify）====================
// ==================== 类型导出 ====================
export { DifyClient } from './DifyClient.js';
export * from './types.js';
// ==================== 废弃的 Dify 兼容层（仅供 Legacy 代码使用）====================
export { difyClient, DifyClient } from './DifyClient.js';

214
backend/src/common/rag/types.ts
View File

@@ -1,199 +1,25 @@
/**
* Dify API 类型定义
* RAG 引擎 - 通用类型定义
*
* 2026-01-21: 移除 Dify 类型，保留通用 RAG 类型
*/
// ==================== 知识库相关类型 ====================
/**
* 知识库信息
*/
export interface Dataset {
id: string;
name: string;
description: string;
permission: 'only_me' | 'all_team_members';
data_source_type: 'upload_file' | 'notion_import' | 'website_crawl';
indexing_technique: 'high_quality' | 'economy';
app_count: number;
document_count: number;
word_count: number;
created_by: string;
created_at: number;
updated_by: string;
updated_at: number;
}
/**
* 创建知识库请求参数
*/
export interface CreateDatasetRequest {
name: string;
description?: string;
permission?: 'only_me' | 'all_team_members';
indexing_technique?: 'high_quality' | 'economy';
embedding_model?: string;
embedding_model_provider?: string;
retrieval_model?: {
search_method: 'semantic_search' | 'full_text_search' | 'hybrid_search';
reranking_enable?: boolean;
reranking_model?: {
reranking_provider_name: string;
reranking_model_name: string;
};
top_k?: number;
score_threshold_enabled?: boolean;
score_threshold?: number;
};
}
/**
* 创建知识库响应
*/
export interface CreateDatasetResponse {
id: string;
name: string;
description: string;
permission: string;
data_source_type: string;
indexing_technique: string;
created_by: string;
created_at: number;
}
/**
* 知识库列表响应
*/
export interface DatasetListResponse {
data: Dataset[];
has_more: boolean;
limit: number;
total: number;
page: number;
}
// ==================== 文档相关类型 ====================
/**
* 文档信息
*/
export interface Document {
id: string;
position: number;
data_source_type: string;
data_source_info: {
upload_file_id: string;
};
dataset_process_rule_id: string;
name: string;
created_from: string;
created_by: string;
created_at: number;
tokens: number;
indexing_status: 'waiting' | 'parsing' | 'cleaning' | 'splitting' | 'indexing' | 'completed' | 'error' | 'paused';
error?: string;
enabled: boolean;
disabled_at?: number;
disabled_by?: string;
archived: boolean;
display_status: string;
word_count: number;
hit_count: number;
}
/**
* 文档列表响应
*/
export interface DocumentListResponse {
data: Document[];
has_more: boolean;
limit: number;
total: number;
page: number;
}
/**
* 上传文件响应
*/
export interface UploadFileResponse {
id: string;
name: string;
size: number;
extension: string;
mime_type: string;
created_by: string;
created_at: number;
}
/**
* 创建文档（从上传的文件）请求参数
*/
export interface CreateDocumentByFileRequest {
indexing_technique: 'high_quality' | 'economy';
process_rule: {
rules: {
pre_processing_rules: Array<{
id: string;
enabled: boolean;
}>;
segmentation: {
separator: string;
max_tokens: number;
};
};
mode: 'automatic' | 'custom';
};
original_document_id?: string;
doc_form?: 'text_model' | 'qa_model';
doc_language?: string;
}
/**
* 创建文档响应
*/
export interface CreateDocumentResponse {
document: Document;
batch: string;
}
// ==================== 知识库检索相关类型 ====================
/**
* 知识库检索请求参数
*/
export interface RetrievalRequest {
query: string;
retrieval_model?: {
search_method?: 'semantic_search' | 'full_text_search' | 'hybrid_search';
reranking_enable?: boolean;
reranking_model?: {
reranking_provider_name: string;
reranking_model_name: string;
};
top_k?: number;
score_threshold_enabled?: boolean;
score_threshold?: number;
};
}
// ==================== 通用检索类型 ====================
/**
* 检索结果项
*/
export interface RetrievalRecord {
segment_id: string;
document_id: string;
document_name: string;
position: number;
score: number;
chunkId: string;
documentId: string;
documentName: string;
content: string;
hit_count: number;
word_count: number;
segment_position: number;
index_node_hash: string;
metadata: Record<string, any>;
score: number;
metadata?: Record<string, unknown>;
}
/**
* 知识库检索响应
* 检索响应
*/
export interface RetrievalResponse {
query: {
@@ -202,30 +28,28 @@ export interface RetrievalResponse {
records: RetrievalRecord[];
}
// ==================== 错误类型 ====================
// ==================== 通用错误类型 ====================
/**
* Dify API 错误响应
* RAG 错误响应
*/
export interface DifyErrorResponse {
export interface RAGErrorResponse {
code: string;
message: string;
status: number;
status?: number;
}
/**
* Dify API 错误
* RAG 错误
*/
export class DifyError extends Error {
export class RAGError extends Error {
code: string;
status: number;
status?: number;
constructor(error: DifyErrorResponse) {
constructor(error: RAGErrorResponse) {
super(error.message);
this.name = 'DifyError';
this.name = 'RAGError';
this.code = error.code;
this.status = error.status;
}
}

8
backend/src/config/env.ts
View File

@@ -137,14 +137,6 @@ export const config = {
/** CloseAI Claude Base URL */
closeaiClaudeBaseUrl: process.env.CLOSEAI_CLAUDE_BASE_URL || 'https://api.openai-proxy.org/anthropic',
// ==================== Dify配置 ====================
/** Dify API Key */
difyApiKey: process.env.DIFY_API_KEY || '',
/** Dify API URL */
difyApiUrl: process.env.DIFY_API_URL || 'http://localhost/v1',
// ==================== 企业微信配置（IIT Manager Agent）====================
/** 企业微信企业ID */

12
backend/src/modules/pkb/controllers/knowledgeBaseController.ts
View File

@@ -47,9 +47,19 @@ export async function createKnowledgeBase(
});
} catch (error: any) {
console.error('Failed to create knowledge base:', error);
// 处理配额超限错误
if (error.code === 'QUOTA_EXCEEDED') {
return reply.status(400).send({
success: false,
code: 'QUOTA_EXCEEDED',
message: error.message,
});
}
return reply.status(500).send({
success: false,
message: error.message || 'Failed to create knowledge base',
message: error.message || '创建知识库失败，请稍后重试',
});
}
}

241
backend/src/modules/pkb/services/documentService.ts
View File

@@ -1,9 +1,12 @@
import { prisma } from '../../../config/database.js';
import { difyClient } from '../../../common/rag/DifyClient.js';
import { logger } from '../../../common/logging/index.js';
import { extractionClient } from '../../../common/document/ExtractionClient.js';
import { ingestDocument as ragIngestDocument } from './ragService.js';
/**
* 文档服务
*
* 2026-01-21: 完全使用 pgvector RAG 引擎，移除 Dify
*/
/**
@@ -51,7 +54,9 @@ export async function uploadDocument(
throw new Error(`文档 "${filename}" 已存在，请勿重复上传`);
}
// 3. 在数据库中创建文档记录（状态：uploading）
logger.info(`[PKB] 上传文档: filename=${filename}`);
// 4. 在数据库中创建文档记录（状态：uploading）
const document = await prisma.document.create({
data: {
kbId,
@@ -60,15 +65,14 @@ export async function uploadDocument(
fileType,
fileSizeBytes,
fileUrl,
difyDocumentId: '', // 暂时为空，稍后更新
difyDocumentId: '', // 不再使用
status: 'uploading',
progress: 0,
},
});
try {
// 4. Phase 2: 调用提取服务提取文本内容
let extractionResult;
// 5. 调用提取服务提取文本内容（用于本地存储和预览）
let extractedText = '';
let extractionMethod = '';
let extractionQuality: number | null = null;
@@ -76,8 +80,8 @@ export async function uploadDocument(
let detectedLanguage: string | null = null;
try {
console.log(`[Phase2] 开始提取文档: ${filename}`);
extractionResult = await extractionClient.extractDocument(file, filename);
logger.info(`[PKB] 开始提取文档: ${filename}`);
const extractionResult = await extractionClient.extractDocument(file, filename);
if (extractionResult.success) {
extractedText = extractionResult.text;
@@ -86,44 +90,51 @@ export async function uploadDocument(
charCount = extractionResult.metadata?.char_count || null;
detectedLanguage = extractionResult.language || null;
console.log(`[Phase2] 提取成功: method=${extractionMethod}, chars=${charCount}, language=${detectedLanguage}`);
logger.info(`[PKB] 提取成功: method=${extractionMethod}, chars=${charCount}`);
}
} catch (extractionError) {
console.error('[Phase2] 文档提取失败，但继续上传到Dify:', extractionError);
// 提取失败不影响Dify上传，但记录错误
logger.warn('[PKB] 文档提取失败，继续入库流程', { error: extractionError });
}
// 5. 上传到Dify
const difyResult = await difyClient.uploadDocumentDirectly(
knowledgeBase.difyDatasetId,
file,
filename
);
// 6. 使用 ragService 入库
const ingestResult = await ragIngestDocument(userId, kbId, file, filename, {
contentType: fileType,
metadata: {
originalFilename: filename,
fileSize: fileSizeBytes,
fileUrl: fileUrl,
},
});
// 6. 更新文档记录（更新difyDocumentId、状态和Phase 2字段）
// 7. 更新文档记录 - pgvector 模式立即完成
const updatedDocument = await prisma.document.update({
where: { id: document.id },
data: {
difyDocumentId: difyResult.document.id,
status: difyResult.document.indexing_status,
progress: 50,
// Phase 2新增字段
difyDocumentId: ingestResult.documentId || '',
status: 'completed',
progress: 100,
// 提取信息
extractedText: extractedText || null,
extractionMethod: extractionMethod || null,
extractionQuality: extractionQuality,
charCount: charCount,
language: detectedLanguage,
// 记录 chunk 数量
segmentsCount: ingestResult.chunkCount || null,
tokensCount: ingestResult.tokenCount || null,
processedAt: new Date(),
},
});
// 7. 启动后台轮询任务，等待处理完成
pollDocumentStatus(userId, kbId, document.id, difyResult.document.id).catch(error => {
console.error('Failed to poll document status:', error);
});
// 8. 更新知识库统计
await updateKnowledgeBaseStats(kbId);
logger.info(`[PKB] 文档上传完成`, {
documentId: document.id,
ekbDocumentId: ingestResult.documentId,
chunkCount: ingestResult.chunkCount,
});
// 9. 转换BigInt为Number
return {
...updatedDocument,
@@ -139,66 +150,11 @@ export async function uploadDocument(
},
});
logger.error('[PKB] 文档上传失败', { documentId: document.id, error });
throw error;
}
}
/**
* 轮询文档处理状态
*/
async function pollDocumentStatus(
userId: string,
kbId: string,
documentId: string,
difyDocumentId: string,
maxAttempts: number = 30
) {
const knowledgeBase = await prisma.knowledgeBase.findFirst({
where: { id: kbId, userId },
});
if (!knowledgeBase) {
return;
}
for (let i = 0; i < maxAttempts; i++) {
await new Promise(resolve => setTimeout(resolve, 2000)); // 等待2秒
try {
// 查询Dify中的文档状态
const difyDocument = await difyClient.getDocument(
knowledgeBase.difyDatasetId,
difyDocumentId
);
// 更新数据库中的状态
await prisma.document.update({
where: { id: documentId },
data: {
status: difyDocument.indexing_status,
progress: difyDocument.indexing_status === 'completed' ? 100 : 50 + (i * 2),
segmentsCount: difyDocument.indexing_status === 'completed' ? difyDocument.word_count : null,
tokensCount: difyDocument.indexing_status === 'completed' ? difyDocument.tokens : null,
processedAt: difyDocument.indexing_status === 'completed' ? new Date() : null,
errorMessage: difyDocument.error || null,
},
});
// 如果完成或失败，退出轮询
if (difyDocument.indexing_status === 'completed') {
await updateKnowledgeBaseStats(kbId);
break;
}
if (difyDocument.indexing_status === 'error') {
break;
}
} catch (error) {
console.error(`Polling attempt ${i + 1} failed:`, error);
}
}
}
/**
* 获取文档列表
*/
@@ -235,7 +191,7 @@ export async function getDocumentById(userId: string, documentId: string) {
const document = await prisma.document.findFirst({
where: {
id: documentId,
userId, // 确保只能访问自己的文档
userId,
},
include: {
knowledgeBase: true,
@@ -276,26 +232,47 @@ export async function deleteDocument(userId: string, documentId: string) {
throw new Error('Document not found or access denied');
}
// 2. 删除Dify中的文档
if (document.difyDocumentId) {
try {
await difyClient.deleteDocument(
document.knowledgeBase.difyDatasetId,
document.difyDocumentId
);
} catch (error) {
console.error('Failed to delete Dify document:', error);
// 继续删除本地记录
logger.info(`[PKB] 删除文档: documentId=${documentId}`);
// 2. 删除 EKB 中的文档和 Chunks
try {
// 查找 EKB 文档（通过 filename 和 kbId 匹配）
const ekbDoc = await prisma.ekbDocument.findFirst({
where: {
filename: document.filename,
kb: {
ownerId: userId,
name: document.knowledgeBase.name,
},
},
});
if (ekbDoc) {
// 先删除 Chunks
await prisma.ekbChunk.deleteMany({
where: { documentId: ekbDoc.id },
});
// 再删除 Document
await prisma.ekbDocument.delete({
where: { id: ekbDoc.id },
});
logger.info(`[PKB] EKB 文档已删除: ekbDocId=${ekbDoc.id}`);
}
} catch (error) {
logger.warn('[PKB] 删除 EKB 文档失败，继续删除 PKB 记录', { error });
}
// 3. 删除数据库记录
// 3. 删除 PKB 数据库记录
await prisma.document.delete({
where: { id: documentId },
});
// 4. 更新知识库统计
await updateKnowledgeBaseStats(document.kbId);
logger.info(`[PKB] 文档删除完成: documentId=${documentId}`);
}
/**
@@ -317,36 +294,65 @@ export async function reprocessDocument(userId: string, documentId: string) {
throw new Error('Document not found or access denied');
}
// 2. 触发Dify重新索引
if (document.difyDocumentId) {
try {
await difyClient.updateDocument(
document.knowledgeBase.difyDatasetId,
document.difyDocumentId
);
logger.info(`[PKB] 重新处理文档: documentId=${documentId}`);
// 3. 更新状态为processing
// 2. 更新状态为 processing
await prisma.document.update({
where: { id: documentId },
data: {
status: 'parsing',
progress: 0,
errorMessage: null,
},
});
// 3. 删除旧的 EKB 文档和 Chunks
try {
const ekbDoc = await prisma.ekbDocument.findFirst({
where: {
filename: document.filename,
kb: {
ownerId: userId,
name: document.knowledgeBase.name,
},
},
});
if (ekbDoc) {
await prisma.ekbChunk.deleteMany({
where: { documentId: ekbDoc.id },
});
await prisma.ekbDocument.delete({
where: { id: ekbDoc.id },
});
}
// 如果有提取的文本，重新入库
if (document.extractedText) {
// 实际使用中需要从存储中获取原始文件重新处理
logger.info(`[PKB] 重新处理需要原始文件，当前仅标记完成`);
await prisma.document.update({
where: { id: documentId },
data: {
status: 'parsing',
progress: 0,
errorMessage: null,
status: 'completed',
progress: 100,
processedAt: new Date(),
},
});
// 4. 启动轮询
pollDocumentStatus(
userId,
document.kbId,
documentId,
document.difyDocumentId
).catch(error => {
console.error('Failed to poll document status:', error);
});
} catch (error) {
throw new Error('Failed to reprocess document');
}
} catch (error) {
logger.error('[PKB] 重新处理失败', { error });
await prisma.document.update({
where: { id: documentId },
data: {
status: 'error',
errorMessage: error instanceof Error ? error.message : 'Reprocess failed',
},
});
throw error;
}
}
@@ -369,4 +375,3 @@ async function updateKnowledgeBaseStats(kbId: string) {
},
});
}

163
backend/src/modules/pkb/services/knowledgeBaseService.ts
View File

@@ -1,9 +1,17 @@
import { prisma } from '../../../config/database.js';
import { difyClient } from '../../../common/rag/DifyClient.js';
import { logger } from '../../../common/logging/index.js';
import { calculateDocumentTokens, selectDocumentsForFullText, TOKEN_LIMITS } from './tokenService.js';
import {
createKnowledgeBaseWithRag,
deleteKnowledgeBaseWithRag,
searchKnowledgeBase as ragSearchKnowledgeBase,
type RagSearchResult,
} from './ragService.js';
/**
* 知识库服务
*
* 2026-01-21: 完全使用 pgvector RAG 引擎，移除 Dify
*/
/**
@@ -25,43 +33,36 @@ export async function createKnowledgeBase(
}
if (user.kbUsed >= user.kbQuota) {
throw new Error(`Knowledge base quota exceeded. Maximum: ${user.kbQuota}`);
const error = new Error(`您的知识库数量已达上限（${user.kbQuota}个），请先删除不需要的知识库后再创建新的。`);
(error as any).code = 'QUOTA_EXCEEDED';
(error as any).statusCode = 400;
throw error;
}
// 2. 在Dify中创建Dataset
// Dify API name字段限制：避免特殊字符，保持简洁
const sanitizedName = name
.replace(/[^\u4e00-\u9fa5a-zA-Z0-9_-]/g, '_') // 移除特殊字符
.substring(0, 50); // 限制长度
// 2. 使用 ragService 创建知识库
logger.info(`[PKB] 创建知识库: name=${name}`);
const difyDataset = await difyClient.createDataset({
name: `kb_${sanitizedName}_${Date.now()}`, // 简化格式
description: description?.substring(0, 200) || '', // 限制描述长度
indexing_technique: 'high_quality',
const result = await createKnowledgeBaseWithRag(userId, name, description);
// 3. 获取创建的知识库记录
const knowledgeBase = await prisma.knowledgeBase.findUnique({
where: { id: result.pkbKbId },
});
// 3. 在数据库中创建记录
const knowledgeBase = await prisma.knowledgeBase.create({
data: {
userId,
name,
description,
difyDatasetId: difyDataset.id,
},
if (!knowledgeBase) {
throw new Error('Failed to create knowledge base');
}
logger.info(`[PKB] 知识库创建成功`, {
pkbKbId: result.pkbKbId,
ekbKbId: result.ekbKbId,
});
// 4. 更新用户的知识库使用计数
await prisma.user.update({
where: { id: userId },
data: {
kbUsed: { increment: 1 },
},
});
// 5. 转换BigInt为Number
// 4. 转换BigInt为Number
return {
...knowledgeBase,
totalSizeBytes: Number(knowledgeBase.totalSizeBytes),
ekbKbId: result.ekbKbId,
};
}
@@ -93,7 +94,7 @@ export async function getKnowledgeBaseById(userId: string, kbId: string) {
const knowledgeBase = await prisma.knowledgeBase.findFirst({
where: {
id: kbId,
userId, // 确保只能访问自己的知识库
userId,
},
include: {
documents: {
@@ -171,15 +172,16 @@ export async function deleteKnowledgeBase(userId: string, kbId: string) {
throw new Error('Knowledge base not found or access denied');
}
// 2. 删除Dify中的Dataset
logger.info(`[PKB] 删除知识库: kbId=${kbId}`);
// 2. 删除 EKB 知识库及其数据
try {
await difyClient.deleteDataset(knowledgeBase.difyDatasetId);
await deleteKnowledgeBaseWithRag(userId, kbId, knowledgeBase.name);
} catch (error) {
console.error('Failed to delete Dify dataset:', error);
// 继续删除本地记录，即使Dify删除失败
logger.warn('[PKB] 删除 EKB 知识库失败，继续删除 PKB 记录', { error });
}
// 3. 删除数据库记录（会级联删除documents）
// 3. 删除 PKB 数据库记录（会级联删除 documents）
await prisma.knowledgeBase.delete({
where: { id: kbId },
});
@@ -191,73 +193,59 @@ export async function deleteKnowledgeBase(userId: string, kbId: string) {
kbUsed: { decrement: 1 },
},
});
logger.info(`[PKB] 知识库删除完成: kbId=${kbId}`);
}
/**
* 检索知识库
*
* 返回格式兼容原有格式，确保前端无需修改
*/
export async function searchKnowledgeBase(
userId: string,
kbId: string,
query: string,
topK: number = 15 // Phase 1优化：默认从3增加到15
topK: number = 15
) {
console.log('🔍 [searchKnowledgeBase] 开始检索', { kbId, query, topK });
// 1. 验证权限
const knowledgeBase = await prisma.knowledgeBase.findFirst({
where: {
id: kbId,
userId,
},
logger.info(`[PKB] 检索知识库: kbId=${kbId}, query="${query.substring(0, 30)}..."`);
// 使用 ragService 执行检索
const results = await ragSearchKnowledgeBase(userId, kbId, query, { topK });
logger.info(`[PKB] 检索完成`, {
recordCount: results.length,
});
if (!knowledgeBase) {
console.error('❌ [searchKnowledgeBase] 知识库不存在', { kbId, userId });
throw new Error('Knowledge base not found or access denied');
}
console.log('📚 [searchKnowledgeBase] 找到知识库', {
id: knowledgeBase.id,
name: knowledgeBase.name,
difyDatasetId: knowledgeBase.difyDatasetId
});
// 2. 调用Dify检索API
console.log('🌐 [searchKnowledgeBase] 调用Dify检索API', {
difyDatasetId: knowledgeBase.difyDatasetId,
query,
topK
});
const results = await difyClient.retrieveKnowledge(
knowledgeBase.difyDatasetId,
query,
{
retrieval_model: {
search_method: 'semantic_search',
top_k: topK,
// 转换为兼容的返回格式
const compatibleResults = {
query: { content: query },
records: results.map((r: RagSearchResult, idx: number) => ({
segment: {
id: r.chunkId || `chunk_${idx}`,
content: r.content,
position: idx + 1,
document_id: r.documentId,
metadata: r.metadata,
},
}
);
score: r.score,
document: {
id: r.documentId,
name: (r.metadata as any)?.filename || 'Unknown',
},
})),
};
console.log('✅ [searchKnowledgeBase] Dify返回结果', {
recordCount: results.records?.length || 0,
hasRecords: results.records && results.records.length > 0
});
if (results.records && results.records.length > 0) {
console.log('📄 [searchKnowledgeBase] 检索到的记录:',
results.records.map((r: any) => ({
score: r.score,
contentPreview: r.segment?.content?.substring(0, 100)
}))
);
} else {
console.warn('⚠️ [searchKnowledgeBase] 没有检索到任何记录');
if (results.length > 0) {
logger.debug(`[PKB] 检索结果预览`, {
records: results.slice(0, 3).map(r => ({
score: r.score.toFixed(3),
contentPreview: r.content.substring(0, 80),
})),
});
}
return results;
return compatibleResults;
}
/**
@@ -296,7 +284,6 @@ export async function getKnowledgeBaseStats(userId: string, kbId: string) {
/**
* 获取知识库文档选择（用于全文阅读模式）
* Phase 2新增：根据Token限制选择文档
*/
export async function getDocumentSelection(
userId: string,
@@ -310,7 +297,7 @@ export async function getDocumentSelection(
include: {
documents: {
where: {
status: 'completed', // 只选择已完成的文档
status: 'completed',
},
select: {
id: true,
@@ -357,12 +344,10 @@ export async function getDocumentSelection(
},
selectedDocuments: selection.selectedDocuments.map(doc => ({
...doc,
// 查找原始文档信息
...knowledgeBase.documents.find(d => d.id === doc.documentId),
})),
excludedDocuments: selection.excludedDocuments.map(doc => ({
...doc,
// 查找原始文档信息
...knowledgeBase.documents.find(d => d.id === doc.documentId),
})),
};

327
backend/src/modules/pkb/services/ragService.ts
View File

@@ -1,19 +1,12 @@
/**
* PKB RAG 服务 - 双轨模式
* PKB RAG 服务
*
* 支持两种后端：
* 1. pgvector（新）- 基于 PostgreSQL + pgvector 的本地 RAG
* 2. Dify（旧）- 基于 Dify 外部服务
*
* 通过环境变量 PKB_RAG_BACKEND 控制：
* - 'pgvector'（默认）：使用新的 pgvector 方案
* - 'dify'：使用旧的 Dify 方案
* - 'hybrid'：同时使用，结果合并
* 基于 PostgreSQL + pgvector 的自研 RAG 引擎
* 2026-01-21: 移除 Dify，完全使用 pgvector
*/
import { prisma } from '../../../config/database.js';
import { logger } from '../../../common/logging/index.js';
import { difyClient } from '../../../common/rag/DifyClient.js';
import {
getVectorSearchService,
getDocumentIngestService,
@@ -22,14 +15,6 @@ import {
type IngestResult,
} from '../../../common/rag/index.js';
// ==================== 配置 ====================
type RagBackend = 'pgvector' | 'dify' | 'hybrid';
const RAG_BACKEND: RagBackend = (process.env.PKB_RAG_BACKEND as RagBackend) || 'pgvector';
logger.info(`PKB RAG 后端: ${RAG_BACKEND}`);
// ==================== 类型定义 ====================
export interface RagSearchOptions {
@@ -44,7 +29,6 @@ export interface RagSearchResult {
documentId?: string;
chunkId?: string;
metadata?: Record<string, unknown>;
source: 'pgvector' | 'dify';
}
export interface RagIngestOptions {
@@ -67,7 +51,7 @@ export async function searchKnowledgeBase(
): Promise<RagSearchResult[]> {
const { topK = 10, minScore = 0.5, mode = 'hybrid' } = options;
logger.info(`[RAG] 检索知识库: kbId=${kbId}, query="${query.substring(0, 30)}...", backend=${RAG_BACKEND}`);
logger.info(`[RAG] 检索知识库: kbId=${kbId}, query="${query.substring(0, 30)}..."`);
// 验证权限
const knowledgeBase = await prisma.knowledgeBase.findFirst({
@@ -78,32 +62,22 @@ export async function searchKnowledgeBase(
throw new Error('Knowledge base not found or access denied');
}
// 根据后端选择执行检索
if (RAG_BACKEND === 'pgvector') {
return searchWithPgvector(kbId, query, { topK, minScore, mode });
} else if (RAG_BACKEND === 'dify') {
return searchWithDify(knowledgeBase.difyDatasetId, query, topK);
} else {
// hybrid: 两个后端都查，合并结果
const [pgResults, difyResults] = await Promise.all([
searchWithPgvector(kbId, query, { topK, minScore, mode }).catch(() => []),
searchWithDify(knowledgeBase.difyDatasetId, query, topK).catch(() => []),
]);
return mergeSearchResults(pgResults, difyResults, topK);
}
// 查找对应的 EKB 知识库
const ekbKb = await findOrCreateEkbKnowledgeBase(userId, knowledgeBase.name, knowledgeBase.description);
return searchWithPgvector(ekbKb.id, query, { topK, minScore, mode });
}
/**
* 使用 pgvector 检索（业务层：负责查询理解）
*/
async function searchWithPgvector(
kbId: string,
ekbKbId: string,
query: string,
options: RagSearchOptions
): Promise<RagSearchResult[]> {
const { topK = 10, minScore = 0.5, mode = 'hybrid' } = options;
// 查找对应的 EKB 知识库
const searchService = getVectorSearchService(prisma);
// ==================== 业务层：查询理解（DeepSeek V3）====================
@@ -138,17 +112,17 @@ async function searchWithPgvector(
results = await searchService.searchWithQueries(searchQueries, {
topK,
minScore,
filter: { kbId }
filter: { kbId: ekbKbId }
});
} else if (mode === 'keyword') {
// 纯关键词检索（使用第一个翻译结果）
const keywordQuery = searchQueries[searchQueries.length - 1]; // 优先用英文
results = await searchService.keywordSearch(keywordQuery, { topK, filter: { kbId } });
results = await searchService.keywordSearch(keywordQuery, { topK, filter: { kbId: ekbKbId } });
} else {
// 混合检索：向量 + 关键词
// 对每个查询词都执行混合检索，然后融合
const allResults = await Promise.all(
searchQueries.map(q => searchService.hybridSearch(q, { topK: topK * 2, filter: { kbId } }))
searchQueries.map(q => searchService.hybridSearch(q, { topK: topK * 2, filter: { kbId: ekbKbId } }))
);
// RRF 融合多个查询的结果
@@ -161,7 +135,6 @@ async function searchWithPgvector(
documentId: r.documentId,
chunkId: r.chunkId,
metadata: r.metadata,
source: 'pgvector' as const,
}));
}
@@ -197,58 +170,6 @@ function fuseMultiQueryResults(
}));
}
/**
* 使用 Dify 检索
*/
async function searchWithDify(
difyDatasetId: string,
query: string,
topK: number
): Promise<RagSearchResult[]> {
const results = await difyClient.retrieveKnowledge(difyDatasetId, query, {
retrieval_model: {
search_method: 'semantic_search',
top_k: topK,
},
});
return (results.records || []).map((r: any) => ({
content: r.segment?.content || '',
score: r.score || 0,
metadata: r.segment?.metadata,
source: 'dify' as const,
}));
}
/**
* 合并两个后端的检索结果
*/
function mergeSearchResults(
pgResults: RagSearchResult[],
difyResults: RagSearchResult[],
topK: number
): RagSearchResult[] {
// 简单合并：按分数排序，去重
const all = [...pgResults, ...difyResults];
// 按分数降序排序
all.sort((a, b) => b.score - a.score);
// 去重（基于内容相似度，简化为前100字符比较）
const seen = new Set<string>();
const unique: RagSearchResult[] = [];
for (const result of all) {
const key = result.content.substring(0, 100);
if (!seen.has(key)) {
seen.add(key);
unique.push(result);
}
}
return unique.slice(0, topK);
}
// ==================== 入库服务 ====================
/**
@@ -261,7 +182,7 @@ export async function ingestDocument(
filename: string,
options: RagIngestOptions = {}
): Promise<IngestResult> {
logger.info(`[RAG] 入库文档: kbId=${kbId}, filename=${filename}, backend=${RAG_BACKEND}`);
logger.info(`[RAG] 入库文档: kbId=${kbId}, filename=${filename}`);
// 验证权限
const knowledgeBase = await prisma.knowledgeBase.findFirst({
@@ -272,80 +193,53 @@ export async function ingestDocument(
throw new Error('Knowledge base not found or access denied');
}
if (RAG_BACKEND === 'pgvector' || RAG_BACKEND === 'hybrid') {
// 使用新的 pgvector 入库流程
const ingestService = getDocumentIngestService(prisma);
const result = await ingestService.ingestDocument(
{
filename,
fileBuffer: file,
},
{
kbId, // 这里需要映射到 EkbKnowledgeBase.id
contentType: options.contentType,
tags: options.tags,
metadata: options.metadata,
generateSummary: options.generateSummary,
}
);
// 查找或创建对应的 EKB 知识库
const ekbKb = await findOrCreateEkbKnowledgeBase(userId, knowledgeBase.name, knowledgeBase.description);
logger.info(`[RAG] PKB->EKB 映射: pkbKbId=${kbId} -> ekbKbId=${ekbKb.id}`);
// 如果是 hybrid 模式，同时上传到 Dify
if (RAG_BACKEND === 'hybrid') {
try {
await difyClient.uploadDocumentDirectly(
knowledgeBase.difyDatasetId,
file,
filename
);
} catch (error) {
logger.warn('Dify 上传失败，但 pgvector 已成功', { error });
}
// 使用 pgvector 入库
const ingestService = getDocumentIngestService(prisma);
const result = await ingestService.ingestDocument(
{
filename,
fileBuffer: file,
},
{
kbId: ekbKb.id,
contentType: options.contentType,
tags: options.tags,
metadata: options.metadata,
generateSummary: options.generateSummary,
}
);
return result;
} else {
// 纯 Dify 模式
const difyResult = await difyClient.uploadDocumentDirectly(
knowledgeBase.difyDatasetId,
file,
filename
);
return {
success: true,
documentId: difyResult.document.id,
};
}
return result;
}
// ==================== 知识库管理 ====================
/**
* 创建知识库（双轨）
* 查找或创建 EKB 知识库
*/
export async function createKnowledgeBaseWithRag(
async function findOrCreateEkbKnowledgeBase(
userId: string,
name: string,
description?: string
): Promise<{ pkbKbId: string; ekbKbId?: string; difyDatasetId?: string }> {
let difyDatasetId: string | undefined;
let ekbKbId: string | undefined;
description?: string | null
) {
// 查找已存在的 EKB 知识库
let ekbKb = await prisma.ekbKnowledgeBase.findFirst({
where: {
ownerId: userId,
name: name,
},
});
// 1. 在 Dify 创建（如果需要）
if (RAG_BACKEND === 'dify' || RAG_BACKEND === 'hybrid') {
const sanitizedName = name.replace(/[^\u4e00-\u9fa5a-zA-Z0-9_-]/g, '_').substring(0, 50);
const difyDataset = await difyClient.createDataset({
name: `kb_${sanitizedName}_${Date.now()}`,
description: description?.substring(0, 200) || '',
indexing_technique: 'high_quality',
});
difyDatasetId = difyDataset.id;
}
// 2. 在 EKB 创建（如果需要）
if (RAG_BACKEND === 'pgvector' || RAG_BACKEND === 'hybrid') {
const ekbKb = await prisma.ekbKnowledgeBase.create({
// 如果不存在，创建一个
if (!ekbKb) {
logger.info(`[RAG] 创建 EKB 知识库: name=${name}`);
ekbKb = await prisma.ekbKnowledgeBase.create({
data: {
name,
description,
@@ -354,35 +248,97 @@ export async function createKnowledgeBaseWithRag(
config: {},
},
});
ekbKbId = ekbKb.id;
logger.info(`[RAG] EKB 知识库已创建: ekbKbId=${ekbKb.id}`);
}
// 3. 在 PKB 创建主记录
return ekbKb;
}
/**
* 创建知识库
*/
export async function createKnowledgeBaseWithRag(
userId: string,
name: string,
description?: string
): Promise<{ pkbKbId: string; ekbKbId: string }> {
// 1. 在 EKB 创建知识库
const ekbKb = await prisma.ekbKnowledgeBase.create({
data: {
name,
description,
type: 'USER',
ownerId: userId,
config: {},
},
});
// 2. 在 PKB 创建主记录
const pkbKb = await prisma.knowledgeBase.create({
data: {
userId,
name,
description,
difyDatasetId: difyDatasetId || '',
// 可以添加 ekbKbId 字段关联，或通过 metadata 存储
difyDatasetId: '', // 不再使用，保留为空
},
});
// 4. 更新用户配额
// 3. 更新用户配额
await prisma.user.update({
where: { id: userId },
data: { kbUsed: { increment: 1 } },
});
logger.info(`[RAG] 知识库创建成功: pkbKbId=${pkbKb.id}, ekbKbId=${ekbKb.id}`);
return {
pkbKbId: pkbKb.id,
ekbKbId,
difyDatasetId,
ekbKbId: ekbKb.id,
};
}
/**
* 获取知识库统计（双轨）
* 删除知识库
*/
export async function deleteKnowledgeBaseWithRag(
userId: string,
kbId: string,
knowledgeBaseName: string
): Promise<void> {
// 1. 删除 EKB 知识库及其所有文档和 chunks
const ekbKb = await prisma.ekbKnowledgeBase.findFirst({
where: {
ownerId: userId,
name: knowledgeBaseName,
},
});
if (ekbKb) {
// 删除所有 chunks
await prisma.ekbChunk.deleteMany({
where: {
document: {
kbId: ekbKb.id,
},
},
});
// 删除所有 documents
await prisma.ekbDocument.deleteMany({
where: { kbId: ekbKb.id },
});
// 删除 knowledge base
await prisma.ekbKnowledgeBase.delete({
where: { id: ekbKb.id },
});
logger.info(`[RAG] EKB 知识库已删除: ekbKbId=${ekbKb.id}`);
}
}
/**
* 获取知识库统计
*/
export async function getKnowledgeBaseStats(
userId: string,
@@ -390,7 +346,7 @@ export async function getKnowledgeBaseStats(
): Promise<{
documentCount: number;
totalTokens: number;
backend: RagBackend;
chunkCount: number;
}> {
const knowledgeBase = await prisma.knowledgeBase.findFirst({
where: { id: kbId, userId },
@@ -407,34 +363,35 @@ export async function getKnowledgeBaseStats(
totalTokens: knowledgeBase.documents.reduce((sum, d) => sum + (d.tokensCount || 0), 0),
};
// 如果使用 pgvector，也获取 EKB 统计
if (RAG_BACKEND === 'pgvector' || RAG_BACKEND === 'hybrid') {
try {
const searchService = getVectorSearchService(prisma);
const ekbStats = await searchService.getKnowledgeBaseStats(kbId);
// 获取 EKB 统计
try {
const ekbKb = await prisma.ekbKnowledgeBase.findFirst({
where: {
ownerId: userId,
name: knowledgeBase.name,
},
});
if (ekbKb) {
const chunkCount = await prisma.ekbChunk.count({
where: {
document: {
kbId: ekbKb.id,
},
},
});
return {
documentCount: Math.max(pkbStats.documentCount, ekbStats.documentCount),
totalTokens: Math.max(pkbStats.totalTokens, ekbStats.totalTokens),
backend: RAG_BACKEND,
...pkbStats,
chunkCount,
};
} catch {
// EKB 统计失败，返回 PKB 统计
}
} catch (error) {
logger.warn('[RAG] 获取 EKB 统计失败', { error });
}
return {
...pkbStats,
backend: RAG_BACKEND,
chunkCount: 0,
};
}
// ==================== 导出当前后端配置 ====================
export function getCurrentBackend(): RagBackend {
return RAG_BACKEND;
}
export { RAG_BACKEND };