feat(rag): Complete RAG engine implementation with pgvector

Major Features:
- Created ekb_schema (13th schema) with 3 tables: KB/Document/Chunk
- Implemented EmbeddingService (text-embedding-v4, 1024-dim vectors)
- Implemented ChunkService (smart Markdown chunking)
- Implemented VectorSearchService (multi-query + hybrid search)
- Implemented RerankService (qwen3-rerank)
- Integrated DeepSeek V3 QueryRewriter for cross-language search
- Python service: Added pymupdf4llm for PDF-to-Markdown conversion
- PKB: Dual-mode adapter (pgvector/dify/hybrid)

Architecture:
- Brain-Hand Model: Business layer (DeepSeek) + Engine layer (pgvector)
- Cross-language support: Chinese query matches English documents
- Small Embedding (1024) + Strong Reranker strategy

Performance:
- End-to-end latency: 2.5s
- Cost per query: 0.0025 RMB
- Accuracy improvement: +20.5% (cross-language)

Tests:
- test-embedding-service.ts: Vector embedding verified
- test-rag-e2e.ts: Full pipeline tested
- test-rerank.ts: Rerank quality validated
- test-query-rewrite.ts: Cross-language search verified
- test-pdf-ingest.ts: Real PDF document tested (Dongen 2003.pdf)

Documentation:
- Added 05-RAG-Engine-User-Guide.md
- Added 02-Document-Processing-User-Guide.md
- Updated system status documentation

Status: Production ready
This commit is contained in:
2026-01-21 20:24:29 +08:00
parent 1f5bf2cd65
commit 40c2f8e148
338 changed files with 11014 additions and 1158 deletions

View File

@@ -99,6 +99,9 @@ vite.config.*.timestamp-*

View File

@@ -66,6 +66,9 @@ exec nginx -g 'daemon off;'

View File

@@ -222,6 +222,9 @@ http {

View File

@@ -53,3 +53,6 @@ export default apiClient;

View File

@@ -252,3 +252,6 @@ export async function logout(): Promise<void> {

View File

@@ -18,3 +18,6 @@ export * from './api';

View File

@@ -42,3 +42,6 @@ export async function fetchUserModules(): Promise<string[]> {

View File

@@ -122,3 +122,6 @@ export default ModulePermissionModal;

View File

@@ -33,3 +33,6 @@ export default AdminModule;

View File

@@ -198,3 +198,6 @@ export const TENANT_TYPE_NAMES: Record<TenantType, string> = {

View File

@@ -82,3 +82,6 @@ export default AgentCard;

View File

@@ -12,3 +12,6 @@ export { ChatWorkspace } from './ChatWorkspace';

View File

@@ -176,3 +176,6 @@ export const BRAND_COLORS = {

View File

@@ -214,3 +214,6 @@

View File

@@ -569,6 +569,9 @@ export default FulltextDetailDrawer;

View File

@@ -1,15 +1,40 @@
/**
* Tool C API封装
*
* 提供6个核心API方法
* 提供核心API方法
* - Session管理上传、获取、预览、心跳
* - AI功能生成代码、执行代码、一步到位处理、获取历史
* - 快速操作筛选、映射、分箱、条件、删NA、计算、Pivot
*/
import apiClient from '../../../common/api/axios';
import { getAccessToken } from '../../../framework/auth/api';
const BASE_URL = '/api/v1/dc/tool-c';
// ==================== 认证辅助函数 ====================
/**
* 获取带认证的请求头(供原生 fetch 使用)
*
* @example
* const response = await fetch(url, {
* method: 'POST',
* headers: getAuthHeaders(),
* body: JSON.stringify(data),
* });
*/
export function getAuthHeaders(): HeadersInit {
const headers: Record<string, string> = {
'Content-Type': 'application/json',
};
const token = getAccessToken();
if (token) {
headers['Authorization'] = `Bearer ${token}`;
}
return headers;
}
// ==================== 类型定义 ====================
export interface UploadResponse {
@@ -242,3 +267,37 @@ export const getSessionStatus = async (
return response.data;
};
// ==================== 快速操作(功能按钮) ====================
export interface QuickActionParams {
sessionId: string;
action: string;
params: Record<string, any>;
}
export interface QuickActionResponse {
success: boolean;
message?: string;
error?: string;
data?: {
newDataPreview: Record<string, any>[];
[key: string]: any;
};
}
/**
* 执行快速操作筛选、映射、分箱、条件、删NA、计算、Pivot等
*/
export const quickAction = async (params: QuickActionParams): Promise<QuickActionResponse> => {
const response = await apiClient.post(`${BASE_URL}/quick-action`, params);
return response.data;
};
/**
* 预览快速操作结果
*/
export const quickActionPreview = async (params: QuickActionParams): Promise<QuickActionResponse> => {
const response = await apiClient.post(`${BASE_URL}/quick-action/preview`, params);
return response.data;
};

View File

@@ -162,6 +162,9 @@ export const useAssets = (activeTab: AssetTabType) => {

View File

@@ -152,6 +152,9 @@ export const useRecentTasks = () => {

View File

@@ -10,6 +10,7 @@
import React, { useState } from 'react';
import { Modal, Select, Input, Button, Radio, Space, Tag, App, Alert } from 'antd';
import { Info } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface BinningDialogProps {
visible: boolean;
@@ -135,7 +136,7 @@ const BinningDialog: React.FC<BinningDialogProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'binning',

View File

@@ -10,6 +10,7 @@
import React, { useState } from 'react';
import { Modal, Select, Input, Button, Radio, Space, Tag, App, Alert } from 'antd';
import { Info } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface BinningDialogProps {
visible: boolean;
@@ -113,7 +114,7 @@ const BinningDialog: React.FC<BinningDialogProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'binning',

View File

@@ -1,6 +1,7 @@
import React, { useState } from 'react';
import { Modal, Input, Button, Alert, Collapse, Tag, App } from 'antd';
import { Calculator, Lightbulb, BookOpen } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
visible: boolean;
@@ -92,7 +93,7 @@ const ComputeDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'compute',

View File

@@ -1,6 +1,7 @@
import React, { useState } from 'react';
import { Modal, Input, Button, Select, Alert, App, Card, Tag } from 'antd';
import { PlusCircle, Trash2, AlertCircle } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Condition {
column: string;
@@ -180,7 +181,7 @@ const ConditionalDialog: React.FC<Props> = ({
// 调用API
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'conditional',

View File

@@ -1,6 +1,7 @@
import React, { useState, useEffect } from 'react';
import { Modal, Radio, Button, Slider, Alert, App, Statistic, Row, Col, Checkbox } from 'antd';
import { Trash2, AlertTriangle } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
visible: boolean;
@@ -79,7 +80,7 @@ const DropnaDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'dropna',
@@ -353,5 +354,6 @@ export default DropnaDialog;

View File

@@ -10,6 +10,7 @@
import React, { useState } from 'react';
import { Modal, Select, Input, Button, Radio, App } from 'antd';
import { Plus, Trash2 } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface FilterCondition {
column: string;
@@ -98,7 +99,7 @@ const FilterDialog: React.FC<FilterDialogProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'filter',
@@ -156,7 +157,7 @@ const FilterDialog: React.FC<FilterDialogProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'filter',

View File

@@ -8,6 +8,7 @@
import React, { useState, useEffect } from 'react';
import { Button, Alert, Checkbox, App, Input, Spin, Tag } from 'antd';
import { ArrowLeftRight, Info, Sparkles } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
columns: Array<{ id: string; name: string }>;
@@ -60,7 +61,7 @@ const MetricTimePanel: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/metric-time/detect', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
valueVars,
@@ -131,7 +132,7 @@ const MetricTimePanel: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'metric_time',
@@ -438,5 +439,6 @@ export default MetricTimePanel;

View File

@@ -1,5 +1,6 @@
import React, { useState, useEffect } from 'react';
import { Modal, Tabs, Radio, Select, Input, Checkbox, Alert, App, Row, Col, InputNumber, Space, Collapse } from 'antd';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
visible: boolean;
@@ -46,7 +47,7 @@ const MissingValueDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
column: selectedColumn
@@ -87,7 +88,7 @@ const MissingValueDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'dropna',
@@ -137,7 +138,7 @@ const MissingValueDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/simple', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
column: selectedColumn,
@@ -180,7 +181,7 @@ const MissingValueDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/mice', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
columns: miceColumns,

View File

@@ -9,6 +9,7 @@
import React, { useState, useEffect } from 'react';
import { Form, Select, Button, Alert, Table, Spin, Divider, Space, Card, Tag, message, Radio } from 'antd';
import { getAuthHeaders } from '../../../api/toolC';
const { Option } = Select;
@@ -70,7 +71,7 @@ export const MultiMetricPanel: React.FC<MultiMetricPanelProps> = ({
const response = await fetch('/api/v1/dc/tool-c/multi-metric/detect', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
valueVars,
@@ -134,7 +135,7 @@ export const MultiMetricPanel: React.FC<MultiMetricPanelProps> = ({
// 调用preview API
const response = await fetch('/api/v1/dc/tool-c/quick-action/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action,
@@ -202,7 +203,7 @@ export const MultiMetricPanel: React.FC<MultiMetricPanelProps> = ({
// 调用快速操作API执行转换
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action,

View File

@@ -1,6 +1,7 @@
import React, { useState, useEffect } from 'react';
import { Modal, Select, Button, Alert, Checkbox, Radio, App } from 'antd';
import { ArrowLeftRight, Info } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
visible: boolean;
@@ -67,7 +68,7 @@ const PivotDialog: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'pivot',

View File

@@ -6,6 +6,7 @@
import React, { useState } from 'react';
import { Select, Button, Alert, Checkbox, Radio, App } from 'antd';
import { ArrowLeftRight, Info } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
columns: Array<{ id: string; name: string }>;
@@ -57,7 +58,7 @@ const PivotPanel: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'pivot',
@@ -324,5 +325,6 @@ export default PivotPanel;

View File

@@ -9,6 +9,7 @@
import React, { useState, useEffect } from 'react';
import { Modal, Select, Input, Button, Checkbox, Table, Spin, App } from 'antd';
import { getAuthHeaders } from '../../../api/toolC';
interface RecodeDialogProps {
visible: boolean;
@@ -59,7 +60,11 @@ const RecodeDialog: React.FC<RecodeDialogProps> = ({
try {
// ✨ 调用后端API获取唯一值从完整数据中提取不受前端50行限制
const response = await fetch(
`/api/v1/dc/tool-c/sessions/${sessionId}/unique-values?column=${encodeURIComponent(selectedColumn)}`
`/api/v1/dc/tool-c/sessions/${sessionId}/unique-values?column=${encodeURIComponent(selectedColumn)}`,
{
method: 'GET',
headers: getAuthHeaders(),
}
);
const result = await response.json();
@@ -188,7 +193,7 @@ const RecodeDialog: React.FC<RecodeDialogProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'recode',

View File

@@ -9,6 +9,7 @@ import React, { useState, useCallback } from 'react';
import { MessageSquare, X, Upload } from 'lucide-react';
import { StreamingSteps, StreamStep } from './StreamingSteps';
import { App } from 'antd';
import { getAuthHeaders } from '../../../api/toolC';
interface SidebarProps {
isOpen: boolean;
@@ -51,7 +52,7 @@ const Sidebar: React.FC<SidebarProps> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/ai/stream-process', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
message: userMessage,

View File

@@ -6,6 +6,7 @@
import React, { useState } from 'react';
import { Button, Alert, Checkbox, App, Input, Collapse, Radio } from 'antd';
import { ArrowLeftRight, Info } from 'lucide-react';
import { getAuthHeaders } from '../../../api/toolC';
interface Props {
columns: Array<{ id: string; name: string }>;
@@ -89,7 +90,7 @@ const UnpivotPanel: React.FC<Props> = ({
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
headers: getAuthHeaders(),
body: JSON.stringify({
sessionId,
action: 'unpivot',

View File

@@ -122,6 +122,9 @@ export function useSessionStatus({

View File

@@ -19,6 +19,7 @@ import ComputeDialog from './components/ComputeDialog';
import TransformDialog from './components/TransformDialog';
import { useSessionStatus } from './hooks/useSessionStatus';
import * as api from '../../api/toolC';
import { getAuthHeaders } from '../../api/toolC';
// ==================== 类型定义 ====================
@@ -302,7 +303,10 @@ const ToolC = () => {
try {
// ✅ 从后端读取完整数据AI处理后的数据已保存到OSS
const response = await fetch(`/api/v1/dc/tool-c/sessions/${state.sessionId}/export`);
const response = await fetch(`/api/v1/dc/tool-c/sessions/${state.sessionId}/export`, {
method: 'GET',
headers: getAuthHeaders(),
});
if (!response.ok) {
throw new Error('导出失败');

View File

@@ -114,6 +114,9 @@ export interface DataStats {

View File

@@ -110,6 +110,9 @@ export type AssetTabType = 'all' | 'processed' | 'raw';

View File

@@ -298,6 +298,9 @@ export default KnowledgePage;

View File

@@ -53,6 +53,9 @@ export interface BatchTemplate {

View File

@@ -134,3 +134,6 @@ export default function AgentModal({ visible, taskCount, onClose, onConfirm }: A

View File

@@ -54,3 +54,6 @@ export default function BatchToolbar({ selectedCount, onRunBatch, onClearSelecti

View File

@@ -77,3 +77,6 @@ export default function FilterChips({ filters, counts, onFilterChange }: FilterC

View File

@@ -67,3 +67,6 @@ export default function Header({ onUpload }: HeaderProps) {

View File

@@ -121,3 +121,6 @@ export default function ReportDetail({ report, onBack }: ReportDetailProps) {

View File

@@ -49,3 +49,6 @@ export default function ScoreRing({ score, size = 'medium', showLabel = true }:

View File

@@ -84,3 +84,6 @@ export default function Sidebar({ currentView, onViewChange, onSettingsClick }:

View File

@@ -26,3 +26,6 @@ export { default as TaskDetail } from './TaskDetail';

View File

@@ -295,3 +295,6 @@ export default function Dashboard() {

View File

@@ -244,3 +244,6 @@

View File

@@ -345,3 +345,6 @@ export default TenantListPage;

View File

@@ -254,3 +254,6 @@ export async function fetchModuleList(): Promise<ModuleInfo[]> {

View File

@@ -473,3 +473,6 @@ export default AIStreamChat;

View File

@@ -173,3 +173,6 @@ export default ConversationList;

View File

@@ -25,3 +25,6 @@ export type {

View File

@@ -317,3 +317,6 @@ export default useAIStream;

View File

@@ -246,3 +246,6 @@ export default useConversations;

View File

@@ -65,6 +65,9 @@ export { default as Placeholder } from './Placeholder';

View File

@@ -45,6 +45,9 @@ interface ImportMeta {