HaHafeng
40c2f8e148
Major Features: - Created ekb_schema (13th schema) with 3 tables: KB/Document/Chunk - Implemented EmbeddingService (text-embedding-v4, 1024-dim vectors) - Implemented ChunkService (smart Markdown chunking) - Implemented VectorSearchService (multi-query + hybrid search) - Implemented RerankService (qwen3-rerank) - Integrated DeepSeek V3 QueryRewriter for cross-language search - Python service: Added pymupdf4llm for PDF-to-Markdown conversion - PKB: Dual-mode adapter (pgvector/dify/hybrid) Architecture: - Brain-Hand Model: Business layer (DeepSeek) + Engine layer (pgvector) - Cross-language support: Chinese query matches English documents - Small Embedding (1024) + Strong Reranker strategy Performance: - End-to-end latency: 2.5s - Cost per query: 0.0025 RMB - Accuracy improvement: +20.5% (cross-language) Tests: - test-embedding-service.ts: Vector embedding verified - test-rag-e2e.ts: Full pipeline tested - test-rerank.ts: Rerank quality validated - test-query-rewrite.ts: Cross-language search verified - test-pdf-ingest.ts: Real PDF document tested (Dongen 2003.pdf) Documentation: - Added 05-RAG-Engine-User-Guide.md - Added 02-Document-Processing-User-Guide.md - Updated system status documentation Status: Production ready
123 lines
2.3 KiB
SQL
123 lines
2.3 KiB
SQL
-- ==================== Postgres-Only 改造：手动迁移 ====================
|
||
-- 文件: 001_add_postgres_cache_and_checkpoint.sql
|
||
-- 目的: 添加缓存表和断点续传字段
|
||
-- 日期: 2025-12-13
|
||
-- 说明: 避免Prisma migrate的shadow database问题，手动添加所需表和字段
|
||
|
||
-- ==================== 1. 创建缓存表 (AppCache) ====================
|
||
|
||
CREATE TABLE IF NOT EXISTS platform_schema.app_cache (
|
||
id SERIAL PRIMARY KEY,
|
||
key VARCHAR(500) UNIQUE NOT NULL,
|
||
value JSONB NOT NULL,
|
||
expires_at TIMESTAMP NOT NULL,
|
||
created_at TIMESTAMP DEFAULT NOW()
|
||
);
|
||
|
||
-- 创建索引（优化过期查询和key查询）
|
||
CREATE INDEX IF NOT EXISTS idx_app_cache_expires
|
||
ON platform_schema.app_cache(expires_at);
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_app_cache_key_expires
|
||
ON platform_schema.app_cache(key, expires_at);
|
||
|
||
-- ==================== 2. 为AslScreeningTask添加新字段 ====================
|
||
|
||
-- 任务拆分支持字段
|
||
ALTER TABLE asl_schema.screening_tasks
|
||
ADD COLUMN IF NOT EXISTS total_batches INTEGER DEFAULT 1,
|
||
ADD COLUMN IF NOT EXISTS processed_batches INTEGER DEFAULT 0,
|
||
ADD COLUMN IF NOT EXISTS current_batch_index INTEGER DEFAULT 0;
|
||
|
||
-- 断点续传支持字段
|
||
ALTER TABLE asl_schema.screening_tasks
|
||
ADD COLUMN IF NOT EXISTS current_index INTEGER DEFAULT 0,
|
||
ADD COLUMN IF NOT EXISTS last_checkpoint TIMESTAMP,
|
||
ADD COLUMN IF NOT EXISTS checkpoint_data JSONB;
|
||
|
||
-- ==================== 3. 验证创建结果 ====================
|
||
|
||
-- 查看app_cache表结构
|
||
SELECT
|
||
column_name,
|
||
data_type,
|
||
is_nullable,
|
||
column_default
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'platform_schema'
|
||
AND table_name = 'app_cache'
|
||
ORDER BY ordinal_position;
|
||
|
||
-- 查看screening_tasks新增字段
|
||
SELECT
|
||
column_name,
|
||
data_type,
|
||
is_nullable,
|
||
column_default
|
||
FROM information_schema.columns
|
||
WHERE table_schema = 'asl_schema'
|
||
AND table_name = 'screening_tasks'
|
||
AND column_name IN (
|
||
'total_batches', 'processed_batches', 'current_batch_index',
|
||
'current_index', 'last_checkpoint', 'checkpoint_data'
|
||
)
|
||
ORDER BY ordinal_position;
|
||
|
||
-- ==================== 完成 ====================
|
||
-- ✅ 缓存表已创建
|
||
-- ✅ 任务拆分字段已添加
|
||
-- ✅ 断点续传字段已添加
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|