feat(iit): Implement real-time quality control system

Summary:

- Add 4 new database tables: iit_field_metadata, iit_qc_logs, iit_record_summary, iit_qc_project_stats

- Implement pg-boss debounce mechanism in WebhookController

- Refactor QC Worker for dual output: QC logs + record summary

- Enhance HardRuleEngine to support form-based rule filtering

- Create QcService for QC data queries

- Optimize ChatService with new intents: query_enrollment, query_qc_status

- Add admin batch operations: one-click full QC + one-click full summary

- Create IIT Admin management module: project config, QC rules, user mapping

Status: Code complete, pending end-to-end testing
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-02-07 21:56:11 +08:00
parent 0c590854b5
commit 5db4a7064c
74 changed files with 13383 additions and 2129 deletions

View File

@@ -1,3 +1,20 @@
-- CreateSchemas (added for shadow database compatibility)
CREATE SCHEMA IF NOT EXISTS "admin_schema";
CREATE SCHEMA IF NOT EXISTS "agent_schema";
CREATE SCHEMA IF NOT EXISTS "aia_schema";
CREATE SCHEMA IF NOT EXISTS "asl_schema";
CREATE SCHEMA IF NOT EXISTS "capability_schema";
CREATE SCHEMA IF NOT EXISTS "common_schema";
CREATE SCHEMA IF NOT EXISTS "dc_schema";
CREATE SCHEMA IF NOT EXISTS "ekb_schema";
CREATE SCHEMA IF NOT EXISTS "iit_schema";
CREATE SCHEMA IF NOT EXISTS "pkb_schema";
CREATE SCHEMA IF NOT EXISTS "platform_schema";
CREATE SCHEMA IF NOT EXISTS "protocol_schema";
CREATE SCHEMA IF NOT EXISTS "rvw_schema";
CREATE SCHEMA IF NOT EXISTS "ssa_schema";
CREATE SCHEMA IF NOT EXISTS "st_schema";
-- CreateTable
CREATE TABLE "users" (
"id" TEXT NOT NULL,

View File

@@ -1,3 +1,22 @@
-- CreateTable (merged from create_tool_c_session.sql for shadow database compatibility)
CREATE TABLE IF NOT EXISTS "dc_schema"."dc_tool_c_sessions" (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id VARCHAR(255) NOT NULL,
file_name VARCHAR(500) NOT NULL,
file_key VARCHAR(500) NOT NULL,
total_rows INTEGER NOT NULL,
total_cols INTEGER NOT NULL,
columns JSONB NOT NULL,
encoding VARCHAR(50),
file_size INTEGER NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_dc_tool_c_sessions_user_id ON "dc_schema"."dc_tool_c_sessions"(user_id);
CREATE INDEX IF NOT EXISTS idx_dc_tool_c_sessions_expires_at ON "dc_schema"."dc_tool_c_sessions"(expires_at);
-- AlterTable
-- 添加 column_mapping 字段到 dc_tool_c_sessions 表
-- 用于解决表头特殊字符问题

View File

@@ -0,0 +1,162 @@
-- IIT Manager Agent V2.9.1 数据库迁移
-- 创建 8 张新表用于支持完整的 Agent 功能
-- 日期: 2026-02-07
-- CreateTable: iit_skills (Skill 配置存储)
CREATE TABLE "iit_schema"."skills" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"skill_type" TEXT NOT NULL,
"name" TEXT NOT NULL,
"description" TEXT,
"config" JSONB NOT NULL,
"is_active" BOOLEAN NOT NULL DEFAULT true,
"version" INTEGER NOT NULL DEFAULT 1,
"trigger_type" TEXT NOT NULL DEFAULT 'webhook',
"cron_schedule" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "skills_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_field_mapping (字段名映射字典)
CREATE TABLE "iit_schema"."field_mapping" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"alias_name" TEXT NOT NULL,
"actual_name" TEXT NOT NULL,
"field_type" TEXT,
"field_label" TEXT,
"validation" JSONB,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "field_mapping_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_conversation_history (对话历史/流水账)
CREATE TABLE "iit_schema"."conversation_history" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"record_id" TEXT,
"role" TEXT NOT NULL,
"content" TEXT NOT NULL,
"intent" TEXT,
"entities" JSONB,
"feedback" TEXT,
"feedback_reason" TEXT,
"embedding" vector(1536),
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "conversation_history_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_project_memory (项目级热记忆)
CREATE TABLE "iit_schema"."project_memory" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"content" TEXT NOT NULL,
"last_updated_by" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "project_memory_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_weekly_reports (周报归档/历史书)
CREATE TABLE "iit_schema"."weekly_reports" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"week_number" INTEGER NOT NULL,
"week_start" TIMESTAMP(3) NOT NULL,
"week_end" TIMESTAMP(3) NOT NULL,
"summary" TEXT NOT NULL,
"metrics" JSONB,
"created_by" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "weekly_reports_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_agent_trace (ReAct 推理轨迹)
CREATE TABLE "iit_schema"."agent_trace" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"query" TEXT NOT NULL,
"intent_type" TEXT,
"trace" JSONB NOT NULL,
"token_usage" INTEGER,
"duration" INTEGER,
"success" BOOLEAN NOT NULL DEFAULT true,
"error_msg" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "agent_trace_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_pii_audit_log (PII 脱敏审计日志)
CREATE TABLE "iit_schema"."pii_audit_log" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"user_id" TEXT NOT NULL,
"session_id" TEXT NOT NULL,
"original_hash" TEXT NOT NULL,
"masked_payload" TEXT NOT NULL,
"masking_map" TEXT NOT NULL,
"pii_count" INTEGER NOT NULL,
"pii_types" TEXT[],
"llm_provider" TEXT NOT NULL,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "pii_audit_log_pkey" PRIMARY KEY ("id")
);
-- CreateTable: iit_form_templates (表单模板)
CREATE TABLE "iit_schema"."form_templates" (
"id" TEXT NOT NULL,
"project_id" TEXT NOT NULL,
"form_name" TEXT NOT NULL,
"field_schema" JSONB NOT NULL,
"keywords" TEXT[],
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL,
CONSTRAINT "form_templates_pkey" PRIMARY KEY ("id")
);
-- CreateIndex: iit_skills
CREATE INDEX "idx_iit_skill_project" ON "iit_schema"."skills"("project_id");
CREATE INDEX "idx_iit_skill_active" ON "iit_schema"."skills"("is_active");
CREATE UNIQUE INDEX "unique_iit_skill_project_type" ON "iit_schema"."skills"("project_id", "skill_type");
-- CreateIndex: iit_field_mapping
CREATE INDEX "idx_iit_field_mapping_project" ON "iit_schema"."field_mapping"("project_id");
CREATE UNIQUE INDEX "unique_iit_field_mapping" ON "iit_schema"."field_mapping"("project_id", "alias_name");
-- CreateIndex: iit_conversation_history
CREATE INDEX "idx_iit_conv_project_user" ON "iit_schema"."conversation_history"("project_id", "user_id");
CREATE INDEX "idx_iit_conv_project_record" ON "iit_schema"."conversation_history"("project_id", "record_id");
CREATE INDEX "idx_iit_conv_created" ON "iit_schema"."conversation_history"("created_at");
-- CreateIndex: iit_project_memory
CREATE UNIQUE INDEX "project_memory_project_id_key" ON "iit_schema"."project_memory"("project_id");
-- CreateIndex: iit_weekly_reports
CREATE INDEX "idx_iit_weekly_report_project" ON "iit_schema"."weekly_reports"("project_id");
CREATE UNIQUE INDEX "unique_iit_weekly_report" ON "iit_schema"."weekly_reports"("project_id", "week_number");
-- CreateIndex: iit_agent_trace
CREATE INDEX "idx_iit_trace_project_time" ON "iit_schema"."agent_trace"("project_id", "created_at");
CREATE INDEX "idx_iit_trace_user" ON "iit_schema"."agent_trace"("user_id");
-- CreateIndex: iit_pii_audit_log
CREATE INDEX "idx_iit_pii_project_user" ON "iit_schema"."pii_audit_log"("project_id", "user_id");
CREATE INDEX "idx_iit_pii_session" ON "iit_schema"."pii_audit_log"("session_id");
CREATE INDEX "idx_iit_pii_created" ON "iit_schema"."pii_audit_log"("created_at");
-- CreateIndex: iit_form_templates
CREATE INDEX "idx_iit_form_template_project" ON "iit_schema"."form_templates"("project_id");
CREATE UNIQUE INDEX "unique_iit_form_template" ON "iit_schema"."form_templates"("project_id", "form_name");

View File

@@ -1,105 +0,0 @@
-- 创建 Tool C Session 表
-- 日期: 2025-12-06
-- 用途: 科研数据编辑器会话管理
CREATE TABLE IF NOT EXISTS dc_schema.dc_tool_c_sessions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id VARCHAR(255) NOT NULL,
file_name VARCHAR(500) NOT NULL,
file_key VARCHAR(500) NOT NULL,
-- 数据元信息
total_rows INTEGER NOT NULL,
total_cols INTEGER NOT NULL,
columns JSONB NOT NULL,
encoding VARCHAR(50),
file_size INTEGER NOT NULL,
-- 时间戳
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL
);
-- 创建索引
CREATE INDEX IF NOT EXISTS idx_dc_tool_c_sessions_user_id ON dc_schema.dc_tool_c_sessions(user_id);
CREATE INDEX IF NOT EXISTS idx_dc_tool_c_sessions_expires_at ON dc_schema.dc_tool_c_sessions(expires_at);
-- 添加注释
COMMENT ON TABLE dc_schema.dc_tool_c_sessions IS 'Tool C (科研数据编辑器) Session会话表';
COMMENT ON COLUMN dc_schema.dc_tool_c_sessions.file_key IS 'OSS存储路径: dc/tool-c/sessions/{timestamp}-{fileName}';
COMMENT ON COLUMN dc_schema.dc_tool_c_sessions.columns IS '列名数组 ["age", "gender", "diagnosis"]';
COMMENT ON COLUMN dc_schema.dc_tool_c_sessions.expires_at IS '过期时间创建后10分钟';

View File

@@ -1,71 +0,0 @@
-- ============================================================
-- EKB Schema 索引创建脚本
-- 执行时机prisma migrate 之后手动执行
-- 参考文档docs/02-通用能力层/03-RAG引擎/04-数据模型设计.md
-- ============================================================
-- 1. 确保 pgvector 扩展已启用
CREATE EXTENSION IF NOT EXISTS vector;
-- 2. 确保 pg_bigm 扩展已启用(中文关键词检索)
CREATE EXTENSION IF NOT EXISTS pg_bigm;
-- ===== MVP 阶段必须创建 =====
-- 3. HNSW 向量索引(语义检索核心)
-- 参数说明m=16 每层最大连接数ef_construction=64 构建时搜索范围
CREATE INDEX IF NOT EXISTS idx_ekb_chunk_embedding
ON "ekb_schema"."ekb_chunk"
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
-- ===== Phase 2 阶段使用(可预创建)=====
-- 4. pg_bigm 中文关键词索引
CREATE INDEX IF NOT EXISTS idx_ekb_chunk_content_bigm
ON "ekb_schema"."ekb_chunk"
USING gin (content gin_bigm_ops);
-- 5. 文档摘要关键词索引
CREATE INDEX IF NOT EXISTS idx_ekb_doc_summary_bigm
ON "ekb_schema"."ekb_document"
USING gin (summary gin_bigm_ops);
-- 6. 全文内容关键词索引
CREATE INDEX IF NOT EXISTS idx_ekb_doc_text_bigm
ON "ekb_schema"."ekb_document"
USING gin (extracted_text gin_bigm_ops);
-- ===== Phase 3 阶段使用(可预创建)=====
-- 7. JSONB GIN 索引metadata 查询加速)
CREATE INDEX IF NOT EXISTS idx_ekb_doc_metadata_gin
ON "ekb_schema"."ekb_document"
USING gin (metadata jsonb_path_ops);
-- 8. JSONB GIN 索引structuredData 查询加速)
CREATE INDEX IF NOT EXISTS idx_ekb_doc_structured_gin
ON "ekb_schema"."ekb_document"
USING gin (structured_data jsonb_path_ops);
-- 9. 标签数组索引
CREATE INDEX IF NOT EXISTS idx_ekb_doc_tags_gin
ON "ekb_schema"."ekb_document"
USING gin (tags);
-- 10. 切片元数据索引
CREATE INDEX IF NOT EXISTS idx_ekb_chunk_metadata_gin
ON "ekb_schema"."ekb_chunk"
USING gin (metadata jsonb_path_ops);
-- ===== 验证索引创建 =====
-- SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'ekb_schema';

View File

@@ -1,38 +0,0 @@
-- ============================================================
-- EKB Schema MVP 索引创建脚本
-- 执行时机prisma db push 之后手动执行
-- 说明MVP 阶段只创建 HNSW 向量索引pg_bigm 索引在 Phase 2 创建
-- ============================================================
-- 1. 确保 pgvector 扩展已启用
CREATE EXTENSION IF NOT EXISTS vector;
-- 2. HNSW 向量索引(语义检索核心)
-- 参数说明m=16 每层最大连接数ef_construction=64 构建时搜索范围
CREATE INDEX IF NOT EXISTS idx_ekb_chunk_embedding
ON "ekb_schema"."ekb_chunk"
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
-- 3. JSONB GIN 索引(可选,提升查询性能)
CREATE INDEX IF NOT EXISTS idx_ekb_doc_metadata_gin
ON "ekb_schema"."ekb_document"
USING gin (metadata jsonb_path_ops);
CREATE INDEX IF NOT EXISTS idx_ekb_doc_structured_gin
ON "ekb_schema"."ekb_document"
USING gin (structured_data jsonb_path_ops);
-- 4. 标签数组索引
CREATE INDEX IF NOT EXISTS idx_ekb_doc_tags_gin
ON "ekb_schema"."ekb_document"
USING gin (tags);

View File

@@ -1,141 +0,0 @@
-- =====================================================
-- 全文复筛数据库迁移脚本(手动执行)
-- Schema: asl_schema
-- 日期: 2025-11-23
-- 说明: 只操作asl_schema不影响其他schema
-- =====================================================
-- 1. 修改 literatures 表,添加全文复筛相关字段
ALTER TABLE asl_schema.literatures
ADD COLUMN IF NOT EXISTS stage TEXT DEFAULT 'imported',
ADD COLUMN IF NOT EXISTS has_pdf BOOLEAN DEFAULT false,
ADD COLUMN IF NOT EXISTS pdf_storage_type TEXT,
ADD COLUMN IF NOT EXISTS pdf_storage_ref TEXT,
ADD COLUMN IF NOT EXISTS pdf_status TEXT DEFAULT 'pending',
ADD COLUMN IF NOT EXISTS pdf_uploaded_at TIMESTAMP(3),
ADD COLUMN IF NOT EXISTS full_text_storage_type TEXT,
ADD COLUMN IF NOT EXISTS full_text_storage_ref TEXT,
ADD COLUMN IF NOT EXISTS full_text_url TEXT,
ADD COLUMN IF NOT EXISTS full_text_format TEXT,
ADD COLUMN IF NOT EXISTS full_text_source TEXT,
ADD COLUMN IF NOT EXISTS full_text_token_count INTEGER,
ADD COLUMN IF NOT EXISTS full_text_extracted_at TIMESTAMP(3);
-- 添加索引
CREATE INDEX IF NOT EXISTS idx_literatures_stage ON asl_schema.literatures(stage);
CREATE INDEX IF NOT EXISTS idx_literatures_has_pdf ON asl_schema.literatures(has_pdf);
CREATE INDEX IF NOT EXISTS idx_literatures_pdf_status ON asl_schema.literatures(pdf_status);
-- 2. 创建 fulltext_screening_tasks 表
CREATE TABLE IF NOT EXISTS asl_schema.fulltext_screening_tasks (
id TEXT PRIMARY KEY,
project_id TEXT NOT NULL,
model_a TEXT NOT NULL,
model_b TEXT NOT NULL,
prompt_version TEXT,
status TEXT NOT NULL DEFAULT 'pending',
total_count INTEGER NOT NULL DEFAULT 0,
processed_count INTEGER NOT NULL DEFAULT 0,
success_count INTEGER NOT NULL DEFAULT 0,
failed_count INTEGER NOT NULL DEFAULT 0,
degraded_count INTEGER NOT NULL DEFAULT 0,
total_tokens INTEGER DEFAULT 0,
total_cost DOUBLE PRECISION DEFAULT 0,
started_at TIMESTAMP(3),
completed_at TIMESTAMP(3),
estimated_end_at TIMESTAMP(3),
error_message TEXT,
error_stack TEXT,
created_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_fulltext_task_project FOREIGN KEY (project_id)
REFERENCES asl_schema.screening_projects(id) ON DELETE CASCADE
);
-- 添加索引
CREATE INDEX IF NOT EXISTS idx_fulltext_tasks_project_id ON asl_schema.fulltext_screening_tasks(project_id);
CREATE INDEX IF NOT EXISTS idx_fulltext_tasks_status ON asl_schema.fulltext_screening_tasks(status);
CREATE INDEX IF NOT EXISTS idx_fulltext_tasks_created_at ON asl_schema.fulltext_screening_tasks(created_at);
-- 3. 创建 fulltext_screening_results 表
CREATE TABLE IF NOT EXISTS asl_schema.fulltext_screening_results (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL,
project_id TEXT NOT NULL,
literature_id TEXT NOT NULL,
-- Model A (DeepSeek-V3) 结果
model_a_name TEXT,
model_a_status TEXT,
model_a_fields JSONB,
model_a_overall JSONB,
model_a_processing_log JSONB,
model_a_verification JSONB,
model_a_tokens INTEGER,
model_a_cost DOUBLE PRECISION,
model_a_error TEXT,
-- Model B (Qwen-Max) 结果
model_b_name TEXT,
model_b_status TEXT,
model_b_fields JSONB,
model_b_overall JSONB,
model_b_processing_log JSONB,
model_b_verification JSONB,
model_b_tokens INTEGER,
model_b_cost DOUBLE PRECISION,
model_b_error TEXT,
-- 验证结果
medical_logic_issues JSONB,
evidence_chain_issues JSONB,
-- 冲突检测
is_conflict BOOLEAN DEFAULT false,
conflict_severity TEXT,
conflict_fields TEXT[],
conflict_details JSONB,
review_priority INTEGER DEFAULT 50,
review_deadline TIMESTAMP(3),
-- 人工复核
final_decision TEXT,
final_decision_by TEXT,
final_decision_at TIMESTAMP(3),
exclusion_reason TEXT,
review_notes TEXT,
-- 处理状态
processing_status TEXT DEFAULT 'pending',
is_degraded BOOLEAN DEFAULT false,
degraded_model TEXT,
-- 元数据
processed_at TIMESTAMP(3),
prompt_version TEXT,
raw_output_a JSONB,
raw_output_b JSONB,
created_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT fk_fulltext_result_task FOREIGN KEY (task_id)
REFERENCES asl_schema.fulltext_screening_tasks(id) ON DELETE CASCADE,
CONSTRAINT fk_fulltext_result_project FOREIGN KEY (project_id)
REFERENCES asl_schema.screening_projects(id) ON DELETE CASCADE,
CONSTRAINT fk_fulltext_result_literature FOREIGN KEY (literature_id)
REFERENCES asl_schema.literatures(id) ON DELETE CASCADE,
CONSTRAINT unique_project_literature_fulltext UNIQUE (project_id, literature_id)
);
-- 添加索引
CREATE INDEX IF NOT EXISTS idx_fulltext_results_task_id ON asl_schema.fulltext_screening_results(task_id);
CREATE INDEX IF NOT EXISTS idx_fulltext_results_project_id ON asl_schema.fulltext_screening_results(project_id);
CREATE INDEX IF NOT EXISTS idx_fulltext_results_literature_id ON asl_schema.fulltext_screening_results(literature_id);
CREATE INDEX IF NOT EXISTS idx_fulltext_results_is_conflict ON asl_schema.fulltext_screening_results(is_conflict);
CREATE INDEX IF NOT EXISTS idx_fulltext_results_final_decision ON asl_schema.fulltext_screening_results(final_decision);
CREATE INDEX IF NOT EXISTS idx_fulltext_results_review_priority ON asl_schema.fulltext_screening_results(review_priority);
-- 完成
SELECT 'Migration completed successfully!' AS status;