feat(platform): Fix pg-boss queue conflict and add safety standards
Summary: - Fix pg-boss queue conflict (duplicate key violation on queue_pkey) - Add global error listener to prevent process crash - Reduce connection pool from 10 to 4 - Add graceful shutdown handling (SIGTERM/SIGINT) - Fix researchWorker recursive call bug in catch block - Make screeningWorker idempotent using upsert Security Standards (v1.1): - Prohibit recursive retry in Worker catch blocks - Prohibit payload bloat (only store fileKey/ID in job.data) - Require Worker idempotency (upsert + unique constraint) - Recommend task-specific expireInSeconds settings - Document graceful shutdown pattern New Features: - PKB signed URL endpoint for document preview/download - pg_bigm installation guide for Docker - Dockerfile.postgres-with-extensions for pgvector + pg_bigm Documentation: - Update Postgres-Only async task processing guide (v1.1) - Add troubleshooting SQL queries - Update safety checklist Tested: Local verification passed
This commit is contained in:
@@ -91,3 +91,4 @@ export async function moduleRoutes(fastify: FastifyInstance) {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -121,3 +121,4 @@ export interface PaginatedResponse<T> {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -168,3 +168,4 @@ export const ROLE_DISPLAY_NAMES: Record<UserRole, string> = {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -243,3 +243,4 @@ async function matchIntent(query: string): Promise<{
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -97,3 +97,4 @@ export async function uploadAttachment(
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -26,3 +26,4 @@ export { aiaRoutes };
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -368,5 +368,6 @@ runTests().catch((error) => {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -309,5 +309,6 @@ runTest()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -347,5 +347,6 @@ Content-Type: application/json
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -257,7 +257,7 @@ async function processScreeningBatchWithCheckpoint(
|
||||
literature.publicationYear ? Number(literature.publicationYear) : undefined
|
||||
);
|
||||
|
||||
// 保存结果（使用 screeningService.ts 相同的映射方式）
|
||||
// 保存结果（使用 upsert 保证幂等性，任务重试时覆盖而不是重复创建）
|
||||
const dbResult = {
|
||||
projectId,
|
||||
literatureId: literature.id,
|
||||
@@ -294,8 +294,16 @@ async function processScreeningBatchWithCheckpoint(
|
||||
finalDecision: screeningResult.finalDecision,
|
||||
};
|
||||
|
||||
await prisma.aslScreeningResult.create({
|
||||
data: dbResult,
|
||||
// 🛡️ 使用 upsert 实现幂等性（利用 unique_project_literature 约束）
|
||||
await prisma.aslScreeningResult.upsert({
|
||||
where: {
|
||||
projectId_literatureId: {
|
||||
projectId,
|
||||
literatureId: literature.id,
|
||||
},
|
||||
},
|
||||
create: dbResult,
|
||||
update: dbResult,
|
||||
});
|
||||
|
||||
successCount++;
|
||||
|
||||
@@ -67,12 +67,8 @@ export function registerResearchWorker() {
|
||||
error: error.message,
|
||||
});
|
||||
|
||||
// 更新任务状态为失败
|
||||
try {
|
||||
await researchService.executeSearch(taskId, query);
|
||||
} catch {
|
||||
// 忽略
|
||||
}
|
||||
// ❌ 已移除错误的 executeSearch 调用
|
||||
// 任务失败后 pg-boss 会自动重试（最多3次）
|
||||
|
||||
return {
|
||||
success: false,
|
||||
|
||||
@@ -283,5 +283,6 @@ export const conflictDetectionService = new ConflictDetectionService();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -233,5 +233,6 @@ curl -X POST http://localhost:3000/api/v1/dc/tool-c/test/execute \
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -287,5 +287,6 @@ export const streamAIController = new StreamAIController();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -196,5 +196,6 @@ logger.info('[SessionMemory] 会话记忆管理器已启动', {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -130,5 +130,6 @@ checkTableStructure();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -117,5 +117,6 @@ checkProjectConfig().catch(console.error);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -99,5 +99,6 @@ main();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -556,5 +556,6 @@ URL: https://iit.xunzhengyixue.com/api/v1/iit/patient-wechat/callback
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -191,5 +191,6 @@ console.log('');
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -508,5 +508,6 @@ export const patientWechatService = new PatientWechatService();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -153,5 +153,6 @@ testDifyIntegration().catch(error => {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -182,5 +182,6 @@ testIitDatabase()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -168,5 +168,6 @@ if (hasError) {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -194,5 +194,6 @@ async function testUrlVerification() {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -275,5 +275,6 @@ main().catch((error) => {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -159,5 +159,6 @@ Write-Host ""
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -252,5 +252,6 @@ export interface CachedProtocolRules {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import type { FastifyRequest, FastifyReply } from 'fastify';
|
||||
import * as documentService from '../services/documentService.js';
|
||||
import { storage } from '../../../common/storage/index.js';
|
||||
import { OSSAdapter } from '../../../common/storage/OSSAdapter.js';
|
||||
import { randomUUID } from 'crypto';
|
||||
import path from 'path';
|
||||
import { logger } from '../../../common/logging/index.js';
|
||||
|
||||
/**
|
||||
* 获取用户ID（从JWT Token中获取）
|
||||
@@ -374,4 +376,93 @@ export async function getDocumentFullText(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文档签名URL（用于前端预览/下载）
|
||||
*
|
||||
* @description
|
||||
* 生成一个带有过期时间的签名URL，前端可以直接使用该URL：
|
||||
* - 在浏览器中预览 PDF
|
||||
* - 下载文件（会恢复原始文件名）
|
||||
*/
|
||||
export async function getDocumentSignedUrl(
|
||||
request: FastifyRequest<{
|
||||
Params: {
|
||||
id: string;
|
||||
};
|
||||
Querystring: {
|
||||
/** 过期时间（秒），默认3600秒 */
|
||||
expires?: string;
|
||||
/** 是否作为附件下载（添加 Content-Disposition），默认 false */
|
||||
download?: string;
|
||||
};
|
||||
}>,
|
||||
reply: FastifyReply
|
||||
) {
|
||||
try {
|
||||
const { id } = request.params;
|
||||
const expires = parseInt(request.query.expires || '3600', 10);
|
||||
const download = request.query.download === 'true';
|
||||
|
||||
const userId = getUserId(request);
|
||||
|
||||
// 获取文档信息
|
||||
const document = await documentService.getDocumentById(userId, id);
|
||||
|
||||
// 检查是否有存储路径
|
||||
if (!document.storageKey) {
|
||||
logger.warn('[PKB] 文档没有存储路径，可能是旧数据', { documentId: id });
|
||||
return reply.status(404).send({
|
||||
success: false,
|
||||
message: '文档文件不可用，请重新上传',
|
||||
});
|
||||
}
|
||||
|
||||
// 生成签名URL
|
||||
let signedUrl: string;
|
||||
|
||||
// 检查存储适配器类型
|
||||
if (storage instanceof OSSAdapter) {
|
||||
// OSS: 使用带原始文件名的签名URL
|
||||
signedUrl = download
|
||||
? storage.getSignedUrl(document.storageKey, expires, document.filename)
|
||||
: storage.getSignedUrl(document.storageKey, expires);
|
||||
} else {
|
||||
// 本地存储: 使用 getUrl
|
||||
signedUrl = storage.getUrl(document.storageKey);
|
||||
}
|
||||
|
||||
logger.info('[PKB] 生成签名URL', {
|
||||
documentId: id,
|
||||
filename: document.filename,
|
||||
expires,
|
||||
download
|
||||
});
|
||||
|
||||
return reply.send({
|
||||
success: true,
|
||||
data: {
|
||||
documentId: document.id,
|
||||
filename: document.filename,
|
||||
fileType: document.fileType,
|
||||
url: signedUrl,
|
||||
expiresIn: expires,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
logger.error('[PKB] 获取签名URL失败', { error: error.message });
|
||||
|
||||
if (error.message.includes('not found') || error.message.includes('access denied')) {
|
||||
return reply.status(404).send({
|
||||
success: false,
|
||||
message: error.message,
|
||||
});
|
||||
}
|
||||
|
||||
return reply.status(500).send({
|
||||
success: false,
|
||||
message: error.message || 'Failed to get signed URL',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -65,5 +65,6 @@ export default async function healthRoutes(fastify: FastifyInstance) {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ export default async function knowledgeBaseRoutes(fastify: FastifyInstance) {
|
||||
// Phase 2: 获取文档全文
|
||||
fastify.get('/documents/:id/full-text', { preHandler: [authenticate, requireModule('PKB')] }, documentController.getDocumentFullText);
|
||||
|
||||
// 获取文档签名URL（用于预览/下载）
|
||||
// Query: ?expires=3600&download=true
|
||||
fastify.get('/documents/:id/signed-url', { preHandler: [authenticate, requireModule('PKB')] }, documentController.getDocumentSignedUrl);
|
||||
|
||||
// 删除文档
|
||||
fastify.delete('/documents/:id', { preHandler: [authenticate, requireModule('PKB')] }, documentController.deleteDocument);
|
||||
|
||||
|
||||
@@ -143,5 +143,6 @@ Content-Type: application/json
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -128,5 +128,6 @@ Write-Host " - 删除任务: DELETE $BaseUrl/api/v1/rvw/tasks/{taskId}" -Foregr
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -42,5 +42,6 @@ export * from './services/utils.js';
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -133,5 +133,6 @@ export function validateAgentSelection(agents: string[]): void {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user