Files
AIclinicalresearch/backend/src/common/jobs/MemoryQueue.ts
HaHafeng fa72beea6c feat(platform): Complete Postgres-Only architecture refactoring (Phase 1-7)
Major Changes:
- Implement Platform-Only architecture pattern (unified task management)
- Add PostgresCacheAdapter for unified caching (platform_schema.app_cache)
- Add PgBossQueue for job queue management (platform_schema.job)
- Implement CheckpointService using job.data (generic for all modules)
- Add intelligent threshold-based dual-mode processing (THRESHOLD=50)
- Add task splitting mechanism (auto chunk size recommendation)
- Refactor ASL screening service with smart mode selection
- Refactor DC extraction service with smart mode selection
- Register workers for ASL and DC modules

Technical Highlights:
- All task management data stored in platform_schema.job.data (JSONB)
- Business tables remain clean (no task management fields)
- CheckpointService is generic (shared by all modules)
- Zero code duplication (DRY principle)
- Follows 3-layer architecture principle
- Zero additional cost (no Redis needed, save 8400 CNY/year)

Code Statistics:
- New code: ~1750 lines
- Modified code: ~500 lines
- Test code: ~1800 lines
- Documentation: ~3000 lines

Testing:
- Unit tests: 8/8 passed
- Integration tests: 2/2 passed
- Architecture validation: passed
- Linter errors: 0

Files:
- Platform layer: PostgresCacheAdapter, PgBossQueue, CheckpointService, utils
- ASL module: screeningService, screeningWorker
- DC module: ExtractionController, extractionWorker
- Tests: 11 test files
- Docs: Updated 4 key documents

Status: Phase 1-7 completed, Phase 8-9 pending
2025-12-13 16:10:04 +08:00

221 lines
5.3 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Job, JobQueue, JobHandler } from './types.js'
import { randomUUID } from 'crypto'
/**
* 内存队列实现
*
* 适用场景:
* - 本地开发环境
* - 单实例部署
* - 非关键任务(重启会丢失)
*
* 特点:
* - ✅ 简单易用,无需外部依赖
* - ✅ 性能高效
* - ⚠️ 进程重启后任务丢失
* - ⚠️ 不支持多实例共享
*
* @example
* ```typescript
* const queue = new MemoryQueue()
*
* // 注册处理函数
* queue.process('email:send', async (job) => {
* await sendEmail(job.data.to, job.data.subject)
* })
*
* // 创建任务
* const job = await queue.push('email:send', { to: 'user@example.com', subject: 'Hello' })
*
* // 查询任务
* const status = await queue.getJob(job.id)
* ```
*/
export class MemoryQueue implements JobQueue {
private jobs: Map<string, Job> = new Map()
private handlers: Map<string, JobHandler> = new Map()
private processing: boolean = false
/**
* 启动队列MemoryQueue无需启动立即可用
*/
async start(): Promise<void> {
// MemoryQueue不需要初始化已经ready
this.processing = true
}
/**
* 停止队列MemoryQueue无需清理
*/
async stop(): Promise<void> {
// MemoryQueue不需要清理
this.processing = false
}
/**
* 添加任务到队列
*/
async push<T>(type: string, data: T): Promise<Job<T>> {
const job: Job<T> = {
id: randomUUID(),
type,
data,
status: 'pending',
progress: 0,
createdAt: new Date(),
updatedAt: new Date()
}
this.jobs.set(job.id, job)
// 触发任务处理
this.processNextJob()
return job
}
/**
* 注册任务处理函数
*/
process<T>(type: string, handler: JobHandler<T>): void {
this.handlers.set(type, handler)
console.log(`[MemoryQueue] Registered handler for job type: ${type}`)
// 开始处理队列中的待处理任务
this.processNextJob()
}
/**
* 获取任务信息
*/
async getJob(id: string): Promise<Job | null> {
return this.jobs.get(id) || null
}
/**
* 更新任务进度
*/
async updateProgress(id: string, progress: number): Promise<void> {
const job = this.jobs.get(id)
if (job) {
job.progress = Math.min(100, Math.max(0, progress))
job.updatedAt = new Date()
this.jobs.set(id, job)
}
}
/**
* 标记任务为完成
*/
async completeJob(id: string, result: any): Promise<void> {
const job = this.jobs.get(id)
if (job) {
job.status = 'completed'
job.progress = 100
job.result = result
job.completedAt = new Date()
job.updatedAt = new Date()
this.jobs.set(id, job)
console.log(`[MemoryQueue] Job completed: ${id} (type: ${job.type})`)
}
}
/**
* 标记任务为失败
*/
async failJob(id: string, error: string): Promise<void> {
const job = this.jobs.get(id)
if (job) {
job.status = 'failed'
job.error = error
job.completedAt = new Date()
job.updatedAt = new Date()
this.jobs.set(id, job)
console.error(`[MemoryQueue] Job failed: ${id} (type: ${job.type})`, error)
}
}
/**
* 处理下一个待处理任务
*/
private async processNextJob(): Promise<void> {
if (this.processing) return
// 查找第一个待处理的任务
const pendingJob = Array.from(this.jobs.values()).find(
job => job.status === 'pending'
)
if (!pendingJob) return
// 获取对应的处理函数
const handler = this.handlers.get(pendingJob.type)
if (!handler) {
// 没有注册处理函数,跳过
return
}
// 标记为处理中
this.processing = true
pendingJob.status = 'processing'
pendingJob.startedAt = new Date()
pendingJob.updatedAt = new Date()
this.jobs.set(pendingJob.id, pendingJob)
console.log(`[MemoryQueue] Processing job: ${pendingJob.id} (type: ${pendingJob.type})`)
try {
// 执行处理函数
const result = await handler(pendingJob)
// 标记为完成
await this.completeJob(pendingJob.id, result)
} catch (error: any) {
// 标记为失败
await this.failJob(pendingJob.id, error.message)
} finally {
this.processing = false
// 继续处理下一个任务
setImmediate(() => this.processNextJob())
}
}
/**
* 获取队列统计信息
*/
getStats() {
const jobs = Array.from(this.jobs.values())
return {
total: jobs.length,
pending: jobs.filter(j => j.status === 'pending').length,
processing: jobs.filter(j => j.status === 'processing').length,
completed: jobs.filter(j => j.status === 'completed').length,
failed: jobs.filter(j => j.status === 'failed').length
}
}
/**
* 清理已完成的任务(避免内存泄漏)
* 建议定期调用
*/
cleanup(olderThan: Date = new Date(Date.now() - 24 * 60 * 60 * 1000)) {
let removed = 0
for (const [id, job] of this.jobs) {
if (
(job.status === 'completed' || job.status === 'failed') &&
job.completedAt &&
job.completedAt < olderThan
) {
this.jobs.delete(id)
removed++
}
}
console.log(`[MemoryQueue] Cleanup: removed ${removed} old jobs`)
return removed
}
}