Major Changes: - Implement Platform-Only architecture pattern (unified task management) - Add PostgresCacheAdapter for unified caching (platform_schema.app_cache) - Add PgBossQueue for job queue management (platform_schema.job) - Implement CheckpointService using job.data (generic for all modules) - Add intelligent threshold-based dual-mode processing (THRESHOLD=50) - Add task splitting mechanism (auto chunk size recommendation) - Refactor ASL screening service with smart mode selection - Refactor DC extraction service with smart mode selection - Register workers for ASL and DC modules Technical Highlights: - All task management data stored in platform_schema.job.data (JSONB) - Business tables remain clean (no task management fields) - CheckpointService is generic (shared by all modules) - Zero code duplication (DRY principle) - Follows 3-layer architecture principle - Zero additional cost (no Redis needed, save 8400 CNY/year) Code Statistics: - New code: ~1750 lines - Modified code: ~500 lines - Test code: ~1800 lines - Documentation: ~3000 lines Testing: - Unit tests: 8/8 passed - Integration tests: 2/2 passed - Architecture validation: passed - Linter errors: 0 Files: - Platform layer: PostgresCacheAdapter, PgBossQueue, CheckpointService, utils - ASL module: screeningService, screeningWorker - DC module: ExtractionController, extractionWorker - Tests: 11 test files - Docs: Updated 4 key documents Status: Phase 1-7 completed, Phase 8-9 pending
221 lines
5.3 KiB
TypeScript
221 lines
5.3 KiB
TypeScript
import { Job, JobQueue, JobHandler } from './types.js'
|
||
import { randomUUID } from 'crypto'
|
||
|
||
/**
|
||
* 内存队列实现
|
||
*
|
||
* 适用场景:
|
||
* - 本地开发环境
|
||
* - 单实例部署
|
||
* - 非关键任务(重启会丢失)
|
||
*
|
||
* 特点:
|
||
* - ✅ 简单易用,无需外部依赖
|
||
* - ✅ 性能高效
|
||
* - ⚠️ 进程重启后任务丢失
|
||
* - ⚠️ 不支持多实例共享
|
||
*
|
||
* @example
|
||
* ```typescript
|
||
* const queue = new MemoryQueue()
|
||
*
|
||
* // 注册处理函数
|
||
* queue.process('email:send', async (job) => {
|
||
* await sendEmail(job.data.to, job.data.subject)
|
||
* })
|
||
*
|
||
* // 创建任务
|
||
* const job = await queue.push('email:send', { to: 'user@example.com', subject: 'Hello' })
|
||
*
|
||
* // 查询任务
|
||
* const status = await queue.getJob(job.id)
|
||
* ```
|
||
*/
|
||
export class MemoryQueue implements JobQueue {
|
||
private jobs: Map<string, Job> = new Map()
|
||
private handlers: Map<string, JobHandler> = new Map()
|
||
private processing: boolean = false
|
||
|
||
/**
|
||
* 启动队列(MemoryQueue无需启动,立即可用)
|
||
*/
|
||
async start(): Promise<void> {
|
||
// MemoryQueue不需要初始化,已经ready
|
||
this.processing = true
|
||
}
|
||
|
||
/**
|
||
* 停止队列(MemoryQueue无需清理)
|
||
*/
|
||
async stop(): Promise<void> {
|
||
// MemoryQueue不需要清理
|
||
this.processing = false
|
||
}
|
||
|
||
/**
|
||
* 添加任务到队列
|
||
*/
|
||
async push<T>(type: string, data: T): Promise<Job<T>> {
|
||
const job: Job<T> = {
|
||
id: randomUUID(),
|
||
type,
|
||
data,
|
||
status: 'pending',
|
||
progress: 0,
|
||
createdAt: new Date(),
|
||
updatedAt: new Date()
|
||
}
|
||
|
||
this.jobs.set(job.id, job)
|
||
|
||
// 触发任务处理
|
||
this.processNextJob()
|
||
|
||
return job
|
||
}
|
||
|
||
/**
|
||
* 注册任务处理函数
|
||
*/
|
||
process<T>(type: string, handler: JobHandler<T>): void {
|
||
this.handlers.set(type, handler)
|
||
console.log(`[MemoryQueue] Registered handler for job type: ${type}`)
|
||
|
||
// 开始处理队列中的待处理任务
|
||
this.processNextJob()
|
||
}
|
||
|
||
/**
|
||
* 获取任务信息
|
||
*/
|
||
async getJob(id: string): Promise<Job | null> {
|
||
return this.jobs.get(id) || null
|
||
}
|
||
|
||
/**
|
||
* 更新任务进度
|
||
*/
|
||
async updateProgress(id: string, progress: number): Promise<void> {
|
||
const job = this.jobs.get(id)
|
||
if (job) {
|
||
job.progress = Math.min(100, Math.max(0, progress))
|
||
job.updatedAt = new Date()
|
||
this.jobs.set(id, job)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 标记任务为完成
|
||
*/
|
||
async completeJob(id: string, result: any): Promise<void> {
|
||
const job = this.jobs.get(id)
|
||
if (job) {
|
||
job.status = 'completed'
|
||
job.progress = 100
|
||
job.result = result
|
||
job.completedAt = new Date()
|
||
job.updatedAt = new Date()
|
||
this.jobs.set(id, job)
|
||
|
||
console.log(`[MemoryQueue] Job completed: ${id} (type: ${job.type})`)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 标记任务为失败
|
||
*/
|
||
async failJob(id: string, error: string): Promise<void> {
|
||
const job = this.jobs.get(id)
|
||
if (job) {
|
||
job.status = 'failed'
|
||
job.error = error
|
||
job.completedAt = new Date()
|
||
job.updatedAt = new Date()
|
||
this.jobs.set(id, job)
|
||
|
||
console.error(`[MemoryQueue] Job failed: ${id} (type: ${job.type})`, error)
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 处理下一个待处理任务
|
||
*/
|
||
private async processNextJob(): Promise<void> {
|
||
if (this.processing) return
|
||
|
||
// 查找第一个待处理的任务
|
||
const pendingJob = Array.from(this.jobs.values()).find(
|
||
job => job.status === 'pending'
|
||
)
|
||
|
||
if (!pendingJob) return
|
||
|
||
// 获取对应的处理函数
|
||
const handler = this.handlers.get(pendingJob.type)
|
||
if (!handler) {
|
||
// 没有注册处理函数,跳过
|
||
return
|
||
}
|
||
|
||
// 标记为处理中
|
||
this.processing = true
|
||
pendingJob.status = 'processing'
|
||
pendingJob.startedAt = new Date()
|
||
pendingJob.updatedAt = new Date()
|
||
this.jobs.set(pendingJob.id, pendingJob)
|
||
|
||
console.log(`[MemoryQueue] Processing job: ${pendingJob.id} (type: ${pendingJob.type})`)
|
||
|
||
try {
|
||
// 执行处理函数
|
||
const result = await handler(pendingJob)
|
||
|
||
// 标记为完成
|
||
await this.completeJob(pendingJob.id, result)
|
||
} catch (error: any) {
|
||
// 标记为失败
|
||
await this.failJob(pendingJob.id, error.message)
|
||
} finally {
|
||
this.processing = false
|
||
|
||
// 继续处理下一个任务
|
||
setImmediate(() => this.processNextJob())
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 获取队列统计信息
|
||
*/
|
||
getStats() {
|
||
const jobs = Array.from(this.jobs.values())
|
||
return {
|
||
total: jobs.length,
|
||
pending: jobs.filter(j => j.status === 'pending').length,
|
||
processing: jobs.filter(j => j.status === 'processing').length,
|
||
completed: jobs.filter(j => j.status === 'completed').length,
|
||
failed: jobs.filter(j => j.status === 'failed').length
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 清理已完成的任务(避免内存泄漏)
|
||
* 建议定期调用
|
||
*/
|
||
cleanup(olderThan: Date = new Date(Date.now() - 24 * 60 * 60 * 1000)) {
|
||
let removed = 0
|
||
for (const [id, job] of this.jobs) {
|
||
if (
|
||
(job.status === 'completed' || job.status === 'failed') &&
|
||
job.completedAt &&
|
||
job.completedAt < olderThan
|
||
) {
|
||
this.jobs.delete(id)
|
||
removed++
|
||
}
|
||
}
|
||
console.log(`[MemoryQueue] Cleanup: removed ${removed} old jobs`)
|
||
return removed
|
||
}
|
||
}
|
||
|