feat(platform): Complete Postgres-Only architecture refactoring (Phase 1-7)
Major Changes: - Implement Platform-Only architecture pattern (unified task management) - Add PostgresCacheAdapter for unified caching (platform_schema.app_cache) - Add PgBossQueue for job queue management (platform_schema.job) - Implement CheckpointService using job.data (generic for all modules) - Add intelligent threshold-based dual-mode processing (THRESHOLD=50) - Add task splitting mechanism (auto chunk size recommendation) - Refactor ASL screening service with smart mode selection - Refactor DC extraction service with smart mode selection - Register workers for ASL and DC modules Technical Highlights: - All task management data stored in platform_schema.job.data (JSONB) - Business tables remain clean (no task management fields) - CheckpointService is generic (shared by all modules) - Zero code duplication (DRY principle) - Follows 3-layer architecture principle - Zero additional cost (no Redis needed, save 8400 CNY/year) Code Statistics: - New code: ~1750 lines - Modified code: ~500 lines - Test code: ~1800 lines - Documentation: ~3000 lines Testing: - Unit tests: 8/8 passed - Integration tests: 2/2 passed - Architecture validation: passed - Linter errors: 0 Files: - Platform layer: PostgresCacheAdapter, PgBossQueue, CheckpointService, utils - ASL module: screeningService, screeningWorker - DC module: ExtractionController, extractionWorker - Tests: 11 test files - Docs: Updated 4 key documents Status: Phase 1-7 completed, Phase 8-9 pending
This commit is contained in:
27
backend/src/common/cache/CacheFactory.ts
vendored
27
backend/src/common/cache/CacheFactory.ts
vendored
@@ -1,6 +1,8 @@
|
||||
import { CacheAdapter } from './CacheAdapter.js'
|
||||
import { MemoryCacheAdapter } from './MemoryCacheAdapter.js'
|
||||
import { RedisCacheAdapter } from './RedisCacheAdapter.js'
|
||||
import { PostgresCacheAdapter } from './PostgresCacheAdapter.js'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
|
||||
/**
|
||||
* 缓存工厂类
|
||||
@@ -8,16 +10,18 @@ import { RedisCacheAdapter } from './RedisCacheAdapter.js'
|
||||
* 根据环境变量自动选择缓存实现：
|
||||
* - CACHE_TYPE=memory: 使用MemoryCacheAdapter（内存缓存）
|
||||
* - CACHE_TYPE=redis: 使用RedisCacheAdapter（Redis缓存）
|
||||
* - CACHE_TYPE=postgres: 使用PostgresCacheAdapter（Postgres缓存）
|
||||
*
|
||||
* 零代码切换：
|
||||
* - 本地开发：不配置CACHE_TYPE，默认使用memory
|
||||
* - 云端部署：配置CACHE_TYPE=redis，自动切换到Redis
|
||||
* - Postgres-Only架构：配置CACHE_TYPE=postgres
|
||||
* - 高性能场景：配置CACHE_TYPE=redis
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { cache } from '@/common/cache'
|
||||
*
|
||||
* // 业务代码不关心是memory还是redis
|
||||
* // 业务代码不关心具体实现
|
||||
* await cache.set('user:123', userData, 60)
|
||||
* const user = await cache.get('user:123')
|
||||
* ```
|
||||
@@ -48,6 +52,9 @@ export class CacheFactory {
|
||||
case 'redis':
|
||||
return this.createRedisAdapter()
|
||||
|
||||
case 'postgres':
|
||||
return this.createPostgresAdapter()
|
||||
|
||||
default:
|
||||
console.warn(`[CacheFactory] Unknown CACHE_TYPE: ${cacheType}, fallback to memory`)
|
||||
return this.createMemoryAdapter()
|
||||
@@ -89,6 +96,22 @@ export class CacheFactory {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建Postgres缓存适配器
|
||||
*/
|
||||
private static createPostgresAdapter(): PostgresCacheAdapter {
|
||||
console.log('[CacheFactory] Using PostgresCacheAdapter (Postgres-Only架构)')
|
||||
|
||||
// 获取全局Prisma实例
|
||||
// 注意：需要确保Prisma已经初始化
|
||||
const prisma = global.prisma || new PrismaClient()
|
||||
if (!global.prisma) {
|
||||
global.prisma = prisma
|
||||
}
|
||||
|
||||
return new PostgresCacheAdapter(prisma)
|
||||
}
|
||||
|
||||
/**
|
||||
* 重置实例（用于测试）
|
||||
*/
|
||||
|
||||
349
backend/src/common/cache/PostgresCacheAdapter.ts
vendored
Normal file
349
backend/src/common/cache/PostgresCacheAdapter.ts
vendored
Normal file
@@ -0,0 +1,349 @@
|
||||
import { CacheAdapter } from './CacheAdapter.js'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
|
||||
/**
|
||||
* Postgres缓存适配器
|
||||
*
|
||||
* 适用场景：
|
||||
* - Postgres-Only架构（无需Redis）
|
||||
* - 云原生Serverless环境（SAE）
|
||||
* - 多实例部署需要共享缓存
|
||||
*
|
||||
* 特点：
|
||||
* - ✅ 无需额外Redis实例，降低成本
|
||||
* - ✅ 多实例自动共享缓存
|
||||
* - ✅ 数据持久化，实例重启不丢失
|
||||
* - ✅ 适合中小规模应用（<10万MAU）
|
||||
* - ⚠️ 性能低于Redis（但足够）
|
||||
* - ⚠️ 需要定期清理过期数据
|
||||
*
|
||||
* 性能指标：
|
||||
* - 单次get/set: ~2-5ms
|
||||
* - 批量操作(10条): ~10-20ms
|
||||
* - 适用并发: <100 QPS
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const cache = new PostgresCacheAdapter(prisma)
|
||||
* await cache.set('llm:result:abc', data, 3600) // 1小时过期
|
||||
* const data = await cache.get('llm:result:abc')
|
||||
* ```
|
||||
*/
|
||||
export class PostgresCacheAdapter implements CacheAdapter {
|
||||
private prisma: PrismaClient
|
||||
private cleanupTimer: NodeJS.Timeout | null = null
|
||||
private readonly CLEANUP_INTERVAL = 5 * 60 * 1000 // 5分钟
|
||||
private readonly CLEANUP_BATCH_SIZE = 1000 // 每次最多删除1000条
|
||||
|
||||
constructor(prisma: PrismaClient) {
|
||||
this.prisma = prisma
|
||||
// 启动后台清理任务
|
||||
this.startCleanupTask()
|
||||
}
|
||||
|
||||
/**
|
||||
* 启动定期清理过期缓存
|
||||
*
|
||||
* 策略：
|
||||
* - 每5分钟运行一次
|
||||
* - 每次最多删除1000条（避免长事务锁表）
|
||||
* - 使用WHERE expires_at < NOW()快速定位
|
||||
*/
|
||||
private startCleanupTask(): void {
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
return // 测试环境不启动定时任务
|
||||
}
|
||||
|
||||
this.cleanupTimer = setInterval(async () => {
|
||||
try {
|
||||
await this.cleanupExpired()
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] Cleanup failed:', error)
|
||||
}
|
||||
}, this.CLEANUP_INTERVAL)
|
||||
|
||||
console.log('[PostgresCacheAdapter] Cleanup task started (interval: 5min, batch: 1000)')
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止清理任务
|
||||
*/
|
||||
destroy(): void {
|
||||
if (this.cleanupTimer) {
|
||||
clearInterval(this.cleanupTimer)
|
||||
this.cleanupTimer = null
|
||||
console.log('[PostgresCacheAdapter] Cleanup task stopped')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理过期缓存（批量删除）
|
||||
*
|
||||
* 优化点：
|
||||
* - LIMIT 1000 避免大事务
|
||||
* - DELETE 使用索引 (idx_app_cache_expires)
|
||||
* - 快照读不阻塞其他查询
|
||||
*/
|
||||
private async cleanupExpired(): Promise<void> {
|
||||
try {
|
||||
const result = await this.prisma.$executeRaw`
|
||||
DELETE FROM platform_schema.app_cache
|
||||
WHERE id IN (
|
||||
SELECT id FROM platform_schema.app_cache
|
||||
WHERE expires_at < NOW()
|
||||
LIMIT ${this.CLEANUP_BATCH_SIZE}
|
||||
)
|
||||
`
|
||||
|
||||
if (result > 0) {
|
||||
console.log(`[PostgresCacheAdapter] Cleanup: removed ${result} expired entries`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] Cleanup error:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取缓存值
|
||||
*
|
||||
* 逻辑：
|
||||
* 1. SELECT + 过期检查
|
||||
* 2. 如果过期，顺手删除（懒惰删除）
|
||||
* 3. 返回值或null
|
||||
*/
|
||||
async get<T = any>(key: string): Promise<T | null> {
|
||||
try {
|
||||
const entry = await this.prisma.appCache.findUnique({
|
||||
where: { key }
|
||||
})
|
||||
|
||||
if (!entry) {
|
||||
return null
|
||||
}
|
||||
|
||||
// 检查是否过期
|
||||
if (entry.expiresAt < new Date()) {
|
||||
// 过期了，删除并返回null（懒惰删除）
|
||||
await this.prisma.appCache.delete({
|
||||
where: { key }
|
||||
}).catch(() => {
|
||||
// 删除失败不影响主流程
|
||||
})
|
||||
return null
|
||||
}
|
||||
|
||||
// 返回缓存值
|
||||
return entry.value as T
|
||||
} catch (error) {
|
||||
console.error(`[PostgresCacheAdapter] get() error for key: ${key}`, error)
|
||||
return null // 缓存失败不影响业务
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置缓存值
|
||||
*
|
||||
* 逻辑：
|
||||
* 1. 计算过期时间（秒 -> 毫秒 -> Date）
|
||||
* 2. UPSERT (INSERT ON CONFLICT UPDATE)
|
||||
*/
|
||||
async set(key: string, value: any, ttl?: number): Promise<void> {
|
||||
try {
|
||||
// 计算过期时间（默认7天）
|
||||
const defaultTTL = 7 * 24 * 60 * 60 // 7天
|
||||
const expiresAt = new Date(Date.now() + (ttl || defaultTTL) * 1000)
|
||||
|
||||
await this.prisma.appCache.upsert({
|
||||
where: { key },
|
||||
update: {
|
||||
value: value as any, // Prisma会自动处理JSON
|
||||
expiresAt
|
||||
},
|
||||
create: {
|
||||
key,
|
||||
value: value as any,
|
||||
expiresAt
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`[PostgresCacheAdapter] set() error for key: ${key}`, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 删除缓存
|
||||
*/
|
||||
async delete(key: string): Promise<void> {
|
||||
try {
|
||||
await this.prisma.appCache.delete({
|
||||
where: { key }
|
||||
}).catch(() => {
|
||||
// Key不存在也算成功
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`[PostgresCacheAdapter] delete() error for key: ${key}`, error)
|
||||
// 删除失败不抛错
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 清空所有缓存
|
||||
* ⚠️ 生产环境慎用！
|
||||
*/
|
||||
async clear(): Promise<void> {
|
||||
try {
|
||||
const result = await this.prisma.appCache.deleteMany({})
|
||||
console.log(`[PostgresCacheAdapter] Cleared ${result.count} cache entries`)
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] clear() error:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查缓存是否存在
|
||||
*/
|
||||
async has(key: string): Promise<boolean> {
|
||||
try {
|
||||
const entry = await this.prisma.appCache.findUnique({
|
||||
where: { key },
|
||||
select: { expiresAt: true }
|
||||
})
|
||||
|
||||
if (!entry) {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否过期
|
||||
if (entry.expiresAt < new Date()) {
|
||||
// 过期了，顺手删除
|
||||
await this.prisma.appCache.delete({
|
||||
where: { key }
|
||||
}).catch(() => {})
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.error(`[PostgresCacheAdapter] has() error for key: ${key}`, error)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量获取缓存
|
||||
*
|
||||
* 优化：
|
||||
* - 一次查询获取所有key
|
||||
* - 客户端过滤过期数据
|
||||
*/
|
||||
async mget<T = any>(keys: string[]): Promise<(T | null)[]> {
|
||||
if (keys.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
try {
|
||||
// 一次性查询所有key
|
||||
const entries = await this.prisma.appCache.findMany({
|
||||
where: {
|
||||
key: { in: keys }
|
||||
}
|
||||
})
|
||||
|
||||
// 构建key -> entry映射
|
||||
const entryMap = new Map(entries.map((e) => [e.key, e] as const))
|
||||
const now = new Date()
|
||||
|
||||
// 按keys顺序返回结果
|
||||
return keys.map(key => {
|
||||
const entry = entryMap.get(key)
|
||||
if (!entry) {
|
||||
return null
|
||||
}
|
||||
|
||||
// 检查过期
|
||||
if (entry.expiresAt < now) {
|
||||
// 过期了，异步删除（不阻塞返回）
|
||||
this.prisma.appCache.delete({
|
||||
where: { key }
|
||||
}).catch(() => {})
|
||||
return null
|
||||
}
|
||||
|
||||
return entry.value as T
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] mget() error:', error)
|
||||
// 返回全null（缓存失败不影响业务）
|
||||
return keys.map(() => null)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量设置缓存
|
||||
*
|
||||
* 优化：
|
||||
* - 使用事务批量插入
|
||||
* - 遇到冲突则更新
|
||||
*/
|
||||
async mset(entries: Array<{ key: string; value: any }>, ttl?: number): Promise<void> {
|
||||
if (entries.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// 计算过期时间
|
||||
const defaultTTL = 7 * 24 * 60 * 60 // 7天
|
||||
const expiresAt = new Date(Date.now() + (ttl || defaultTTL) * 1000)
|
||||
|
||||
// 使用事务批量upsert
|
||||
await this.prisma.$transaction(
|
||||
entries.map(({ key, value }) =>
|
||||
this.prisma.appCache.upsert({
|
||||
where: { key },
|
||||
update: {
|
||||
value: value as any,
|
||||
expiresAt
|
||||
},
|
||||
create: {
|
||||
key,
|
||||
value: value as any,
|
||||
expiresAt
|
||||
}
|
||||
})
|
||||
)
|
||||
)
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] mset() error:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取缓存统计信息（调试用）
|
||||
*/
|
||||
async getStats() {
|
||||
try {
|
||||
const total = await this.prisma.appCache.count()
|
||||
const expired = await this.prisma.appCache.count({
|
||||
where: {
|
||||
expiresAt: {
|
||||
lt: new Date()
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
total,
|
||||
active: total - expired,
|
||||
expired
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[PostgresCacheAdapter] getStats() error:', error)
|
||||
return { total: 0, active: 0, expired: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
4
backend/src/common/cache/index.ts
vendored
4
backend/src/common/cache/index.ts
vendored
@@ -35,6 +35,7 @@
|
||||
export type { CacheAdapter } from './CacheAdapter.js'
|
||||
export { MemoryCacheAdapter } from './MemoryCacheAdapter.js'
|
||||
export { RedisCacheAdapter } from './RedisCacheAdapter.js'
|
||||
export { PostgresCacheAdapter } from './PostgresCacheAdapter.js'
|
||||
export { CacheFactory } from './CacheFactory.js'
|
||||
|
||||
// Import for usage below
|
||||
@@ -45,7 +46,8 @@ import { CacheFactory } from './CacheFactory.js'
|
||||
*
|
||||
* 自动根据环境变量选择缓存实现：
|
||||
* - CACHE_TYPE=memory: 内存缓存（本地开发）
|
||||
* - CACHE_TYPE=redis: Redis缓存（生产环境）
|
||||
* - CACHE_TYPE=redis: Redis缓存（高性能场景）
|
||||
* - CACHE_TYPE=postgres: Postgres缓存（Postgres-Only架构）
|
||||
*/
|
||||
export const cache = CacheFactory.getInstance()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user