Develop/AIclinicalresearch
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): integrate Alibaba Cloud OSS for file persistence - Add OSSAdapter and LocalAdapter with StorageFactory pattern - Integrate PKB module with OSS upload - Rename difyDocumentId to storageKey - Create 4 OSS buckets and development specification

Browse Source
This commit is contained in:
HaHafeng
2026-01-22 22:02:20 +08:00
parent 483c62fb6f
commit 9c96f75c52
309 changed files with 4583 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/common/auth/jwt.service.ts
View File

@@ -199,3 +199,6 @@ export const jwtService = new JWTService();

3
backend/src/common/jobs/utils.ts
View File

@@ -327,6 +327,9 @@ export function getBatchItems<T>(

3
backend/src/common/prompt/prompt.types.ts
View File

@@ -82,3 +82,6 @@ export interface VariableValidation {

3
backend/src/common/rag/ChunkService.ts
View File

@@ -352,3 +352,6 @@ export function chunkMarkdown(markdown: string, config?: ChunkConfig): TextChunk
export default ChunkService;

3
backend/src/common/rag/DifyClient.ts
View File

@@ -48,3 +48,6 @@ class DeprecatedDifyClient {
export const difyClient = new DeprecatedDifyClient();
export const DifyClient = DeprecatedDifyClient;

407
backend/src/common/storage/OSSAdapter.ts
View File

@@ -1,135 +1,372 @@
import { StorageAdapter } from './StorageAdapter.js'
// import OSS from 'ali-oss' // ⚠️ 需要安装：npm install ali-oss
import OSS from 'ali-oss'
/**
* 阿里云OSS适配器
* 阿里云OSS存储适配器
*
* 适用场景：
* - 云端SaaS部署（阿里云Serverless）
* - 高可用、高并发场景
* - 需要CDN加速
* 支持功能：
* - 文件上传/下载/删除
* - 私有Bucket签名URL
* - 内网Endpoint（SAE部署零流量费）
* - 静态Bucket公开访问
*
* 配置要求：
* - OSS_REGION: OSS地域（如：oss-cn-hangzhou）
* - OSS_BUCKET: OSS Bucket名称
* - OSS_ACCESS_KEY_ID: AccessKey ID
* - OSS_ACCESS_KEY_SECRET: AccessKey Secret
* Bucket策略：
* - ai-clinical-data: 私有，核心数据（文献、病历、报告）
* - ai-clinical-static: 公共读，静态资源（头像、Logo、图片）
*
* @example
* ```typescript
* const adapter = new OSSAdapter({
* region: 'oss-cn-hangzhou',
* bucket: 'aiclinical-prod',
* region: 'oss-cn-beijing',
* bucket: 'ai-clinical-data',
* accessKeyId: process.env.OSS_ACCESS_KEY_ID!,
* accessKeySecret: process.env.OSS_ACCESS_KEY_SECRET!
* accessKeySecret: process.env.OSS_ACCESS_KEY_SECRET!,
* internal: true // SAE内网
* })
* await adapter.upload('literature/123.pdf', buffer)
* ```
*
* ⚠️ 当前为预留实现，待云端部署时完善
* // 上传文件
* const url = await adapter.upload('pkb/tenant123/user456/doc.pdf', buffer)
*
* // 获取签名URL（私有Bucket）
* const signedUrl = adapter.getSignedUrl('pkb/tenant123/user456/doc.pdf')
* ```
*/
export interface OSSAdapterConfig {
/** OSS地域，如 oss-cn-beijing */
region: string
/** Bucket名称 */
bucket: string
/** AccessKey ID */
accessKeyId: string
/** AccessKey Secret */
accessKeySecret: string
/** 是否使用内网Endpoint（SAE部署必须为true） */
internal?: boolean
/** 签名URL过期时间（秒），默认3600 */
signedUrlExpires?: number
}
export class OSSAdapter implements StorageAdapter {
// private readonly client: OSS
private readonly client: OSS
private readonly bucket: string
private readonly region: string
private readonly internal: boolean
private readonly signedUrlExpires: number
constructor(config: {
region: string
bucket: string
accessKeyId: string
accessKeySecret: string
}) {
constructor(config: OSSAdapterConfig) {
this.region = config.region
this.bucket = config.bucket
this.internal = config.internal ?? false
this.signedUrlExpires = config.signedUrlExpires ?? 3600
// ⚠️ TODO: 待安装 ali-oss 后取消注释
// this.client = new OSS({
// region: config.region,
// bucket: config.bucket,
// accessKeyId: config.accessKeyId,
// accessKeySecret: config.accessKeySecret
// })
// 构建Endpoint
// 内网：oss-cn-beijing-internal.aliyuncs.com（SAE零流量费）
// 公网：oss-cn-beijing.aliyuncs.com（本地开发）
const endpoint = this.internal
? `${config.region}-internal.aliyuncs.com`
: `${config.region}.aliyuncs.com`
this.client = new OSS({
region: config.region,
bucket: config.bucket,
accessKeyId: config.accessKeyId,
accessKeySecret: config.accessKeySecret,
endpoint,
// 禁用自动URL编码，避免中文文件名问题
secure: true,
})
console.log(`[OSSAdapter] Initialized: bucket=${config.bucket}, region=${config.region}, internal=${this.internal}`)
}
/**
* 上传文件到OSS
*
* @param key 存储路径，如 pkb/tenant123/user456/doc.pdf
* @param buffer 文件内容
* @returns 文件访问URL（私有Bucket返回签名URL）
*/
async upload(_key: string, _buffer: Buffer): Promise<string> {
// ⚠️ TODO: 待实现
// const result = await this.client.put(key, buffer)
// return result.url
async upload(key: string, buffer: Buffer): Promise<string> {
try {
const normalizedKey = this.normalizeKey(key)
// 使用 put 方法上传 Buffer（适合 < 30MB 文件）
const result = await this.client.put(normalizedKey, buffer)
console.log(`[OSSAdapter] Upload success: ${normalizedKey}, size=${buffer.length}`)
// 返回签名URL（假设是私有Bucket）
return this.getSignedUrl(normalizedKey)
} catch (error) {
console.error(`[OSSAdapter] Upload failed: ${key}`, error)
throw new Error(`OSS上传失败: ${key}`)
}
}
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
/**
* 流式上传（适合大文件 > 30MB）
*
* @param key 存储路径
* @param stream 可读流
* @returns 文件访问URL
*/
async uploadStream(key: string, stream: NodeJS.ReadableStream): Promise<string> {
try {
const normalizedKey = this.normalizeKey(key)
// 使用 putStream 方法上传流
const result = await this.client.putStream(normalizedKey, stream)
console.log(`[OSSAdapter] Stream upload success: ${normalizedKey}`)
return this.getSignedUrl(normalizedKey)
} catch (error) {
console.error(`[OSSAdapter] Stream upload failed: ${key}`, error)
throw new Error(`OSS流式上传失败: ${key}`)
}
}
/**
* 从OSS下载文件
*
* @param key 存储路径
* @returns 文件内容 Buffer
*/
async download(_key: string): Promise<Buffer> {
// ⚠️ TODO: 待实现
// const result = await this.client.get(key)
// return result.content as Buffer
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
async download(key: string): Promise<Buffer> {
try {
const normalizedKey = this.normalizeKey(key)
const result = await this.client.get(normalizedKey)
// result.content 可能是 Buffer 或 string
if (Buffer.isBuffer(result.content)) {
return result.content
}
// 如果是字符串，转换为 Buffer
return Buffer.from(result.content as string)
} catch (error: any) {
// 处理文件不存在的情况
if (error.code === 'NoSuchKey') {
throw new Error(`文件不存在: ${key}`)
}
console.error(`[OSSAdapter] Download failed: ${key}`, error)
throw new Error(`OSS下载失败: ${key}`)
}
}
/**
* 从OSS删除文件
*
* @param key 存储路径
*/
async delete(_key: string): Promise<void> {
// ⚠️ TODO: 待实现
// await this.client.delete(key)
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
async delete(key: string): Promise<void> {
try {
const normalizedKey = this.normalizeKey(key)
await this.client.delete(normalizedKey)
console.log(`[OSSAdapter] Delete success: ${normalizedKey}`)
} catch (error: any) {
// 删除不存在的文件不报错（幂等）
if (error.code === 'NoSuchKey') {
console.log(`[OSSAdapter] Delete skipped (not exists): ${key}`)
return
}
console.error(`[OSSAdapter] Delete failed: ${key}`, error)
throw new Error(`OSS删除失败: ${key}`)
}
}
/**
* 获取文件访问URL
*
* 私有Bucket：返回签名URL
* 公共Bucket：返回直接URL
*
* @param key 存储路径
* @returns 文件访问URL
*/
getUrl(key: string): string {
// 返回OSS公开访问URL
// 格式：https://{bucket}.{region}.aliyuncs.com/{key}
return `https://${this.bucket}.${this.region}.aliyuncs.com/${key}`
// 默认返回签名URL（适合私有Bucket）
return this.getSignedUrl(key)
}
/**
* 获取签名URL（私有Bucket访问）
*
* @param key 存储路径
* @param expires 过期时间（秒），默认使用配置值
* @param originalFilename 原始文件名（可选，用于下载时恢复文件名）
* @returns 签名URL
*/
getSignedUrl(key: string, expires?: number, originalFilename?: string): string {
const normalizedKey = this.normalizeKey(key)
const expireTime = expires ?? this.signedUrlExpires
// 生成签名URL选项
const options: any = {
expires: expireTime,
}
// 如果提供了原始文件名，添加 Content-Disposition 头
// 下载时浏览器会使用这个文件名
if (originalFilename) {
// RFC 5987 编码，支持中文文件名
const encodedFilename = encodeURIComponent(originalFilename)
options.response = {
'content-disposition': `attachment; filename*=UTF-8''${encodedFilename}`,
}
}
// 生成签名URL
const url = this.client.signatureUrl(normalizedKey, options)
return url
}
/**
* 获取公开访问URL（适合静态资源Bucket）
*
* 格式：https://{bucket}.{region}.aliyuncs.com/{key}
*
* @param key 存储路径
* @returns 公开URL
*/
getPublicUrl(key: string): string {
const normalizedKey = this.normalizeKey(key)
return `https://${this.bucket}.${this.region}.aliyuncs.com/${normalizedKey}`
}
/**
* 检查文件是否存在
*
* @param key 存储路径
* @returns 是否存在
*/
async exists(_key: string): Promise<boolean> {
// ⚠️ TODO: 待实现
// try {
// await this.client.head(key)
// return true
// } catch (error) {
// return false
// }
async exists(key: string): Promise<boolean> {
try {
const normalizedKey = this.normalizeKey(key)
await this.client.head(normalizedKey)
return true
} catch (error: any) {
if (error.code === 'NoSuchKey') {
return false
}
// 其他错误抛出
throw error
}
}
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
/**
* 获取文件元信息
*
* @param key 存储路径
* @returns 文件元信息（大小、类型、最后修改时间等）
*/
async getMetadata(key: string): Promise<{
size: number
contentType: string
lastModified: Date
} | null> {
try {
const normalizedKey = this.normalizeKey(key)
const result = await this.client.head(normalizedKey)
return {
size: parseInt(result.res.headers['content-length'] as string, 10),
contentType: result.res.headers['content-type'] as string,
lastModified: new Date(result.res.headers['last-modified'] as string),
}
} catch (error: any) {
if (error.code === 'NoSuchKey') {
return null
}
throw error
}
}
/**
* 批量删除文件
*
* @param keys 存储路径数组
*/
async deleteMany(keys: string[]): Promise<void> {
if (keys.length === 0) return
try {
const normalizedKeys = keys.map(k => this.normalizeKey(k))
// OSS支持批量删除，最多1000个
await this.client.deleteMulti(normalizedKeys, {
quiet: true, // 静默模式，不返回删除结果
})
console.log(`[OSSAdapter] Batch delete success: ${keys.length} files`)
} catch (error) {
console.error(`[OSSAdapter] Batch delete failed`, error)
throw new Error(`OSS批量删除失败`)
}
}
/**
* 列出目录下的文件
*
* @param prefix 目录前缀，如 pkb/tenant123/
* @param maxKeys 最大返回数量
* @returns 文件列表
*/
async list(prefix: string, maxKeys: number = 1000): Promise<{
key: string
size: number
lastModified: Date
}[]> {
try {
const normalizedPrefix = this.normalizeKey(prefix)
const result = await this.client.list({
prefix: normalizedPrefix,
'max-keys': maxKeys,
}, {})
return (result.objects || []).map(obj => ({
key: obj.name,
size: obj.size,
lastModified: new Date(obj.lastModified),
}))
} catch (error) {
console.error(`[OSSAdapter] List failed: ${prefix}`, error)
throw new Error(`OSS列表失败: ${prefix}`)
}
}
/**
* 复制文件
*
* @param sourceKey 源路径
* @param targetKey 目标路径
*/
async copy(sourceKey: string, targetKey: string): Promise<void> {
try {
const normalizedSource = this.normalizeKey(sourceKey)
const normalizedTarget = this.normalizeKey(targetKey)
await this.client.copy(normalizedTarget, normalizedSource)
console.log(`[OSSAdapter] Copy success: ${sourceKey} -> ${targetKey}`)
} catch (error) {
console.error(`[OSSAdapter] Copy failed: ${sourceKey} -> ${targetKey}`, error)
throw new Error(`OSS复制失败`)
}
}
/**
* 规范化Key（移除开头的斜杠）
*/
private normalizeKey(key: string): string {
return key.replace(/^\/+/, '')
}
}
/**
* ⚠️ 实施说明：
*
* 1. 安装依赖：
* npm install ali-oss
* npm install -D @types/ali-oss
*
* 2. 取消注释代码：
* - import OSS from 'ali-oss'
* - new OSS({ ... })
* - 所有方法的实现代码
*
* 3. 配置环境变量：
* OSS_REGION=oss-cn-hangzhou
* OSS_BUCKET=aiclinical-prod
* OSS_ACCESS_KEY_ID=your-access-key-id
* OSS_ACCESS_KEY_SECRET=your-access-key-secret
*
* 4. 测试：
* - 上传小文件
* - 下载文件
* - 删除文件
* - 检查文件是否存在
*/

133
backend/src/common/storage/StorageFactory.ts
View File

@@ -1,6 +1,6 @@
import { StorageAdapter } from './StorageAdapter.js'
import { LocalAdapter } from './LocalAdapter.js'
import { OSSAdapter } from './OSSAdapter.js'
import { OSSAdapter, OSSAdapterConfig } from './OSSAdapter.js'
/**
* 存储工厂类
@@ -9,23 +9,39 @@ import { OSSAdapter } from './OSSAdapter.js'
* - STORAGE_TYPE=local: 使用LocalAdapter（本地文件系统）
* - STORAGE_TYPE=oss: 使用OSSAdapter（阿里云OSS）
*
* 支持双Bucket策略：
* - Data Bucket（私有）: 核心数据，文献、病历、报告
* - Static Bucket（公共读）: 静态资源，头像、Logo、图片
*
* 零代码切换：
* - 本地开发：不配置STORAGE_TYPE，默认使用local
* - 云端部署：配置STORAGE_TYPE=oss，自动切换到OSS
*
* @example
* ```typescript
* import { storage } from '@/common/storage'
* import { storage, staticStorage } from '@/common/storage'
*
* // 业务代码不关心是local还是oss
* const url = await storage.upload('literature/123.pdf', buffer)
* // 上传私有文件（自动使用签名URL）
* const url = await storage.upload('pkb/tenant/user/doc.pdf', buffer)
*
* // 上传静态资源（公开访问）
* const avatarUrl = await staticStorage.upload('avatars/user123.jpg', buffer)
* ```
*/
export class StorageFactory {
/** 主存储实例（私有数据） */
private static instance: StorageAdapter | null = null
/** 静态存储实例（公共资源） */
private static staticInstance: StorageAdapter | null = null
/**
* 获取存储适配器实例（单例模式）
* 获取主存储适配器实例（单例模式）
*
* 用于存储私有数据：
* - PKB文献文件
* - 审稿PDF报告
* - 临床数据文件
*/
static getInstance(): StorageAdapter {
if (!this.instance) {
@@ -35,7 +51,22 @@ export class StorageFactory {
}
/**
* 创建存储适配器
* 获取静态资源存储适配器实例（单例模式）
*
* 用于存储公共资源：
* - 用户头像
* - 系统Logo
* - RAG引用的图片
*/
static getStaticInstance(): StorageAdapter {
if (!this.staticInstance) {
this.staticInstance = this.createStaticAdapter()
}
return this.staticInstance
}
/**
* 创建主存储适配器（私有数据）
*/
private static createAdapter(): StorageAdapter {
const storageType = process.env.STORAGE_TYPE || 'local'
@@ -54,41 +85,99 @@ export class StorageFactory {
}
/**
* 创建本地适配器
* 创建静态资源存储适配器
*/
private static createLocalAdapter(): LocalAdapter {
const baseDir = process.env.LOCAL_STORAGE_DIR || 'uploads'
const baseUrl = process.env.LOCAL_STORAGE_URL || 'http://localhost:3001/uploads'
console.log(`[StorageFactory] Using LocalAdapter (baseDir: ${baseDir})`)
return new LocalAdapter(baseDir, baseUrl)
private static createStaticAdapter(): StorageAdapter {
const storageType = process.env.STORAGE_TYPE || 'local'
switch (storageType) {
case 'local':
// 本地模式下，静态资源也用同一个目录
return this.createLocalAdapter('uploads/static', 'http://localhost:3001/uploads/static')
case 'oss':
return this.createOSSStaticAdapter()
default:
return this.createLocalAdapter('uploads/static', 'http://localhost:3001/uploads/static')
}
}
/**
* 创建OSS适配器
* 创建本地适配器
*/
private static createLocalAdapter(
baseDir?: string,
baseUrl?: string
): LocalAdapter {
const dir = baseDir || process.env.LOCAL_STORAGE_DIR || 'uploads'
const url = baseUrl || process.env.LOCAL_STORAGE_URL || 'http://localhost:3001/uploads'
console.log(`[StorageFactory] Using LocalAdapter (baseDir: ${dir})`)
return new LocalAdapter(dir, url)
}
/**
* 创建OSS适配器（私有数据Bucket）
*/
private static createOSSAdapter(): OSSAdapter {
const config = this.getOSSConfig()
console.log(`[StorageFactory] Using OSSAdapter (region: ${config.region}, bucket: ${config.bucket}, internal: ${config.internal})`)
return new OSSAdapter(config)
}
/**
* 创建OSS静态资源适配器（公共Bucket）
*/
private static createOSSStaticAdapter(): OSSAdapter {
const baseConfig = this.getOSSConfig()
// 静态资源使用独立的Bucket
const staticBucket = process.env.OSS_BUCKET_STATIC || `${baseConfig.bucket}-static`
console.log(`[StorageFactory] Using OSSAdapter for static (bucket: ${staticBucket})`)
return new OSSAdapter({
...baseConfig,
bucket: staticBucket,
})
}
/**
* 获取OSS配置（带验证）
*/
private static getOSSConfig(): OSSAdapterConfig {
const region = process.env.OSS_REGION
const bucket = process.env.OSS_BUCKET
const accessKeyId = process.env.OSS_ACCESS_KEY_ID
const accessKeySecret = process.env.OSS_ACCESS_KEY_SECRET
const internal = process.env.OSS_INTERNAL === 'true'
const signedUrlExpires = parseInt(process.env.OSS_SIGNED_URL_EXPIRES || '3600', 10)
// 验证必需的环境变量
if (!region || !bucket || !accessKeyId || !accessKeySecret) {
const missing = []
if (!region) missing.push('OSS_REGION')
if (!bucket) missing.push('OSS_BUCKET')
if (!accessKeyId) missing.push('OSS_ACCESS_KEY_ID')
if (!accessKeySecret) missing.push('OSS_ACCESS_KEY_SECRET')
throw new Error(
'[StorageFactory] OSS configuration incomplete. Required: OSS_REGION, OSS_BUCKET, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET'
`[StorageFactory] OSS configuration incomplete. Missing: ${missing.join(', ')}`
)
}
console.log(`[StorageFactory] Using OSSAdapter (region: ${region}, bucket: ${bucket})`)
return new OSSAdapter({
return {
region,
bucket,
accessKeyId,
accessKeySecret
})
accessKeySecret,
internal,
signedUrlExpires,
}
}
/**
@@ -96,6 +185,6 @@ export class StorageFactory {
*/
static reset(): void {
this.instance = null
this.staticInstance = null
}
}

45
backend/src/common/storage/index.ts
View File

@@ -3,40 +3,63 @@
*
* 提供平台级的文件存储能力，支持本地和云端无缝切换。
*
* 双Bucket策略：
* - storage: 私有数据（文献、病历、报告），使用签名URL
* - staticStorage: 静态资源（头像、Logo、图片），公开访问
*
* @module storage
*
* @example
* ```typescript
* // 方式1：使用单例（推荐）
* import { storage } from '@/common/storage'
* const url = await storage.upload('literature/123.pdf', buffer)
* import { storage, staticStorage } from '@/common/storage'
*
* // 方式2：直接使用适配器
* import { LocalAdapter } from '@/common/storage'
* const adapter = new LocalAdapter()
* const url = await adapter.upload('literature/123.pdf', buffer)
* // 上传私有文件
* const url = await storage.upload('pkb/tenant/user/doc.pdf', buffer)
*
* // 方式3：使用工厂
* // 上传静态资源
* const avatarUrl = await staticStorage.upload('avatars/user123.jpg', buffer)
*
* // 方式2：使用工厂
* import { StorageFactory } from '@/common/storage'
* const storage = StorageFactory.getInstance()
* const url = await storage.upload('literature/123.pdf', buffer)
* const dataStorage = StorageFactory.getInstance()
* const staticStorage = StorageFactory.getStaticInstance()
* ```
*/
export type { StorageAdapter } from './StorageAdapter.js'
export { LocalAdapter } from './LocalAdapter.js'
export { OSSAdapter } from './OSSAdapter.js'
export type { OSSAdapterConfig } from './OSSAdapter.js'
export { StorageFactory } from './StorageFactory.js'
// Import for usage below
import { StorageFactory } from './StorageFactory.js'
/**
* 全局存储实例（推荐使用）
* 全局主存储实例（私有数据）
*
* 自动根据环境变量选择存储实现：
* - STORAGE_TYPE=local: 本地文件系统
* - STORAGE_TYPE=oss: 阿里云OSS
* - STORAGE_TYPE=oss: 阿里云OSS（私有Bucket，签名URL）
*
* 使用场景：
* - PKB文献文件
* - 审稿PDF报告
* - 临床数据文件
*/
export const storage = StorageFactory.getInstance()
/**
* 全局静态资源存储实例
*
* 自动根据环境变量选择存储实现：
* - STORAGE_TYPE=local: 本地文件系统
* - STORAGE_TYPE=oss: 阿里云OSS（公共读Bucket）
*
* 使用场景：
* - 用户头像
* - 系统Logo
* - RAG引用的图片
*/
export const staticStorage = StorageFactory.getStaticInstance()

3
backend/src/common/streaming/OpenAIStreamAdapter.ts
View File

@@ -203,3 +203,6 @@ export function createOpenAIStreamAdapter(

3
backend/src/common/streaming/StreamingService.ts
View File

@@ -209,3 +209,6 @@ export async function streamChat(

3
backend/src/common/streaming/index.ts
View File

@@ -27,3 +27,6 @@ export { THINKING_TAGS } from './types';

3
backend/src/common/streaming/types.ts
View File

@@ -102,3 +102,6 @@ export type SSEEventType =