feat(storage): integrate Alibaba Cloud OSS for file persistence - Add OSSAdapter and LocalAdapter with StorageFactory pattern - Integrate PKB module with OSS upload - Rename difyDocumentId to storageKey - Create 4 OSS buckets and development specification

This commit is contained in:
2026-01-22 22:02:20 +08:00
parent 483c62fb6f
commit 9c96f75c52
309 changed files with 4583 additions and 172 deletions

View File

@@ -199,3 +199,6 @@ export const jwtService = new JWTService();

View File

@@ -327,6 +327,9 @@ export function getBatchItems<T>(

View File

@@ -82,3 +82,6 @@ export interface VariableValidation {

View File

@@ -352,3 +352,6 @@ export function chunkMarkdown(markdown: string, config?: ChunkConfig): TextChunk
export default ChunkService;

View File

@@ -48,3 +48,6 @@ class DeprecatedDifyClient {
export const difyClient = new DeprecatedDifyClient();
export const DifyClient = DeprecatedDifyClient;

View File

@@ -1,135 +1,372 @@
import { StorageAdapter } from './StorageAdapter.js'
// import OSS from 'ali-oss' // ⚠️ 需要安装npm install ali-oss
import OSS from 'ali-oss'
/**
* 阿里云OSS适配器
* 阿里云OSS存储适配器
*
* 适用场景
* - 云端SaaS部署阿里云Serverless
* - 高可用、高并发场景
* - 需要CDN加速
* 支持功能
* - 文件上传/下载/删除
* - 私有Bucket签名URL
* - 内网EndpointSAE部署零流量费
* - 静态Bucket公开访问
*
* 配置要求
* - OSS_REGION: OSS地域oss-cn-hangzhou
* - OSS_BUCKET: OSS Bucket名称
* - OSS_ACCESS_KEY_ID: AccessKey ID
* - OSS_ACCESS_KEY_SECRET: AccessKey Secret
* Bucket策略
* - ai-clinical-data: 私有,核心数据(文献、病历、报告
* - ai-clinical-static: 公共读静态资源头像、Logo、图片
*
* @example
* ```typescript
* const adapter = new OSSAdapter({
* region: 'oss-cn-hangzhou',
* bucket: 'aiclinical-prod',
* region: 'oss-cn-beijing',
* bucket: 'ai-clinical-data',
* accessKeyId: process.env.OSS_ACCESS_KEY_ID!,
* accessKeySecret: process.env.OSS_ACCESS_KEY_SECRET!
* accessKeySecret: process.env.OSS_ACCESS_KEY_SECRET!,
* internal: true // SAE内网
* })
* await adapter.upload('literature/123.pdf', buffer)
* ```
*
* ⚠️ 当前为预留实现,待云端部署时完善
* // 上传文件
* const url = await adapter.upload('pkb/tenant123/user456/doc.pdf', buffer)
*
* // 获取签名URL私有Bucket
* const signedUrl = adapter.getSignedUrl('pkb/tenant123/user456/doc.pdf')
* ```
*/
export interface OSSAdapterConfig {
/** OSS地域如 oss-cn-beijing */
region: string
/** Bucket名称 */
bucket: string
/** AccessKey ID */
accessKeyId: string
/** AccessKey Secret */
accessKeySecret: string
/** 是否使用内网EndpointSAE部署必须为true */
internal?: boolean
/** 签名URL过期时间默认3600 */
signedUrlExpires?: number
}
export class OSSAdapter implements StorageAdapter {
// private readonly client: OSS
private readonly client: OSS
private readonly bucket: string
private readonly region: string
private readonly internal: boolean
private readonly signedUrlExpires: number
constructor(config: {
region: string
bucket: string
accessKeyId: string
accessKeySecret: string
}) {
constructor(config: OSSAdapterConfig) {
this.region = config.region
this.bucket = config.bucket
this.internal = config.internal ?? false
this.signedUrlExpires = config.signedUrlExpires ?? 3600
// ⚠️ TODO: 待安装 ali-oss 后取消注释
// this.client = new OSS({
// region: config.region,
// bucket: config.bucket,
// accessKeyId: config.accessKeyId,
// accessKeySecret: config.accessKeySecret
// })
// 构建Endpoint
// 内网oss-cn-beijing-internal.aliyuncs.comSAE零流量费
// 公网oss-cn-beijing.aliyuncs.com本地开发
const endpoint = this.internal
? `${config.region}-internal.aliyuncs.com`
: `${config.region}.aliyuncs.com`
this.client = new OSS({
region: config.region,
bucket: config.bucket,
accessKeyId: config.accessKeyId,
accessKeySecret: config.accessKeySecret,
endpoint,
// 禁用自动URL编码避免中文文件名问题
secure: true,
})
console.log(`[OSSAdapter] Initialized: bucket=${config.bucket}, region=${config.region}, internal=${this.internal}`)
}
/**
* 上传文件到OSS
*
* @param key 存储路径,如 pkb/tenant123/user456/doc.pdf
* @param buffer 文件内容
* @returns 文件访问URL私有Bucket返回签名URL
*/
async upload(_key: string, _buffer: Buffer): Promise<string> {
// ⚠️ TODO: 待实现
// const result = await this.client.put(key, buffer)
// return result.url
async upload(key: string, buffer: Buffer): Promise<string> {
try {
const normalizedKey = this.normalizeKey(key)
// 使用 put 方法上传 Buffer适合 < 30MB 文件)
const result = await this.client.put(normalizedKey, buffer)
console.log(`[OSSAdapter] Upload success: ${normalizedKey}, size=${buffer.length}`)
// 返回签名URL假设是私有Bucket
return this.getSignedUrl(normalizedKey)
} catch (error) {
console.error(`[OSSAdapter] Upload failed: ${key}`, error)
throw new Error(`OSS上传失败: ${key}`)
}
}
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
/**
* 流式上传(适合大文件 > 30MB
*
* @param key 存储路径
* @param stream 可读流
* @returns 文件访问URL
*/
async uploadStream(key: string, stream: NodeJS.ReadableStream): Promise<string> {
try {
const normalizedKey = this.normalizeKey(key)
// 使用 putStream 方法上传流
const result = await this.client.putStream(normalizedKey, stream)
console.log(`[OSSAdapter] Stream upload success: ${normalizedKey}`)
return this.getSignedUrl(normalizedKey)
} catch (error) {
console.error(`[OSSAdapter] Stream upload failed: ${key}`, error)
throw new Error(`OSS流式上传失败: ${key}`)
}
}
/**
* 从OSS下载文件
*
* @param key 存储路径
* @returns 文件内容 Buffer
*/
async download(_key: string): Promise<Buffer> {
// ⚠️ TODO: 待实现
// const result = await this.client.get(key)
// return result.content as Buffer
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
async download(key: string): Promise<Buffer> {
try {
const normalizedKey = this.normalizeKey(key)
const result = await this.client.get(normalizedKey)
// result.content 可能是 Buffer 或 string
if (Buffer.isBuffer(result.content)) {
return result.content
}
// 如果是字符串,转换为 Buffer
return Buffer.from(result.content as string)
} catch (error: any) {
// 处理文件不存在的情况
if (error.code === 'NoSuchKey') {
throw new Error(`文件不存在: ${key}`)
}
console.error(`[OSSAdapter] Download failed: ${key}`, error)
throw new Error(`OSS下载失败: ${key}`)
}
}
/**
* 从OSS删除文件
*
* @param key 存储路径
*/
async delete(_key: string): Promise<void> {
// ⚠️ TODO: 待实现
// await this.client.delete(key)
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
async delete(key: string): Promise<void> {
try {
const normalizedKey = this.normalizeKey(key)
await this.client.delete(normalizedKey)
console.log(`[OSSAdapter] Delete success: ${normalizedKey}`)
} catch (error: any) {
// 删除不存在的文件不报错(幂等)
if (error.code === 'NoSuchKey') {
console.log(`[OSSAdapter] Delete skipped (not exists): ${key}`)
return
}
console.error(`[OSSAdapter] Delete failed: ${key}`, error)
throw new Error(`OSS删除失败: ${key}`)
}
}
/**
* 获取文件访问URL
*
* 私有Bucket返回签名URL
* 公共Bucket返回直接URL
*
* @param key 存储路径
* @returns 文件访问URL
*/
getUrl(key: string): string {
// 返回OSS公开访问URL
// 格式https://{bucket}.{region}.aliyuncs.com/{key}
return `https://${this.bucket}.${this.region}.aliyuncs.com/${key}`
// 默认返回签名URL适合私有Bucket
return this.getSignedUrl(key)
}
/**
* 获取签名URL私有Bucket访问
*
* @param key 存储路径
* @param expires 过期时间(秒),默认使用配置值
* @param originalFilename 原始文件名(可选,用于下载时恢复文件名)
* @returns 签名URL
*/
getSignedUrl(key: string, expires?: number, originalFilename?: string): string {
const normalizedKey = this.normalizeKey(key)
const expireTime = expires ?? this.signedUrlExpires
// 生成签名URL选项
const options: any = {
expires: expireTime,
}
// 如果提供了原始文件名,添加 Content-Disposition 头
// 下载时浏览器会使用这个文件名
if (originalFilename) {
// RFC 5987 编码,支持中文文件名
const encodedFilename = encodeURIComponent(originalFilename)
options.response = {
'content-disposition': `attachment; filename*=UTF-8''${encodedFilename}`,
}
}
// 生成签名URL
const url = this.client.signatureUrl(normalizedKey, options)
return url
}
/**
* 获取公开访问URL适合静态资源Bucket
*
* 格式https://{bucket}.{region}.aliyuncs.com/{key}
*
* @param key 存储路径
* @returns 公开URL
*/
getPublicUrl(key: string): string {
const normalizedKey = this.normalizeKey(key)
return `https://${this.bucket}.${this.region}.aliyuncs.com/${normalizedKey}`
}
/**
* 检查文件是否存在
*
* @param key 存储路径
* @returns 是否存在
*/
async exists(_key: string): Promise<boolean> {
// ⚠️ TODO: 待实现
// try {
// await this.client.head(key)
// return true
// } catch (error) {
// return false
// }
async exists(key: string): Promise<boolean> {
try {
const normalizedKey = this.normalizeKey(key)
await this.client.head(normalizedKey)
return true
} catch (error: any) {
if (error.code === 'NoSuchKey') {
return false
}
// 其他错误抛出
throw error
}
}
throw new Error('[OSSAdapter] Not implemented yet. Please install ali-oss and configure OSS.')
/**
* 获取文件元信息
*
* @param key 存储路径
* @returns 文件元信息(大小、类型、最后修改时间等)
*/
async getMetadata(key: string): Promise<{
size: number
contentType: string
lastModified: Date
} | null> {
try {
const normalizedKey = this.normalizeKey(key)
const result = await this.client.head(normalizedKey)
return {
size: parseInt(result.res.headers['content-length'] as string, 10),
contentType: result.res.headers['content-type'] as string,
lastModified: new Date(result.res.headers['last-modified'] as string),
}
} catch (error: any) {
if (error.code === 'NoSuchKey') {
return null
}
throw error
}
}
/**
* 批量删除文件
*
* @param keys 存储路径数组
*/
async deleteMany(keys: string[]): Promise<void> {
if (keys.length === 0) return
try {
const normalizedKeys = keys.map(k => this.normalizeKey(k))
// OSS支持批量删除最多1000个
await this.client.deleteMulti(normalizedKeys, {
quiet: true, // 静默模式,不返回删除结果
})
console.log(`[OSSAdapter] Batch delete success: ${keys.length} files`)
} catch (error) {
console.error(`[OSSAdapter] Batch delete failed`, error)
throw new Error(`OSS批量删除失败`)
}
}
/**
* 列出目录下的文件
*
* @param prefix 目录前缀,如 pkb/tenant123/
* @param maxKeys 最大返回数量
* @returns 文件列表
*/
async list(prefix: string, maxKeys: number = 1000): Promise<{
key: string
size: number
lastModified: Date
}[]> {
try {
const normalizedPrefix = this.normalizeKey(prefix)
const result = await this.client.list({
prefix: normalizedPrefix,
'max-keys': maxKeys,
}, {})
return (result.objects || []).map(obj => ({
key: obj.name,
size: obj.size,
lastModified: new Date(obj.lastModified),
}))
} catch (error) {
console.error(`[OSSAdapter] List failed: ${prefix}`, error)
throw new Error(`OSS列表失败: ${prefix}`)
}
}
/**
* 复制文件
*
* @param sourceKey 源路径
* @param targetKey 目标路径
*/
async copy(sourceKey: string, targetKey: string): Promise<void> {
try {
const normalizedSource = this.normalizeKey(sourceKey)
const normalizedTarget = this.normalizeKey(targetKey)
await this.client.copy(normalizedTarget, normalizedSource)
console.log(`[OSSAdapter] Copy success: ${sourceKey} -> ${targetKey}`)
} catch (error) {
console.error(`[OSSAdapter] Copy failed: ${sourceKey} -> ${targetKey}`, error)
throw new Error(`OSS复制失败`)
}
}
/**
* 规范化Key移除开头的斜杠
*/
private normalizeKey(key: string): string {
return key.replace(/^\/+/, '')
}
}
/**
* ⚠️ 实施说明:
*
* 1. 安装依赖:
* npm install ali-oss
* npm install -D @types/ali-oss
*
* 2. 取消注释代码:
* - import OSS from 'ali-oss'
* - new OSS({ ... })
* - 所有方法的实现代码
*
* 3. 配置环境变量:
* OSS_REGION=oss-cn-hangzhou
* OSS_BUCKET=aiclinical-prod
* OSS_ACCESS_KEY_ID=your-access-key-id
* OSS_ACCESS_KEY_SECRET=your-access-key-secret
*
* 4. 测试:
* - 上传小文件
* - 下载文件
* - 删除文件
* - 检查文件是否存在
*/

View File

@@ -1,6 +1,6 @@
import { StorageAdapter } from './StorageAdapter.js'
import { LocalAdapter } from './LocalAdapter.js'
import { OSSAdapter } from './OSSAdapter.js'
import { OSSAdapter, OSSAdapterConfig } from './OSSAdapter.js'
/**
* 存储工厂类
@@ -9,23 +9,39 @@ import { OSSAdapter } from './OSSAdapter.js'
* - STORAGE_TYPE=local: 使用LocalAdapter本地文件系统
* - STORAGE_TYPE=oss: 使用OSSAdapter阿里云OSS
*
* 支持双Bucket策略
* - Data Bucket私有: 核心数据,文献、病历、报告
* - Static Bucket公共读: 静态资源头像、Logo、图片
*
* 零代码切换:
* - 本地开发不配置STORAGE_TYPE默认使用local
* - 云端部署配置STORAGE_TYPE=oss自动切换到OSS
*
* @example
* ```typescript
* import { storage } from '@/common/storage'
* import { storage, staticStorage } from '@/common/storage'
*
* // 业务代码不关心是local还是oss
* const url = await storage.upload('literature/123.pdf', buffer)
* // 上传私有文件自动使用签名URL
* const url = await storage.upload('pkb/tenant/user/doc.pdf', buffer)
*
* // 上传静态资源(公开访问)
* const avatarUrl = await staticStorage.upload('avatars/user123.jpg', buffer)
* ```
*/
export class StorageFactory {
/** 主存储实例(私有数据) */
private static instance: StorageAdapter | null = null
/** 静态存储实例(公共资源) */
private static staticInstance: StorageAdapter | null = null
/**
* 获取存储适配器实例(单例模式)
* 获取存储适配器实例(单例模式)
*
* 用于存储私有数据:
* - PKB文献文件
* - 审稿PDF报告
* - 临床数据文件
*/
static getInstance(): StorageAdapter {
if (!this.instance) {
@@ -35,7 +51,22 @@ export class StorageFactory {
}
/**
* 创建存储适配器
* 获取静态资源存储适配器实例(单例模式)
*
* 用于存储公共资源:
* - 用户头像
* - 系统Logo
* - RAG引用的图片
*/
static getStaticInstance(): StorageAdapter {
if (!this.staticInstance) {
this.staticInstance = this.createStaticAdapter()
}
return this.staticInstance
}
/**
* 创建主存储适配器(私有数据)
*/
private static createAdapter(): StorageAdapter {
const storageType = process.env.STORAGE_TYPE || 'local'
@@ -54,41 +85,99 @@ export class StorageFactory {
}
/**
* 创建本地适配器
* 创建静态资源存储适配器
*/
private static createLocalAdapter(): LocalAdapter {
const baseDir = process.env.LOCAL_STORAGE_DIR || 'uploads'
const baseUrl = process.env.LOCAL_STORAGE_URL || 'http://localhost:3001/uploads'
console.log(`[StorageFactory] Using LocalAdapter (baseDir: ${baseDir})`)
return new LocalAdapter(baseDir, baseUrl)
private static createStaticAdapter(): StorageAdapter {
const storageType = process.env.STORAGE_TYPE || 'local'
switch (storageType) {
case 'local':
// 本地模式下,静态资源也用同一个目录
return this.createLocalAdapter('uploads/static', 'http://localhost:3001/uploads/static')
case 'oss':
return this.createOSSStaticAdapter()
default:
return this.createLocalAdapter('uploads/static', 'http://localhost:3001/uploads/static')
}
}
/**
* 创建OSS适配器
* 创建本地适配器
*/
private static createLocalAdapter(
baseDir?: string,
baseUrl?: string
): LocalAdapter {
const dir = baseDir || process.env.LOCAL_STORAGE_DIR || 'uploads'
const url = baseUrl || process.env.LOCAL_STORAGE_URL || 'http://localhost:3001/uploads'
console.log(`[StorageFactory] Using LocalAdapter (baseDir: ${dir})`)
return new LocalAdapter(dir, url)
}
/**
* 创建OSS适配器私有数据Bucket
*/
private static createOSSAdapter(): OSSAdapter {
const config = this.getOSSConfig()
console.log(`[StorageFactory] Using OSSAdapter (region: ${config.region}, bucket: ${config.bucket}, internal: ${config.internal})`)
return new OSSAdapter(config)
}
/**
* 创建OSS静态资源适配器公共Bucket
*/
private static createOSSStaticAdapter(): OSSAdapter {
const baseConfig = this.getOSSConfig()
// 静态资源使用独立的Bucket
const staticBucket = process.env.OSS_BUCKET_STATIC || `${baseConfig.bucket}-static`
console.log(`[StorageFactory] Using OSSAdapter for static (bucket: ${staticBucket})`)
return new OSSAdapter({
...baseConfig,
bucket: staticBucket,
})
}
/**
* 获取OSS配置带验证
*/
private static getOSSConfig(): OSSAdapterConfig {
const region = process.env.OSS_REGION
const bucket = process.env.OSS_BUCKET
const accessKeyId = process.env.OSS_ACCESS_KEY_ID
const accessKeySecret = process.env.OSS_ACCESS_KEY_SECRET
const internal = process.env.OSS_INTERNAL === 'true'
const signedUrlExpires = parseInt(process.env.OSS_SIGNED_URL_EXPIRES || '3600', 10)
// 验证必需的环境变量
if (!region || !bucket || !accessKeyId || !accessKeySecret) {
const missing = []
if (!region) missing.push('OSS_REGION')
if (!bucket) missing.push('OSS_BUCKET')
if (!accessKeyId) missing.push('OSS_ACCESS_KEY_ID')
if (!accessKeySecret) missing.push('OSS_ACCESS_KEY_SECRET')
throw new Error(
'[StorageFactory] OSS configuration incomplete. Required: OSS_REGION, OSS_BUCKET, OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET'
`[StorageFactory] OSS configuration incomplete. Missing: ${missing.join(', ')}`
)
}
console.log(`[StorageFactory] Using OSSAdapter (region: ${region}, bucket: ${bucket})`)
return new OSSAdapter({
return {
region,
bucket,
accessKeyId,
accessKeySecret
})
accessKeySecret,
internal,
signedUrlExpires,
}
}
/**
@@ -96,6 +185,6 @@ export class StorageFactory {
*/
static reset(): void {
this.instance = null
this.staticInstance = null
}
}

View File

@@ -3,40 +3,63 @@
*
* 提供平台级的文件存储能力,支持本地和云端无缝切换。
*
* 双Bucket策略
* - storage: 私有数据文献、病历、报告使用签名URL
* - staticStorage: 静态资源头像、Logo、图片公开访问
*
* @module storage
*
* @example
* ```typescript
* // 方式1使用单例推荐
* import { storage } from '@/common/storage'
* const url = await storage.upload('literature/123.pdf', buffer)
* import { storage, staticStorage } from '@/common/storage'
*
* // 方式2直接使用适配器
* import { LocalAdapter } from '@/common/storage'
* const adapter = new LocalAdapter()
* const url = await adapter.upload('literature/123.pdf', buffer)
* // 上传私有文件
* const url = await storage.upload('pkb/tenant/user/doc.pdf', buffer)
*
* // 方式3使用工厂
* // 上传静态资源
* const avatarUrl = await staticStorage.upload('avatars/user123.jpg', buffer)
*
* // 方式2使用工厂
* import { StorageFactory } from '@/common/storage'
* const storage = StorageFactory.getInstance()
* const url = await storage.upload('literature/123.pdf', buffer)
* const dataStorage = StorageFactory.getInstance()
* const staticStorage = StorageFactory.getStaticInstance()
* ```
*/
export type { StorageAdapter } from './StorageAdapter.js'
export { LocalAdapter } from './LocalAdapter.js'
export { OSSAdapter } from './OSSAdapter.js'
export type { OSSAdapterConfig } from './OSSAdapter.js'
export { StorageFactory } from './StorageFactory.js'
// Import for usage below
import { StorageFactory } from './StorageFactory.js'
/**
* 全局存储实例(推荐使用
* 全局存储实例(私有数据
*
* 自动根据环境变量选择存储实现:
* - STORAGE_TYPE=local: 本地文件系统
* - STORAGE_TYPE=oss: 阿里云OSS
* - STORAGE_TYPE=oss: 阿里云OSS私有Bucket签名URL
*
* 使用场景:
* - PKB文献文件
* - 审稿PDF报告
* - 临床数据文件
*/
export const storage = StorageFactory.getInstance()
/**
* 全局静态资源存储实例
*
* 自动根据环境变量选择存储实现:
* - STORAGE_TYPE=local: 本地文件系统
* - STORAGE_TYPE=oss: 阿里云OSS公共读Bucket
*
* 使用场景:
* - 用户头像
* - 系统Logo
* - RAG引用的图片
*/
export const staticStorage = StorageFactory.getStaticInstance()

View File

@@ -203,3 +203,6 @@ export function createOpenAIStreamAdapter(

View File

@@ -209,3 +209,6 @@ export async function streamChat(

View File

@@ -27,3 +27,6 @@ export { THINKING_TAGS } from './types';

View File

@@ -102,3 +102,6 @@ export type SSEEventType =

View File

@@ -69,14 +69,23 @@ export const config = {
/** 阿里云OSS地域 */
ossRegion: process.env.OSS_REGION,
/** 阿里云OSS Bucket名称 */
/** 阿里云OSS Bucket名称(私有数据) */
ossBucket: process.env.OSS_BUCKET,
/** 阿里云OSS 静态资源Bucket名称公共读 */
ossBucketStatic: process.env.OSS_BUCKET_STATIC,
/** 阿里云OSS AccessKey ID */
ossAccessKeyId: process.env.OSS_ACCESS_KEY_ID,
/** 阿里云OSS AccessKey Secret */
ossAccessKeySecret: process.env.OSS_ACCESS_KEY_SECRET,
/** 是否使用OSS内网EndpointSAE部署必须为true */
ossInternal: process.env.OSS_INTERNAL === 'true',
/** OSS签名URL过期时间 */
ossSignedUrlExpires: parseInt(process.env.OSS_SIGNED_URL_EXPIRES || '3600', 10),
// ==================== 缓存配置(平台基础设施)====================
@@ -150,8 +159,8 @@ export const config = {
// ==================== 文件上传配置Legacy兼容====================
/** 文件上传大小限制 */
uploadMaxSize: parseInt(process.env.UPLOAD_MAX_SIZE || '10485760', 10), // 10MB
/** 文件上传大小限制30MB */
uploadMaxSize: parseInt(process.env.UPLOAD_MAX_SIZE || '31457280', 10), // 30MB
/** 文件上传目录Legacy兼容新模块使用storage */
uploadDir: process.env.UPLOAD_DIR || './uploads',
@@ -185,6 +194,16 @@ export function validateEnv(): void {
if (!config.ossBucket) errors.push('OSS_BUCKET is required when STORAGE_TYPE=oss')
if (!config.ossAccessKeyId) errors.push('OSS_ACCESS_KEY_ID is required when STORAGE_TYPE=oss')
if (!config.ossAccessKeySecret) errors.push('OSS_ACCESS_KEY_SECRET is required when STORAGE_TYPE=oss')
// 可选配置警告
if (!config.ossBucketStatic) {
warnings.push('OSS_BUCKET_STATIC not set, static resources will use main bucket')
}
// 生产环境必须使用内网
if (config.nodeEnv === 'production' && !config.ossInternal) {
warnings.push('OSS_INTERNAL should be true in production (SAE内网访问免流量费)')
}
}
// 如果使用Redis验证Redis配置

View File

@@ -275,11 +275,12 @@ export class ChatController {
select: {
id: true,
filename: true,
difyDocumentId: true,
storageKey: true, // 原 difyDocumentId,现为 OSS 路径
},
});
const difyDocIds = documents.map(d => d.difyDocumentId).filter(Boolean);
// Legacy: 此代码已废弃Dify 已移除
const difyDocIds = documents.map(d => d.storageKey).filter(Boolean);
console.log(`📄 [ChatController] 目标Dify文档ID:`, difyDocIds);
// 过滤结果

View File

@@ -48,7 +48,7 @@ export async function uploadDocument(
fileType,
fileSizeBytes,
fileUrl,
difyDocumentId: '', // 暂时为空,稍后更新
storageKey: '', // 原 difyDocumentId现为 OSS 路径Legacy 代码已废弃)
status: 'uploading',
progress: 0,
},
@@ -88,11 +88,11 @@ export async function uploadDocument(
filename
);
// 6. 更新文档记录(更新difyDocumentId、状态和Phase 2字段
// 6. 更新文档记录(Legacy此代码已废弃Dify 已移除
const updatedDocument = await prisma.document.update({
where: { id: document.id },
data: {
difyDocumentId: difyResult.document.id,
storageKey: difyResult.document.id, // Legacy: 原为 difyDocumentId
status: difyResult.document.indexing_status,
progress: 50,
// Phase 2新增字段
@@ -138,7 +138,7 @@ async function pollDocumentStatus(
userId: string,
kbId: string,
documentId: string,
difyDocumentId: string,
legacyDifyDocId: string, // Legacy: 原为 difyDocumentId
maxAttempts: number = 30
) {
const knowledgeBase = await prisma.knowledgeBase.findFirst({
@@ -156,7 +156,7 @@ async function pollDocumentStatus(
// 查询Dify中的文档状态
const difyDocument = await difyClient.getDocument(
knowledgeBase.difyDatasetId,
difyDocumentId
legacyDifyDocId
);
// 更新数据库中的状态
@@ -264,12 +264,12 @@ export async function deleteDocument(userId: string, documentId: string) {
throw new Error('Document not found or access denied');
}
// 2. 删除Dify中的文档
if (document.difyDocumentId) {
// 2. 删除Dify中的文档Legacy此代码已废弃Dify 已移除)
if (document.storageKey) {
try {
await difyClient.deleteDocument(
document.knowledgeBase.difyDatasetId,
document.difyDocumentId
document.storageKey // Legacy: 原为 difyDocumentId
);
} catch (error) {
console.error('Failed to delete Dify document:', error);
@@ -305,12 +305,12 @@ export async function reprocessDocument(userId: string, documentId: string) {
throw new Error('Document not found or access denied');
}
// 2. 触发Dify重新索引
if (document.difyDocumentId) {
// 2. 触发Dify重新索引Legacy此代码已废弃Dify 已移除)
if (document.storageKey) {
try {
await difyClient.updateDocument(
document.knowledgeBase.difyDatasetId,
document.difyDocumentId
document.storageKey // Legacy: 原为 difyDocumentId
);
// 3. 更新状态为processing
@@ -328,7 +328,7 @@ export async function reprocessDocument(userId: string, documentId: string) {
userId,
document.kbId,
documentId,
document.difyDocumentId
document.storageKey // Legacy: 原为 difyDocumentId
).catch(error => {
console.error('Failed to poll document status:', error);
});

View File

@@ -88,3 +88,6 @@ export async function moduleRoutes(fastify: FastifyInstance) {

View File

@@ -118,3 +118,6 @@ export interface PaginatedResponse<T> {

View File

@@ -165,3 +165,6 @@ export const ROLE_DISPLAY_NAMES: Record<UserRole, string> = {

View File

@@ -240,3 +240,6 @@ async function matchIntent(query: string): Promise<{

View File

@@ -94,3 +94,6 @@ export async function uploadAttachment(

View File

@@ -23,3 +23,6 @@ export { aiaRoutes };

View File

@@ -363,6 +363,9 @@ runTests().catch((error) => {

View File

@@ -342,6 +342,9 @@ Content-Type: application/json

View File

@@ -278,6 +278,9 @@ export const conflictDetectionService = new ConflictDetectionService();

View File

@@ -228,6 +228,9 @@ curl -X POST http://localhost:3000/api/v1/dc/tool-c/test/execute \

View File

@@ -282,6 +282,9 @@ export const streamAIController = new StreamAIController();

View File

@@ -191,6 +191,9 @@ logger.info('[SessionMemory] 会话记忆管理器已启动', {

View File

@@ -125,6 +125,9 @@ checkTableStructure();

View File

@@ -112,6 +112,9 @@ checkProjectConfig().catch(console.error);

View File

@@ -94,6 +94,9 @@ main();

View File

@@ -551,6 +551,9 @@ URL: https://iit.xunzhengyixue.com/api/v1/iit/patient-wechat/callback

View File

@@ -186,6 +186,9 @@ console.log('');

View File

@@ -503,6 +503,9 @@ export const patientWechatService = new PatientWechatService();

View File

@@ -148,6 +148,9 @@ testDifyIntegration().catch(error => {

View File

@@ -177,6 +177,9 @@ testIitDatabase()

View File

@@ -163,6 +163,9 @@ if (hasError) {

View File

@@ -189,6 +189,9 @@ async function testUrlVerification() {

View File

@@ -270,6 +270,9 @@ main().catch((error) => {

View File

@@ -154,6 +154,9 @@ Write-Host ""

View File

@@ -247,6 +247,9 @@ export interface CachedProtocolRules {

View File

@@ -1,5 +1,8 @@
import type { FastifyRequest, FastifyReply } from 'fastify';
import * as documentService from '../services/documentService.js';
import { storage } from '../../../common/storage/index.js';
import { randomUUID } from 'crypto';
import path from 'path';
/**
* 获取用户ID从JWT Token中获取
@@ -12,6 +15,30 @@ function getUserId(request: FastifyRequest): string {
return userId;
}
/**
* 获取租户ID从JWT Token中获取
*/
function getTenantId(request: FastifyRequest): string {
const tenantId = (request as any).user?.tenantId;
// 如果没有租户ID使用默认值
return tenantId || 'default';
}
/**
* 生成 PKB 文档存储 Key
* 格式tenants/{tenantId}/users/{userId}/pkb/{kbId}/{uuid}.{ext}
*/
function generatePkbStorageKey(
tenantId: string,
userId: string,
kbId: string,
filename: string
): string {
const uuid = randomUUID().replace(/-/g, '').substring(0, 16);
const ext = path.extname(filename).toLowerCase();
return `tenants/${tenantId}/users/${userId}/pkb/${kbId}/${uuid}${ext}`;
}
/**
* 上传文档
*/
@@ -45,15 +72,15 @@ export async function uploadDocument(
const fileType = data.mimetype;
const fileSizeBytes = file.length;
// 文件大小限制(10MB
const maxSize = 10 * 1024 * 1024;
console.log(`📊 文件大小: ${(fileSizeBytes / 1024 / 1024).toFixed(2)}MB (限制: 10MB)`);
// 文件大小限制(30MB - 按 OSS 规范
const maxSize = 30 * 1024 * 1024;
console.log(`📊 文件大小: ${(fileSizeBytes / 1024 / 1024).toFixed(2)}MB (限制: 30MB)`);
if (fileSizeBytes > maxSize) {
console.error(`❌ 文件太大: ${(fileSizeBytes / 1024 / 1024).toFixed(2)}MB`);
return reply.status(400).send({
success: false,
message: 'File size exceeds 10MB limit',
message: 'File size exceeds 30MB limit',
});
}
@@ -75,9 +102,30 @@ export async function uploadDocument(
});
}
// 上传文档这里fileUrl暂时为空实际应该上传到对象存储
console.log(`⚙️ 调用文档服务上传文件...`);
// 获取用户信息
const userId = getUserId(request);
const tenantId = getTenantId(request);
// 生成 OSS 存储 Key包含 kbId
const storageKey = generatePkbStorageKey(tenantId, userId, kbId, filename);
console.log(`📦 OSS 存储路径: ${storageKey}`);
// 上传到 OSS
console.log(`☁️ 上传文件到存储服务...`);
let fileUrl = '';
try {
fileUrl = await storage.upload(storageKey, file);
console.log(`✅ 文件已上传到存储服务`);
} catch (storageError) {
console.error(`❌ 存储服务上传失败:`, storageError);
return reply.status(500).send({
success: false,
message: 'Failed to upload file to storage',
});
}
// 调用文档服务处理(传入 storageKey
console.log(`⚙️ 调用文档服务处理文件...`);
const document = await documentService.uploadDocument(
userId,
kbId,
@@ -85,7 +133,8 @@ export async function uploadDocument(
filename,
fileType,
fileSizeBytes,
'' // fileUrl - 可以上传到OSS后填入
fileUrl,
storageKey // 新增:存储路径
);
console.log(`✅ 文档上传成功: ${document.id}`);

View File

@@ -60,6 +60,9 @@ export default async function healthRoutes(fastify: FastifyInstance) {

View File

@@ -2,6 +2,7 @@ import { prisma } from '../../../config/database.js';
import { logger } from '../../../common/logging/index.js';
import { extractionClient } from '../../../common/document/ExtractionClient.js';
import { ingestDocument as ragIngestDocument } from './ragService.js';
import { storage } from '../../../common/storage/index.js';
/**
* 文档服务
@@ -11,6 +12,15 @@ import { ingestDocument as ragIngestDocument } from './ragService.js';
/**
* 上传文档到知识库
*
* @param userId - 用户ID
* @param kbId - 知识库ID
* @param file - 文件内容 Buffer
* @param filename - 原始文件名
* @param fileType - MIME 类型
* @param fileSizeBytes - 文件大小(字节)
* @param fileUrl - 文件访问 URL签名URL
* @param storageKey - OSS 存储路径(新增)
*/
export async function uploadDocument(
userId: string,
@@ -19,7 +29,8 @@ export async function uploadDocument(
filename: string,
fileType: string,
fileSizeBytes: number,
fileUrl: string
fileUrl: string,
storageKey?: string // 新增OSS 存储路径
) {
// 1. 验证知识库权限
const knowledgeBase = await prisma.knowledgeBase.findFirst({
@@ -65,7 +76,7 @@ export async function uploadDocument(
fileType,
fileSizeBytes,
fileUrl,
difyDocumentId: '', // 不再使用
storageKey: storageKey || '', // OSS 存储路径
status: 'uploading',
progress: 0,
},
@@ -107,10 +118,10 @@ export async function uploadDocument(
});
// 7. 更新文档记录 - pgvector 模式立即完成
// 注意storageKey 已在创建时设置,这里不需要更新
const updatedDocument = await prisma.document.update({
where: { id: document.id },
data: {
difyDocumentId: ingestResult.documentId || '',
status: 'completed',
progress: 100,
// 提取信息
@@ -234,13 +245,23 @@ export async function deleteDocument(userId: string, documentId: string) {
logger.info(`[PKB] 删除文档: documentId=${documentId}`);
// 1.5 删除 OSS 文件
if (document.storageKey) {
try {
await storage.delete(document.storageKey);
logger.info(`[PKB] OSS 文件已删除: ${document.storageKey}`);
} catch (storageError) {
logger.warn(`[PKB] OSS 文件删除失败,继续删除数据库记录`, { storageError });
}
}
// 2. 删除 EKB 中的文档和 Chunks
try {
// 查找 EKB 文档(通过 filename 和 kbId 匹配)
const ekbDoc = await prisma.ekbDocument.findFirst({
where: {
filename: document.filename,
kb: {
knowledgeBase: {
ownerId: userId,
name: document.knowledgeBase.name,
},
@@ -311,7 +332,7 @@ export async function reprocessDocument(userId: string, documentId: string) {
const ekbDoc = await prisma.ekbDocument.findFirst({
where: {
filename: document.filename,
kb: {
knowledgeBase: {
ownerId: userId,
name: document.knowledgeBase.name,
},

View File

@@ -140,5 +140,8 @@ Content-Type: application/json

View File

@@ -125,5 +125,8 @@ Write-Host " - 删除任务: DELETE $BaseUrl/api/v1/rvw/tasks/{taskId}" -Foregr

View File

@@ -39,5 +39,8 @@ export * from './services/utils.js';

View File

@@ -130,5 +130,8 @@ export function validateAgentSelection(agents: string[]): void {

View File

@@ -428,6 +428,9 @@ SET session_replication_role = 'origin';

View File

@@ -110,3 +110,6 @@ async function testCrossLanguageSearch() {
testCrossLanguageSearch();

View File

@@ -172,3 +172,6 @@ async function testQueryRewrite() {
testQueryRewrite();

View File

@@ -118,3 +118,6 @@ async function testRerank() {
testRerank();

View File

@@ -130,6 +130,9 @@ WHERE key = 'verify_test';

View File

@@ -273,6 +273,9 @@ verifyDatabase()

View File

@@ -63,6 +63,9 @@ export {}