Develop/AIclinicalresearch
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dc): Add multi-metric transformation feature (direction 1+2)

Browse Source 
Summary:
- Implement intelligent multi-metric grouping detection algorithm
- Add direction 1: timepoint-as-row, metric-as-column (analysis format)
- Add direction 2: timepoint-as-column, metric-as-row (display format)
- Fix column name pattern detection (FMA___ issue)
- Maintain original Record ID order in output
- Add full-select/clear buttons in UI
- Integrate into TransformDialog with Radio selection
- Update 3 documentation files

Technical Details:
- Python: detect_metric_groups(), apply_multi_metric_to_long(), apply_multi_metric_to_matrix()
- Backend: 3 new methods in QuickActionService
- Frontend: MultiMetricPanel.tsx (531 lines)
- Total: ~1460 lines of new code

Status: Fully tested and verified, ready for production
This commit is contained in:
HaHafeng
2025-12-21 15:06:15 +08:00
parent 8be8cdcf53
commit 9b81aef9a7
123 changed files with 4781 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/common/jobs/utils.ts
View File

@@ -281,3 +281,9 @@ export function getBatchItems<T>(

6
backend/src/modules/asl/fulltext-screening/__tests__/api-integration-test.ts
View File

@@ -310,6 +310,12 @@ runTests().catch((error) => {

6
backend/src/modules/asl/fulltext-screening/__tests__/e2e-real-test-v2.ts
View File

@@ -251,6 +251,12 @@ runTest()

6
backend/src/modules/asl/fulltext-screening/__tests__/fulltext-screening-api.http
View File

@@ -289,6 +289,12 @@ Content-Type: application/json

6
backend/src/modules/asl/fulltext-screening/services/ExcelExporter.ts
View File

@@ -368,6 +368,12 @@ export class ExcelExporter {

6
backend/src/modules/dc/tool-b/services/ConflictDetectionService.ts
View File

@@ -225,6 +225,12 @@ export const conflictDetectionService = new ConflictDetectionService();

6
backend/src/modules/dc/tool-b/services/TemplateService.ts
View File

@@ -253,6 +253,12 @@ export const templateService = new TemplateService();

6
backend/src/modules/dc/tool-b/workers/extractionWorker.ts
View File

@@ -390,3 +390,9 @@ async function countCompletedBatches(taskId: string): Promise<number> {
}

6
backend/src/modules/dc/tool-c/README.md
View File

@@ -182,3 +182,9 @@ curl -X POST http://localhost:3000/api/v1/dc/tool-c/test/execute \

168
backend/src/modules/dc/tool-c/controllers/QuickActionController.ts
View File

@@ -25,7 +25,7 @@ import { prisma } from '../../../../config/database.js';
interface QuickActionRequest {
sessionId: string;
action: 'filter' | 'recode' | 'binning' | 'conditional' | 'dropna' | 'dedup';
action: 'filter' | 'recode' | 'binning' | 'conditional' | 'dropna' | 'dedup' | 'compute' | 'pivot' | 'unpivot' | 'metric_time' | 'multi_metric_to_long' | 'multi_metric_to_matrix';
params: any;
userId?: string;
}
@@ -105,6 +105,18 @@ export class QuickActionController {
case 'pivot':
actionDescription = 'Pivot转换';
break;
case 'unpivot':
actionDescription = 'Unpivot转换（宽→长表）';
break;
case 'metric_time':
actionDescription = '指标-时间表转换';
break;
case 'multi_metric_to_long':
actionDescription = '多指标转长表';
break;
case 'multi_metric_to_matrix':
actionDescription = '多指标转矩阵';
break;
default:
logger.warn(`[QuickAction] 不支持的操作: ${action}`);
return reply.code(400).send({
@@ -184,6 +196,22 @@ export class QuickActionController {
pivotValueOrder
);
break;
case 'unpivot':
// Unpivot不需要columnMapping，直接执行
executeResult = await quickActionService.executeUnpivot(fullData, params);
break;
case 'metric_time':
// 指标-时间表转换
executeResult = await quickActionService.executeMetricTime(fullData, params);
break;
case 'multi_metric_to_long':
// 多指标转长表
executeResult = await quickActionService.executeMultiMetricToLong(fullData, params);
break;
case 'multi_metric_to_matrix':
// 多指标转矩阵
executeResult = await quickActionService.executeMultiMetricToMatrix(fullData, params);
break;
}
if (!executeResult.success) {
@@ -340,9 +368,27 @@ export class QuickActionController {
case 'dropna':
executeResult = await quickActionService.executeDropna(fullData, params);
break;
case 'compute':
executeResult = await quickActionService.executeCompute(fullData, params);
break;
case 'dedup':
// TODO: 实现去重功能
return reply.code(400).send({ success: false, error: '去重功能尚未实现' });
case 'pivot':
executeResult = await quickActionService.executePivot(fullData, params);
break;
case 'unpivot':
executeResult = await quickActionService.executeUnpivot(fullData, params);
break;
case 'metric_time':
executeResult = await quickActionService.executeMetricTime(fullData, params);
break;
case 'multi_metric_to_long':
executeResult = await quickActionService.executeMultiMetricToLong(fullData, params);
break;
case 'multi_metric_to_matrix':
executeResult = await quickActionService.executeMultiMetricToMatrix(fullData, params);
break;
default:
return reply.code(400).send({ success: false, error: '不支持的操作' });
}
@@ -361,14 +407,29 @@ export class QuickActionController {
const newRows = resultData.length;
let estimatedChange = '';
if (action === 'filter' || action === 'dropna') {
estimatedChange = `将保留 ${newRows} 行（删除 ${originalRows - newRows} 行）`;
} else if (action === 'recode' || action === 'binning' || action === 'conditional' || action === 'compute') {
estimatedChange = `将新增 1 列`;
} else if (action === 'pivot') {
const originalCols = Object.keys(fullData[0] || {}).length;
const newCols = Object.keys(resultData[0] || {}).length;
estimatedChange = `行数: ${originalRows} → ${newRows}, 列数: ${originalCols} → ${newCols}`;
switch (action) {
case 'filter':
case 'dropna':
estimatedChange = `将保留 ${newRows} 行（删除 ${originalRows - newRows} 行）`;
break;
case 'recode':
case 'binning':
case 'conditional':
case 'compute':
estimatedChange = `将新增 1 列`;
break;
case 'pivot':
case 'unpivot':
case 'metric_time':
case 'multi_metric_to_long':
case 'multi_metric_to_matrix': {
const originalCols = Object.keys(fullData[0] || {}).length;
const newCols = Object.keys(resultData[0] || {}).length;
estimatedChange = `行数: ${originalRows} → ${newRows}, 列数: ${originalCols} → ${newCols}`;
break;
}
default:
estimatedChange = `操作完成`;
}
return reply.code(200).send({
@@ -541,6 +602,95 @@ export class QuickActionController {
});
}
}
/**
* POST /api/v1/dc/tool-c/metric-time/detect
* 检测指标-时间表转换模式
*/
async handleMetricTimeDetect(request: FastifyRequest, reply: FastifyReply) {
try {
const { sessionId, valueVars } = request.body as { sessionId: string; valueVars: string[] };
logger.info(`[QuickAction] 检测指标-时间表模式: session=${sessionId}, ${valueVars?.length || 0} 列`);
// 验证参数
if (!valueVars || valueVars.length < 2) {
return reply.code(400).send({
success: false,
error: '至少需要2列才能检测模式'
});
}
// 调用Service检测模式
const result = await quickActionService.detectMetricTimePattern(valueVars);
if (!result.success) {
return reply.code(500).send({
success: false,
error: result.error || '模式检测失败'
});
}
return reply.code(200).send({
success: true,
pattern: result.pattern,
execution_time: result.execution_time
});
} catch (error: any) {
logger.error(`[QuickAction] 模式检测失败: ${error.message}`);
return reply.code(500).send({
success: false,
error: error.message
});
}
}
/**
* POST /api/v1/dc/tool-c/multi-metric/detect
* 检测多指标分组
*/
async handleMultiMetricDetect(request: FastifyRequest, reply: FastifyReply) {
try {
const { sessionId, valueVars, separators } = request.body as {
sessionId: string;
valueVars: string[];
separators?: string[];
};
logger.info(`[QuickAction] 检测多指标分组: session=${sessionId}, ${valueVars?.length || 0} 列`);
// 验证参数
if (!valueVars || valueVars.length < 2) {
return reply.code(400).send({
success: false,
error: '至少需要2列才能检测分组'
});
}
// 调用Service检测分组
const result = await quickActionService.detectMultiMetricGroups(valueVars, separators);
if (!result.success) {
return reply.code(500).send({
success: false,
error: result.message || '分组检测失败'
});
}
return reply.code(200).send({
success: true,
grouping: result
});
} catch (error: any) {
logger.error(`[QuickAction] 多指标分组检测失败: ${error.message}`);
return reply.code(500).send({
success: false,
error: error.message
});
}
}
}
// ==================== 导出单例 ====================

6
backend/src/modules/dc/tool-c/controllers/StreamAIController.ts
View File

@@ -236,3 +236,9 @@ export const streamAIController = new StreamAIController();

14
backend/src/modules/dc/tool-c/routes/index.ts
View File

@@ -133,5 +133,19 @@ export async function toolCRoutes(fastify: FastifyInstance) {
fastify.post('/fillna/mice', {
handler: quickActionController.handleFillnaMice.bind(quickActionController),
});
// ✨ 指标-时间表转换（新增）
// 检测指标-时间表转换模式
fastify.post('/metric-time/detect', {
handler: quickActionController.handleMetricTimeDetect.bind(quickActionController),
});
// ✨ 多指标转换（新增）
// 检测多指标分组
fastify.post('/multi-metric/detect', {
handler: quickActionController.handleMultiMetricDetect.bind(quickActionController),
});
}

265
backend/src/modules/dc/tool-c/services/QuickActionService.ts
View File

@@ -77,6 +77,49 @@ interface PivotParams {
unusedAggMethod?: 'first' | 'mode' | 'mean'; // ✨ 新增：未选择列的聚合方式
}
interface UnpivotParams {
idVars: string[]; // ID列（保持不变的列）
valueVars: string[]; // 值列（需要转换的列）
varName: string; // 变量名列名
valueName: string; // 值列名
parseColumnNames?: boolean; // 是否解析列名
separator?: string; // 分隔符
metricName?: string; // 指标列名
timeName?: string; // 时间列名
dropna?: boolean; // 是否删除缺失值行
}
interface MetricTimeParams {
idVars: string[]; // ID列（保持不变的列）
valueVars: string[]; // 值列（同一指标的多个时间点）
metricName?: string; // 指标名称（可选，自动检测）
separator?: string; // 分隔符（可选，自动检测）
timepointColName?: string; // 时间点列名
}
interface MultiMetricToLongParams {
idVars: string[]; // ID列
valueVars: string[]; // 值列（多个指标的多个时间点）
separators?: string[]; // 可选的分隔符列表
eventColName?: string; // 时间点列名（默认 'Event_Name'）
}
interface MultiMetricToMatrixParams {
idVars: string[]; // ID列
valueVars: string[]; // 值列（多个指标的多个时间点）
separators?: string[]; // 可选的分隔符列表
metricColName?: string; // 指标列名（默认 '指标名'）
}
interface MetricGrouping {
success: boolean;
metric_groups?: Record<string, string[]>; // 指标分组
separator?: string; // 检测到的分隔符
timepoints?: string[]; // 时间点列表
confidence?: number; // 置信度
message?: string;
}
interface FillnaSimpleParams {
column: string;
newColumnName: string;
@@ -100,6 +143,7 @@ interface OperationResult {
error?: string;
message?: string;
stats?: any;
pattern?: any; // ✨ 新增：用于指标-时间表模式检测
}
// ==================== 服务类 ====================
@@ -359,6 +403,209 @@ export class QuickActionService {
}
}
/**
* 执行Unpivot（宽表转长表）
*/
async executeUnpivot(data: any[], params: UnpivotParams): Promise<OperationResult> {
try {
logger.info(`[QuickActionService] 调用Unpivot API: ${params.idVars.length} ID列 × ${params.valueVars.length} 值列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/unpivot`, {
data,
id_vars: params.idVars,
value_vars: params.valueVars,
var_name: params.varName || '变量',
value_name: params.valueName || '值',
parse_column_names: params.parseColumnNames || false,
separator: params.separator || '_',
metric_name: params.metricName,
time_name: params.timeName,
dropna: params.dropna || false,
}, {
timeout: 60000,
});
logger.info(`[QuickActionService] Unpivot成功: ${response.data.result_shape?.[0] || 0} 行`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] Unpivot失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
error: error.message || 'Unpivot失败',
};
}
}
/**
* 检测指标-时间表转换模式
*/
async detectMetricTimePattern(valueVars: string[]): Promise<OperationResult> {
try {
logger.info(`[QuickActionService] 检测指标-时间表模式: ${valueVars.length} 列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/metric-time/detect`, {
value_vars: valueVars,
}, {
timeout: 10000,
});
logger.info(`[QuickActionService] 模式检测成功`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] 模式检测失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
error: error.message || '模式检测失败',
};
}
}
/**
* 执行指标-时间表转换
*/
async executeMetricTime(data: any[], params: MetricTimeParams): Promise<OperationResult> {
try {
logger.info(`[QuickActionService] 调用指标-时间表转换API: ${params.idVars.length} ID列 × ${params.valueVars.length} 值列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/metric-time`, {
data,
id_vars: params.idVars,
value_vars: params.valueVars,
metric_name: params.metricName,
separator: params.separator,
timepoint_col_name: params.timepointColName || '时间点',
}, {
timeout: 60000,
});
logger.info(`[QuickActionService] 指标-时间表转换成功: ${response.data.result_shape?.[0] || 0} 行`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] 指标-时间表转换失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
error: error.message || '指标-时间表转换失败',
};
}
}
/**
* 检测多指标分组
*/
async detectMultiMetricGroups(valueVars: string[], separators?: string[]): Promise<MetricGrouping> {
try {
logger.info(`[QuickActionService] 调用多指标分组检测API: ${valueVars.length} 列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/multi-metric/detect`, {
value_vars: valueVars,
separators: separators,
}, {
timeout: 10000,
});
logger.info(`[QuickActionService] 多指标分组检测成功: ${Object.keys(response.data.metric_groups || {}).length} 个指标`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] 多指标分组检测失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
message: error.message || '多指标分组检测失败',
};
}
}
/**
* 执行多指标转长表（时间点为行，指标为列）
*/
async executeMultiMetricToLong(data: any[], params: MultiMetricToLongParams): Promise<OperationResult> {
try {
logger.info(`[QuickActionService] 调用多指标转长表API: ${params.idVars.length} ID列 × ${params.valueVars.length} 值列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/multi-metric/to-long`, {
data,
id_vars: params.idVars,
value_vars: params.valueVars,
separators: params.separators,
event_col_name: params.eventColName || 'Event_Name',
}, {
timeout: 60000,
});
logger.info(`[QuickActionService] 多指标转长表成功: ${response.data.result_shape?.[0] || 0} 行`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] 多指标转长表失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
error: error.message || '多指标转长表失败',
};
}
}
/**
* 执行多指标转矩阵（时间点为列，指标为行）
*/
async executeMultiMetricToMatrix(data: any[], params: MultiMetricToMatrixParams): Promise<OperationResult> {
try {
logger.info(`[QuickActionService] 调用多指标转矩阵API: ${params.idVars.length} ID列 × ${params.valueVars.length} 值列`);
const response = await axios.post(`${PYTHON_SERVICE_URL}/api/operations/multi-metric/to-matrix`, {
data,
id_vars: params.idVars,
value_vars: params.valueVars,
separators: params.separators,
metric_col_name: params.metricColName || '指标名',
}, {
timeout: 60000,
});
logger.info(`[QuickActionService] 多指标转矩阵成功: ${response.data.result_shape?.[0] || 0} 行`);
return response.data;
} catch (error: any) {
logger.error(`[QuickActionService] 多指标转矩阵失败: ${error.message}`);
if (error.response?.data) {
return error.response.data;
}
return {
success: false,
error: error.message || '多指标转矩阵失败',
};
}
}
/**
* 获取列的缺失值统计
*/
@@ -463,3 +710,21 @@ export class QuickActionService {
export const quickActionService = new QuickActionService();
// ==================== 导出类型 ====================
export type {
FilterParams,
RecodeParams,
BinningParams,
ConditionalParams,
PivotParams,
UnpivotParams,
MetricTimeParams,
MultiMetricToLongParams,
MultiMetricToMatrixParams,
MetricGrouping,
FillnaSimpleParams,
FillnaMiceParams,
OperationResult,
};

6
backend/src/tests/README.md
View File

@@ -382,3 +382,9 @@ SET session_replication_role = 'origin';
**作者：** AI Clinical Research Team

6
backend/src/tests/verify-test1-database.sql
View File

@@ -84,3 +84,9 @@ WHERE key = 'verify_test';
\echo '=========================================='

6
backend/src/tests/verify-test1-database.ts
View File

@@ -227,3 +227,9 @@ verifyDatabase()
});

6
backend/src/types/global.d.ts vendored
View File

@@ -17,3 +17,9 @@ export {}