Develop/AIclinicalresearch
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dc/tool-c): Fix special character handling and improve UX

Browse Source 
Major fixes:
- Fix pivot transformation with special characters in column names
- Fix compute column validation for Chinese punctuation
- Fix recode dialog to fetch unique values from full dataset via new API
- Add column mapping mechanism to handle special characters

Database migration:
- Add column_mapping field to dc_tool_c_sessions table
- Migration file: 20251208_add_column_mapping

UX improvements:
- Darken table grid lines for better visibility
- Reduce column width by 40% with tooltip support
- Insert new columns next to source columns
- Preserve original row order after operations
- Add notice about 50-row preview limit

Modified files:
- Backend: SessionService, SessionController, QuickActionService, routes
- Python: pivot.py, compute.py, recode.py, binning.py, conditional.py
- Frontend: DataGrid, RecodeDialog, index.tsx, ag-grid-custom.css
- Database: schema.prisma, migration SQL

Status: Code complete, database migrated, ready for testing
This commit is contained in:
HaHafeng
2025-12-08 23:20:55 +08:00
parent f729699510
commit 91cab452d1
90 changed files with 735 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/migrations/add_data_stats_to_tool_c_session.sql
View File

@@ -22,3 +22,4 @@ WHERE table_schema = 'dc_schema'
\echo '✅ 字段 data_stats 已成功添加到 dc_tool_c_sessions 表'

10
backend/prisma/migrations/20251208_add_column_mapping/migration.sql Normal file
View File

@@ -0,0 +1,10 @@
-- AlterTable
-- 添加 column_mapping 字段到 dc_tool_c_sessions 表
-- 用于解决表头特殊字符问题
ALTER TABLE "dc_schema"."dc_tool_c_sessions"
ADD COLUMN IF NOT EXISTS "column_mapping" JSONB;
-- 添加注释
COMMENT ON COLUMN "dc_schema"."dc_tool_c_sessions"."column_mapping" IS '列名映射：[{originalName, safeName, displayName}] 解决特殊字符问题';

1
backend/prisma/migrations/create_tool_c_session.sql
View File

@@ -34,3 +34,4 @@ COMMENT ON COLUMN dc_schema.dc_tool_c_sessions.expires_at IS '过期时间（创

1
backend/prisma/schema.prisma
View File

@@ -860,6 +860,7 @@ model DcToolCSession {
totalRows Int @map("total_rows")
totalCols Int @map("total_cols")
columns Json @map("columns") // ["age", "gender", "diagnosis"] 列名数组
columnMapping Json? @map("column_mapping") // ✨ 列名映射：[{originalName, safeName, displayName}] 解决特殊字符问题
encoding String? @map("encoding") // 文件编码 utf-8, gbk等
fileSize Int @map("file_size") // 文件大小（字节）

1
backend/recover-code-from-cursor-db.js
View File

@@ -184,3 +184,4 @@ function extractCodeBlocks(obj, blocks = []) {

1
backend/scripts/check-dc-tables.mjs
View File

@@ -203,3 +203,4 @@ checkDCTables();

1
backend/scripts/create-tool-c-ai-history-table.mjs
View File

@@ -155,3 +155,4 @@ createAiHistoryTable()

1
backend/scripts/create-tool-c-table.js
View File

@@ -142,3 +142,4 @@ createToolCTable()

1
backend/scripts/create-tool-c-table.mjs
View File

@@ -139,3 +139,4 @@ createToolCTable()

1
backend/src/modules/asl/fulltext-screening/__tests__/api-integration-test.ts
View File

@@ -307,3 +307,4 @@ runTests().catch((error) => {

1
backend/src/modules/asl/fulltext-screening/__tests__/e2e-real-test-v2.ts
View File

@@ -248,3 +248,4 @@ runTest()

1
backend/src/modules/asl/fulltext-screening/__tests__/fulltext-screening-api.http
View File

@@ -286,3 +286,4 @@ Content-Type: application/json

1
backend/src/modules/asl/fulltext-screening/services/ExcelExporter.ts
View File

@@ -365,3 +365,4 @@ export class ExcelExporter {

1
backend/src/modules/dc/tool-b/services/ConflictDetectionService.ts
View File

@@ -222,3 +222,4 @@ export const conflictDetectionService = new ConflictDetectionService();

1
backend/src/modules/dc/tool-b/services/TemplateService.ts
View File

@@ -250,3 +250,4 @@ export const templateService = new TemplateService();

1
backend/src/modules/dc/tool-c/README.md
View File

@@ -172,3 +172,4 @@ curl -X POST http://localhost:3000/api/v1/dc/tool-c/test/execute \

67
backend/src/modules/dc/tool-c/controllers/SessionController.ts
View File

@@ -25,6 +25,10 @@ interface SessionIdParams {
id: string;
}
interface GetUniqueValuesQuery {
column: string;
}
// ==================== 控制器 ====================
export class SessionController {
@@ -362,6 +366,69 @@ export class SessionController {
});
}
}
/**
* ✨ 获取列的唯一值（用于数值映射）
*
* GET /api/v1/dc/tool-c/sessions/:id/unique-values?column=xxx
*/
async getUniqueValues(
request: FastifyRequest<{ Params: SessionIdParams; Querystring: GetUniqueValuesQuery }>,
reply: FastifyReply
) {
try {
const { id } = request.params;
const { column } = request.query;
if (!column) {
return reply.code(400).send({
success: false,
error: '缺少column参数',
});
}
logger.info(`[SessionController] 获取唯一值: session=${id}, column=${column}`);
// 1. 获取完整数据
const data = await sessionService.getFullData(id);
// 2. 提取唯一值（去除空值和首尾空格）
const values = data.map((row) => row[column]);
const cleanedValues = values.map((val) => {
if (val === null || val === undefined || val === '') return null;
// 如果是字符串，去除首尾空格
return typeof val === 'string' ? val.trim() : val;
});
// 3. 去重
const uniqueValues = Array.from(new Set(cleanedValues))
.filter((v) => v !== null && v !== '' && v !== '(空白)')
.sort(); // 排序，方便查看
logger.info(`[SessionController] 唯一值数量: ${uniqueValues.length}`);
// 4. 返回结果
return reply.send({
success: true,
data: {
column,
uniqueValues,
count: uniqueValues.length,
},
});
} catch (error: any) {
logger.error(`[SessionController] 获取唯一值失败: ${error.message}`);
const statusCode = error.message.includes('不存在') || error.message.includes('过期')
? 404
: 500;
return reply.code(statusCode).send({
success: false,
error: error.message || '获取唯一值失败',
});
}
}
}
// ==================== 导出单例实例 ====================

1
backend/src/modules/dc/tool-c/controllers/StreamAIController.ts
View File

@@ -226,3 +226,4 @@ export class StreamAIController {
export const streamAIController = new StreamAIController();

5
backend/src/modules/dc/tool-c/routes/index.ts
View File

@@ -61,6 +61,11 @@ export async function toolCRoutes(fastify: FastifyInstance) {
handler: sessionController.updateHeartbeat.bind(sessionController),
});
// ✨ 获取列的唯一值（用于数值映射）
fastify.get('/sessions/:id/unique-values', {
handler: sessionController.getUniqueValues.bind(sessionController),
});
// ==================== AI代码生成路由（Day 3） ====================
// 生成代码（不执行）

39
backend/src/modules/dc/tool-c/services/SessionService.ts
View File

@@ -18,6 +18,12 @@ import * as xlsx from 'xlsx';
// ==================== 类型定义 ====================
interface ColumnMapping {
originalName: string;
safeName: string;
displayName: string;
}
interface SessionData {
id: string;
userId: string;
@@ -26,6 +32,7 @@ interface SessionData {
totalRows: number;
totalCols: number;
columns: string[];
columnMapping?: ColumnMapping[]; // ✨ 新增：列名映射
encoding: string | null;
fileSize: number;
createdAt: Date;
@@ -102,8 +109,12 @@ export class SessionService {
const totalRows = data.length;
const totalCols = Object.keys(data[0] || {}).length;
const columns = Object.keys(data[0] || {});
// ✨ 生成列名映射（解决特殊字符问题）
const columnMapping = this.generateColumnMapping(columns);
logger.info(`[SessionService] 解析完成: ${totalRows}行 x ${totalCols}列`);
logger.info(`[SessionService] 列名映射: ${columnMapping.length}个列`);
// 4. 上传到OSS（使用平台storage服务）
const timestamp = Date.now();
@@ -130,6 +141,7 @@ export class SessionService {
totalRows,
totalCols,
columns: columns, // Prisma会自动转换为JSONB
columnMapping: JSON.parse(JSON.stringify(columnMapping)), // ✨ 存储列名映射
encoding: 'utf-8', // 默认utf-8，后续可扩展检测
fileSize: fileBuffer.length,
dataStats: JSON.parse(JSON.stringify(dataStats)), // ✨ 存储统计信息（转换为JSON）
@@ -370,12 +382,15 @@ export class SessionService {
// 4. 更新Session元数据
const newColumns = Object.keys(processedData[0] || {});
const newColumnMapping = this.generateColumnMapping(newColumns); // ✨ 重新生成列名映射
await prisma.dcToolCSession.update({
where: { id: sessionId },
data: {
totalRows: processedData.length,
totalCols: newColumns.length,
columns: newColumns,
columnMapping: JSON.parse(JSON.stringify(newColumnMapping)), // ✨ 更新列名映射
updatedAt: new Date(),
},
});
@@ -517,6 +532,29 @@ export class SessionService {
};
}
/**
* ✨ 生成安全的列名映射
*
* 解决特殊字符问题：表头包含括号、等号等特殊字符会导致Python处理失败
*
* @param originalColumns - 原始列名数组
* @returns 列名映射数组
*/
private generateColumnMapping(originalColumns: string[]): ColumnMapping[] {
return originalColumns.map((originalName, index) => {
// 安全列名：col_0, col_1, col_2...
const safeName = `col_${index}`;
// 显示名称：用于前端展示（保持原始名称）
const displayName = originalName;
return {
originalName,
safeName,
displayName,
};
});
}
/**
* 检测列的数据类型
*
@@ -571,6 +609,7 @@ export class SessionService {
totalRows: session.totalRows,
totalCols: session.totalCols,
columns: session.columns as string[],
columnMapping: session.columnMapping as ColumnMapping[] | undefined, // ✨ 返回列名映射
encoding: session.encoding,
fileSize: session.fileSize,
createdAt: session.createdAt,

1
backend/sync-dc-database.ps1
View File

@@ -30,3 +30,4 @@ Write-Host "✅ 完成！" -ForegroundColor Green

1
backend/test-tool-c-advanced-scenarios.mjs
View File

@@ -317,3 +317,4 @@ runAdvancedTests().catch(error => {
});

1
backend/test-tool-c-day2.mjs
View File

@@ -383,3 +383,4 @@ runAllTests()

1
backend/test-tool-c-day3.mjs
View File

@@ -341,3 +341,4 @@ runAllTests()