fix(dc/tool-c): Fix special character handling and improve UX
Major fixes: - Fix pivot transformation with special characters in column names - Fix compute column validation for Chinese punctuation - Fix recode dialog to fetch unique values from full dataset via new API - Add column mapping mechanism to handle special characters Database migration: - Add column_mapping field to dc_tool_c_sessions table - Migration file: 20251208_add_column_mapping UX improvements: - Darken table grid lines for better visibility - Reduce column width by 40% with tooltip support - Insert new columns next to source columns - Preserve original row order after operations - Add notice about 50-row preview limit Modified files: - Backend: SessionService, SessionController, QuickActionService, routes - Python: pivot.py, compute.py, recode.py, binning.py, conditional.py - Frontend: DataGrid, RecodeDialog, index.tsx, ag-grid-custom.css - Database: schema.prisma, migration SQL Status: Code complete, database migrated, ready for testing
This commit is contained in:
@@ -307,3 +307,4 @@ runTests().catch((error) => {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -248,3 +248,4 @@ runTest()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -286,3 +286,4 @@ Content-Type: application/json
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -365,3 +365,4 @@ export class ExcelExporter {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -222,3 +222,4 @@ export const conflictDetectionService = new ConflictDetectionService();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -250,3 +250,4 @@ export const templateService = new TemplateService();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -172,3 +172,4 @@ curl -X POST http://localhost:3000/api/v1/dc/tool-c/test/execute \
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,10 @@ interface SessionIdParams {
|
||||
id: string;
|
||||
}
|
||||
|
||||
interface GetUniqueValuesQuery {
|
||||
column: string;
|
||||
}
|
||||
|
||||
// ==================== 控制器 ====================
|
||||
|
||||
export class SessionController {
|
||||
@@ -362,6 +366,69 @@ export class SessionController {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ✨ 获取列的唯一值（用于数值映射）
|
||||
*
|
||||
* GET /api/v1/dc/tool-c/sessions/:id/unique-values?column=xxx
|
||||
*/
|
||||
async getUniqueValues(
|
||||
request: FastifyRequest<{ Params: SessionIdParams; Querystring: GetUniqueValuesQuery }>,
|
||||
reply: FastifyReply
|
||||
) {
|
||||
try {
|
||||
const { id } = request.params;
|
||||
const { column } = request.query;
|
||||
|
||||
if (!column) {
|
||||
return reply.code(400).send({
|
||||
success: false,
|
||||
error: '缺少column参数',
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(`[SessionController] 获取唯一值: session=${id}, column=${column}`);
|
||||
|
||||
// 1. 获取完整数据
|
||||
const data = await sessionService.getFullData(id);
|
||||
|
||||
// 2. 提取唯一值（去除空值和首尾空格）
|
||||
const values = data.map((row) => row[column]);
|
||||
const cleanedValues = values.map((val) => {
|
||||
if (val === null || val === undefined || val === '') return null;
|
||||
// 如果是字符串，去除首尾空格
|
||||
return typeof val === 'string' ? val.trim() : val;
|
||||
});
|
||||
|
||||
// 3. 去重
|
||||
const uniqueValues = Array.from(new Set(cleanedValues))
|
||||
.filter((v) => v !== null && v !== '' && v !== '(空白)')
|
||||
.sort(); // 排序，方便查看
|
||||
|
||||
logger.info(`[SessionController] 唯一值数量: ${uniqueValues.length}`);
|
||||
|
||||
// 4. 返回结果
|
||||
return reply.send({
|
||||
success: true,
|
||||
data: {
|
||||
column,
|
||||
uniqueValues,
|
||||
count: uniqueValues.length,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
logger.error(`[SessionController] 获取唯一值失败: ${error.message}`);
|
||||
|
||||
const statusCode = error.message.includes('不存在') || error.message.includes('过期')
|
||||
? 404
|
||||
: 500;
|
||||
|
||||
return reply.code(statusCode).send({
|
||||
success: false,
|
||||
error: error.message || '获取唯一值失败',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 导出单例实例 ====================
|
||||
|
||||
@@ -226,3 +226,4 @@ export class StreamAIController {
|
||||
export const streamAIController = new StreamAIController();
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -61,6 +61,11 @@ export async function toolCRoutes(fastify: FastifyInstance) {
|
||||
handler: sessionController.updateHeartbeat.bind(sessionController),
|
||||
});
|
||||
|
||||
// ✨ 获取列的唯一值（用于数值映射）
|
||||
fastify.get('/sessions/:id/unique-values', {
|
||||
handler: sessionController.getUniqueValues.bind(sessionController),
|
||||
});
|
||||
|
||||
// ==================== AI代码生成路由（Day 3） ====================
|
||||
|
||||
// 生成代码（不执行）
|
||||
|
||||
@@ -18,6 +18,12 @@ import * as xlsx from 'xlsx';
|
||||
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
interface ColumnMapping {
|
||||
originalName: string;
|
||||
safeName: string;
|
||||
displayName: string;
|
||||
}
|
||||
|
||||
interface SessionData {
|
||||
id: string;
|
||||
userId: string;
|
||||
@@ -26,6 +32,7 @@ interface SessionData {
|
||||
totalRows: number;
|
||||
totalCols: number;
|
||||
columns: string[];
|
||||
columnMapping?: ColumnMapping[]; // ✨ 新增：列名映射
|
||||
encoding: string | null;
|
||||
fileSize: number;
|
||||
createdAt: Date;
|
||||
@@ -102,8 +109,12 @@ export class SessionService {
|
||||
const totalRows = data.length;
|
||||
const totalCols = Object.keys(data[0] || {}).length;
|
||||
const columns = Object.keys(data[0] || {});
|
||||
|
||||
// ✨ 生成列名映射（解决特殊字符问题）
|
||||
const columnMapping = this.generateColumnMapping(columns);
|
||||
|
||||
logger.info(`[SessionService] 解析完成: ${totalRows}行 x ${totalCols}列`);
|
||||
logger.info(`[SessionService] 列名映射: ${columnMapping.length}个列`);
|
||||
|
||||
// 4. 上传到OSS（使用平台storage服务）
|
||||
const timestamp = Date.now();
|
||||
@@ -130,6 +141,7 @@ export class SessionService {
|
||||
totalRows,
|
||||
totalCols,
|
||||
columns: columns, // Prisma会自动转换为JSONB
|
||||
columnMapping: JSON.parse(JSON.stringify(columnMapping)), // ✨ 存储列名映射
|
||||
encoding: 'utf-8', // 默认utf-8，后续可扩展检测
|
||||
fileSize: fileBuffer.length,
|
||||
dataStats: JSON.parse(JSON.stringify(dataStats)), // ✨ 存储统计信息（转换为JSON）
|
||||
@@ -370,12 +382,15 @@ export class SessionService {
|
||||
|
||||
// 4. 更新Session元数据
|
||||
const newColumns = Object.keys(processedData[0] || {});
|
||||
const newColumnMapping = this.generateColumnMapping(newColumns); // ✨ 重新生成列名映射
|
||||
|
||||
await prisma.dcToolCSession.update({
|
||||
where: { id: sessionId },
|
||||
data: {
|
||||
totalRows: processedData.length,
|
||||
totalCols: newColumns.length,
|
||||
columns: newColumns,
|
||||
columnMapping: JSON.parse(JSON.stringify(newColumnMapping)), // ✨ 更新列名映射
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
});
|
||||
@@ -517,6 +532,29 @@ export class SessionService {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* ✨ 生成安全的列名映射
|
||||
*
|
||||
* 解决特殊字符问题：表头包含括号、等号等特殊字符会导致Python处理失败
|
||||
*
|
||||
* @param originalColumns - 原始列名数组
|
||||
* @returns 列名映射数组
|
||||
*/
|
||||
private generateColumnMapping(originalColumns: string[]): ColumnMapping[] {
|
||||
return originalColumns.map((originalName, index) => {
|
||||
// 安全列名：col_0, col_1, col_2...
|
||||
const safeName = `col_${index}`;
|
||||
// 显示名称：用于前端展示（保持原始名称）
|
||||
const displayName = originalName;
|
||||
|
||||
return {
|
||||
originalName,
|
||||
safeName,
|
||||
displayName,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测列的数据类型
|
||||
*
|
||||
@@ -571,6 +609,7 @@ export class SessionService {
|
||||
totalRows: session.totalRows,
|
||||
totalCols: session.totalCols,
|
||||
columns: session.columns as string[],
|
||||
columnMapping: session.columnMapping as ColumnMapping[] | undefined, // ✨ 返回列名映射
|
||||
encoding: session.encoding,
|
||||
fileSize: session.fileSize,
|
||||
createdAt: session.createdAt,
|
||||
|
||||
Reference in New Issue
Block a user