Develop/AIclinicalresearch
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dc/tool-c): Add missing value imputation feature with 6 methods and MICE

Browse Source 
Major features:
1. Missing value imputation (6 simple methods + MICE):
   - Mean/Median/Mode/Constant imputation
   - Forward fill (ffill) and Backward fill (bfill) for time series
   - MICE multivariate imputation (in progress, shape issue to fix)

2. Auto precision detection:
   - Automatically match decimal places of original data
   - Prevent false precision (e.g. 13.57 instead of 13.566716417910449)

3. Categorical variable detection:
   - Auto-detect and skip categorical columns in MICE
   - Show warnings for unsuitable columns
   - Suggest mode imputation for categorical data

4. UI improvements:
   - Rename button: "Delete Missing" to "Missing Value Handling"
   - Remove standalone "Dedup" and "MICE" buttons
   - 3-tab dialog: Delete / Fill / Advanced Fill
   - Display column statistics and recommended methods
   - Extended warning messages (8 seconds for skipped columns)

5. Bug fixes:
   - Fix sessionService.updateSessionData -> saveProcessedData
   - Fix OperationResult interface (add message and stats)
   - Fix Toolbar button labels and removal

Modified files:
Python: operations/fillna.py (new, 556 lines), main.py (3 new endpoints)
Backend: QuickActionService.ts, QuickActionController.ts, routes/index.ts
Frontend: MissingValueDialog.tsx (new, 437 lines), Toolbar.tsx, index.tsx
Tests: test_fillna_operations.py (774 lines), test scripts and docs
Docs: 5 documentation files updated

Known issues:
- MICE imputation has DataFrame shape mismatch issue (under debugging)
- Workaround: Use 6 simple imputation methods first

Status: Development complete, MICE debugging in progress
Lines added: ~2000 lines across 3 tiers
This commit is contained in:
HaHafeng
2025-12-10 13:06:00 +08:00
parent f4f1d09837
commit 74cf346453
102 changed files with 3806 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
frontend-v2/src/modules/asl/components/FulltextDetailDrawer.tsx
View File

@@ -517,4 +517,6 @@ export default FulltextDetailDrawer;

2
frontend-v2/src/modules/asl/hooks/useFulltextResults.ts
View File

@@ -116,4 +116,6 @@ export function useFulltextResults({

2
frontend-v2/src/modules/asl/hooks/useFulltextTask.ts
View File

@@ -79,4 +79,6 @@ export function useFulltextTask({

2
frontend-v2/src/modules/asl/pages/FulltextResults.tsx
View File

@@ -470,4 +470,6 @@ export default FulltextResults;

2
frontend-v2/src/modules/dc/hooks/useAssets.ts
View File

@@ -111,3 +111,5 @@ export const useAssets = (activeTab: AssetTabType) => {

2
frontend-v2/src/modules/dc/hooks/useRecentTasks.ts
View File

@@ -101,3 +101,5 @@ export const useRecentTasks = () => {

2
frontend-v2/src/modules/dc/pages/tool-c/components/BinningDialog_improved.tsx
View File

@@ -337,3 +337,5 @@ export default BinningDialog;

2
frontend-v2/src/modules/dc/pages/tool-c/components/DropnaDialog.tsx
View File

@@ -300,3 +300,5 @@ export default DropnaDialog;

436
frontend-v2/src/modules/dc/pages/tool-c/components/MissingValueDialog.tsx Normal file
View File

@@ -0,0 +1,436 @@
import React, { useState, useEffect } from 'react';
import { Modal, Tabs, Radio, Select, Input, Checkbox, Alert, App, Row, Col, InputNumber, Space } from 'antd';
interface Props {
visible: boolean;
onClose: () => void;
onApply: (newData: any[]) => void;
columns: Array<{ id: string; name: string }>;
sessionId: string | null;
}
const MissingValueDialog: React.FC<Props> = ({
visible,
onClose,
onApply,
columns,
sessionId,
}) => {
const { message } = App.useApp();
const [activeTab, setActiveTab] = useState('delete');
const [loading, setLoading] = useState(false);
// Tab 1: 删除相关状态
const [deleteMethod, setDeleteMethod] = useState<'row' | 'column'>('row');
const [threshold, setThreshold] = useState(50);
const [selectedColumns] = useState<string[]>([]);
// Tab 2: 简单填补相关状态
const [selectedColumn, setSelectedColumn] = useState('');
const [newColumnName, setNewColumnName] = useState('');
const [fillMethod, setFillMethod] = useState<'mean' | 'median' | 'mode' | 'constant' | 'ffill' | 'bfill'>('median');
const [fillValue, setFillValue] = useState<any>('');
const [columnStats, setColumnStats] = useState<any>(null);
// Tab 3: MICE相关状态
const [miceColumns, setMiceColumns] = useState<string[]>([]);
const [nIterations, setNIterations] = useState(10);
const [randomState, setRandomState] = useState(42);
// 当选择列变化时，自动生成新列名和获取统计
useEffect(() => {
const fetchColumnStats = async () => {
if (!sessionId || !selectedColumn) return;
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/stats', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId,
column: selectedColumn
})
});
const result = await response.json();
if (result.success) {
setColumnStats(result.stats);
}
} catch (error) {
console.error('获取统计信息失败:', error);
}
};
if (selectedColumn && activeTab === 'fill') {
setNewColumnName(`${selectedColumn}_填补`);
fetchColumnStats();
}
}, [selectedColumn, activeTab, sessionId]);
// 当统计信息更新时，自动设置推荐的填补方法
useEffect(() => {
if (columnStats && columnStats.recommended_method) {
setFillMethod(columnStats.recommended_method as any);
}
}, [columnStats]);
// 执行删除
const handleDelete = async () => {
if (!sessionId) {
message.error('Session ID不存在');
return;
}
setLoading(true);
try {
const response = await fetch('/api/v1/dc/tool-c/quick-action', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId,
action: 'dropna',
params: {
method: deleteMethod,
threshold,
columns: selectedColumns
}
})
});
const result = await response.json();
if (result.success) {
message.success('删除成功');
onApply(result.data.newDataPreview);
onClose();
} else {
message.error(result.error || '删除失败');
}
} catch (error: any) {
message.error(error.message || '删除失败');
} finally {
setLoading(false);
}
};
// 执行简单填补
const handleFillSimple = async () => {
if (!sessionId || !selectedColumn) {
message.error('请选择列');
return;
}
if (!newColumnName) {
message.error('请输入新列名');
return;
}
if (fillMethod === 'constant' && !fillValue) {
message.error('请输入填补值');
return;
}
setLoading(true);
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/simple', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId,
column: selectedColumn,
newColumnName,
method: fillMethod,
fillValue: fillMethod === 'constant' ? fillValue : undefined
})
});
const result = await response.json();
if (result.success) {
message.success(result.data.message || '填补成功');
onApply(result.data.newDataPreview);
onClose();
} else {
message.error(result.error || '填补失败');
}
} catch (error: any) {
message.error(error.message || '填补失败');
} finally {
setLoading(false);
}
};
// 执行MICE填补
const handleFillMice = async () => {
if (!sessionId) {
message.error('Session ID不存在');
return;
}
if (miceColumns.length === 0) {
message.error('请至少选择一列');
return;
}
setLoading(true);
try {
const response = await fetch('/api/v1/dc/tool-c/fillna/mice', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
sessionId,
columns: miceColumns,
nIterations,
randomState
})
});
const result = await response.json();
if (result.success) {
const msg = result.data.message || 'MICE填补成功';
// 如果消息中包含"跳过"，使用warning而不是success，并延长显示时间
if (msg.includes('跳过') || msg.includes('分类变量')) {
message.warning(msg, 8); // 显示8秒
} else {
message.success(msg, 4); // 显示4秒
}
onApply(result.data.newDataPreview);
onClose();
} else {
message.error(result.error || 'MICE填补失败');
}
} catch (error: any) {
message.error(error.message || 'MICE填补失败');
} finally {
setLoading(false);
}
};
const handleOk = () => {
if (activeTab === 'delete') {
handleDelete();
} else if (activeTab === 'fill') {
handleFillSimple();
} else if (activeTab === 'mice') {
handleFillMice();
}
};
return (
<Modal
title="缺失值处理"
open={visible}
onCancel={onClose}
onOk={handleOk}
okText={activeTab === 'delete' ? '执行删除' : activeTab === 'fill' ? '执行填补' : '执行MICE填补'}
cancelText="取消"
width={680}
confirmLoading={loading}
>
<Tabs
activeKey={activeTab}
onChange={setActiveTab}
items={[
{
key: 'delete',
label: '删除',
children: (
<div className="space-y-4">
<Alert
message="删除含有缺失值的行或列（原始数据将被修改）"
type="warning"
showIcon
/>
<div>
<div className="mb-2 font-medium">删除方式：</div>
<Radio.Group value={deleteMethod} onChange={(e) => setDeleteMethod(e.target.value)}>
<Space direction="vertical">
<Radio value="row">删除含有缺失值的行</Radio>
<Radio value="column">删除缺失率过高的列（阈值：{threshold}%）</Radio>
</Space>
</Radio.Group>
</div>
{deleteMethod === 'column' && (
<div>
<div className="mb-2">缺失率阈值：{threshold}%</div>
<InputNumber
min={0}
max={100}
value={threshold}
onChange={(val) => setThreshold(val || 50)}
style={{ width: '100%' }}
/>
</div>
)}
</div>
),
},
{
key: 'fill',
label: '填补',
children: (
<div className="space-y-4">
<Alert
message="💡 新列将创建在原列旁边，便于对比"
type="info"
showIcon
/>
<div>
<div className="mb-2 font-medium">原始列：</div>
<Select
value={selectedColumn}
onChange={setSelectedColumn}
placeholder="选择要填补的列"
style={{ width: '100%' }}
options={columns.map(col => ({
label: col.name,
value: col.name
}))}
/>
</div>
<div>
<div className="mb-2 font-medium">新列名：</div>
<Input
value={newColumnName}
onChange={(e) => setNewColumnName(e.target.value)}
placeholder="如：体重_填补"
/>
</div>
{columnStats && (
<div className="p-3 bg-gray-50 rounded">
<div className="text-sm space-y-1">
<div>• 缺失：{columnStats.missing_count}个（{columnStats.missing_rate}%）</div>
<div>• 有效值：{columnStats.valid_count}个</div>
{columnStats.mean !== null && <div>• 均值：{columnStats.mean?.toFixed(2)}</div>}
{columnStats.median !== null && <div>• 中位数：{columnStats.median?.toFixed(2)}</div>}
<div>• 推荐方法：<span className="text-blue-600">
{columnStats.recommended_method === 'mean' && '均值填补'}
{columnStats.recommended_method === 'median' && '中位数填补'}
{columnStats.recommended_method === 'mode' && '众数填补'}
{columnStats.recommended_method === 'constant' && '固定值填补'}
{columnStats.recommended_method === 'ffill' && '前向填充'}
{columnStats.recommended_method === 'bfill' && '后向填充'}
</span></div>
</div>
</div>
)}
<div>
<div className="mb-2 font-medium">填补方法：</div>
<Radio.Group value={fillMethod} onChange={(e) => setFillMethod(e.target.value)}>
<Space direction="vertical">
<Radio value="mean">均值填补（适合正态分布）</Radio>
<Radio value="median">中位数填补（适合偏态分布）</Radio>
<Radio value="mode">众数填补（适合分类变量）</Radio>
<Radio value="constant">固定值填补</Radio>
<Radio value="ffill">前向填充（时间序列）</Radio>
<Radio value="bfill">后向填充（时间序列）</Radio>
</Space>
</Radio.Group>
</div>
{fillMethod === 'constant' && (
<div>
<div className="mb-2 font-medium">填补值：</div>
<Input
value={fillValue}
onChange={(e) => setFillValue(e.target.value)}
placeholder="输入固定值"
/>
</div>
)}
</div>
),
},
{
key: 'mice',
label: '高级填补',
children: (
<div className="space-y-4">
<Alert
message="⭐ MICE多重插补 - 医学研究高质量填补的首选方法"
type="success"
showIcon
description="MICE会根据其他变量的值来预测缺失值，适合缺失率5%-30%、需要考虑变量间相关性的场景。"
/>
<Alert
message="⚠️ 重要：MICE仅适用于数值列"
type="warning"
showIcon
description={
<div className="text-sm">
<div>• ✅ 适合：年龄、体重、血压、评分等数值列</div>
<div>• ❌ 不适合：婚姻状况、性别、职业等分类列（请使用"众数填补"）</div>
<div className="mt-2 text-orange-600">分类列如果被选中，会自动跳过并创建原样副本列</div>
</div>
}
/>
<div>
<div className="mb-2 font-medium">选择要填补的列（可多选）：</div>
<Checkbox.Group
value={miceColumns}
onChange={setMiceColumns}
style={{ width: '100%' }}
>
<Space direction="vertical">
{columns.map(col => (
<Checkbox key={col.id} value={col.name}>
{col.name}
</Checkbox>
))}
</Space>
</Checkbox.Group>
</div>
<div className="p-3 bg-blue-50 rounded">
<div className="text-sm space-y-1">
<div>• 新列命名：原列名 + "_MICE"</div>
<div>• 新列位置：紧邻各原列</div>
<div>• 计算时间：10万行约1分钟</div>
</div>
</div>
<Row gutter={16}>
<Col span={12}>
<div className="mb-2">迭代次数：</div>
<InputNumber
min={5}
max={50}
value={nIterations}
onChange={(val) => setNIterations(val || 10)}
style={{ width: '100%' }}
/>
</Col>
<Col span={12}>
<div className="mb-2">随机种子：</div>
<InputNumber
value={randomState}
onChange={(val) => setRandomState(val || 42)}
style={{ width: '100%' }}
/>
</Col>
</Row>
</div>
),
},
]}
/>
</Modal>
);
};
export default MissingValueDialog;

16
frontend-v2/src/modules/dc/pages/tool-c/components/Toolbar.tsx
View File

@@ -113,7 +113,7 @@ const Toolbar: React.FC<ToolbarProps> = ({
/>
<ToolbarButton
icon={Trash2}
label="删除缺失值"
label="缺失值处理"
colorClass="text-red-600 bg-red-50 hover:bg-red-100"
onClick={onDropnaClick}
disabled={!sessionId}
@@ -125,13 +125,6 @@ const Toolbar: React.FC<ToolbarProps> = ({
onClick={onComputeClick}
disabled={!sessionId}
/>
<ToolbarButton
icon={FileSearch}
label="去重"
colorClass="text-orange-600 bg-orange-50 hover:bg-orange-100"
onClick={onDedupClick}
disabled={true}
/>
<div className="w-[1px] h-8 bg-slate-200 mx-2"></div>
@@ -143,13 +136,6 @@ const Toolbar: React.FC<ToolbarProps> = ({
onClick={onPivotClick}
disabled={!sessionId}
/>
<ToolbarButton
icon={CalendarClock}
label="多重插补"
colorClass="text-rose-600 bg-rose-50 hover:bg-rose-100"
onClick={onMiceClick}
disabled={true}
/>
<div className="flex-1"></div>

4
frontend-v2/src/modules/dc/pages/tool-c/index.tsx
View File

@@ -14,7 +14,7 @@ import FilterDialog from './components/FilterDialog';
import RecodeDialog from './components/RecodeDialog';
import BinningDialog from './components/BinningDialog';
import ConditionalDialog from './components/ConditionalDialog';
import DropnaDialog from './components/DropnaDialog';
import MissingValueDialog from './components/MissingValueDialog';
import ComputeDialog from './components/ComputeDialog';
import PivotDialog from './components/PivotDialog';
import * as api from '../../api/toolC';
@@ -342,7 +342,7 @@ const ToolC = () => {
onApply={handleQuickActionDataUpdate}
/>
<DropnaDialog
<MissingValueDialog
visible={state.dropnaDialogVisible}
columns={state.columns}
data={state.data}

2
frontend-v2/src/modules/dc/pages/tool-c/types/index.ts
View File

@@ -63,3 +63,5 @@ export interface DataStats {

2
frontend-v2/src/modules/dc/types/portal.ts
View File

@@ -59,3 +59,5 @@ export type AssetTabType = 'all' | 'processed' | 'raw';

2
frontend-v2/src/shared/components/index.ts
View File

@@ -14,3 +14,5 @@ export { default as Placeholder } from './Placeholder';