feat(rvw): Implement RVW V2.0 Data Forensics Module - Day 6 StatValidator
Summary: - Implement L2 Statistical Validator (CI-P consistency, T-test reverse) - Implement L2.5 Consistency Forensics (SE Triangle, SD>Mean check) - Add error/warning severity classification with tolerance thresholds - Support 5+ CI formats parsing (parentheses, brackets, 95% CI prefix) - Complete Python forensics service (types, config, validator, extractor) V2.0 Development Progress (Week 2 Day 6): - Day 1-5: Python service setup, Word table extraction, L1 arithmetic validator - Day 6: L2 StatValidator + L2.5 consistency forensics (promoted from V2.1) Test Results: - Unit tests: 4/4 passed (CI-P, SE Triangle, SD>Mean, T-test) - Real document tests: 5/5 successful, 2 reasonable WARNINGs Status: Day 6 completed, ready for Day 7 (Skills Framework) Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
839
extraction_service/forensics/validator.py
Normal file
839
extraction_service/forensics/validator.py
Normal file
@@ -0,0 +1,839 @@
|
||||
"""
|
||||
数据侦探模块 - 验证器
|
||||
|
||||
包含 L1 算术验证、L2 统计验证、L2.5 一致性取证。
|
||||
|
||||
L1 算术验证：
|
||||
- n (%) 格式验证
|
||||
- Sum/Total 校验
|
||||
- 容错逻辑
|
||||
|
||||
L2 统计验证：
|
||||
- T 检验 P 值逆向验证
|
||||
- 卡方检验 P 值逆向验证
|
||||
- CI vs P 值逻辑检查
|
||||
|
||||
L2.5 一致性取证（终审提权）：
|
||||
- SE 三角验证（回归系数 CI↔P 一致性）
|
||||
- SD > Mean 检查（正值指标启发式规则）
|
||||
"""
|
||||
|
||||
import re
|
||||
import math
|
||||
from typing import List, Optional, Tuple
|
||||
from loguru import logger
|
||||
|
||||
# scipy 用于统计计算
|
||||
try:
|
||||
from scipy import stats
|
||||
SCIPY_AVAILABLE = True
|
||||
except ImportError:
|
||||
SCIPY_AVAILABLE = False
|
||||
logger.warning("scipy 未安装，L2 统计验证将受限")
|
||||
|
||||
from .types import (
|
||||
TableData,
|
||||
Issue,
|
||||
Severity,
|
||||
IssueType,
|
||||
CellLocation,
|
||||
ForensicsConfig,
|
||||
)
|
||||
from .config import (
|
||||
PERCENT_PATTERN,
|
||||
PVALUE_PATTERN,
|
||||
CI_PATTERN,
|
||||
MEAN_SD_PATTERN,
|
||||
MEAN_SD_PAREN_PATTERN,
|
||||
CI_PATTERNS,
|
||||
EFFECT_SIZE_PATTERN,
|
||||
DEFAULT_TOLERANCE_PERCENT,
|
||||
PVALUE_ERROR_THRESHOLD,
|
||||
PVALUE_WARNING_THRESHOLD,
|
||||
STAT_RELATIVE_TOLERANCE,
|
||||
)
|
||||
|
||||
|
||||
class ArithmeticValidator:
|
||||
"""
|
||||
L1 算术自洽性验证器
|
||||
|
||||
验证表格中的数值计算是否正确：
|
||||
- n (%) 格式中的百分比是否等于 n/N
|
||||
- Total/Sum 行是否等于其他行之和
|
||||
"""
|
||||
|
||||
def __init__(self, config: ForensicsConfig):
|
||||
self.config = config
|
||||
self.tolerance = config.tolerance_percent
|
||||
|
||||
def validate(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
验证表格的算术一致性
|
||||
|
||||
Args:
|
||||
table: 要验证的表格数据
|
||||
|
||||
Returns:
|
||||
发现的问题列表
|
||||
"""
|
||||
if table.skipped or not table.data:
|
||||
return []
|
||||
|
||||
issues: List[Issue] = []
|
||||
|
||||
# 1. 验证 n (%) 格式
|
||||
percent_issues = self._validate_percent_format(table)
|
||||
issues.extend(percent_issues)
|
||||
|
||||
# 2. 验证 Sum/Total 行
|
||||
sum_issues = self._validate_sum_rows(table)
|
||||
issues.extend(sum_issues)
|
||||
|
||||
# 更新表格的 issues
|
||||
table.issues.extend(issues)
|
||||
|
||||
logger.debug(f"表格 {table.id} 算术验证完成: {len(issues)} 个问题")
|
||||
|
||||
return issues
|
||||
|
||||
def _validate_percent_format(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
验证 n (%) 格式
|
||||
|
||||
查找形如 "45 (50.0%)" 的单元格，验证百分比是否正确。
|
||||
需要从表头或同行找到总数 N。
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
data = table.data
|
||||
|
||||
if len(data) < 2: # 至少需要表头和一行数据
|
||||
return issues
|
||||
|
||||
# 尝试从表头识别 N 列（如 "n", "N", "Total", "合计"）
|
||||
header = data[0]
|
||||
n_col_indices = self._find_n_columns(header)
|
||||
|
||||
for row_idx, row in enumerate(data[1:], start=2): # 从第2行开始（数据行）
|
||||
for col_idx, cell in enumerate(row, start=1):
|
||||
# 查找 n (%) 格式
|
||||
match = PERCENT_PATTERN.search(cell)
|
||||
if match:
|
||||
n_value = float(match.group(1))
|
||||
reported_percent = float(match.group(2))
|
||||
|
||||
# 尝试找到对应的 N 值
|
||||
total_n = self._find_total_n(data, row_idx - 1, col_idx - 1, n_col_indices)
|
||||
|
||||
if total_n is not None and total_n > 0:
|
||||
# 计算实际百分比
|
||||
calculated_percent = (n_value / total_n) * 100
|
||||
|
||||
# 检查差异
|
||||
diff = abs(calculated_percent - reported_percent)
|
||||
if diff > self.tolerance:
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.ARITHMETIC_PERCENT,
|
||||
message=f"百分比计算错误: 报告值 {reported_percent}%，计算值 {calculated_percent:.1f}% (n={n_value}, N={total_n})",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=col_idx
|
||||
),
|
||||
evidence={
|
||||
"n": n_value,
|
||||
"N": total_n,
|
||||
"reported_percent": reported_percent,
|
||||
"calculated_percent": round(calculated_percent, 2),
|
||||
"difference": round(diff, 2)
|
||||
}
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
def _find_n_columns(self, header: List[str]) -> List[int]:
|
||||
"""
|
||||
从表头识别可能包含 N 值的列索引
|
||||
"""
|
||||
n_keywords = ["n", "total", "合计", "总数", "all", "sum"]
|
||||
indices = []
|
||||
|
||||
for idx, cell in enumerate(header):
|
||||
cell_lower = cell.lower().strip()
|
||||
for keyword in n_keywords:
|
||||
if keyword in cell_lower:
|
||||
indices.append(idx)
|
||||
break
|
||||
|
||||
return indices
|
||||
|
||||
def _find_total_n(
|
||||
self,
|
||||
data: List[List[str]],
|
||||
row_idx: int,
|
||||
col_idx: int,
|
||||
n_col_indices: List[int]
|
||||
) -> Optional[float]:
|
||||
"""
|
||||
查找对应的总数 N
|
||||
|
||||
策略：
|
||||
1. 首先检查同行的 N 列
|
||||
2. 如果没有，检查表头行对应位置
|
||||
3. 尝试解析同列第一个纯数字
|
||||
"""
|
||||
row = data[row_idx]
|
||||
|
||||
# 策略 1：检查同行的 N 列
|
||||
for n_col in n_col_indices:
|
||||
if n_col < len(row):
|
||||
n_val = self._parse_number(row[n_col])
|
||||
if n_val is not None and n_val > 0:
|
||||
return n_val
|
||||
|
||||
# 策略 2：检查同列的第一行（可能是 N 值）
|
||||
if row_idx > 0:
|
||||
first_data_row = data[1] if len(data) > 1 else None
|
||||
if first_data_row and col_idx < len(first_data_row):
|
||||
# 检查是否该列第一行就是数字（Total N）
|
||||
n_val = self._parse_number(first_data_row[col_idx])
|
||||
if n_val is not None and n_val > 0:
|
||||
return n_val
|
||||
|
||||
# 策略 3：尝试从同行其他单元格累加
|
||||
# 这是一个启发式方法，可能不准确
|
||||
|
||||
return None
|
||||
|
||||
def _parse_number(self, text: str) -> Optional[float]:
|
||||
"""
|
||||
从文本中解析数字
|
||||
|
||||
处理：
|
||||
- 纯数字 "45"
|
||||
- 带逗号 "1,234"
|
||||
- 带空格 "1 234"
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
# 移除常见分隔符
|
||||
cleaned = text.strip().replace(",", "").replace(" ", "")
|
||||
|
||||
# 尝试提取第一个数字
|
||||
match = re.match(r"^(\d+(?:\.\d+)?)", cleaned)
|
||||
if match:
|
||||
try:
|
||||
return float(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _validate_sum_rows(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
验证 Sum/Total 行
|
||||
|
||||
查找标记为 "Total", "Sum", "合计" 的行，验证其值是否等于上方各行之和。
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
data = table.data
|
||||
|
||||
if len(data) < 3: # 至少需要表头、数据行和合计行
|
||||
return issues
|
||||
|
||||
# 查找 Total/Sum 行
|
||||
total_keywords = ["total", "sum", "合计", "总计", "总和", "all"]
|
||||
|
||||
for row_idx, row in enumerate(data[1:], start=2): # 跳过表头
|
||||
first_cell = row[0].lower().strip() if row else ""
|
||||
|
||||
is_total_row = any(kw in first_cell for kw in total_keywords)
|
||||
|
||||
if is_total_row:
|
||||
# 验证每个数值列
|
||||
for col_idx, cell in enumerate(row[1:], start=2): # 跳过第一列
|
||||
total_val = self._parse_number(cell)
|
||||
if total_val is None:
|
||||
continue
|
||||
|
||||
# 计算上方各行的和
|
||||
column_sum = 0.0
|
||||
valid_sum = True
|
||||
|
||||
for prev_row_idx in range(1, row_idx - 1): # 从第一个数据行到当前行的上一行
|
||||
if col_idx - 1 < len(data[prev_row_idx]):
|
||||
prev_cell = data[prev_row_idx][col_idx - 1]
|
||||
prev_val = self._parse_number(prev_cell)
|
||||
if prev_val is not None:
|
||||
column_sum += prev_val
|
||||
else:
|
||||
# 如果有非数字单元格，跳过验证
|
||||
valid_sum = False
|
||||
break
|
||||
|
||||
if valid_sum and column_sum > 0:
|
||||
diff = abs(total_val - column_sum)
|
||||
# 允许小数点误差
|
||||
if diff > 0.5: # 容错 0.5
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.ARITHMETIC_SUM,
|
||||
message=f"合计行计算错误: 报告值 {total_val}，计算值 {column_sum}",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=col_idx
|
||||
),
|
||||
evidence={
|
||||
"reported_total": total_val,
|
||||
"calculated_sum": column_sum,
|
||||
"difference": round(diff, 2)
|
||||
}
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
class StatValidator:
|
||||
"""
|
||||
L2 统计学复核验证器 + L2.5 一致性取证
|
||||
|
||||
验证统计检验结果的合理性：
|
||||
- T 检验 P 值逆向验证
|
||||
- 卡方检验 P 值逆向验证（基于频数表）
|
||||
- CI 与 P 值逻辑一致性检查
|
||||
- SE 三角验证（回归系数 CI↔P 一致性）
|
||||
- SD > Mean 检查（正值指标启发式规则）
|
||||
"""
|
||||
|
||||
def __init__(self, config: ForensicsConfig):
|
||||
self.config = config
|
||||
|
||||
def validate(self, table: TableData, full_text: str) -> List[Issue]:
|
||||
"""
|
||||
验证表格的统计学一致性
|
||||
|
||||
Args:
|
||||
table: 要验证的表格数据
|
||||
full_text: 文档全文（用于方法识别）
|
||||
|
||||
Returns:
|
||||
发现的问题列表
|
||||
"""
|
||||
if table.skipped or not table.data:
|
||||
return []
|
||||
|
||||
# 仅在 L1_L2 模式下执行
|
||||
if self.config.check_level != "L1_L2":
|
||||
return []
|
||||
|
||||
issues: List[Issue] = []
|
||||
|
||||
# 1. CI vs P 值逻辑检查（基础）
|
||||
ci_issues = self._validate_ci_pvalue_consistency(table)
|
||||
issues.extend(ci_issues)
|
||||
|
||||
# 2. T 检验逆向验证
|
||||
if SCIPY_AVAILABLE:
|
||||
ttest_issues = self._validate_ttest(table)
|
||||
issues.extend(ttest_issues)
|
||||
|
||||
# 3. SE 三角验证（终审提权：回归系数 CI↔P 一致性）
|
||||
se_issues = self._validate_se_triangle(table)
|
||||
issues.extend(se_issues)
|
||||
|
||||
# 4. SD > Mean 检查（终审提权：启发式规则）
|
||||
sd_issues = self._validate_sd_greater_mean(table)
|
||||
issues.extend(sd_issues)
|
||||
|
||||
# 更新表格的 issues
|
||||
table.issues.extend(issues)
|
||||
|
||||
logger.debug(f"表格 {table.id} 统计验证完成: {len(issues)} 个问题")
|
||||
|
||||
return issues
|
||||
|
||||
def _validate_ci_pvalue_consistency(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
验证 CI 与 P 值的逻辑一致性
|
||||
|
||||
黄金法则：
|
||||
- 若 95% CI 跨越 1.0（如 0.8-1.2）→ P 值必须 ≥ 0.05
|
||||
- 若 95% CI 不跨越 1.0（如 1.1-1.5）→ P 值必须 < 0.05
|
||||
|
||||
违反此规则 = 数据逻辑矛盾
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
data = table.data
|
||||
|
||||
for row_idx, row in enumerate(data[1:], start=2):
|
||||
row_text = " ".join(row)
|
||||
|
||||
# 查找 CI（使用增强的 CI 解析）
|
||||
ci_result = self._parse_ci(row_text)
|
||||
if ci_result is None:
|
||||
continue
|
||||
|
||||
ci_lower, ci_upper = ci_result
|
||||
|
||||
# 查找 P 值
|
||||
pvalue = self._parse_pvalue(row_text)
|
||||
if pvalue is None:
|
||||
continue
|
||||
|
||||
# 检查逻辑一致性
|
||||
ci_crosses_one = ci_lower <= 1.0 <= ci_upper
|
||||
p_significant = pvalue < 0.05
|
||||
|
||||
# 矛盾情况
|
||||
if ci_crosses_one and p_significant:
|
||||
# CI 跨越 1 但 P < 0.05，矛盾
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.STAT_CI_PVALUE_CONFLICT,
|
||||
message=f"CI 与 P 值逻辑矛盾: 95% CI ({ci_lower}-{ci_upper}) 跨越 1.0，但 P={pvalue} < 0.05",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1 # 整行问题
|
||||
),
|
||||
evidence={
|
||||
"ci_lower": ci_lower,
|
||||
"ci_upper": ci_upper,
|
||||
"ci_crosses_one": ci_crosses_one,
|
||||
"pvalue": pvalue,
|
||||
"p_significant": p_significant
|
||||
}
|
||||
))
|
||||
elif not ci_crosses_one and not p_significant:
|
||||
# CI 不跨越 1 但 P ≥ 0.05，矛盾
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.STAT_CI_PVALUE_CONFLICT,
|
||||
message=f"CI 与 P 值逻辑矛盾: 95% CI ({ci_lower}-{ci_upper}) 不跨越 1.0，但 P={pvalue} ≥ 0.05",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1
|
||||
),
|
||||
evidence={
|
||||
"ci_lower": ci_lower,
|
||||
"ci_upper": ci_upper,
|
||||
"ci_crosses_one": ci_crosses_one,
|
||||
"pvalue": pvalue,
|
||||
"p_significant": p_significant
|
||||
}
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
def _validate_ttest(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
T 检验逆向验证
|
||||
|
||||
从表格中提取 M±SD, n 信息，反推 t 值和 P 值，
|
||||
与报告的 P 值进行对比。
|
||||
|
||||
公式: t = (M1 - M2) / sqrt(SD1²/n1 + SD2²/n2)
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
|
||||
if not SCIPY_AVAILABLE:
|
||||
return issues
|
||||
|
||||
data = table.data
|
||||
if len(data) < 2:
|
||||
return issues
|
||||
|
||||
# 查找包含组比较数据的行
|
||||
for row_idx, row in enumerate(data[1:], start=2):
|
||||
# 尝试提取同一行中的两组数据
|
||||
mean_sd_matches = list(MEAN_SD_PATTERN.finditer(" ".join(row)))
|
||||
|
||||
if len(mean_sd_matches) >= 2:
|
||||
# 找到至少两组 Mean±SD 数据
|
||||
try:
|
||||
m1, sd1 = float(mean_sd_matches[0].group(1)), float(mean_sd_matches[0].group(2))
|
||||
m2, sd2 = float(mean_sd_matches[1].group(1)), float(mean_sd_matches[1].group(2))
|
||||
|
||||
# 提取 P 值
|
||||
row_text = " ".join(row)
|
||||
pvalue = self._parse_pvalue(row_text)
|
||||
|
||||
if pvalue is None:
|
||||
continue
|
||||
|
||||
# 尝试从表头获取样本量（简化处理，假设 n=30）
|
||||
# 实际实现需要更复杂的表格解析
|
||||
n1, n2 = self._estimate_sample_sizes(table, row_idx)
|
||||
|
||||
if n1 is None or n2 is None:
|
||||
continue
|
||||
|
||||
# 计算 t 值
|
||||
se = math.sqrt(sd1**2/n1 + sd2**2/n2)
|
||||
if se == 0:
|
||||
continue
|
||||
|
||||
t_calc = abs(m1 - m2) / se
|
||||
df = n1 + n2 - 2
|
||||
|
||||
# 计算 P 值
|
||||
p_calc = 2 * (1 - stats.t.cdf(t_calc, df))
|
||||
|
||||
# 比较 P 值
|
||||
p_diff = abs(p_calc - pvalue)
|
||||
|
||||
if p_diff > PVALUE_ERROR_THRESHOLD:
|
||||
# 严重矛盾
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.STAT_TTEST_PVALUE,
|
||||
message=f"T 检验 P 值不一致: 报告 P={pvalue}，计算 P={p_calc:.4f}（差异 {p_diff:.3f}）",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1
|
||||
),
|
||||
evidence={
|
||||
"group1": {"mean": m1, "sd": sd1, "n": n1},
|
||||
"group2": {"mean": m2, "sd": sd2, "n": n2},
|
||||
"t_calculated": round(t_calc, 3),
|
||||
"df": df,
|
||||
"p_calculated": round(p_calc, 4),
|
||||
"p_reported": pvalue,
|
||||
"p_difference": round(p_diff, 4)
|
||||
}
|
||||
))
|
||||
elif p_diff > PVALUE_WARNING_THRESHOLD:
|
||||
# 可能是舍入误差
|
||||
issues.append(Issue(
|
||||
severity=Severity.WARNING,
|
||||
type=IssueType.STAT_TTEST_PVALUE,
|
||||
message=f"T 检验 P 值轻微偏差: 报告 P={pvalue}，计算 P={p_calc:.4f}（可能是舍入误差）",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1
|
||||
),
|
||||
evidence={
|
||||
"p_calculated": round(p_calc, 4),
|
||||
"p_reported": pvalue,
|
||||
"p_difference": round(p_diff, 4)
|
||||
}
|
||||
))
|
||||
|
||||
except (ValueError, TypeError, ZeroDivisionError) as e:
|
||||
logger.debug(f"T 检验验证失败: {e}")
|
||||
continue
|
||||
|
||||
return issues
|
||||
|
||||
def _validate_se_triangle(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
SE 三角验证（终审提权）
|
||||
|
||||
用于 Logistic 回归、Cox 回归等场景。
|
||||
|
||||
原理：
|
||||
- SE = (ln(CI_upper) - ln(CI_lower)) / 3.92
|
||||
- Z = ln(OR) / SE
|
||||
- P_calculated = 2 * (1 - norm.cdf(|Z|))
|
||||
|
||||
若报告的 P 值与计算的 P 值严重不一致，则存在问题。
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
data = table.data
|
||||
|
||||
if not SCIPY_AVAILABLE:
|
||||
return issues
|
||||
|
||||
for row_idx, row in enumerate(data[1:], start=2):
|
||||
row_text = " ".join(row)
|
||||
|
||||
# 查找 OR/HR/RR
|
||||
effect_match = EFFECT_SIZE_PATTERN.search(row_text)
|
||||
if not effect_match:
|
||||
continue
|
||||
|
||||
try:
|
||||
effect_size = float(effect_match.group(1))
|
||||
if effect_size <= 0:
|
||||
continue
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# 查找 CI
|
||||
ci_result = self._parse_ci(row_text)
|
||||
if ci_result is None:
|
||||
continue
|
||||
|
||||
ci_lower, ci_upper = ci_result
|
||||
|
||||
# 确保 CI 有效（正数且 lower < upper）
|
||||
if ci_lower <= 0 or ci_upper <= 0 or ci_lower >= ci_upper:
|
||||
continue
|
||||
|
||||
# 查找报告的 P 值
|
||||
pvalue = self._parse_pvalue(row_text)
|
||||
if pvalue is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
# SE 三角计算
|
||||
ln_effect = math.log(effect_size)
|
||||
ln_ci_lower = math.log(ci_lower)
|
||||
ln_ci_upper = math.log(ci_upper)
|
||||
|
||||
# SE = (ln(CI_upper) - ln(CI_lower)) / 3.92 (for 95% CI)
|
||||
se = (ln_ci_upper - ln_ci_lower) / 3.92
|
||||
|
||||
if se <= 0:
|
||||
continue
|
||||
|
||||
# Z = ln(OR) / SE
|
||||
z = abs(ln_effect) / se
|
||||
|
||||
# P = 2 * (1 - norm.cdf(|Z|))
|
||||
p_calc = 2 * (1 - stats.norm.cdf(z))
|
||||
|
||||
# 比较 P 值
|
||||
p_diff = abs(p_calc - pvalue)
|
||||
|
||||
if p_diff > PVALUE_ERROR_THRESHOLD:
|
||||
# 严重矛盾
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.STAT_SE_TRIANGLE,
|
||||
message=f"SE 三角验证不一致: 报告 P={pvalue}，由 CI 反推 P={p_calc:.4f}（差异 {p_diff:.3f}）",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1
|
||||
),
|
||||
evidence={
|
||||
"effect_size": effect_size,
|
||||
"ci_lower": ci_lower,
|
||||
"ci_upper": ci_upper,
|
||||
"se_calculated": round(se, 4),
|
||||
"z_calculated": round(z, 3),
|
||||
"p_calculated": round(p_calc, 4),
|
||||
"p_reported": pvalue,
|
||||
"p_difference": round(p_diff, 4)
|
||||
}
|
||||
))
|
||||
elif p_diff > PVALUE_WARNING_THRESHOLD:
|
||||
# 轻微偏差，可能是舍入误差
|
||||
issues.append(Issue(
|
||||
severity=Severity.WARNING,
|
||||
type=IssueType.STAT_SE_TRIANGLE,
|
||||
message=f"SE 三角验证轻微偏差: 报告 P={pvalue}，由 CI 反推 P={p_calc:.4f}（可能是舍入误差）",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=1
|
||||
),
|
||||
evidence={
|
||||
"effect_size": effect_size,
|
||||
"p_calculated": round(p_calc, 4),
|
||||
"p_reported": pvalue,
|
||||
"p_difference": round(p_diff, 4)
|
||||
}
|
||||
))
|
||||
|
||||
except (ValueError, ZeroDivisionError, TypeError) as e:
|
||||
logger.debug(f"SE 三角验证失败: {e}")
|
||||
continue
|
||||
|
||||
return issues
|
||||
|
||||
def _validate_sd_greater_mean(self, table: TableData) -> List[Issue]:
|
||||
"""
|
||||
SD > Mean 启发式检查（终审提权）
|
||||
|
||||
对于正值指标（如年龄、体重、血压、实验室指标），
|
||||
SD > Mean 通常是不合理的，可能暗示数据问题。
|
||||
|
||||
例外情况：
|
||||
- 差值指标（可正可负）
|
||||
- 某些偏态分布指标
|
||||
"""
|
||||
issues: List[Issue] = []
|
||||
data = table.data
|
||||
|
||||
# 识别表头，判断哪些列是正值指标
|
||||
if len(data) < 2:
|
||||
return issues
|
||||
|
||||
header = data[0]
|
||||
|
||||
# 正值指标的关键词（这些指标通常不应有 SD > Mean）
|
||||
positive_indicators = [
|
||||
"age", "年龄", "weight", "体重", "bmi", "height", "身高",
|
||||
"sbp", "dbp", "血压", "heart rate", "心率", "pulse", "脉搏",
|
||||
"wbc", "rbc", "hgb", "plt", "白细胞", "红细胞", "血红蛋白", "血小板",
|
||||
"creatinine", "肌酐", "bun", "尿素氮", "glucose", "血糖",
|
||||
"alt", "ast", "转氨酶", "bilirubin", "胆红素",
|
||||
"cost", "费用", "time", "时间", "duration", "持续"
|
||||
]
|
||||
|
||||
for row_idx, row in enumerate(data[1:], start=2):
|
||||
for col_idx, cell in enumerate(row, start=1):
|
||||
# 检查 Mean±SD 格式
|
||||
match = MEAN_SD_PATTERN.search(cell)
|
||||
if not match:
|
||||
# 尝试括号格式
|
||||
match = MEAN_SD_PAREN_PATTERN.search(cell)
|
||||
|
||||
if not match:
|
||||
continue
|
||||
|
||||
try:
|
||||
mean_val = float(match.group(1))
|
||||
sd_val = float(match.group(2))
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# 检查 SD > Mean（仅对 mean > 0 的情况）
|
||||
if mean_val > 0 and sd_val > mean_val:
|
||||
# 检查是否是正值指标（通过表头或行首判断）
|
||||
context_text = ""
|
||||
if col_idx - 1 < len(header):
|
||||
context_text += header[col_idx - 1].lower()
|
||||
if len(row) > 0:
|
||||
context_text += " " + row[0].lower()
|
||||
|
||||
# 判断是否是已知的正值指标
|
||||
is_positive_indicator = any(kw in context_text for kw in positive_indicators)
|
||||
|
||||
# 计算 CV（变异系数）
|
||||
cv = sd_val / mean_val if mean_val != 0 else 0
|
||||
|
||||
if is_positive_indicator:
|
||||
# 已知正值指标，SD > Mean 是错误
|
||||
issues.append(Issue(
|
||||
severity=Severity.ERROR,
|
||||
type=IssueType.STAT_SD_GREATER_MEAN,
|
||||
message=f"SD 大于 Mean 异常: {mean_val}±{sd_val}，CV={cv:.1%}，该指标通常为正值",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=col_idx
|
||||
),
|
||||
evidence={
|
||||
"mean": mean_val,
|
||||
"sd": sd_val,
|
||||
"cv": round(cv, 3),
|
||||
"context": context_text[:50]
|
||||
}
|
||||
))
|
||||
else:
|
||||
# 未确定的指标，给出警告
|
||||
issues.append(Issue(
|
||||
severity=Severity.WARNING,
|
||||
type=IssueType.STAT_SD_GREATER_MEAN,
|
||||
message=f"SD 大于 Mean: {mean_val}±{sd_val}，CV={cv:.1%}，建议核查数据分布",
|
||||
location=CellLocation(
|
||||
table_id=table.id,
|
||||
row=row_idx,
|
||||
col=col_idx
|
||||
),
|
||||
evidence={
|
||||
"mean": mean_val,
|
||||
"sd": sd_val,
|
||||
"cv": round(cv, 3)
|
||||
}
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
# ==================== 辅助方法 ====================
|
||||
|
||||
def _parse_ci(self, text: str) -> Optional[Tuple[float, float]]:
|
||||
"""
|
||||
解析 CI 字符串，支持多种格式（终审建议）
|
||||
|
||||
支持格式：
|
||||
- 2.5 (1.1-3.5)
|
||||
- 2.5 (1.1, 3.5)
|
||||
- 2.5 [1.1; 3.5]
|
||||
- 95% CI: 1.1-3.5
|
||||
- 95% CI 1.1 to 3.5
|
||||
"""
|
||||
for pattern in CI_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
try:
|
||||
lower = float(match.group(1))
|
||||
upper = float(match.group(2))
|
||||
if lower < upper: # 基本合理性检查
|
||||
return lower, upper
|
||||
except (ValueError, TypeError, IndexError):
|
||||
continue
|
||||
|
||||
# 回退到原始的 CI_PATTERN
|
||||
match = CI_PATTERN.search(text)
|
||||
if match:
|
||||
try:
|
||||
lower = float(match.group(1))
|
||||
upper = float(match.group(2))
|
||||
if lower < upper:
|
||||
return lower, upper
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _parse_pvalue(self, text: str) -> Optional[float]:
|
||||
"""
|
||||
解析 P 值
|
||||
|
||||
处理：
|
||||
- P=0.05
|
||||
- P<0.001
|
||||
- P>0.05
|
||||
- p值=0.05
|
||||
"""
|
||||
match = PVALUE_PATTERN.search(text)
|
||||
if match:
|
||||
try:
|
||||
return float(match.group(1))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def _estimate_sample_sizes(
|
||||
self,
|
||||
table: TableData,
|
||||
row_idx: int
|
||||
) -> Tuple[Optional[int], Optional[int]]:
|
||||
"""
|
||||
尝试从表格中估计样本量
|
||||
|
||||
策略：
|
||||
1. 查找表头中的 n 值
|
||||
2. 查找 "(n=XX)" 格式
|
||||
3. 默认返回 None
|
||||
"""
|
||||
data = table.data
|
||||
header = data[0] if data else []
|
||||
|
||||
# 从表头查找 (n=XX) 格式
|
||||
n_pattern = re.compile(r"\(?\s*n\s*[=:]\s*(\d+)\s*\)?", re.IGNORECASE)
|
||||
|
||||
n_values = []
|
||||
for cell in header:
|
||||
match = n_pattern.search(cell)
|
||||
if match:
|
||||
try:
|
||||
n_values.append(int(match.group(1)))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if len(n_values) >= 2:
|
||||
return n_values[0], n_values[1]
|
||||
|
||||
# 如果找不到，返回 None（不进行验证）
|
||||
return None, None
|
||||
Reference in New Issue
Block a user