738 lines
33 KiB
Python
738 lines
33 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import joblib
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.inspection import permutation_importance
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
SRC_ROOT = PROJECT_ROOT / "src"
|
|
VENDOR_ROOT = PROJECT_ROOT / "vendor"
|
|
VENV_SITE_PACKAGES = sorted((PROJECT_ROOT / ".venv" / "lib").glob("python*/site-packages"))
|
|
|
|
for candidate in [*VENV_SITE_PACKAGES, SRC_ROOT, VENDOR_ROOT]:
|
|
candidate_str = str(candidate)
|
|
if candidate.exists() and candidate_str not in sys.path:
|
|
sys.path.append(candidate_str)
|
|
|
|
from evaluation.aggregate_scores import aggregate_object_scores, compute_aggregate_metrics
|
|
from evaluation.metrics import bootstrap_metric_interval, compute_binary_metrics, optimize_threshold
|
|
from evaluation.visualization import (
|
|
save_bar_plot,
|
|
save_confusion_matrix,
|
|
save_example_rois,
|
|
save_pipeline_overview,
|
|
save_pr_roc_summary,
|
|
save_ranking_curve,
|
|
)
|
|
from models.baseline_pipeline import (
|
|
FEATURE_GROUPS,
|
|
KEY_COLUMNS,
|
|
build_model,
|
|
feature_columns_by_group,
|
|
fit_predict_probability,
|
|
merge_task_with_features,
|
|
)
|
|
from utils.project import CHECKPOINTS_ROOT, CROPS_ROOT, FEATURES_ROOT, FIGURES_ROOT, PREDICTIONS_ROOT, REPORTS_ROOT, TABLES_ROOT, ensure_output_layout, write_markdown
|
|
|
|
|
|
TASK_FILES = {
|
|
"a41_clean_binary": PROJECT_ROOT / "outputs" / "dataset_index" / "task_a41_clean_binary.parquet",
|
|
"a42_strict_binary": PROJECT_ROOT / "outputs" / "dataset_index" / "task_a42_strict_binary.parquet",
|
|
"a43_full_realistic_binary": PROJECT_ROOT / "outputs" / "dataset_index" / "task_a43_full_realistic_binary.parquet",
|
|
}
|
|
MODEL_NAMES = [
|
|
"logistic_regression",
|
|
"linear_svm",
|
|
"rbf_svm",
|
|
"random_forest",
|
|
"xgboost",
|
|
]
|
|
|
|
|
|
def _pick_feature_path(base_name: str) -> Path:
|
|
candidates = [
|
|
FEATURES_ROOT / f"{base_name}.parquet",
|
|
FEATURES_ROOT / f"{base_name}_fast.parquet",
|
|
FEATURES_ROOT / f"{base_name}_lite.parquet",
|
|
]
|
|
for candidate in candidates:
|
|
if candidate.exists():
|
|
return candidate
|
|
raise FileNotFoundError(f"None of the candidate feature files exist for {base_name}: {candidates}")
|
|
|
|
|
|
def _load_required_artifacts() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict[str, pd.DataFrame]]:
|
|
features_combined = pd.read_parquet(_pick_feature_path("features_combined"))
|
|
features_bbox = pd.read_parquet(_pick_feature_path("features_bbox"))
|
|
features_mask = pd.read_parquet(_pick_feature_path("features_mask"))
|
|
tasks = {task_name: pd.read_parquet(path) for task_name, path in TASK_FILES.items()}
|
|
return features_combined, features_bbox, features_mask, tasks
|
|
|
|
|
|
def _split_frame(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
|
train_df = df.loc[df["split_name"] == "train"].copy()
|
|
val_df = df.loc[df["split_name"] == "val"].copy()
|
|
test_df = df.loc[df["split_name"] == "test"].copy()
|
|
if min(len(train_df), len(val_df), len(test_df)) == 0:
|
|
raise ValueError("One of the splits is empty.")
|
|
return train_df, val_df, test_df
|
|
|
|
|
|
def _cap_frame(frame: pd.DataFrame, max_rows: int, random_state: int) -> pd.DataFrame:
|
|
if len(frame) <= max_rows:
|
|
return frame
|
|
sampled_groups: list[pd.DataFrame] = []
|
|
remaining = max_rows
|
|
groups = list(frame.groupby("target_binary", sort=True))
|
|
for index, (_, group) in enumerate(groups):
|
|
if index == len(groups) - 1:
|
|
n_take = min(len(group), remaining)
|
|
else:
|
|
n_take = max(1, int(round(max_rows * len(group) / len(frame))))
|
|
n_take = min(len(group), n_take)
|
|
remaining -= n_take
|
|
sampled_groups.append(group.sample(n=n_take, random_state=random_state))
|
|
return pd.concat(sampled_groups, ignore_index=True)
|
|
|
|
|
|
def _select_best_row(leaderboard_df: pd.DataFrame, task_name: str | None = None) -> pd.Series:
|
|
frame = leaderboard_df.copy()
|
|
if task_name is not None:
|
|
frame = frame.loc[frame["task_name"] == task_name].copy()
|
|
if len(frame) == 0:
|
|
raise ValueError("Leaderboard is empty for selection.")
|
|
return frame.sort_values(
|
|
["validation_pr_auc", "test_pr_auc", "test_recall_blast"],
|
|
ascending=[False, False, False],
|
|
).iloc[0]
|
|
|
|
|
|
def run_a6() -> dict[str, str]:
|
|
ensure_output_layout()
|
|
features_combined, features_bbox, features_mask, tasks = _load_required_artifacts()
|
|
feature_groups = feature_columns_by_group(features_combined)
|
|
|
|
metrics_rows: list[dict[str, object]] = []
|
|
prediction_rows: list[dict[str, object]] = []
|
|
checkpoint_rows: list[dict[str, object]] = []
|
|
|
|
for task_name, task_df in tasks.items():
|
|
merged_df = merge_task_with_features(task_df, features_combined)
|
|
train_df, val_df, test_df = _split_frame(merged_df)
|
|
train_df = _cap_frame(train_df, max_rows=20000, random_state=42)
|
|
val_df = _cap_frame(val_df, max_rows=5000, random_state=43)
|
|
scale_pos_weight = float((train_df["target_binary"].eq(0).sum()) / max(train_df["target_binary"].eq(1).sum(), 1))
|
|
|
|
for model_name in MODEL_NAMES:
|
|
for feature_group_name, feature_columns in feature_groups.items():
|
|
X_train = train_df[feature_columns]
|
|
y_train = train_df["target_binary"].astype(int)
|
|
X_val = val_df[feature_columns]
|
|
y_val = val_df["target_binary"].astype(int)
|
|
X_test = test_df[feature_columns]
|
|
y_test = test_df["target_binary"].astype(int)
|
|
|
|
pipeline, spec = build_model(model_name, scale_pos_weight=scale_pos_weight, use_class_weight=True)
|
|
val_score = fit_predict_probability(pipeline, X_train, y_train, X_val)
|
|
threshold = optimize_threshold(y_val.to_numpy(), val_score)
|
|
test_score = np.asarray(pipeline.predict_proba(X_test)[:, 1], dtype=float) if hasattr(pipeline, "predict_proba") else fit_predict_probability(pipeline, X_train, y_train, X_test)
|
|
|
|
val_metrics = compute_binary_metrics(y_val.to_numpy(), val_score, threshold)
|
|
test_metrics = compute_binary_metrics(y_test.to_numpy(), test_score, threshold)
|
|
ci = bootstrap_metric_interval(y_test.to_numpy(), test_score, threshold, n_bootstrap=60, random_state=42)
|
|
|
|
run_id = f"{task_name}__{model_name}__{feature_group_name}"
|
|
checkpoint_path = CHECKPOINTS_ROOT / f"{run_id}.joblib"
|
|
joblib.dump(
|
|
{
|
|
"pipeline": pipeline,
|
|
"feature_columns": feature_columns,
|
|
"threshold": threshold,
|
|
"task_name": task_name,
|
|
"model_name": model_name,
|
|
"feature_group": feature_group_name,
|
|
},
|
|
checkpoint_path,
|
|
)
|
|
checkpoint_rows.append({"run_id": run_id, "checkpoint_path": str(checkpoint_path)})
|
|
|
|
y_pred = (test_score >= threshold).astype(int)
|
|
split_rows = test_df[KEY_COLUMNS + ["split_name", "target_binary"]].copy()
|
|
split_rows["run_id"] = run_id
|
|
split_rows["task_name"] = task_name
|
|
split_rows["model_name"] = model_name
|
|
split_rows["implementation"] = spec.implementation
|
|
split_rows["feature_group"] = feature_group_name
|
|
split_rows["threshold"] = threshold
|
|
split_rows["split_name"] = "test"
|
|
split_rows["y_true"] = y_test.to_numpy()
|
|
split_rows["y_score"] = test_score
|
|
split_rows["y_pred"] = y_pred
|
|
prediction_rows.extend(split_rows.to_dict("records"))
|
|
|
|
metrics_rows.append(
|
|
{
|
|
"run_id": run_id,
|
|
"task_name": task_name,
|
|
"model_name": model_name,
|
|
"implementation": spec.implementation,
|
|
"feature_group": feature_group_name,
|
|
"n_features": int(len(feature_columns)),
|
|
"threshold": threshold,
|
|
"validation_pr_auc": val_metrics.pr_auc,
|
|
"validation_roc_auc": val_metrics.roc_auc,
|
|
"validation_recall_blast": val_metrics.recall_blast,
|
|
"test_pr_auc": test_metrics.pr_auc,
|
|
"test_pr_auc_ci_low": ci["pr_auc"][0],
|
|
"test_pr_auc_ci_high": ci["pr_auc"][1],
|
|
"test_roc_auc": test_metrics.roc_auc,
|
|
"test_roc_auc_ci_low": ci["roc_auc"][0],
|
|
"test_roc_auc_ci_high": ci["roc_auc"][1],
|
|
"test_balanced_accuracy": test_metrics.balanced_accuracy,
|
|
"test_balanced_accuracy_ci_low": ci["balanced_accuracy"][0],
|
|
"test_balanced_accuracy_ci_high": ci["balanced_accuracy"][1],
|
|
"test_macro_f1": test_metrics.macro_f1,
|
|
"test_macro_f1_ci_low": ci["macro_f1"][0],
|
|
"test_macro_f1_ci_high": ci["macro_f1"][1],
|
|
"test_recall_blast": test_metrics.recall_blast,
|
|
"test_recall_blast_ci_low": ci["recall_blast"][0],
|
|
"test_recall_blast_ci_high": ci["recall_blast"][1],
|
|
"test_specificity": test_metrics.specificity,
|
|
"test_specificity_ci_low": ci["specificity"][0],
|
|
"test_specificity_ci_high": ci["specificity"][1],
|
|
"test_matthews_corrcoef": test_metrics.matthews_corrcoef,
|
|
"test_matthews_corrcoef_ci_low": ci["matthews_corrcoef"][0],
|
|
"test_matthews_corrcoef_ci_high": ci["matthews_corrcoef"][1],
|
|
"test_brier_score": test_metrics.brier_score,
|
|
"test_brier_score_ci_low": ci["brier_score"][0],
|
|
"test_brier_score_ci_high": ci["brier_score"][1],
|
|
"test_expected_calibration_error": test_metrics.expected_calibration_error,
|
|
"test_expected_calibration_error_ci_low": ci["expected_calibration_error"][0],
|
|
"test_expected_calibration_error_ci_high": ci["expected_calibration_error"][1],
|
|
}
|
|
)
|
|
|
|
metrics_df = pd.DataFrame(metrics_rows).sort_values(
|
|
["task_name", "test_pr_auc", "test_recall_blast"],
|
|
ascending=[True, False, False],
|
|
)
|
|
predictions_df = pd.DataFrame(prediction_rows)
|
|
|
|
if len(metrics_df) != len(TASK_FILES) * len(MODEL_NAMES) * len(FEATURE_GROUPS):
|
|
raise ValueError("A6 did not produce the expected number of model runs.")
|
|
if predictions_df.empty:
|
|
raise ValueError("A6 predictions are empty.")
|
|
|
|
object_metrics_path = TABLES_ROOT / "object_level_metrics_baselines.csv"
|
|
leaderboard_path = TABLES_ROOT / "baseline_model_leaderboard.csv"
|
|
feature_ablation_path = TABLES_ROOT / "feature_group_ablation.csv"
|
|
predictions_path = PREDICTIONS_ROOT / "object_level_baseline_predictions.parquet"
|
|
checkpoint_manifest_path = TABLES_ROOT / "checkpoint_manifest.csv"
|
|
|
|
metrics_df.to_csv(object_metrics_path, index=False)
|
|
metrics_df.to_csv(leaderboard_path, index=False)
|
|
metrics_df.to_csv(feature_ablation_path, index=False)
|
|
predictions_df.to_parquet(predictions_path, index=False)
|
|
pd.DataFrame(checkpoint_rows).to_csv(checkpoint_manifest_path, index=False)
|
|
|
|
save_pr_roc_summary(predictions_df, FIGURES_ROOT / "pr_curves_baselines.png", FIGURES_ROOT / "roc_curves_baselines.png")
|
|
best_full_row = _select_best_row(metrics_df, task_name="a43_full_realistic_binary")
|
|
best_test_predictions = predictions_df.loc[
|
|
(predictions_df["run_id"] == best_full_row["run_id"]) & (predictions_df["split_name"] == "test")
|
|
].copy()
|
|
save_confusion_matrix(
|
|
best_test_predictions["y_true"].to_numpy(),
|
|
best_test_predictions["y_pred"].to_numpy(),
|
|
FIGURES_ROOT / "confusion_matrix_best_baseline.png",
|
|
title=f"Best baseline: {best_full_row['model_name']} | {best_full_row['feature_group']}",
|
|
)
|
|
|
|
report_lines = [
|
|
"# A6 Baseline QC",
|
|
"",
|
|
f"- Total baseline runs: `{len(metrics_df)}`",
|
|
f"- Prediction rows saved: `{len(predictions_df)}`",
|
|
f"- Leaderboard path: `{leaderboard_path}`",
|
|
f"- Best full-realistic run: `{best_full_row['run_id']}`",
|
|
]
|
|
write_markdown(REPORTS_ROOT / "a6_baseline_qc.md", report_lines)
|
|
|
|
return {
|
|
"object_level_metrics_csv": str(object_metrics_path),
|
|
"feature_group_ablation_csv": str(feature_ablation_path),
|
|
"object_level_predictions_parquet": str(predictions_path),
|
|
"baseline_model_leaderboard_csv": str(leaderboard_path),
|
|
"pr_curves_png": str(FIGURES_ROOT / "pr_curves_baselines.png"),
|
|
"roc_curves_png": str(FIGURES_ROOT / "roc_curves_baselines.png"),
|
|
"confusion_matrix_png": str(FIGURES_ROOT / "confusion_matrix_best_baseline.png"),
|
|
"a6_qc_report_md": str(REPORTS_ROOT / "a6_baseline_qc.md"),
|
|
}
|
|
|
|
|
|
def run_a7(leaderboard_df: pd.DataFrame, full_task_df: pd.DataFrame, features_combined: pd.DataFrame) -> None:
|
|
best_row = _select_best_row(leaderboard_df, task_name="a43_full_realistic_binary")
|
|
feature_group = best_row["feature_group"]
|
|
feature_columns = feature_columns_by_group(features_combined)[feature_group]
|
|
|
|
merged_df = merge_task_with_features(full_task_df, features_combined)
|
|
train_df, val_df, test_df = _split_frame(merged_df)
|
|
train_df = _cap_frame(train_df, max_rows=20000, random_state=42)
|
|
val_df = _cap_frame(val_df, max_rows=5000, random_state=43)
|
|
if len(train_df) > 10000:
|
|
train_df = _cap_frame(train_df, max_rows=10000, random_state=44)
|
|
scale_pos_weight = float((train_df["target_binary"].eq(0).sum()) / max(train_df["target_binary"].eq(1).sum(), 1))
|
|
pipeline, spec = build_model(str(best_row["model_name"]), scale_pos_weight=scale_pos_weight, use_class_weight=True)
|
|
X_train = train_df[feature_columns]
|
|
y_train = train_df["target_binary"].astype(int)
|
|
if len(test_df) > 2000:
|
|
test_df = test_df.sample(n=2000, random_state=42)
|
|
X_test = test_df[feature_columns]
|
|
y_test = test_df["target_binary"].astype(int)
|
|
pipeline.fit(X_train, y_train)
|
|
|
|
importance = permutation_importance(
|
|
pipeline,
|
|
X_test,
|
|
y_test,
|
|
n_repeats=3,
|
|
random_state=42,
|
|
n_jobs=1,
|
|
scoring="average_precision",
|
|
)
|
|
stability_rows: list[dict[str, object]] = []
|
|
for repeat_idx in range(3):
|
|
sample_df = train_df.sample(frac=0.6, replace=True, random_state=42 + repeat_idx)
|
|
sample_pipeline, _ = build_model(str(best_row["model_name"]), scale_pos_weight=scale_pos_weight, use_class_weight=True)
|
|
sample_pipeline.fit(sample_df[feature_columns], sample_df["target_binary"].astype(int))
|
|
sample_importance = permutation_importance(
|
|
sample_pipeline,
|
|
X_test,
|
|
y_test,
|
|
n_repeats=2,
|
|
random_state=100 + repeat_idx,
|
|
n_jobs=1,
|
|
scoring="average_precision",
|
|
)
|
|
ranking = np.argsort(-sample_importance.importances_mean)[:20]
|
|
for rank_position, index in enumerate(ranking, start=1):
|
|
stability_rows.append(
|
|
{
|
|
"repeat_id": repeat_idx,
|
|
"feature_name": feature_columns[index],
|
|
"importance_mean": float(sample_importance.importances_mean[index]),
|
|
"importance_std": float(sample_importance.importances_std[index]),
|
|
"rank_position": rank_position,
|
|
}
|
|
)
|
|
|
|
stability_df = pd.DataFrame(stability_rows)
|
|
top_features_df = (
|
|
stability_df.groupby("feature_name")
|
|
.agg(
|
|
top20_frequency=("feature_name", "size"),
|
|
mean_importance=("importance_mean", "mean"),
|
|
best_rank=("rank_position", "min"),
|
|
)
|
|
.reset_index()
|
|
.sort_values(["top20_frequency", "mean_importance"], ascending=[False, False])
|
|
)
|
|
top_features_path = TABLES_ROOT / "top_features_stability.csv"
|
|
top_features_df.to_csv(top_features_path, index=False)
|
|
|
|
import matplotlib.pyplot as plt
|
|
|
|
top_plot_df = top_features_df.head(20).iloc[::-1]
|
|
fig, ax = plt.subplots(figsize=(10, 8))
|
|
ax.barh(top_plot_df["feature_name"], top_plot_df["top20_frequency"], color="#4c72b0")
|
|
ax.set_title("Feature stability across repeated importance runs")
|
|
ax.set_xlabel("Top-20 frequency")
|
|
fig.tight_layout()
|
|
fig.savefig(FIGURES_ROOT / "feature_importance_stability.png", dpi=180)
|
|
plt.close(fig)
|
|
|
|
shap_like_df = pd.DataFrame(
|
|
{
|
|
"feature_name": feature_columns,
|
|
"importance_mean": importance.importances_mean,
|
|
"importance_std": importance.importances_std,
|
|
}
|
|
).sort_values("importance_mean", ascending=False)
|
|
top_shap_df = shap_like_df.head(20).iloc[::-1]
|
|
fig, ax = plt.subplots(figsize=(10, 8))
|
|
ax.barh(top_shap_df["feature_name"], top_shap_df["importance_mean"], xerr=top_shap_df["importance_std"], color="#55a868")
|
|
ax.set_title("Top permutation-importance features for the best model")
|
|
ax.set_xlabel("Importance")
|
|
fig.tight_layout()
|
|
fig.savefig(FIGURES_ROOT / "shap_summary_top_model.png", dpi=180)
|
|
plt.close(fig)
|
|
|
|
notes_lines = [
|
|
"# Feature Interpretation Notes",
|
|
"",
|
|
f"- Selected model: `{best_row['run_id']}`",
|
|
f"- Implementation: `{spec.implementation}`",
|
|
"",
|
|
"## Stable top features",
|
|
"",
|
|
]
|
|
for _, row in top_features_df.head(15).iterrows():
|
|
notes_lines.append(
|
|
f"- `{row['feature_name']}`: frequency `{int(row['top20_frequency'])}`, mean importance `{row['mean_importance']:.6f}`."
|
|
)
|
|
write_markdown(REPORTS_ROOT / "feature_interpretation_notes.md", notes_lines)
|
|
|
|
|
|
def _evaluate_single_configuration(
|
|
task_name: str,
|
|
task_df: pd.DataFrame,
|
|
feature_df: pd.DataFrame,
|
|
feature_columns: list[str],
|
|
model_name: str,
|
|
use_class_weight: bool,
|
|
label: str,
|
|
return_prediction_df: bool = False,
|
|
) -> dict[str, object] | tuple[dict[str, object], pd.DataFrame]:
|
|
merged_df = merge_task_with_features(task_df, feature_df)
|
|
train_df, val_df, test_df = _split_frame(merged_df)
|
|
train_df = _cap_frame(train_df, max_rows=20000, random_state=42)
|
|
val_df = _cap_frame(val_df, max_rows=5000, random_state=43)
|
|
scale_pos_weight = float((train_df["target_binary"].eq(0).sum()) / max(train_df["target_binary"].eq(1).sum(), 1))
|
|
pipeline, spec = build_model(model_name, scale_pos_weight=scale_pos_weight, use_class_weight=use_class_weight)
|
|
val_score = fit_predict_probability(pipeline, train_df[feature_columns], train_df["target_binary"], val_df[feature_columns])
|
|
threshold = optimize_threshold(val_df["target_binary"].to_numpy(), val_score)
|
|
test_score = np.asarray(pipeline.predict_proba(test_df[feature_columns])[:, 1], dtype=float) if hasattr(pipeline, "predict_proba") else fit_predict_probability(pipeline, train_df[feature_columns], train_df["target_binary"], test_df[feature_columns])
|
|
metrics = compute_binary_metrics(test_df["target_binary"].to_numpy(), test_score, threshold)
|
|
result = {
|
|
"task_name": task_name,
|
|
"label": label,
|
|
"model_name": model_name,
|
|
"implementation": spec.implementation,
|
|
"use_class_weight": use_class_weight,
|
|
"n_rows": int(len(test_df)),
|
|
"n_features": int(len(feature_columns)),
|
|
"pr_auc": metrics.pr_auc,
|
|
"roc_auc": metrics.roc_auc,
|
|
"balanced_accuracy": metrics.balanced_accuracy,
|
|
"macro_f1": metrics.macro_f1,
|
|
"recall_blast": metrics.recall_blast,
|
|
"specificity": metrics.specificity,
|
|
"matthews_corrcoef": metrics.matthews_corrcoef,
|
|
"brier_score": metrics.brier_score,
|
|
"expected_calibration_error": metrics.expected_calibration_error,
|
|
}
|
|
if not return_prediction_df:
|
|
return result
|
|
|
|
prediction_df = test_df[KEY_COLUMNS + ["split_name", "target_binary"]].copy()
|
|
prediction_df["task_name"] = task_name
|
|
prediction_df["label"] = label
|
|
prediction_df["model_name"] = model_name
|
|
prediction_df["implementation"] = spec.implementation
|
|
prediction_df["use_class_weight"] = use_class_weight
|
|
prediction_df["threshold"] = float(threshold)
|
|
prediction_df["y_true"] = test_df["target_binary"].astype(int).to_numpy()
|
|
prediction_df["y_score"] = test_score.astype(float)
|
|
prediction_df["y_pred"] = (test_score >= threshold).astype(int)
|
|
return result, prediction_df
|
|
|
|
|
|
def run_a8_mask_usage(
|
|
leaderboard_df: pd.DataFrame,
|
|
tasks: dict[str, pd.DataFrame],
|
|
features_combined: pd.DataFrame,
|
|
features_bbox: pd.DataFrame,
|
|
features_mask: pd.DataFrame,
|
|
) -> pd.DataFrame:
|
|
best_full_row = _select_best_row(leaderboard_df, task_name="a43_full_realistic_binary")
|
|
best_model_name = str(best_full_row["model_name"])
|
|
combined_full_columns = feature_columns_by_group(features_combined)["full"]
|
|
bbox_full_columns = [column for column in features_bbox.columns if column not in KEY_COLUMNS]
|
|
mask_full_columns = [column for column in features_mask.columns if column not in KEY_COLUMNS]
|
|
|
|
mask_ablation_rows: list[dict[str, object]] = []
|
|
mask_prediction_frames: list[pd.DataFrame] = []
|
|
for task_name, task_df in tasks.items():
|
|
mask_keys = set(
|
|
tuple(row)
|
|
for row in features_mask[KEY_COLUMNS[:3]].drop_duplicates().itertuples(index=False, name=None)
|
|
)
|
|
task_mask_ready = task_df.loc[
|
|
task_df[KEY_COLUMNS[:3]].apply(tuple, axis=1).isin(mask_keys)
|
|
].copy()
|
|
for label, feature_df, feature_columns in [
|
|
("bbox_only", features_bbox, bbox_full_columns),
|
|
("mask_only", features_mask, mask_full_columns),
|
|
("bbox_plus_mask", features_combined, combined_full_columns),
|
|
]:
|
|
metrics_row, prediction_df = _evaluate_single_configuration(
|
|
task_name=task_name,
|
|
task_df=task_mask_ready,
|
|
feature_df=feature_df,
|
|
feature_columns=feature_columns,
|
|
model_name=best_model_name,
|
|
use_class_weight=True,
|
|
label=label,
|
|
return_prediction_df=True,
|
|
)
|
|
mask_ablation_rows.append(metrics_row)
|
|
mask_prediction_frames.append(prediction_df)
|
|
mask_ablation_df = pd.DataFrame(mask_ablation_rows)
|
|
mask_ablation_df.to_csv(TABLES_ROOT / "ablation_mask_usage.csv", index=False)
|
|
if mask_prediction_frames:
|
|
pd.concat(mask_prediction_frames, ignore_index=True).to_parquet(
|
|
PREDICTIONS_ROOT / "h2_mask_usage_predictions.parquet",
|
|
index=False,
|
|
)
|
|
return mask_ablation_df
|
|
|
|
|
|
def run_a8_dataset_regimes(
|
|
leaderboard_df: pd.DataFrame,
|
|
tasks: dict[str, pd.DataFrame],
|
|
features_combined: pd.DataFrame,
|
|
) -> pd.DataFrame:
|
|
best_full_row = _select_best_row(leaderboard_df, task_name="a43_full_realistic_binary")
|
|
best_model_name = str(best_full_row["model_name"])
|
|
combined_full_columns = feature_columns_by_group(features_combined)["full"]
|
|
|
|
dataset_ablation_rows: list[dict[str, object]] = []
|
|
for task_name, task_df in tasks.items():
|
|
dataset_ablation_rows.append(
|
|
_evaluate_single_configuration(
|
|
task_name=task_name,
|
|
task_df=task_df,
|
|
feature_df=features_combined,
|
|
feature_columns=combined_full_columns,
|
|
model_name=best_model_name,
|
|
use_class_weight=True,
|
|
label="combined_with_class_weight",
|
|
)
|
|
)
|
|
dataset_ablation_rows.append(
|
|
_evaluate_single_configuration(
|
|
task_name="a43_full_realistic_binary",
|
|
task_df=tasks["a43_full_realistic_binary"],
|
|
feature_df=features_combined,
|
|
feature_columns=combined_full_columns,
|
|
model_name=best_model_name,
|
|
use_class_weight=False,
|
|
label="combined_without_class_weight",
|
|
)
|
|
)
|
|
top_features_df = pd.read_csv(TABLES_ROOT / "top_features_stability.csv").head(30)
|
|
selected_columns = [column for column in top_features_df["feature_name"].tolist() if column in combined_full_columns]
|
|
if not selected_columns:
|
|
selected_columns = combined_full_columns[: min(30, len(combined_full_columns))]
|
|
dataset_ablation_rows.append(
|
|
_evaluate_single_configuration(
|
|
task_name="a43_full_realistic_binary",
|
|
task_df=tasks["a43_full_realistic_binary"],
|
|
feature_df=features_combined,
|
|
feature_columns=selected_columns,
|
|
model_name=best_model_name,
|
|
use_class_weight=True,
|
|
label="combined_top30_features",
|
|
)
|
|
)
|
|
dataset_ablation_df = pd.DataFrame(dataset_ablation_rows)
|
|
dataset_ablation_df.to_csv(TABLES_ROOT / "ablation_dataset_regimes.csv", index=False)
|
|
return dataset_ablation_df
|
|
|
|
|
|
def save_a8_summary_plot(mask_ablation_df: pd.DataFrame, dataset_ablation_df: pd.DataFrame) -> str:
|
|
ablation_plot_df = pd.concat(
|
|
[
|
|
mask_ablation_df.assign(ablation_family="mask_usage"),
|
|
dataset_ablation_df.assign(ablation_family="dataset_regime"),
|
|
],
|
|
ignore_index=True,
|
|
)
|
|
save_bar_plot(
|
|
ablation_plot_df[["label", "pr_auc"]].drop_duplicates(),
|
|
x="label",
|
|
y="pr_auc",
|
|
hue=None,
|
|
path=FIGURES_ROOT / "ablation_summary.png",
|
|
title="A8 ablation summary (PR-AUC)",
|
|
rotation=35,
|
|
)
|
|
return str(FIGURES_ROOT / "ablation_summary.png")
|
|
|
|
|
|
def run_a8(
|
|
leaderboard_df: pd.DataFrame,
|
|
tasks: dict[str, pd.DataFrame],
|
|
features_combined: pd.DataFrame,
|
|
features_bbox: pd.DataFrame,
|
|
features_mask: pd.DataFrame,
|
|
) -> dict[str, str]:
|
|
mask_ablation_df = run_a8_mask_usage(
|
|
leaderboard_df=leaderboard_df,
|
|
tasks=tasks,
|
|
features_combined=features_combined,
|
|
features_bbox=features_bbox,
|
|
features_mask=features_mask,
|
|
)
|
|
dataset_ablation_df = run_a8_dataset_regimes(
|
|
leaderboard_df=leaderboard_df,
|
|
tasks=tasks,
|
|
features_combined=features_combined,
|
|
)
|
|
plot_path = save_a8_summary_plot(mask_ablation_df, dataset_ablation_df)
|
|
|
|
return {
|
|
"ablation_mask_usage_csv": str(TABLES_ROOT / "ablation_mask_usage.csv"),
|
|
"ablation_dataset_regimes_csv": str(TABLES_ROOT / "ablation_dataset_regimes.csv"),
|
|
"ablation_summary_png": plot_path,
|
|
}
|
|
|
|
|
|
def run_a6_a8() -> dict[str, dict[str, str]]:
|
|
result_a6 = run_a6()
|
|
features_combined, features_bbox, features_mask, tasks = _load_required_artifacts()
|
|
leaderboard_df = pd.read_csv(TABLES_ROOT / "baseline_model_leaderboard.csv")
|
|
run_a7(leaderboard_df, tasks["a43_full_realistic_binary"], features_combined)
|
|
result_a8 = run_a8(leaderboard_df, tasks, features_combined, features_bbox, features_mask)
|
|
return {"a6": result_a6, "a8": result_a8}
|
|
|
|
|
|
def run_a10_a11() -> dict[str, str]:
|
|
leaderboard_df = pd.read_csv(TABLES_ROOT / "baseline_model_leaderboard.csv")
|
|
predictions_df = pd.read_parquet(PREDICTIONS_ROOT / "object_level_baseline_predictions.parquet")
|
|
best_full_row = _select_best_row(leaderboard_df, task_name="a43_full_realistic_binary")
|
|
best_predictions = predictions_df.loc[
|
|
(predictions_df["run_id"] == best_full_row["run_id"]) & (predictions_df["split_name"] == "test")
|
|
].copy()
|
|
if best_predictions.empty:
|
|
raise ValueError("No object-level predictions found for A10.")
|
|
|
|
image_score_df = aggregate_object_scores(
|
|
best_predictions.assign(image_contains_blast=best_predictions["y_true"]),
|
|
threshold=float(best_full_row["threshold"]),
|
|
group_columns=["patient_id", "image_id"],
|
|
target_column="image_contains_blast",
|
|
)
|
|
patient_score_df = aggregate_object_scores(
|
|
best_predictions.assign(patient_contains_blast=best_predictions["y_true"]),
|
|
threshold=float(best_full_row["threshold"]),
|
|
group_columns=["patient_id"],
|
|
target_column="patient_contains_blast",
|
|
)
|
|
image_metrics_df = compute_aggregate_metrics(image_score_df, target_column="image_contains_blast")
|
|
patient_metrics_df = compute_aggregate_metrics(patient_score_df, target_column="patient_contains_blast")
|
|
|
|
image_scores_path = PREDICTIONS_ROOT / "image_level_scores.parquet"
|
|
patient_scores_path = PREDICTIONS_ROOT / "patient_level_scores.parquet"
|
|
image_metrics_path = TABLES_ROOT / "image_level_metrics.csv"
|
|
patient_metrics_path = TABLES_ROOT / "patient_level_ranking_metrics.csv"
|
|
image_score_df.to_parquet(image_scores_path, index=False)
|
|
patient_score_df.to_parquet(patient_scores_path, index=False)
|
|
image_metrics_df.to_csv(image_metrics_path, index=False)
|
|
patient_metrics_df.to_csv(patient_metrics_path, index=False)
|
|
|
|
save_ranking_curve(
|
|
image_score_df,
|
|
target_column="image_contains_blast",
|
|
path=FIGURES_ROOT / "image_ranking_recall_curve.png",
|
|
title="Image-level ranking recall",
|
|
)
|
|
save_ranking_curve(
|
|
patient_score_df,
|
|
target_column="patient_contains_blast",
|
|
path=FIGURES_ROOT / "patient_ranking_curve.png",
|
|
title="Patient-level ranking recall",
|
|
)
|
|
|
|
subset_summary_df = pd.read_csv(TABLES_ROOT / "subset_summary.csv").assign(section="subset_summary")
|
|
split_summary_df = pd.read_csv(TABLES_ROOT / "split_summary.csv").assign(section="split_summary")
|
|
task_prevalence_df = pd.read_csv(TABLES_ROOT / "task_prevalence_summary.csv").assign(section="task_prevalence")
|
|
dataset_summary_df = pd.concat(
|
|
[
|
|
subset_summary_df.rename(columns={"subset_name": "name"}),
|
|
split_summary_df.rename(columns={"split_name": "name"}),
|
|
task_prevalence_df.rename(columns={"task_name": "name"}),
|
|
],
|
|
ignore_index=True,
|
|
sort=False,
|
|
)
|
|
dataset_summary_df.to_csv(TABLES_ROOT / "dataset_summary.csv", index=False)
|
|
|
|
save_pipeline_overview(FIGURES_ROOT / "pipeline_overview.png")
|
|
roi_manifest = pd.read_parquet(CROPS_ROOT / "roi_manifest.parquet")
|
|
example_rows = roi_manifest.loc[roi_manifest["mask_crop_path"].notna()].head(4)
|
|
save_example_rois(
|
|
[Path(path) for path in example_rows["bbox_crop_path"].tolist()],
|
|
[Path(path) for path in example_rows["mask_crop_path"].tolist()],
|
|
FIGURES_ROOT / "example_rois.png",
|
|
)
|
|
|
|
mask_plot_df = subset_summary_df[["subset_name", "mask_ready_ratio"]].copy()
|
|
save_bar_plot(
|
|
mask_plot_df,
|
|
x="subset_name",
|
|
y="mask_ready_ratio",
|
|
hue=None,
|
|
path=FIGURES_ROOT / "mask_coverage.png",
|
|
title="Mask coverage by subset",
|
|
)
|
|
|
|
figure_manifest = pd.DataFrame(
|
|
[
|
|
{"figure_name": "pipeline_overview", "path": str(FIGURES_ROOT / "pipeline_overview.png"), "stage": "A11"},
|
|
{"figure_name": "example_rois", "path": str(FIGURES_ROOT / "example_rois.png"), "stage": "A11"},
|
|
{"figure_name": "mask_coverage", "path": str(FIGURES_ROOT / "mask_coverage.png"), "stage": "A11"},
|
|
{"figure_name": "pr_curves_baselines", "path": str(FIGURES_ROOT / "pr_curves_baselines.png"), "stage": "A6"},
|
|
{"figure_name": "roc_curves_baselines", "path": str(FIGURES_ROOT / "roc_curves_baselines.png"), "stage": "A6"},
|
|
{"figure_name": "feature_importance_stability", "path": str(FIGURES_ROOT / "feature_importance_stability.png"), "stage": "A7"},
|
|
{"figure_name": "ablation_summary", "path": str(FIGURES_ROOT / "ablation_summary.png"), "stage": "A8"},
|
|
{"figure_name": "image_ranking_recall_curve", "path": str(FIGURES_ROOT / "image_ranking_recall_curve.png"), "stage": "A10"},
|
|
{"figure_name": "patient_ranking_curve", "path": str(FIGURES_ROOT / "patient_ranking_curve.png"), "stage": "A10"},
|
|
]
|
|
)
|
|
figure_manifest.to_csv(TABLES_ROOT / "figure_manifest.csv", index=False)
|
|
|
|
table_manifest = pd.DataFrame(
|
|
[
|
|
{"table_name": path.stem, "path": str(path)}
|
|
for path in sorted(TABLES_ROOT.glob("*.csv"))
|
|
]
|
|
)
|
|
table_manifest.to_csv(TABLES_ROOT / "table_manifest.csv", index=False)
|
|
|
|
report_lines = [
|
|
"# A10-A11 QC",
|
|
"",
|
|
f"- Best full-realistic baseline: `{best_full_row['run_id']}`",
|
|
f"- Image-level rows: `{len(image_score_df)}`",
|
|
f"- Patient-level rows: `{len(patient_score_df)}`",
|
|
f"- Figure manifest rows: `{len(figure_manifest)}`",
|
|
f"- Table manifest rows: `{len(table_manifest)}`",
|
|
]
|
|
write_markdown(REPORTS_ROOT / "a10_a11_qc.md", report_lines)
|
|
|
|
return {
|
|
"image_level_metrics_csv": str(image_metrics_path),
|
|
"patient_level_ranking_metrics_csv": str(patient_metrics_path),
|
|
"image_level_scores_parquet": str(image_scores_path),
|
|
"patient_level_scores_parquet": str(patient_scores_path),
|
|
"figure_manifest_csv": str(TABLES_ROOT / "figure_manifest.csv"),
|
|
"table_manifest_csv": str(TABLES_ROOT / "table_manifest.csv"),
|
|
"a10_a11_qc_md": str(REPORTS_ROOT / "a10_a11_qc.md"),
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
result_a6_a8 = run_a6_a8()
|
|
result_a10_a11 = run_a10_a11()
|
|
print(json.dumps({"a6_a8": result_a6_a8, "a10_a11": result_a10_a11}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|