95 lines
3.3 KiB
Python
95 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
SRC_ROOT = PROJECT_ROOT / "src"
|
|
VENDOR_ROOT = PROJECT_ROOT / "vendor"
|
|
VENV_SITE_PACKAGES = sorted((PROJECT_ROOT / ".venv" / "lib").glob("python*/site-packages"))
|
|
|
|
for candidate in [*VENV_SITE_PACKAGES, SRC_ROOT, VENDOR_ROOT]:
|
|
candidate_str = str(candidate)
|
|
if candidate.exists() and candidate_str not in sys.path:
|
|
sys.path.append(candidate_str)
|
|
|
|
from utils.project import CROPS_ROOT, DATASET_INDEX_ROOT, REPORTS_ROOT, ensure_output_layout, write_markdown
|
|
|
|
|
|
KEY_COLUMNS = [
|
|
"patient_id",
|
|
"image_id",
|
|
"object_id_within_image",
|
|
"class_id",
|
|
"class_name",
|
|
"has_mask",
|
|
]
|
|
|
|
|
|
def _cap_frame(frame: pd.DataFrame, max_rows: int, random_state: int) -> pd.DataFrame:
|
|
if len(frame) <= max_rows:
|
|
return frame
|
|
sampled_groups: list[pd.DataFrame] = []
|
|
remaining = max_rows
|
|
groups = list(frame.groupby("target_binary", sort=True))
|
|
for index, (_, group) in enumerate(groups):
|
|
if index == len(groups) - 1:
|
|
n_take = min(len(group), remaining)
|
|
else:
|
|
n_take = max(1, int(round(max_rows * len(group) / len(frame))))
|
|
n_take = min(len(group), n_take)
|
|
remaining -= n_take
|
|
sampled_groups.append(group.sample(n=n_take, random_state=random_state))
|
|
return pd.concat(sampled_groups, ignore_index=True)
|
|
|
|
|
|
def build_modeling_workset() -> dict[str, str]:
|
|
ensure_output_layout()
|
|
full_task = pd.read_parquet(DATASET_INDEX_ROOT / "task_a43_full_realistic_binary.parquet")
|
|
roi_manifest = pd.read_parquet(CROPS_ROOT / "roi_manifest.parquet")
|
|
|
|
train_df = _cap_frame(full_task.loc[full_task["split_name"] == "train"].copy(), max_rows=20000, random_state=42)
|
|
val_df = _cap_frame(full_task.loc[full_task["split_name"] == "val"].copy(), max_rows=5000, random_state=43)
|
|
test_df = full_task.loc[full_task["split_name"] == "test"].copy()
|
|
workset_df = pd.concat([train_df, val_df, test_df], ignore_index=True)
|
|
|
|
workset_keys = workset_df[KEY_COLUMNS].drop_duplicates()
|
|
workset_roi = roi_manifest.merge(workset_keys, on=KEY_COLUMNS, how="inner", validate="one_to_one")
|
|
|
|
workset_task_path = DATASET_INDEX_ROOT / "task_a43_full_realistic_modeling_workset.parquet"
|
|
workset_roi_path = CROPS_ROOT / "roi_manifest_modeling_workset.parquet"
|
|
workset_df.to_parquet(workset_task_path, index=False)
|
|
workset_roi.to_parquet(workset_roi_path, index=False)
|
|
|
|
report_lines = [
|
|
"# Modeling Workset QC",
|
|
"",
|
|
f"- Train rows: `{len(train_df)}`",
|
|
f"- Validation rows: `{len(val_df)}`",
|
|
f"- Test rows: `{len(test_df)}`",
|
|
f"- Total workset rows: `{len(workset_df)}`",
|
|
f"- ROI rows matched: `{len(workset_roi)}`",
|
|
f"- Task artifact: `{workset_task_path}`",
|
|
f"- ROI artifact: `{workset_roi_path}`",
|
|
]
|
|
report_path = REPORTS_ROOT / "modeling_workset_qc.md"
|
|
write_markdown(report_path, report_lines)
|
|
|
|
return {
|
|
"task_workset_parquet": str(workset_task_path),
|
|
"roi_workset_parquet": str(workset_roi_path),
|
|
"qc_report_md": str(report_path),
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
print(json.dumps(build_modeling_workset(), ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|