IvanChurilov/E01_textural_features_blast_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
IvanChurilov
2026-09-11 08:28:02 +03:00
commit 1f9b2f20bd
64 changed files with 12593 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/dataset/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Dataset utilities for experiment E01."""
src/dataset/build_index.py
+574
View File
@@ -0,0 +1,574 @@
from __future__ import annotations
import argparse
import csv
import json
import sys
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Iterable
PROJECT_ROOT = Path(__file__).resolve().parents[2]
SRC_ROOT = PROJECT_ROOT / "src"
VENDOR_ROOT = PROJECT_ROOT / "vendor"
VENV_SITE_PACKAGES = sorted((PROJECT_ROOT / ".venv" / "lib").glob("python*/site-packages"))
for candidate in [*VENV_SITE_PACKAGES, SRC_ROOT, VENDOR_ROOT]:
candidate_str = str(candidate)
if candidate.exists() and candidate_str not in sys.path:
sys.path.append(candidate_str)
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ModuleNotFoundError as exc:
raise SystemExit(
"pyarrow is required for A0 because the plan requires parquet outputs. "
"Install it into the experiment environment before running this script."
) from exc
from dataset.class_mapping import mapping_signature, normalize_mapping_for_json, parse_predefined_classes
from dataset.parse_detection_labels import parse_label_file
from dataset.parse_masks import build_mask_index
MAIN_PROTOCOL_EXCLUDES = {"part-1"}
QUALITY_OR_ARTIFACT_CLASS_IDS = {12, 14, 15}
IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".bmp", ".tif", ".tiff"}
EXPECTED_SUMMARY = {
"raw_top_level_dirs_data": 123,
"raw_top_level_dirs_masks": 123,
"raw_images": 12521,
"raw_label_files": 12489,
"raw_objects": 86873,
"raw_mask_files": 86835,
"main_patient_dirs": 122,
"main_images": 12488,
"main_label_files": 12456,
"main_objects": 86396,
"main_empty_label_files": 26,
"main_images_without_labels": 32,
"main_mask_files": 86360,
"main_images_with_any_mask": 12428,
"main_images_without_masks": 60,
"main_checked_images_equal_label_mask_counts": 12395,
"main_images_with_exact_label_mask_order_match": 12395,
"main_float_class_rows": 4991,
"main_blast_patients": 100,
"main_nonblast_patients": 22,
"main_images_with_blast": 8618,
"main_images_only_blast": 3508,
"main_images_blast_mixed": 5110,
"main_images_without_blast": 3812,
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build A0 dataset index artifacts for experiment E01.")
parser.add_argument(
"--data-root",
type=Path,
default=PROJECT_ROOT.parent / "Data" / "data",
help="Path to the data directory containing patient folders.",
)
parser.add_argument(
"--masks-root",
type=Path,
default=PROJECT_ROOT.parent / "Data" / "data_masks",
help="Path to the mask directory containing patient folders.",
)
parser.add_argument(
"--output-root",
type=Path,
default=PROJECT_ROOT / "outputs",
help="Experiment outputs root.",
)
parser.add_argument(
"--exclude-from-main-protocol",
action="append",
default=[],
help="Top-level directories to exclude from the main patient-level protocol.",
)
return parser.parse_args()
def list_image_files(image_dir: Path) -> dict[str, Path]:
if not image_dir.exists():
return {}
return {
image_path.stem: image_path
for image_path in sorted(image_dir.iterdir())
if image_path.is_file() and image_path.suffix.lower() in IMAGE_SUFFIXES
}
def list_label_files(labels_dir: Path) -> dict[str, Path]:
if not labels_dir.exists():
return {}
return {label_path.stem: label_path for label_path in sorted(labels_dir.glob("*.txt"))}
def ensure_parent(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
def write_csv(rows: list[dict[str, object]], path: Path) -> None:
ensure_parent(path)
fieldnames = sorted({key for row in rows for key in row.keys()})
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
def write_json(payload: dict[str, object], path: Path) -> None:
ensure_parent(path)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def write_parquet(rows: list[dict[str, object]], path: Path) -> None:
ensure_parent(path)
if rows:
table = pa.Table.from_pylist(rows)
else:
table = pa.table({})
pq.write_table(table, path)
def determine_image_status(
has_image: bool,
has_label: bool,
empty_label_file: bool,
label_count: int,
mask_count: int,
class_order_match: bool,
) -> str:
if not has_image and has_label:
return "missing_image"
if has_image and not has_label:
return "missing_label"
if empty_label_file:
return "empty_label_file"
if label_count == 0 and mask_count == 0:
return "no_labels_no_masks"
if mask_count == 0:
return "no_masks"
if label_count != mask_count:
return "mask_count_mismatch"
if not class_order_match:
return "mask_class_order_mismatch"
return "ok"
def build_class_mapping_payload(
patient_class_mappings: dict[str, dict[int, str]],
root_reference_mapping: dict[int, str],
) -> tuple[dict[str, object], dict[tuple[tuple[int, str], ...], str]]:
signatures = Counter(mapping_signature(mapping) for mapping in patient_class_mappings.values())
signature_to_variant_id: dict[tuple[tuple[int, str], ...], str] = {}
for index, (signature, _) in enumerate(
sorted(signatures.items(), key=lambda item: (-item[1], item[0])), start=1
):
signature_to_variant_id[signature] = f"variant_{index:02d}"
canonical_signature = max(signatures.items(), key=lambda item: item[1])[0] if signatures else ()
variant_payload = []
for signature, count in sorted(signatures.items(), key=lambda item: (-item[1], item[0])):
variant_id = signature_to_variant_id[signature]
variant_mapping = {class_id: class_name for class_id, class_name in signature}
variant_payload.append(
{
"variant_id": variant_id,
"patient_count": count,
"patients": sorted(
patient_id
for patient_id, patient_mapping in patient_class_mappings.items()
if mapping_signature(patient_mapping) == signature
),
"mapping": normalize_mapping_for_json(variant_mapping),
}
)
payload = {
"generated_at_utc": datetime.now(timezone.utc).isoformat(),
"canonical_mapping": normalize_mapping_for_json({class_id: class_name for class_id, class_name in canonical_signature}),
"root_reference_mapping": normalize_mapping_for_json(root_reference_mapping),
"mapping_variants": variant_payload,
"main_protocol_excludes": sorted(MAIN_PROTOCOL_EXCLUDES),
"notes": [
"canonical_mapping is selected as the most frequent patient-specific mapping",
"part-1 is excluded from the main patient-level protocol by default",
],
}
return payload, signature_to_variant_id
def summarize_protocol(
patient_inventory_rows: list[dict[str, object]],
image_rows: list[dict[str, object]],
object_rows: list[dict[str, object]],
raw_top_level_dirs_data: int,
raw_top_level_dirs_masks: int,
) -> dict[str, int]:
raw_patients = patient_inventory_rows
main_patients = [row for row in patient_inventory_rows if bool(row["include_in_main_protocol"])]
main_images = [row for row in image_rows if bool(row["include_in_main_protocol"]) and bool(row["has_image"])]
main_objects = [row for row in object_rows if bool(row["include_in_main_protocol"])]
def sum_field(rows: Iterable[dict[str, object]], field: str) -> int:
return sum(int(row.get(field, 0)) for row in rows)
summary = {
"raw_top_level_dirs_data": raw_top_level_dirs_data,
"raw_top_level_dirs_masks": raw_top_level_dirs_masks,
"raw_images": sum_field(raw_patients, "images_count"),
"raw_label_files": sum_field(raw_patients, "label_files_count"),
"raw_objects": sum_field(raw_patients, "objects_count"),
"raw_mask_files": sum_field(raw_patients, "mask_files_count"),
"main_patient_dirs": len(main_patients),
"main_images": len(main_images),
"main_label_files": sum_field(main_patients, "label_files_count"),
"main_objects": len(main_objects),
"main_empty_label_files": sum_field(main_patients, "empty_label_files_count"),
"main_images_without_labels": sum_field(main_patients, "images_without_labels_count"),
"main_mask_files": sum_field(main_patients, "mask_files_count"),
"main_images_with_any_mask": sum_field(main_patients, "images_with_any_mask_count"),
"main_images_without_masks": sum_field(main_patients, "images_without_masks_count"),
"main_checked_images_equal_label_mask_counts": sum_field(
main_patients, "checked_images_equal_label_mask_counts"
),
"main_images_with_exact_label_mask_order_match": sum_field(
main_patients, "images_with_exact_label_mask_order_match"
),
"main_float_class_rows": sum_field(main_patients, "float_class_rows_count"),
"main_blast_patients": sum(1 for row in main_patients if bool(row["patient_has_blast"])),
"main_nonblast_patients": sum(1 for row in main_patients if not bool(row["patient_has_blast"])),
"main_images_with_blast": sum_field(main_patients, "images_with_blast_count"),
"main_images_only_blast": sum_field(main_patients, "images_only_blast_count"),
"main_images_blast_mixed": sum_field(main_patients, "images_blast_mixed_count"),
"main_images_without_blast": sum_field(main_patients, "images_without_blast_count"),
}
return summary
def build_runtime_report(
summary: dict[str, int],
output_paths: dict[str, str],
path: Path,
) -> None:
ensure_parent(path)
lines = [
"# Runtime Data Audit",
"",
f"Generated at: `{datetime.now(timezone.utc).isoformat()}`",
"",
"## Expected Comparison",
"",
"| Metric | Expected | Actual | Status |",
"|---|---:|---:|---|",
]
for key in sorted(EXPECTED_SUMMARY):
actual = summary.get(key, 0)
expected = EXPECTED_SUMMARY[key]
status = "MATCH" if actual == expected else "DIFF"
lines.append(f"| {key} | {expected} | {actual} | {status} |")
lines.extend(
[
"",
"## Output Artifacts",
"",
]
)
for name, artifact_path in sorted(output_paths.items()):
lines.append(f"- `{name}`: `{artifact_path}`")
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def build_index_artifacts(
data_root: Path,
masks_root: Path,
output_root: Path,
main_protocol_excludes: set[str] | None = None,
) -> dict[str, object]:
excludes = set(main_protocol_excludes or set()) | MAIN_PROTOCOL_EXCLUDES
data_dirs = {
path.name: path
for path in sorted(data_root.iterdir())
if path.is_dir() and not path.name.startswith(".")
}
mask_dirs = {
path.name: path
for path in sorted(masks_root.iterdir())
if path.is_dir() and not path.name.startswith(".")
}
top_level_ids = sorted(set(data_dirs) | set(mask_dirs))
patient_class_mappings: dict[str, dict[int, str]] = {}
for top_level_id in top_level_ids:
candidate_paths = [
data_dirs.get(top_level_id, Path()) / "predefined_classes.txt" if top_level_id in data_dirs else None,
mask_dirs.get(top_level_id, Path()) / "predefined_classes.txt" if top_level_id in mask_dirs else None,
]
mapping_path = next((path for path in candidate_paths if path is not None and path.exists()), None)
patient_class_mappings[top_level_id] = parse_predefined_classes(mapping_path) if mapping_path else {}
root_reference_mapping = parse_predefined_classes(data_root / "predefined_classes.txt")
class_mapping_payload, signature_to_variant_id = build_class_mapping_payload(
patient_class_mappings=patient_class_mappings,
root_reference_mapping=root_reference_mapping,
)
patient_inventory_rows: list[dict[str, object]] = []
image_rows: list[dict[str, object]] = []
object_rows: list[dict[str, object]] = []
for patient_id in top_level_ids:
data_dir = data_dirs.get(patient_id)
mask_dir = mask_dirs.get(patient_id)
include_in_main_protocol = patient_id not in excludes
class_mapping = patient_class_mappings.get(patient_id, {})
class_mapping_variant_id = signature_to_variant_id.get(mapping_signature(class_mapping), "variant_unknown")
image_files = list_image_files(data_dir / "images") if data_dir else list_image_files(mask_dir / "images")
label_files = (
list_label_files(data_dir / "images_labels")
if data_dir and (data_dir / "images_labels").exists()
else list_label_files(mask_dir / "images_labels") if mask_dir else {}
)
mask_entries_by_image = (
build_mask_index(mask_dir / "images_results" / "masks_sam") if mask_dir else {}
)
patient_counts = Counter()
patient_counts["images_count"] = len(image_files)
patient_counts["label_files_count"] = len(label_files)
patient_counts["mask_files_count"] = sum(len(entries) for entries in mask_entries_by_image.values())
image_ids = sorted(set(image_files) | set(label_files) | set(mask_entries_by_image))
for image_id in image_ids:
image_path = image_files.get(image_id)
label_path = label_files.get(image_id)
mask_entries = mask_entries_by_image.get(image_id, [])
label_entries = parse_label_file(label_path) if label_path else []
has_image = image_path is not None
has_label = label_path is not None
empty_label_file = has_label and not label_entries
label_count = len(label_entries)
mask_count = len(mask_entries)
contains_blast = any(int(entry["class_id"]) == 7 for entry in label_entries)
contains_non_blast = any(int(entry["class_id"]) != 7 for entry in label_entries)
count_match = has_label and label_count > 0 and label_count == mask_count
class_order_match = count_match and [
int(entry["class_id"]) for entry in label_entries
] == [int(entry["class_id"]) for entry in mask_entries]
has_any_mask = mask_count > 0
if has_image and not has_label:
patient_counts["images_without_labels_count"] += 1
if has_label and not has_image:
patient_counts["labels_without_images_count"] += 1
if empty_label_file:
patient_counts["empty_label_files_count"] += 1
if has_any_mask:
patient_counts["images_with_any_mask_count"] += 1
if has_image and not has_any_mask:
patient_counts["images_without_masks_count"] += 1
if count_match:
patient_counts["checked_images_equal_label_mask_counts"] += 1
if class_order_match:
patient_counts["images_with_exact_label_mask_order_match"] += 1
if has_label and label_count != mask_count:
patient_counts["images_with_count_mismatch"] += 1
if has_label and not empty_label_file:
if contains_blast:
patient_counts["images_with_blast_count"] += 1
if contains_non_blast:
patient_counts["images_blast_mixed_count"] += 1
else:
patient_counts["images_only_blast_count"] += 1
else:
patient_counts["images_without_blast_count"] += 1
image_rows.append(
{
"patient_id": patient_id,
"include_in_main_protocol": include_in_main_protocol,
"image_id": image_id,
"image_path": str(image_path.resolve()) if image_path else None,
"label_path": str(label_path.resolve()) if label_path else None,
"has_image": has_image,
"has_label": has_label,
"empty_label_file": empty_label_file,
"has_any_mask": has_any_mask,
"label_object_count": label_count,
"mask_object_count": mask_count,
"label_mask_count_match": label_count == mask_count if has_label else False,
"label_mask_class_order_match": class_order_match,
"contains_blast": contains_blast,
"contains_non_blast": contains_non_blast,
"blast_object_count": sum(1 for entry in label_entries if int(entry["class_id"]) == 7),
"non_blast_object_count": sum(1 for entry in label_entries if int(entry["class_id"]) != 7),
"image_status": determine_image_status(
has_image=has_image,
has_label=has_label,
empty_label_file=empty_label_file,
label_count=label_count,
mask_count=mask_count,
class_order_match=class_order_match,
),
}
)
for detection_entry in label_entries:
object_id = int(detection_entry["object_id_within_image"])
expected_mask = mask_entries[object_id] if object_id < len(mask_entries) else None
has_mask = expected_mask is not None and int(expected_mask["class_id"]) == int(detection_entry["class_id"])
class_id = int(detection_entry["class_id"])
class_name = class_mapping.get(class_id, f"UNKNOWN_CLASS_{class_id}")
patient_counts["objects_count"] += 1
if bool(detection_entry["class_id_was_float"]):
patient_counts["float_class_rows_count"] += 1
if class_id == 7:
patient_counts["blast_objects_count"] += 1
else:
patient_counts["non_blast_objects_count"] += 1
if not has_mask:
patient_counts["objects_without_masks_count"] += 1
object_rows.append(
{
"patient_id": patient_id,
"include_in_main_protocol": include_in_main_protocol,
"image_id": image_id,
"image_path": str(image_path.resolve()) if image_path else None,
"label_path": str(label_path.resolve()) if label_path else None,
"mask_path": expected_mask["mask_path"] if has_mask else None,
"object_id_within_image": object_id,
"raw_class_id": detection_entry["raw_class_id"],
"class_id": class_id,
"class_name": class_name,
"class_id_was_float": bool(detection_entry["class_id_was_float"]),
"x_center_norm": float(detection_entry["x_center_norm"]),
"y_center_norm": float(detection_entry["y_center_norm"]),
"width_norm": float(detection_entry["width_norm"]),
"height_norm": float(detection_entry["height_norm"]),
"has_mask": has_mask,
"mask_status": "matched" if has_mask else "missing",
"mask_class_id": int(expected_mask["class_id"]) if expected_mask is not None else None,
"label_count_for_image": label_count,
"mask_count_for_image": mask_count,
"image_has_any_mask": has_any_mask,
"is_blast": class_id == 7,
"is_quality_or_artifact": class_id in QUALITY_OR_ARTIFACT_CLASS_IDS,
}
)
patient_inventory_rows.append(
{
"patient_id": patient_id,
"include_in_main_protocol": include_in_main_protocol,
"has_data_dir": data_dir is not None,
"has_mask_dir": mask_dir is not None,
"images_count": patient_counts["images_count"],
"label_files_count": patient_counts["label_files_count"],
"empty_label_files_count": patient_counts["empty_label_files_count"],
"images_without_labels_count": patient_counts["images_without_labels_count"],
"labels_without_images_count": patient_counts["labels_without_images_count"],
"objects_count": patient_counts["objects_count"],
"float_class_rows_count": patient_counts["float_class_rows_count"],
"mask_files_count": patient_counts["mask_files_count"],
"images_with_any_mask_count": patient_counts["images_with_any_mask_count"],
"images_without_masks_count": patient_counts["images_without_masks_count"],
"checked_images_equal_label_mask_counts": patient_counts["checked_images_equal_label_mask_counts"],
"images_with_exact_label_mask_order_match": patient_counts[
"images_with_exact_label_mask_order_match"
],
"images_with_count_mismatch": patient_counts["images_with_count_mismatch"],
"objects_without_masks_count": patient_counts["objects_without_masks_count"],
"blast_objects_count": patient_counts["blast_objects_count"],
"non_blast_objects_count": patient_counts["non_blast_objects_count"],
"patient_has_blast": patient_counts["blast_objects_count"] > 0,
"images_with_blast_count": patient_counts["images_with_blast_count"],
"images_only_blast_count": patient_counts["images_only_blast_count"],
"images_blast_mixed_count": patient_counts["images_blast_mixed_count"],
"images_without_blast_count": patient_counts["images_without_blast_count"],
"class_mapping_size": len(class_mapping),
"class_mapping_variant_id": class_mapping_variant_id,
}
)
dataset_index_dir = output_root / "dataset_index"
reports_dir = output_root / "reports"
patient_inventory_path = dataset_index_dir / "patient_inventory.csv"
image_index_path = dataset_index_dir / "image_index.parquet"
object_index_path = dataset_index_dir / "object_index.parquet"
class_mapping_path = dataset_index_dir / "class_mapping.json"
mask_coverage_report_path = dataset_index_dir / "mask_coverage_report.csv"
runtime_report_path = reports_dir / "data_audit_runtime.md"
write_csv(patient_inventory_rows, patient_inventory_path)
write_parquet(image_rows, image_index_path)
write_parquet(object_rows, object_index_path)
write_json(class_mapping_payload, class_mapping_path)
write_csv(image_rows, mask_coverage_report_path)
summary = summarize_protocol(
patient_inventory_rows=patient_inventory_rows,
image_rows=image_rows,
object_rows=object_rows,
raw_top_level_dirs_data=len(data_dirs),
raw_top_level_dirs_masks=len(mask_dirs),
)
build_runtime_report(
summary=summary,
output_paths={
"patient_inventory_csv": str(patient_inventory_path),
"image_index_parquet": str(image_index_path),
"object_index_parquet": str(object_index_path),
"class_mapping_json": str(class_mapping_path),
"mask_coverage_report_csv": str(mask_coverage_report_path),
},
path=runtime_report_path,
)
return {
"summary": summary,
"paths": {
"patient_inventory_csv": str(patient_inventory_path),
"image_index_parquet": str(image_index_path),
"object_index_parquet": str(object_index_path),
"class_mapping_json": str(class_mapping_path),
"mask_coverage_report_csv": str(mask_coverage_report_path),
"data_audit_runtime_md": str(runtime_report_path),
},
}
def main() -> None:
args = parse_args()
excludes = set(args.exclude_from_main_protocol or []) | MAIN_PROTOCOL_EXCLUDES
result = build_index_artifacts(
data_root=args.data_root,
masks_root=args.masks_root,
output_root=args.output_root,
main_protocol_excludes=excludes,
)
print(json.dumps(result["summary"], ensure_ascii=False, indent=2))
print(json.dumps(result["paths"], ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
src/dataset/build_patient_summary.py
+102
View File
@@ -0,0 +1,102 @@
from __future__ import annotations
import argparse
from pathlib import Path
from utils.project import DATASET_INDEX_ROOT, SPLITS_ROOT, REPORTS_ROOT, ensure_output_layout, write_markdown
import pandas as pd
PATIENT_INVENTORY_PATH = DATASET_INDEX_ROOT / "patient_inventory.csv"
IMAGE_INDEX_PATH = DATASET_INDEX_ROOT / "image_index.parquet"
OBJECT_INDEX_PATH = DATASET_INDEX_ROOT / "object_index.parquet"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build patient summary for A2.")
parser.add_argument("--patient-inventory", type=Path, default=PATIENT_INVENTORY_PATH)
parser.add_argument("--image-index", type=Path, default=IMAGE_INDEX_PATH)
parser.add_argument("--object-index", type=Path, default=OBJECT_INDEX_PATH)
return parser.parse_args()
def build_patient_summary(patient_inventory_path: Path, image_index_path: Path, object_index_path: Path) -> Path:
ensure_output_layout()
inventory_df = pd.read_csv(patient_inventory_path)
image_df = pd.read_parquet(image_index_path)
object_df = pd.read_parquet(object_index_path)
inventory_df = inventory_df.loc[inventory_df["include_in_main_protocol"].eq(True)].copy()
image_df = image_df.loc[image_df["include_in_main_protocol"].eq(True)].copy()
object_df = object_df.loc[object_df["include_in_main_protocol"].eq(True)].copy()
object_grouped = (
object_df.groupby("patient_id")
.agg(
objects_count=("object_id_within_image", "size"),
blast_objects_count=("is_blast", "sum"),
mask_missing_objects_count=("has_mask", lambda s: int((~s).sum())),
quality_or_artifact_objects_count=("is_quality_or_artifact", "sum"),
)
.reset_index()
)
object_grouped["blast_fraction"] = object_grouped["blast_objects_count"] / object_grouped["objects_count"]
object_grouped["mask_missing_fraction"] = (
object_grouped["mask_missing_objects_count"] / object_grouped["objects_count"]
)
image_grouped = (
image_df.loc[image_df["has_image"].eq(True)]
.groupby("patient_id")
.agg(
images_count=("image_id", "nunique"),
images_with_blast_count=("contains_blast", "sum"),
images_without_masks_count=("has_any_mask", lambda s: int((~s).sum())),
)
.reset_index()
)
image_grouped["images_with_blast_fraction"] = (
image_grouped["images_with_blast_count"] / image_grouped["images_count"]
)
image_grouped["images_without_masks_fraction"] = (
image_grouped["images_without_masks_count"] / image_grouped["images_count"]
)
patient_summary_df = (
inventory_df[["patient_id", "class_mapping_variant_id"]]
.merge(object_grouped, on="patient_id", how="left")
.merge(image_grouped, on="patient_id", how="left")
.fillna(0)
)
patient_summary_df["patient_has_blast"] = patient_summary_df["blast_objects_count"] > 0
patient_summary_df["blast_stratum"] = patient_summary_df["patient_has_blast"].map({True: "blast", False: "non_blast"})
patient_summary_df = patient_summary_df.sort_values(["patient_has_blast", "blast_fraction", "patient_id"], ascending=[False, False, True])
output_path = SPLITS_ROOT / "patient_summary.csv"
patient_summary_df.to_csv(output_path, index=False)
report_lines = [
"# A2 Patient Summary QC Report",
"",
f"Generated at: `{pd.Timestamp.utcnow().isoformat()}`",
"",
f"- Patients in main protocol: `{patient_summary_df['patient_id'].nunique()}`",
f"- Patients with blast: `{int(patient_summary_df['patient_has_blast'].sum())}`",
f"- Patients without blast: `{int((~patient_summary_df['patient_has_blast']).sum())}`",
f"- Mean blast fraction: `{patient_summary_df['blast_fraction'].mean():.6f}`",
f"- Mean mask missing fraction: `{patient_summary_df['mask_missing_fraction'].mean():.6f}`",
]
write_markdown(REPORTS_ROOT / "a2_patient_summary_qc.md", report_lines)
return output_path
def main() -> None:
args = parse_args()
output_path = build_patient_summary(args.patient_inventory, args.image_index, args.object_index)
print(output_path)
if __name__ == "__main__":
main()
src/dataset/build_subsets.py
+206
View File
@@ -0,0 +1,206 @@
from __future__ import annotations
import argparse
from pathlib import Path
from utils.project import DATASET_INDEX_ROOT, REPORTS_ROOT, TABLES_ROOT, ensure_output_layout, write_markdown
import numpy as np
import pandas as pd
OBJECT_INDEX_PATH = DATASET_INDEX_ROOT / "object_index.parquet"
IMAGE_INDEX_PATH = DATASET_INDEX_ROOT / "image_index.parquet"
SUBSET_RULES = {
"full_realistic": lambda df: df["include_in_main_protocol"].eq(True) & df["geometry_valid"].eq(True),
"clean_cell": lambda df: df["include_in_main_protocol"].eq(True)
& df["geometry_valid"].eq(True)
& ~df["class_id"].isin([12, 14]),
"strict_morphology": lambda df: df["include_in_main_protocol"].eq(True)
& df["geometry_valid"].eq(True)
& ~df["class_id"].isin([12, 14, 15]),
"mask_ready": lambda df: df["include_in_main_protocol"].eq(True)
& df["geometry_valid"].eq(True)
& df["has_mask"].eq(True),
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build A1 subsets for experiment E01.")
parser.add_argument("--object-index", type=Path, default=OBJECT_INDEX_PATH)
parser.add_argument("--image-index", type=Path, default=IMAGE_INDEX_PATH)
return parser.parse_args()
def _geometry_issue_columns(object_df: pd.DataFrame) -> pd.DataFrame:
issue_df = pd.DataFrame(index=object_df.index)
issue_df["finite"] = np.isfinite(object_df["x_center_norm"]) & np.isfinite(object_df["y_center_norm"])
issue_df["finite"] &= np.isfinite(object_df["width_norm"]) & np.isfinite(object_df["height_norm"])
issue_df["x_in_range"] = object_df["x_center_norm"].between(0, 1, inclusive="both")
issue_df["y_in_range"] = object_df["y_center_norm"].between(0, 1, inclusive="both")
issue_df["width_in_range"] = object_df["width_norm"].between(0, 1, inclusive="right")
issue_df["height_in_range"] = object_df["height_norm"].between(0, 1, inclusive="right")
return issue_df
def _geometry_issue_label(row: pd.Series) -> str:
issues: list[str] = []
if not bool(row["finite"]):
issues.append("non_finite")
if not bool(row["x_in_range"]):
issues.append("x_out_of_range")
if not bool(row["y_in_range"]):
issues.append("y_out_of_range")
if not bool(row["width_in_range"]):
issues.append("width_out_of_range")
if not bool(row["height_in_range"]):
issues.append("height_out_of_range")
return ",".join(issues) if issues else "ok"
def _unique_image_count(frame: pd.DataFrame) -> int:
if frame.empty:
return 0
return int(frame[["patient_id", "image_id"]].drop_duplicates().shape[0])
def build_subsets(object_index_path: Path, image_index_path: Path) -> dict[str, object]:
ensure_output_layout()
object_df = pd.read_parquet(object_index_path)
image_df = pd.read_parquet(image_index_path)
geometry_checks = _geometry_issue_columns(object_df)
object_df = object_df.copy()
object_df["geometry_valid"] = geometry_checks.all(axis=1)
object_df["geometry_issue"] = geometry_checks.apply(_geometry_issue_label, axis=1)
subset_paths: dict[str, str] = {}
summary_rows: list[dict[str, object]] = []
zero_object_patients = sorted(
set(object_df.loc[object_df["include_in_main_protocol"].eq(True), "patient_id"].unique().tolist())
)
invalid_geometry_df = object_df.loc[
object_df["include_in_main_protocol"].eq(True) & object_df["geometry_valid"].ne(True)
].copy()
invalid_geometry_path = DATASET_INDEX_ROOT / "invalid_geometry_objects.csv"
invalid_geometry_df.to_csv(invalid_geometry_path, index=False)
for subset_name, predicate in SUBSET_RULES.items():
subset_df = object_df.loc[predicate(object_df)].copy()
subset_path = DATASET_INDEX_ROOT / f"subset_{subset_name}.parquet"
subset_df.to_parquet(subset_path, index=False)
subset_paths[subset_name] = str(subset_path)
patient_count = subset_df["patient_id"].nunique()
image_count = _unique_image_count(subset_df)
class_counts = subset_df.groupby(["class_id", "class_name"]).size().reset_index(name="count")
top_classes = (
class_counts.sort_values(["count", "class_id"], ascending=[False, True])
.head(10)
.to_dict("records")
)
summary_rows.append(
{
"subset_name": subset_name,
"object_count": int(len(subset_df)),
"patient_count": int(patient_count),
"image_count": int(image_count),
"has_bad_cells": bool(subset_df["class_id"].eq(12).any()),
"has_artifacts": bool(subset_df["class_id"].eq(14).any()),
"has_gumpricht_shadows": bool(subset_df["class_id"].eq(15).any()),
"mask_ready_ratio": float(subset_df["has_mask"].mean()) if len(subset_df) else 0.0,
"top_classes": top_classes,
}
)
summary_df = pd.DataFrame(summary_rows)
summary_df.to_csv(TABLES_ROOT / "subset_summary.csv", index=False)
class_distribution_rows: list[dict[str, object]] = []
for subset_name, subset_path in subset_paths.items():
subset_df = pd.read_parquet(subset_path)
grouped = subset_df.groupby(["class_id", "class_name"]).size().reset_index(name="count")
grouped["subset_name"] = subset_name
class_distribution_rows.extend(grouped.to_dict("records"))
pd.DataFrame(class_distribution_rows).to_csv(TABLES_ROOT / "subset_class_distribution.csv", index=False)
excluded_images = _unique_image_count(image_df.loc[image_df["include_in_main_protocol"].eq(False)].copy())
patient_inventory_df = pd.read_csv(DATASET_INDEX_ROOT / "patient_inventory.csv")
main_protocol_patients = patient_inventory_df.loc[
patient_inventory_df["include_in_main_protocol"].eq(True), "patient_id"
].tolist()
zero_object_patient_dirs = sorted(set(main_protocol_patients) - set(zero_object_patients))
report_lines = [
"# A1 Subset QC Report",
"",
f"Generated at: `{pd.Timestamp.utcnow().isoformat()}`",
"",
"## Summary",
"",
f"- Source object index: `{object_index_path}`",
f"- Source image index: `{image_index_path}`",
f"- Images excluded from main protocol: `{excluded_images}`",
f"- Main protocol patient directories: `{len(main_protocol_patients)}`",
f"- Patient directories with zero labeled objects: `{len(zero_object_patient_dirs)}`",
f"- Zero-object patient directories: `{', '.join(zero_object_patient_dirs) if zero_object_patient_dirs else 'none'}`",
f"- Main-protocol objects with invalid geometry: `{len(invalid_geometry_df)}`",
f"- Invalid-geometry artifact: `{invalid_geometry_path}`",
"",
"## Subset Checks",
"",
"| Subset | Objects | Patients | Images | Has bad cells | Has artifacts | Has gumpricht shadows | Mask-ready ratio |",
"|---|---:|---:|---:|---|---|---|---:|",
]
for row in summary_rows:
report_lines.append(
f"| {row['subset_name']} | {row['object_count']} | {row['patient_count']} | {row['image_count']} | "
f"{row['has_bad_cells']} | {row['has_artifacts']} | {row['has_gumpricht_shadows']} | {row['mask_ready_ratio']:.4f} |"
)
report_lines.extend(
[
"",
"## Geometry validation",
"",
"| Issue | Count |",
"|---|---:|",
]
)
issue_counts = (
invalid_geometry_df["geometry_issue"].value_counts().sort_values(ascending=False).to_dict()
if len(invalid_geometry_df)
else {}
)
for issue, count in issue_counts.items():
report_lines.append(f"| {issue} | {int(count)} |")
report_lines.extend(
[
"",
"Note:",
"Object-level subsets can contain fewer patient ids than the main protocol if a patient directory exists but contains zero labeled objects. "
"This is currently expected for the zero-object directories listed above.",
"All A1 subsets additionally exclude objects with non-finite or out-of-range bounding-box geometry; these rows are preserved in invalid_geometry_objects.csv for auditability.",
]
)
write_markdown(REPORTS_ROOT / "a1_subset_qc.md", report_lines)
return {
"subset_paths": subset_paths,
"invalid_geometry_path": str(invalid_geometry_path),
"summary_path": str(TABLES_ROOT / "subset_summary.csv"),
"class_distribution_path": str(TABLES_ROOT / "subset_class_distribution.csv"),
"qc_report_path": str(REPORTS_ROOT / "a1_subset_qc.md"),
}
def main() -> None:
args = parse_args()
result = build_subsets(args.object_index, args.image_index)
print(result)
if __name__ == "__main__":
main()
src/dataset/build_tasks.py
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from utils.project import DATASET_INDEX_ROOT, REPORTS_ROOT, SPLITS_ROOT, TABLES_ROOT, ensure_output_layout, write_markdown
import pandas as pd
SPLIT_JSON_PATH = SPLITS_ROOT / "split_v1.json"
TASK_SPECS = {
"a41_clean_binary": "subset_clean_cell.parquet",
"a42_strict_binary": "subset_strict_morphology.parquet",
"a43_full_realistic_binary": "subset_full_realistic.parquet",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build binary task datasets for A4.")
parser.add_argument("--split-json", type=Path, default=SPLIT_JSON_PATH)
return parser.parse_args()
def build_tasks(split_json_path: Path) -> dict[str, str]:
ensure_output_layout()
split_payload = json.loads(split_json_path.read_text(encoding="utf-8"))
patient_to_split = {patient_id: "train" for patient_id in split_payload["train_patients"]}
patient_to_split.update({patient_id: "val" for patient_id in split_payload["validation_patients"]})
patient_to_split.update({patient_id: "test" for patient_id in split_payload["test_patients"]})
summary_rows: list[dict[str, object]] = []
output_paths: dict[str, str] = {}
for task_name, subset_file in TASK_SPECS.items():
subset_path = DATASET_INDEX_ROOT / subset_file
task_df = pd.read_parquet(subset_path).copy()
task_df["target_binary"] = task_df["class_id"].eq(7).astype(int)
task_df["split_name"] = task_df["patient_id"].map(patient_to_split)
task_output_path = DATASET_INDEX_ROOT / f"task_{task_name}.parquet"
task_df.to_parquet(task_output_path, index=False)
output_paths[task_name] = str(task_output_path)
class_grouped = (
task_df.groupby(["split_name", "target_binary"])
.size()
.reset_index(name="count")
.sort_values(["split_name", "target_binary"])
)
split_counts = {
row["split_name"]: {
"positive": int(class_grouped.loc[(class_grouped["split_name"] == row["split_name"]) & (class_grouped["target_binary"] == 1), "count"].sum()),
"negative": int(class_grouped.loc[(class_grouped["split_name"] == row["split_name"]) & (class_grouped["target_binary"] == 0), "count"].sum()),
}
for row in class_grouped.to_dict("records")
}
hard_negative_rows = (
task_df.loc[task_df["target_binary"].eq(0)]
.groupby(["class_id", "class_name"])
.size()
.reset_index(name="count")
.sort_values(["count", "class_id"], ascending=[False, True])
.head(10)
)
summary_rows.append(
{
"task_name": task_name,
"task_path": str(task_output_path),
"object_count": int(len(task_df)),
"positive_count": int(task_df["target_binary"].sum()),
"negative_count": int((1 - task_df["target_binary"]).sum()),
"positive_fraction": float(task_df["target_binary"].mean()),
"hard_negative_top10": json.dumps(hard_negative_rows.to_dict("records"), ensure_ascii=False),
"train_positive": split_counts.get("train", {}).get("positive", 0),
"train_negative": split_counts.get("train", {}).get("negative", 0),
"val_positive": split_counts.get("val", {}).get("positive", 0),
"val_negative": split_counts.get("val", {}).get("negative", 0),
"test_positive": split_counts.get("test", {}).get("positive", 0),
"test_negative": split_counts.get("test", {}).get("negative", 0),
}
)
summary_df = pd.DataFrame(summary_rows)
summary_df.to_csv(TABLES_ROOT / "task_prevalence_summary.csv", index=False)
report_lines = [
"# A4 Task Definition Report",
"",
f"Generated at: `{pd.Timestamp.utcnow().isoformat()}`",
"",
"## Task summary",
"",
"| Task | Objects | Positive | Negative | Positive fraction |",
"|---|---:|---:|---:|---:|",
]
for row in summary_rows:
report_lines.append(
f"| {row['task_name']} | {row['object_count']} | {row['positive_count']} | {row['negative_count']} | {row['positive_fraction']:.6f} |"
)
write_markdown(REPORTS_ROOT / "task_definition_report.md", report_lines)
output_paths["task_prevalence_summary_csv"] = str(TABLES_ROOT / "task_prevalence_summary.csv")
output_paths["task_definition_report_md"] = str(REPORTS_ROOT / "task_definition_report.md")
return output_paths
def main() -> None:
args = parse_args()
result = build_tasks(args.split_json)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
src/dataset/class_mapping.py
+41
View File
@@ -0,0 +1,41 @@
from __future__ import annotations
import re
from pathlib import Path
CLASS_WITH_ID_RE = re.compile(r"^\s*(?P<class_id>\d+(?:\.0+)?)\s+(?P<class_name>.+?)\s*$")
def parse_predefined_classes(path: Path) -> dict[int, str]:
"""Parse predefined_classes.txt in either indexed or line-only format."""
mapping: dict[int, str] = {}
next_index = 0
for raw_line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
line = raw_line.strip()
if not line:
continue
match = CLASS_WITH_ID_RE.match(line)
if match:
class_id = int(float(match.group("class_id")))
class_name = match.group("class_name").strip()
else:
class_id = next_index
class_name = line
mapping[class_id] = class_name
next_index = max(next_index, class_id + 1)
return mapping
def mapping_signature(mapping: dict[int, str]) -> tuple[tuple[int, str], ...]:
"""Return a hashable signature for a class mapping."""
return tuple(sorted(mapping.items()))
def normalize_mapping_for_json(mapping: dict[int, str]) -> dict[str, str]:
"""Convert class mapping keys to strings for JSON serialization."""
return {str(class_id): class_name for class_id, class_name in sorted(mapping.items())}
src/dataset/make_group_splits.py
+161
View File
@@ -0,0 +1,161 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from sklearn.model_selection import train_test_split
from utils.project import REPORTS_ROOT, SPLITS_ROOT, TABLES_ROOT, ensure_output_layout, write_json, write_markdown
import pandas as pd
PATIENT_SUMMARY_PATH = SPLITS_ROOT / "patient_summary.csv"
OBJECT_INDEX_PATH = Path(__file__).resolve().parents[2] / "outputs" / "dataset_index" / "object_index.parquet"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Create patient-level split_v1 for A2.")
parser.add_argument("--patient-summary", type=Path, default=PATIENT_SUMMARY_PATH)
parser.add_argument("--object-index", type=Path, default=OBJECT_INDEX_PATH)
parser.add_argument("--random-state", type=int, default=42)
return parser.parse_args()
def _stratified_patient_split(patient_summary_df: pd.DataFrame, random_state: int) -> tuple[list[str], list[str], list[str]]:
patient_ids = patient_summary_df["patient_id"].to_numpy()
strata = patient_summary_df["patient_has_blast"].astype(int).to_numpy()
train_ids, temp_ids, train_y, temp_y = train_test_split(
patient_ids,
strata,
test_size=0.30,
random_state=random_state,
stratify=strata,
)
val_ids, test_ids = train_test_split(
temp_ids,
test_size=0.50,
random_state=random_state,
stratify=temp_y,
)
return sorted(train_ids.tolist()), sorted(val_ids.tolist()), sorted(test_ids.tolist())
def _split_image_counts(object_df: pd.DataFrame) -> pd.Series:
image_counts = (
object_df[["split_name", "patient_id", "image_id"]]
.drop_duplicates()
.groupby("split_name")
.size()
)
return image_counts.astype(int)
def make_group_splits(patient_summary_path: Path, object_index_path: Path, random_state: int = 42) -> dict[str, str]:
ensure_output_layout()
patient_summary_df = pd.read_csv(patient_summary_path)
object_df = pd.read_parquet(object_index_path)
object_df = object_df.loc[object_df["include_in_main_protocol"].eq(True)].copy()
train_ids, val_ids, test_ids = _stratified_patient_split(patient_summary_df, random_state=random_state)
split_payload = {
"version": "split_v1",
"random_state": random_state,
"train_patients": train_ids,
"validation_patients": val_ids,
"test_patients": test_ids,
}
split_json_path = SPLITS_ROOT / "split_v1.json"
write_json(split_json_path, split_payload)
for name, values in [("train", train_ids), ("val", val_ids), ("test", test_ids)]:
path = SPLITS_ROOT / f"split_v1_{name}_patients.txt"
path.write_text("\n".join(values) + "\n", encoding="utf-8")
patient_to_split = {patient_id: "train" for patient_id in train_ids}
patient_to_split.update({patient_id: "val" for patient_id in val_ids})
patient_to_split.update({patient_id: "test" for patient_id in test_ids})
object_df["split_name"] = object_df["patient_id"].map(patient_to_split)
class_distribution = (
object_df.groupby(["split_name", "class_id", "class_name"])
.size()
.reset_index(name="count")
.sort_values(["split_name", "count", "class_id"], ascending=[True, False, True])
)
class_distribution.to_csv(TABLES_ROOT / "class_distribution_by_split.csv", index=False)
split_summary = (
object_df.groupby("split_name")
.agg(
object_count=("object_id_within_image", "size"),
blast_count=("is_blast", "sum"),
active_patient_count=("patient_id", "nunique"),
)
.reset_index()
)
split_summary["image_count"] = split_summary["split_name"].map(_split_image_counts(object_df))
split_summary["blast_fraction"] = split_summary["blast_count"] / split_summary["object_count"]
split_summary["patient_count"] = split_summary["split_name"].map(
{"train": len(train_ids), "val": len(val_ids), "test": len(test_ids)}
)
split_summary["zero_object_patient_count"] = split_summary["patient_count"] - split_summary["active_patient_count"]
split_summary.to_csv(TABLES_ROOT / "split_summary.csv", index=False)
report_lines = [
"# A2 Split QC Report",
"",
f"Generated at: `{pd.Timestamp.utcnow().isoformat()}`",
"",
f"- Random state: `{random_state}`",
f"- Train patients: `{len(train_ids)}`",
f"- Validation patients: `{len(val_ids)}`",
f"- Test patients: `{len(test_ids)}`",
"",
"## Split-level summary",
"",
"| Split | Patients in split list | Patients with objects | Zero-object patients | Images | Objects | Blast objects | Blast fraction |",
"|---|---:|---:|---:|---:|---:|---:|---:|",
]
for row in split_summary.to_dict("records"):
report_lines.append(
f"| {row['split_name']} | {int(row['patient_count'])} | {int(row['active_patient_count'])} | {int(row['zero_object_patient_count'])} | "
f"{int(row['image_count'])} | {int(row['object_count'])} | {int(row['blast_count'])} | {float(row['blast_fraction']):.6f} |"
)
intersections = {
"train_val": sorted(set(train_ids) & set(val_ids)),
"train_test": sorted(set(train_ids) & set(test_ids)),
"val_test": sorted(set(val_ids) & set(test_ids)),
}
report_lines.extend(
[
"",
"## Intersection checks",
"",
f"- train ∩ val: `{len(intersections['train_val'])}`",
f"- train ∩ test: `{len(intersections['train_test'])}`",
f"- val ∩ test: `{len(intersections['val_test'])}`",
]
)
write_markdown(REPORTS_ROOT / "a2_split_qc.md", report_lines)
return {
"split_json": str(split_json_path),
"split_summary_csv": str(TABLES_ROOT / "split_summary.csv"),
"class_distribution_csv": str(TABLES_ROOT / "class_distribution_by_split.csv"),
"qc_report_md": str(REPORTS_ROOT / "a2_split_qc.md"),
}
def main() -> None:
args = parse_args()
result = make_group_splits(args.patient_summary, args.object_index, args.random_state)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
src/dataset/parse_detection_labels.py
+39
View File
@@ -0,0 +1,39 @@
from __future__ import annotations
from pathlib import Path
def normalize_class_id(raw_class_id: str) -> tuple[int, bool]:
"""Normalize class id and report whether it was stored as a float-like token."""
class_id = int(float(raw_class_id))
was_float_like = raw_class_id.strip() != str(class_id)
return class_id, was_float_like
def parse_label_file(label_path: Path) -> list[dict[str, object]]:
"""Parse YOLO-like detection labels into structured rows."""
rows: list[dict[str, object]] = []
for object_id, raw_line in enumerate(
line for line in label_path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()
):
parts = raw_line.split()
if len(parts) < 5:
raise ValueError(f"Malformed detection line in {label_path}: {raw_line!r}")
class_id, class_id_was_float = normalize_class_id(parts[0])
rows.append(
{
"object_id_within_image": object_id,
"raw_class_id": parts[0],
"class_id": class_id,
"class_id_was_float": class_id_was_float,
"x_center_norm": float(parts[1]),
"y_center_norm": float(parts[2]),
"width_norm": float(parts[3]),
"height_norm": float(parts[4]),
"raw_line": raw_line,
}
)
return rows
src/dataset/parse_masks.py
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import re
from pathlib import Path
MASK_FILENAME_RE = re.compile(
r"^(?P<image_id>.+?)_class_(?P<class_id>[^_]+)_obj_(?P<object_id>\d+)\.png$"
)
def parse_mask_filename(mask_path: Path) -> dict[str, object] | None:
"""Parse mask filename into image id, class id and object index."""
match = MASK_FILENAME_RE.match(mask_path.name)
if not match:
return None
return {
"image_id": match.group("image_id"),
"class_id": int(float(match.group("class_id"))),
"object_id_within_image": int(match.group("object_id")),
"mask_path": str(mask_path.resolve()),
}
def build_mask_index(mask_dir: Path) -> dict[str, list[dict[str, object]]]:
"""Build an image_id -> sorted mask entries index."""
index: dict[str, list[dict[str, object]]] = {}
if not mask_dir.exists():
return index
for mask_path in sorted(mask_dir.glob("*.png")):
parsed = parse_mask_filename(mask_path)
if parsed is None:
continue
image_id = str(parsed["image_id"])
index.setdefault(image_id, []).append(parsed)
for image_id, entries in index.items():
entries.sort(key=lambda item: int(item["object_id_within_image"]))
return index