Initial commit: project structure, scripts and documentation
This commit is contained in:
@@ -0,0 +1,518 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install pandas"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install pillow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install torch==2.6.0 torchaudio==2.6.0 torchvision==0.21.0 --index-url https://download.pytorch.org/whl/cu124"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#!/usr/bin/env python3\n",
|
||||
"# =============================================================================\n",
|
||||
"# FeatureService Client — Hybrid Version\n",
|
||||
"# Батчинг + поллинг статуса + пост-обработка CSV\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import csv\n",
|
||||
"import time\n",
|
||||
"import requests\n",
|
||||
"import pandas as pd\n",
|
||||
"from PIL import Image\n",
|
||||
"import numpy as np\n",
|
||||
"import io\n",
|
||||
"import re\n",
|
||||
"from http import HTTPStatus\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# CONFIG\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"# Сервер\n",
|
||||
"#SERVER_IP = \"85.143.113.40\"\n",
|
||||
"SERVER_IP = \"192.168.203.91\"\n",
|
||||
"#SERVER_IP = \"127.0.0.1\"\n",
|
||||
"SERVER_PORT = 14407\n",
|
||||
"BASE_URL = f\"http://{SERVER_IP}:{SERVER_PORT}\"\n",
|
||||
"\n",
|
||||
"# Пути\n",
|
||||
"DIR_PATH = \"../data/\"\n",
|
||||
"RESEARCH_DIR = \"prep/\"\n",
|
||||
"PATH_IMAGES = DIR_PATH + RESEARCH_DIR + \"images\"\n",
|
||||
"PATH_MASKS = DIR_PATH + RESEARCH_DIR + \"images_results/masks_sam\"\n",
|
||||
"PATH_CSV = DIR_PATH + RESEARCH_DIR + \"csv\"\n",
|
||||
"PATH_CLASSES = DIR_PATH + RESEARCH_DIR + \"predefined_classes.txt\"\n",
|
||||
"\n",
|
||||
"# Форматы\n",
|
||||
"IMAGE_EXT = \".jpg\"\n",
|
||||
"MASK_EXT = \".png\"\n",
|
||||
"\n",
|
||||
"# Параметры обработки\n",
|
||||
"BATCH_SIZE = 2 # Сколько задач отправлять параллельно\n",
|
||||
"POLL_INTERVAL = 1 # Интервал опроса статуса (сек)\n",
|
||||
"POLL_TIMEOUT = 300 # Максимальное время ожидания задачи (сек)\n",
|
||||
"REQUEST_TIMEOUT = 120 # Таймаут HTTP-запроса (сек)\n",
|
||||
"DELAY_BETWEEN_TASKS = 0.3 # Пауза между задачами внутри батча\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# GLOBAL STATE\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"in_progress = [] # Очередь задач: [(image, mask, task_id), ...]\n",
|
||||
"processed = [] # Успешно обработанные: [(image, mask, task_id, output_csv), ...]\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# UTILS\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"def ensure_dirs():\n",
|
||||
" \"\"\"Создаёт необходимые директории\"\"\"\n",
|
||||
" for d in [PATH_IMAGES, PATH_MASKS, PATH_CSV]:\n",
|
||||
" os.makedirs(d, exist_ok=True)\n",
|
||||
"\n",
|
||||
"def load_classes():\n",
|
||||
" \"\"\"Загружает словарь классов из файла\"\"\"\n",
|
||||
" classes = {}\n",
|
||||
" try:\n",
|
||||
" with open(PATH_CLASSES, \"r\", encoding=\"utf-8\") as f:\n",
|
||||
" for line in f:\n",
|
||||
" line = line.strip()\n",
|
||||
" if not line:\n",
|
||||
" continue\n",
|
||||
" parts = line.split(maxsplit=1)\n",
|
||||
" if len(parts) == 2:\n",
|
||||
" classes[int(parts[0])] = parts[1].strip()\n",
|
||||
" except FileNotFoundError:\n",
|
||||
" print(f\"⚠️ {PATH_CLASSES} не найден, метки будут 'Unknown'\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"⚠️ Ошибка загрузки классов: {e}\")\n",
|
||||
" return classes\n",
|
||||
"\n",
|
||||
"def get_label(filename, classes):\n",
|
||||
" \"\"\"Извлекает метку из имени файла: _class_7_ → класс 7 → название\"\"\"\n",
|
||||
" match = re.search(r'_class_(\\d+)_', filename)\n",
|
||||
" if match:\n",
|
||||
" class_id = int(match.group(1))\n",
|
||||
" return classes.get(class_id, f\"Class_{class_id}\")\n",
|
||||
" return \"Unknown\"\n",
|
||||
"\n",
|
||||
"def normalize_mask(img):\n",
|
||||
" \"\"\"Нормализует маску: все ненулевые пиксели → 1 (для PyRadiomics)\"\"\"\n",
|
||||
" arr = np.array(img)\n",
|
||||
" arr[arr > 0] = 1\n",
|
||||
" return Image.fromarray(arr.astype(np.uint8), mode='L')\n",
|
||||
"\n",
|
||||
"def match_mask_to_image(image_short_name, masks_list):\n",
|
||||
" \"\"\"\n",
|
||||
" Точное сопоставление маски с изображением.\n",
|
||||
" Ищет маски, начинающиеся с имени изображения.\n",
|
||||
" \"\"\"\n",
|
||||
" matched = []\n",
|
||||
" for mask in masks_list:\n",
|
||||
" if mask.startswith(image_short_name) and mask.endswith(MASK_EXT):\n",
|
||||
" matched.append(mask)\n",
|
||||
" return matched\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# CORE: MODEL COMMUNICATION\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"def launch_model(short_name, image_name, mask_name):\n",
|
||||
" \"\"\"\n",
|
||||
" Отправляет изображение и маску на сервер.\n",
|
||||
" Возвращает task_id или None при ошибке.\n",
|
||||
" \"\"\"\n",
|
||||
" image_path = os.path.join(PATH_IMAGES, image_name)\n",
|
||||
" mask_path = os.path.join(PATH_MASKS, mask_name)\n",
|
||||
" \n",
|
||||
" if not os.path.exists(image_path):\n",
|
||||
" print(f\"✗ Файл не найден: {image_path}\")\n",
|
||||
" return None\n",
|
||||
" if not os.path.exists(mask_path):\n",
|
||||
" print(f\"✗ Файл не найден: {mask_path}\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
" print(f\"🔄 Отправка: {image_name} + {mask_name}\")\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" # Читаем изображение\n",
|
||||
" with open(image_path, 'rb') as img_f:\n",
|
||||
" image_bytes = img_f.read()\n",
|
||||
" \n",
|
||||
" # Нормализуем и читаем маску\n",
|
||||
" with Image.open(mask_path) as mask_img:\n",
|
||||
" normalized = normalize_mask(mask_img)\n",
|
||||
" buf = io.BytesIO()\n",
|
||||
" normalized.save(buf, format='PNG')\n",
|
||||
" buf.seek(0)\n",
|
||||
" mask_bytes = buf.read()\n",
|
||||
" \n",
|
||||
" # Формируем multipart/form-data с правильными MIME-типами\n",
|
||||
" files = [\n",
|
||||
" ('image', ('image.jpg', image_bytes, 'image/jpeg')),\n",
|
||||
" ('mask1', ('mask.png', mask_bytes, 'image/png')),\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" resp = requests.post(\n",
|
||||
" f'{BASE_URL}/texture/feature', \n",
|
||||
" files=files, \n",
|
||||
" timeout=REQUEST_TIMEOUT\n",
|
||||
" )\n",
|
||||
" resp.raise_for_status()\n",
|
||||
" \n",
|
||||
" task_id = resp.text.strip().strip('\"').strip(\"'\")\n",
|
||||
" if not task_id or task_id.lower() == 'error':\n",
|
||||
" print(f\"✗ Сервер вернул ошибку: {resp.text}\")\n",
|
||||
" return None\n",
|
||||
" \n",
|
||||
" print(f\"✓ Задача создана: {task_id}\")\n",
|
||||
" return task_id\n",
|
||||
" \n",
|
||||
" except requests.exceptions.RequestException as e:\n",
|
||||
" print(f\"✗ Сетевая ошибка: {e}\")\n",
|
||||
" return None\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Ошибка обработки: {e}\")\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
"def wait_for_result(task_id, mask_name):\n",
|
||||
" \"\"\"\n",
|
||||
" Поллинг статуса задачи до завершения.\n",
|
||||
" Сохраняет результат в CSV при успехе.\n",
|
||||
" Возвращает путь к сохранённому файлу или None.\n",
|
||||
" \"\"\"\n",
|
||||
" mask_base = os.path.splitext(mask_name)[0]\n",
|
||||
" output_csv = os.path.join(PATH_CSV, f\"{mask_base}.csv\")\n",
|
||||
" \n",
|
||||
" start_time = time.time()\n",
|
||||
" \n",
|
||||
" while True:\n",
|
||||
" # Проверка таймаута\n",
|
||||
" if time.time() - start_time > POLL_TIMEOUT:\n",
|
||||
" print(f\"✗ Таймаут ожидания для {task_id}\")\n",
|
||||
" return None\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" status_resp = requests.get(\n",
|
||||
" f'{BASE_URL}/texture/status?id={task_id}',\n",
|
||||
" timeout=REQUEST_TIMEOUT\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" if status_resp.status_code == HTTPStatus.OK:\n",
|
||||
" print(f\"✓ Задача {task_id} завершена, получаем результат...\")\n",
|
||||
" \n",
|
||||
" result_resp = requests.get(\n",
|
||||
" f'{BASE_URL}/texture/result?id={task_id}',\n",
|
||||
" timeout=REQUEST_TIMEOUT\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" if result_resp.status_code == 200 and 'id not found' not in result_resp.text.lower():\n",
|
||||
" os.makedirs(PATH_CSV, exist_ok=True)\n",
|
||||
" with open(output_csv, \"w\", encoding=\"utf-8\") as f:\n",
|
||||
" f.write(result_resp.text)\n",
|
||||
" print(f\"💾 Сохранено: {output_csv}\")\n",
|
||||
" return output_csv\n",
|
||||
" else:\n",
|
||||
" print(f\"✗ Ошибка получения результата: {result_resp.text[:100]}\")\n",
|
||||
" return None\n",
|
||||
" else:\n",
|
||||
" # Задача ещё в процессе\n",
|
||||
" time.sleep(POLL_INTERVAL)\n",
|
||||
" \n",
|
||||
" except requests.exceptions.RequestException as e:\n",
|
||||
" print(f\"⚠️ Ошибка опроса статуса: {e}\")\n",
|
||||
" time.sleep(POLL_INTERVAL)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"✗ Неожиданная ошибка: {e}\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"def process_batch():\n",
|
||||
" \"\"\"\n",
|
||||
" Обрабатывает очередь in_progress: ждёт завершения всех задач и сохраняет результаты.\n",
|
||||
" \"\"\"\n",
|
||||
" global in_progress, processed\n",
|
||||
" \n",
|
||||
" if not in_progress:\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" print(f\"\\n⏳ Ожидание завершения {len(in_progress)} задач...\")\n",
|
||||
" \n",
|
||||
" for image_name, mask_name, task_id in in_progress:\n",
|
||||
" output_path = wait_for_result(task_id, mask_name)\n",
|
||||
" if output_path:\n",
|
||||
" processed.append((image_name, mask_name, task_id, output_path))\n",
|
||||
" \n",
|
||||
" in_progress.clear()\n",
|
||||
" print(\"✅ Батч обработан\\n\")\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# POST-PROCESSING: CSV MERGE & ENRICH\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"def delete_useless_columns(row):\n",
|
||||
" \"\"\"Удаляет служебные колонки PyRadiomics\"\"\"\n",
|
||||
" diagnostics_cols = [\n",
|
||||
" 'diagnostics_Versions_PyRadiomics', 'diagnostics_Versions_Numpy',\n",
|
||||
" 'diagnostics_Versions_SimpleITK', 'diagnostics_Versions_PyWavelet',\n",
|
||||
" 'diagnostics_Versions_Python', 'diagnostics_Configuration_Settings',\n",
|
||||
" 'diagnostics_Configuration_EnabledImageTypes', 'diagnostics_Image-original_Hash',\n",
|
||||
" 'diagnostics_Image-original_Dimensionality', 'diagnostics_Image-original_Spacing',\n",
|
||||
" 'diagnostics_Image-original_Size', 'diagnostics_Mask-original_CenterOfMassIndex',\n",
|
||||
" 'diagnostics_Mask-original_CenterOfMass', 'diagnostics_Mask-original_BoundingBox',\n",
|
||||
" 'diagnostics_Mask-original_Size', 'diagnostics_Mask-original_Hash',\n",
|
||||
" 'diagnostics_Mask-original_Spacing'\n",
|
||||
" ]\n",
|
||||
" for col in diagnostics_cols:\n",
|
||||
" row.pop(col, None)\n",
|
||||
" return row\n",
|
||||
"\n",
|
||||
"def merge_csv_files():\n",
|
||||
" \"\"\"Объединяет все CSV в result.csv\"\"\"\n",
|
||||
" csv_files = [f for f in os.listdir(PATH_CSV) \n",
|
||||
" if f.endswith('.csv') and f != 'result.csv']\n",
|
||||
" if not csv_files:\n",
|
||||
" print(\"⚠️ Нет CSV-файлов для объединения\")\n",
|
||||
" return None\n",
|
||||
" \n",
|
||||
" print(f\"🔗 Объединяю {len(csv_files)} файлов...\")\n",
|
||||
" all_rows = []\n",
|
||||
" \n",
|
||||
" for csv_file in csv_files:\n",
|
||||
" try:\n",
|
||||
" with open(os.path.join(PATH_CSV, csv_file), 'r', newline='', encoding='utf-8') as f:\n",
|
||||
" for row in csv.DictReader(f):\n",
|
||||
" row = delete_useless_columns(row)\n",
|
||||
" row['source_file'] = csv_file\n",
|
||||
" all_rows.append(row)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"⚠️ Ошибка чтения {csv_file}: {e}\")\n",
|
||||
" \n",
|
||||
" if not all_rows:\n",
|
||||
" print(\"⚠️ Нет данных для сохранения\")\n",
|
||||
" return None\n",
|
||||
" \n",
|
||||
" result_path = os.path.join(PATH_CSV, 'result.csv')\n",
|
||||
" pd.DataFrame(all_rows).to_csv(result_path, index=False, encoding='utf-8')\n",
|
||||
" print(f\"💾 Объединённый файл: {result_path}\")\n",
|
||||
" return result_path\n",
|
||||
"\n",
|
||||
"def add_label_and_path_to_cell(result_path=None):\n",
|
||||
" \"\"\"\n",
|
||||
" Добавляет label и path_to_cell ПЕРВЫМИ двумя столбцами в result.csv\n",
|
||||
" \"\"\"\n",
|
||||
" if result_path is None:\n",
|
||||
" result_path = os.path.join(PATH_CSV, 'result.csv')\n",
|
||||
" \n",
|
||||
" if not os.path.exists(result_path):\n",
|
||||
" print(f\"⚠️ Файл не найден: {result_path}\")\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" classes = load_classes()\n",
|
||||
" rows = []\n",
|
||||
" counter = 1\n",
|
||||
" \n",
|
||||
" print(\"🏷️ Добавляю label и path_to_cell...\")\n",
|
||||
" \n",
|
||||
" with open(result_path, 'r', newline='', encoding='utf-8') as f:\n",
|
||||
" reader = csv.DictReader(f)\n",
|
||||
" \n",
|
||||
" for row in reader:\n",
|
||||
" # Ищем имя файла в разных возможных полях\n",
|
||||
" filename = (row.get('name') or row.get('source_file') or \n",
|
||||
" row.get('path_to_file') or row.get('filename') or '')\n",
|
||||
" \n",
|
||||
" # Извлекаем label\n",
|
||||
" if filename:\n",
|
||||
" basename = os.path.splitext(os.path.basename(filename))[0]\n",
|
||||
" label = get_label(basename, classes)\n",
|
||||
" else:\n",
|
||||
" label = \"Unknown\"\n",
|
||||
" \n",
|
||||
" # Формируем path_to_cell: \"1_F0000001_class_7_obj_0000.png.csv\"\n",
|
||||
" source = row.get('source_file', os.path.basename(filename))\n",
|
||||
" path_to_cell = f\"{counter}_{source}\"\n",
|
||||
" \n",
|
||||
" # Создаём новый dict с label и path_to_cell первыми\n",
|
||||
" new_row = {'label': label, 'path_to_cell': path_to_cell}\n",
|
||||
" for key, value in row.items():\n",
|
||||
" if key not in ('label', 'path_to_cell'):\n",
|
||||
" new_row[key] = value\n",
|
||||
" \n",
|
||||
" rows.append(new_row)\n",
|
||||
" counter += 1\n",
|
||||
" \n",
|
||||
" if rows:\n",
|
||||
" final_columns = ['label', 'path_to_cell'] + [c for c in rows[0].keys() if c not in ['label', 'path_to_cell']]\n",
|
||||
" pd.DataFrame(rows).to_csv(\n",
|
||||
" result_path, \n",
|
||||
" index=False, \n",
|
||||
" encoding='utf-8',\n",
|
||||
" columns=final_columns\n",
|
||||
" )\n",
|
||||
" print(f\"✅ Готово: {result_path}\")\n",
|
||||
" return result_path\n",
|
||||
" else:\n",
|
||||
" print(\"⚠️ Нет строк для обработки\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# MAIN PROCESSING LOOP\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"def process_all_images():\n",
|
||||
" \"\"\"\n",
|
||||
" Основной цикл: сканирует изображения, сопоставляет маски,\n",
|
||||
" отправляет задачи батчами с поллингом статуса.\n",
|
||||
" \"\"\"\n",
|
||||
" global in_progress, counter\n",
|
||||
" \n",
|
||||
" ensure_dirs()\n",
|
||||
" \n",
|
||||
" # Сканируем файлы\n",
|
||||
" images = [f for f in os.listdir(PATH_IMAGES) if f.lower().endswith(IMAGE_EXT)]\n",
|
||||
" masks = [f for f in os.listdir(PATH_MASKS) if f.lower().endswith(MASK_EXT)]\n",
|
||||
" \n",
|
||||
" if not images:\n",
|
||||
" print(f\"⚠️ Нет изображений в {PATH_IMAGES}\")\n",
|
||||
" return\n",
|
||||
" if not masks:\n",
|
||||
" print(f\"⚠️ Нет масок в {PATH_MASKS}\")\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" print(f\"📁 Найдено: {len(images)} изображений, {len(masks)} масок\")\n",
|
||||
" \n",
|
||||
" counter = 0 # Счётчик задач в текущем батче\n",
|
||||
" \n",
|
||||
" for img in images:\n",
|
||||
" short = os.path.splitext(img)[0]\n",
|
||||
" img_masks = match_mask_to_image(short, masks)\n",
|
||||
" \n",
|
||||
" if not img_masks:\n",
|
||||
" print(f\"⚠️ Нет масок для {img}\")\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" for mask in img_masks:\n",
|
||||
" # Если батч заполнен — обрабатываем его\n",
|
||||
" if counter >= BATCH_SIZE:\n",
|
||||
" process_batch()\n",
|
||||
" counter = 0\n",
|
||||
" \n",
|
||||
" # Отправляем задачу\n",
|
||||
" task_id = launch_model(short, img, mask)\n",
|
||||
" if task_id:\n",
|
||||
" in_progress.append((img, mask, task_id))\n",
|
||||
" counter += 1\n",
|
||||
" time.sleep(DELAY_BETWEEN_TASKS) # Не перегружаем сервер\n",
|
||||
" \n",
|
||||
" # Обрабатываем оставшиеся задачи\n",
|
||||
" if in_progress:\n",
|
||||
" process_batch()\n",
|
||||
" \n",
|
||||
" print(f\"\\n🎯 Всего обработано: {len(processed)} задач\")\n",
|
||||
"\n",
|
||||
"# =============================================================================\n",
|
||||
"# ENTRY POINT\n",
|
||||
"# =============================================================================\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" print(f\"🚀 FeatureService Client (Hybrid)\")\n",
|
||||
" print(f\"🌐 Сервер: {BASE_URL}\")\n",
|
||||
" print(f\"📦 BATCH_SIZE: {BATCH_SIZE}, ⏱️ POLL_TIMEOUT: {POLL_TIMEOUT}s\\n\")\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" # 1. Обработка изображений\n",
|
||||
" print(\"=== [1/3] Отправка задач на сервер ===\")\n",
|
||||
" process_all_images()\n",
|
||||
" \n",
|
||||
" if not processed:\n",
|
||||
" print(\"⚠️ Нет успешно обработанных задач. Пропускаю пост-обработку.\")\n",
|
||||
" else:\n",
|
||||
" # 2. Объединение CSV\n",
|
||||
" print(\"\\n=== [2/3] Объединение результатов ===\")\n",
|
||||
" result_path = merge_csv_files()\n",
|
||||
" \n",
|
||||
" # 3. Добавление меток\n",
|
||||
" if result_path:\n",
|
||||
" print(\"\\n=== [3/3] Добавление label и path_to_cell ===\")\n",
|
||||
" add_label_and_path_to_cell(result_path)\n",
|
||||
" \n",
|
||||
" print(f\"\\n✨ Готово. Результат: {PATH_CSV}/result.csv\")\n",
|
||||
" \n",
|
||||
" except KeyboardInterrupt:\n",
|
||||
" print(\"\\n⚠️ Прервано пользователем\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"\\n✗ Критическая ошибка: {e}\")\n",
|
||||
" import traceback\n",
|
||||
" traceback.print_exc()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.13"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "553zXx91g7fz"
|
||||
},
|
||||
"source": [
|
||||
"Пример загрузки данных с сайта Breast Cancer Wisconsin (Original)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "JYb7HU1k9lFC",
|
||||
"outputId": "7a608a7c-9f85-4cc8-9b72-65a67381e960"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"# 1) Ссылка на датасет\n",
|
||||
"dataset_url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data\"\n",
|
||||
"\n",
|
||||
"# 2) Имена столбцов (по документации UCI)\n",
|
||||
"col_names = [\n",
|
||||
" \"Sample_code_number\",\n",
|
||||
" \"Clump_Thickness\",\n",
|
||||
" \"Uniformity_of_Cell_Size\",\n",
|
||||
" \"Uniformity_of_Cell_Shape\",\n",
|
||||
" \"Marginal_Adhesion\",\n",
|
||||
" \"Single_Epithelial_Cell_Size\",\n",
|
||||
" \"Bare_Nuclei\",\n",
|
||||
" \"Bland_Chromatin\",\n",
|
||||
" \"Normal_Nucleoli\",\n",
|
||||
" \"Mitoses\",\n",
|
||||
" \"Class\"\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# 3) Считываем csv\n",
|
||||
"df = pd.read_csv(\n",
|
||||
" dataset_url,\n",
|
||||
" names=col_names,\n",
|
||||
" na_values=[\"?\"],\n",
|
||||
" header=None\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 4) Быстрый осмотр\n",
|
||||
"print(df.head())\n",
|
||||
"print(df.info())\n",
|
||||
"\n",
|
||||
"# 5) Удалим строки с пропусками\n",
|
||||
"df.dropna(inplace=True)\n",
|
||||
"\n",
|
||||
"# 6) Разделим на признаки (X) и целевую переменную (y)\n",
|
||||
"y = df[\"Class\"]\n",
|
||||
"X = df.drop([\"Sample_code_number\", \"Class\"], axis=1)\n",
|
||||
"\n",
|
||||
"# 7) Разделим на train и test\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"\n",
|
||||
"# 8) Обучим модель (DecisionTree)\n",
|
||||
"clf = DecisionTreeClassifier(random_state=42)\n",
|
||||
"clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"# 9) Предсказание и оценка\n",
|
||||
"predictions = clf.predict(X_test)\n",
|
||||
"acc = accuracy_score(y_test, predictions)\n",
|
||||
"print(f\"Accuracy: {acc:.3f}\")\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2t5p4Mt7hKSw"
|
||||
},
|
||||
"source": [
|
||||
"Пример кода с requests + BeautifulSoup. Подставить нужную ссылку"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1L4ZnJL5iDMm"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 321
|
||||
},
|
||||
"id": "SqUDAUG0esKO",
|
||||
"outputId": "150062f1-72e8-4f8e-b006-12ef211609ab"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"from bs4 import BeautifulSoup\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"url = \"https://opendata.some-region.ru/datasets/interesting-table\"\n",
|
||||
"response = requests.get(url)\n",
|
||||
"soup = BeautifulSoup(response.text, \"html.parser\")\n",
|
||||
"\n",
|
||||
"# Предположим, на странице есть <table> с классом \"data-table\"\n",
|
||||
"table_html = soup.find(\"table\", {\"class\": \"data-table\"})\n",
|
||||
"\n",
|
||||
"# Преобразуем в DataFrame, если таблица более-менее стандартная\n",
|
||||
"df_list = pd.read_html(str(table_html)) # возвращает список df\n",
|
||||
"df = df_list[0] # берем первую таблицу\n",
|
||||
"\n",
|
||||
"df.head()\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "dl9b0hqOiFII"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"excel_url = \"https://data.gov.ru/files/dataset.xls\"\n",
|
||||
"df_excel = pd.read_excel(excel_url, sheet_name=\"Sheet1\")\n",
|
||||
"print(df_excel.columns)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "A_13q1K9iMr2"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"json_url = \"https://data.gov.ru/files/dataset.json\"\n",
|
||||
"df_json = pd.read_json(json_url, orient=\"records\")\n",
|
||||
"print(df_json.info())\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,274 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "btIcF4lpu_Xx"
|
||||
},
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"# Пример запроса к TCIA API\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "j4rCtvc9u_XD"
|
||||
},
|
||||
"source": [
|
||||
"**Шаг 1**. The Cancer Imaging Archive (TCIA) предоставляет доступ к медицинским изображениям через API. Вы можете использовать библиотеку requests для загрузки данных из TCIA:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "uR4WCv5yrzNl",
|
||||
"outputId": "0a2a3495-5147-4bed-a209-c800c26cda8d"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"# Пример запроса к TCIA API\n",
|
||||
"url = \"https://services.cancerimagingarchive.net/services/v3/TCIA/query/getSeries?Collection=TCGA-BRCA\"\n",
|
||||
"response = requests.get(url)\n",
|
||||
"\n",
|
||||
"# Проверка ответа\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" print(\"Данные получены успешно\")\n",
|
||||
"else:\n",
|
||||
" print(\"Ошибка получения данных\")\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "AlwoAu-ju7SC"
|
||||
},
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "LuW1z73XvxYF"
|
||||
},
|
||||
"source": [
|
||||
"**Шаг 2.** Данные приходят в JSON формате.\n",
|
||||
"В этом примере запрос отправляется с параметром format=json, чтобы вернуть данные в формате JSON.\n",
|
||||
"Преобразование ответа в JSON:\n",
|
||||
"response.json() позволяет конвертировать ответ в словарь Python, с которым будем работать дальше.\n",
|
||||
"Обработка JSON-ответа:\n",
|
||||
"Сначала печатаем весь JSON для ознакомления."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "0rA31BsAuGeM",
|
||||
"outputId": "fac4ab13-ddbb-41fd-ade9-279df305554a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import requests\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"# Пример запроса к TCIA API, предполагающий JSON формат\n",
|
||||
"url = \"https://services.cancerimagingarchive.net/services/v3/TCIA/query/getSeries?Collection=TCGA-BRCA&format=json\"\n",
|
||||
"response = requests.get(url)\n",
|
||||
"\n",
|
||||
"# Проверка ответа\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" print(\"Данные получены успешно\")\n",
|
||||
"\n",
|
||||
" # Преобразование ответа в JSON\n",
|
||||
" data = response.json()\n",
|
||||
" print(json.dumps(data, indent=4)) # Печать данных в формате JSON с отступами\n",
|
||||
"\n",
|
||||
" # Обработка данных\n",
|
||||
" for item in data:\n",
|
||||
" series_instance_uid = item.get('SeriesInstanceUID', 'N/A')\n",
|
||||
" modality = item.get('Modality', 'N/A')\n",
|
||||
" protocol_name = item.get('ProtocolName', 'N/A')\n",
|
||||
" body_part_examined = item.get('BodyPartExamined', 'N/A')\n",
|
||||
"\n",
|
||||
" # Печать извлеченной информации\n",
|
||||
" print(f\"SeriesInstanceUID: {series_instance_uid}\")\n",
|
||||
" print(f\"Modality: {modality}\")\n",
|
||||
" print(f\"ProtocolName: {protocol_name}\")\n",
|
||||
" print(f\"BodyPartExamined: {body_part_examined}\")\n",
|
||||
" print('-' * 30)\n",
|
||||
"else:\n",
|
||||
" print(\"Ошибка получения данных\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "G3VSxAgX6RbK"
|
||||
},
|
||||
"source": [
|
||||
"**Шаг3**.**Получение идентификаторов для загрузки изображений**\n",
|
||||
"\n",
|
||||
"Используя первый запрос к TCIA API, получили метаданные, такие как SeriesInstanceUID для каждой серии изображений.\n",
|
||||
"Эти идентификаторы нужны для следующего шага — получения ссылок на сами изображения.\n",
|
||||
"Запрос на загрузку изображений с использованием SeriesInstanceUID\n",
|
||||
"Чтобы загрузить все изображения в определенной серии, нужно выполнить запрос к TCIA API с использованием SeriesInstanceUID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "AexV0o2gJahr",
|
||||
"outputId": "06c3167c-931f-4f15-917e-a5fe6d7463d5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import requests\n",
|
||||
"\n",
|
||||
"# Пример использования SeriesInstanceUID\n",
|
||||
"series_instance_uid = \"1.3.6.1.4.1.14519.5.2.1.6450.4002.265963154110062013582166845425\" # Пример идентификатора\n",
|
||||
"\n",
|
||||
"# URL для загрузки изображений\n",
|
||||
"url = f\"https://services.cancerimagingarchive.net/services/v3/TCIA/query/getImage?SeriesInstanceUID={series_instance_uid}\"\n",
|
||||
"response = requests.get(url, stream=True)\n",
|
||||
"\n",
|
||||
"# Путь для сохранения ZIP-файла\n",
|
||||
"output_path = \"/content/images.zip\"\n",
|
||||
"\n",
|
||||
"# Создание необходимых папок, если их нет\n",
|
||||
"folder_path = os.path.dirname(output_path)\n",
|
||||
"os.makedirs(folder_path, exist_ok=True)\n",
|
||||
"\n",
|
||||
"# Проверка ответа и сохранение изображения\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" print(f\"Сохранение файла в {output_path}\")\n",
|
||||
" with open(output_path, \"wb\") as file:\n",
|
||||
" for chunk in response.iter_content(chunk_size=128):\n",
|
||||
" file.write(chunk)\n",
|
||||
" print(f\"Изображения успешно загружены и сохранены как {output_path}\")\n",
|
||||
"else:\n",
|
||||
" print(\"Ошибка при загрузке изображений\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "-PwPLc8aKV9h",
|
||||
"outputId": "821075b5-c777-451e-e84f-6ea3f69c092a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import zipfile\n",
|
||||
"\n",
|
||||
"# Распаковка загруженного ZIP-файла\n",
|
||||
"with zipfile.ZipFile(\"images.zip\", 'r') as zip_ref:\n",
|
||||
" zip_ref.extractall(\"images\") # Папка для распакованных изображений\n",
|
||||
"print(\"Файлы успешно распакованы\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "DVVYma3wM4vf"
|
||||
},
|
||||
"source": [
|
||||
"**Шаг. 4**# Для чтения DICOM файлов используйте pydicom, а для визуализации изображений — matplotlib."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "Tw2PTwQrOlLe",
|
||||
"outputId": "d21f3b3b-c985-45ad-fdc6-a8c36cd10335"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install pydicom\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 839
|
||||
},
|
||||
"id": "OLMmOFkCM59-",
|
||||
"outputId": "f4370e76-ad70-45da-8192-fe2de2232455"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import pydicom\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# Папка с распакованными изображениями\n",
|
||||
"dicom_folder = \"images\"\n",
|
||||
"\n",
|
||||
"# Чтение всех DICOM-файлов в папке и визуализация первого изображения\n",
|
||||
"for filename in os.listdir(dicom_folder):\n",
|
||||
" if filename.endswith(\".dcm\"):\n",
|
||||
" dicom_path = os.path.join(dicom_folder, filename)\n",
|
||||
" dicom_data = pydicom.dcmread(dicom_path)\n",
|
||||
"\n",
|
||||
" # Получение массива пикселей изображения\n",
|
||||
" image = dicom_data.pixel_array\n",
|
||||
"\n",
|
||||
" # Визуализация изображения\n",
|
||||
" plt.imshow(image, cmap=\"gray\")\n",
|
||||
" plt.title(f\"Пациент ID: {dicom_data.PatientID}\")\n",
|
||||
" plt.axis(\"off\")\n",
|
||||
" plt.show()\n",
|
||||
" break # Визуализируем только первое изображение\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "U-9TfVjKQsFV"
|
||||
},
|
||||
"source": [
|
||||
" библиотеки, как OpenCV, scikit-image, и NumPy предоставляют множество инструментов для фильтрации, повышения контраста и удаления шума."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "9B_sc-M_NlTZ"
|
||||
},
|
||||
"source": [
|
||||
"Шаг 1. Устанавливаем библиотеку для работы с загрузкой данных по ссылке"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "OsQErPaDDkmK",
|
||||
"outputId": "49149067-e8b9-42f8-acbe-15b2d80cd79e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install gdown\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "_F6Rmsu4NviH"
|
||||
},
|
||||
"source": [
|
||||
"Шаг 3. Скачиваем и распаковываем файлы,з ная ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "d19wjKXOLIyw",
|
||||
"outputId": "804d657b-dce7-4822-c631-b6a330801207"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import gdown\n",
|
||||
"\n",
|
||||
"# ID файла с Google Диска\n",
|
||||
"file_id = '1qVrpV7vmhIsUxFiH189LmAn0ALbAPrgS'\n",
|
||||
"#https://drive.google.com/file/d/1qVrpV7vmhIsUxFiH189LmAn0ALbAPrgS/view?usp=drive_link\n",
|
||||
"# Формируем URL для скачивания\n",
|
||||
"url = f'https://drive.google.com/uc?id={file_id}'\n",
|
||||
"\n",
|
||||
"# Скачиваем файл в директорию /content/\n",
|
||||
"file_path = '/content/my_file_name.tar'\n",
|
||||
"gdown.download(url, file_path, quiet=False)\n",
|
||||
"\n",
|
||||
"# Распаковываем архив .tar\n",
|
||||
"import tarfile\n",
|
||||
"with tarfile.open(file_path, 'r') as tar:\n",
|
||||
" tar.extractall(path='/content/')\n",
|
||||
" print(\"Архив распакован в директорию /content/\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "YNJiS-a6N4vg"
|
||||
},
|
||||
"source": [
|
||||
"Шаг 4. Просматриваем структуру и описание данных в JSON"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "_G-7skCHPJZ2",
|
||||
"outputId": "f0d8712c-1805-4a5d-c857-3c6b6659fc71"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"# Путь к файлу JSON\n",
|
||||
"json_file_path = '/content/Task08_HepaticVessel/dataset.json'\n",
|
||||
"\n",
|
||||
"# Загружаем содержимое JSON файла\n",
|
||||
"with open(json_file_path, 'r') as f:\n",
|
||||
" dataset_info = json.load(f)\n",
|
||||
"\n",
|
||||
"# Просмотрим содержимое JSON\n",
|
||||
"print(json.dumps(dataset_info, indent=4)) # Красиво выводим содержимое\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4GOfTVYdODLL"
|
||||
},
|
||||
"source": [
|
||||
"Шаг 5. Выводим для просмотра 20 случайных объектов с метками из обучающего набора."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 1000
|
||||
},
|
||||
"id": "CTMaymbpP1IQ",
|
||||
"outputId": "e6ce5b6a-b949-4499-93e5-a7f8816bc888"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import nibabel as nib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Папки с изображениями и метками\n",
|
||||
"images_dir = '/content/Task08_HepaticVessel/imagesTr'\n",
|
||||
"labels_dir = '/content/Task08_HepaticVessel/labelsTr'\n",
|
||||
"\n",
|
||||
"# Фильтруем доступные файлы\n",
|
||||
"available_files = sorted([f for f in os.listdir(images_dir) if not f.startswith('._') and f.endswith('.nii.gz')])\n",
|
||||
"images = [os.path.join(images_dir, file) for file in available_files]\n",
|
||||
"labels = [os.path.join(labels_dir, file.replace('imagesTr', 'labelsTr')) for file in available_files]\n",
|
||||
"\n",
|
||||
"# Проверяем количество файлов\n",
|
||||
"print(f\"Обнаружено {len(images)} изображений и {len(labels)} меток.\")\n",
|
||||
"\n",
|
||||
"# Случайно выбираем 20 изображений\n",
|
||||
"random_indices = random.sample(range(len(images)), min(20, len(images)))\n",
|
||||
"\n",
|
||||
"# Цветовая карта для меток\n",
|
||||
"cmap = plt.cm.colors.ListedColormap(['black', 'green', 'red']) # Цвета: фон, сосуды, опухоли\n",
|
||||
"bounds = [0, 1, 2, 3]\n",
|
||||
"norm = plt.cm.colors.BoundaryNorm(bounds, cmap.N)\n",
|
||||
"\n",
|
||||
"# Визуализация изображений и меток\n",
|
||||
"for idx in random_indices:\n",
|
||||
" try:\n",
|
||||
" # Загружаем изображение и метку\n",
|
||||
" image = nib.load(images[idx]).get_fdata()\n",
|
||||
" label = nib.load(labels[idx]).get_fdata()\n",
|
||||
"\n",
|
||||
" # Поиск срезов с ненулевыми метками\n",
|
||||
" non_empty_slices = [z for z in range(label.shape[2]) if np.unique(label[:, :, z]).size > 1]\n",
|
||||
"\n",
|
||||
" if not non_empty_slices:\n",
|
||||
" print(f\"Все срезы в {images[idx]} содержат только фон.\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" # Выбираем случайный срез с ненулевыми метками\n",
|
||||
" slice_index = random.choice(non_empty_slices)\n",
|
||||
" print(f\"Выбран срез {slice_index} для изображения {images[idx]}\")\n",
|
||||
"\n",
|
||||
" plt.figure(figsize=(10, 5))\n",
|
||||
"\n",
|
||||
" # Отображаем изображение\n",
|
||||
" plt.subplot(1, 2, 1)\n",
|
||||
" plt.imshow(image[:, :, slice_index], cmap='gray')\n",
|
||||
" plt.title(f'Изображение (Slice: {slice_index})')\n",
|
||||
" plt.axis('off')\n",
|
||||
"\n",
|
||||
" # Отображаем метки\n",
|
||||
" plt.subplot(1, 2, 2)\n",
|
||||
" plt.imshow(label[:, :, slice_index], cmap=cmap, norm=norm)\n",
|
||||
" plt.title('Метки')\n",
|
||||
" plt.axis('off')\n",
|
||||
"\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Ошибка при обработке файла {images[idx]}: {e}\")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"accelerator": "GPU",
|
||||
"colab": {
|
||||
"gpuType": "T4",
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,303 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "O6Ly_P7Rs5Pq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Установка библиотеки MedMNIST\n",
|
||||
"!pip install -q medmnist"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "FNCBYGtd5wnF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
"# Импорт библиотек\n",
|
||||
"import numpy as np\n",
|
||||
"import tensorflow as tf\n",
|
||||
"from tensorflow.keras import layers, models\n",
|
||||
"import torchvision.transforms as transforms\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"import medmnist\n",
|
||||
"from medmnist import INFO\n",
|
||||
"\n",
|
||||
"# Загружаем датасет ChestMNIST (multi-label: 14 классов)\n",
|
||||
"data_flag = 'chestmnist'\n",
|
||||
"info = INFO[data_flag]\n",
|
||||
"DataClass = getattr(medmnist, info['python_class'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "onPt-ifPcb70",
|
||||
"outputId": "413ff073-3fae-465c-fb82-2ea52d4bc967"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"# Трансформации: PyTorch-style -> Tensor\n",
|
||||
"transform = transforms.Compose([\n",
|
||||
" transforms.ToTensor()\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"# Загрузка данных\n",
|
||||
"train_dataset = DataClass(split='train', transform=transform, download=True)\n",
|
||||
"test_dataset = DataClass(split='test', transform=transform, download=True)\n",
|
||||
"\n",
|
||||
"# DataLoader для PyTorch-совместимых датасетов\n",
|
||||
"train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
|
||||
"test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)\n",
|
||||
"\n",
|
||||
"# Преобразование в NumPy: (batch, 28, 28, 1), labels в one-hot\n",
|
||||
"def convert_dataset_to_numpy(dataloader):\n",
|
||||
" images_all = []\n",
|
||||
" labels_all = []\n",
|
||||
"\n",
|
||||
" for images, labels in dataloader:\n",
|
||||
" images = images.permute(0, 2, 3, 1).numpy() # (B, H, W, C)\n",
|
||||
" labels = labels.numpy()\n",
|
||||
" images_all.append(images)\n",
|
||||
" labels_all.append(labels)\n",
|
||||
"\n",
|
||||
" X = np.concatenate(images_all, axis=0)\n",
|
||||
" y = np.concatenate(labels_all, axis=0)\n",
|
||||
"\n",
|
||||
" return X, y\n",
|
||||
"\n",
|
||||
"X_train, y_train = convert_dataset_to_numpy(train_loader)\n",
|
||||
"X_test, y_test = convert_dataset_to_numpy(test_loader)\n",
|
||||
"\n",
|
||||
"print(\"X_train shape:\", X_train.shape) # (n_samples, 28, 28, 1)\n",
|
||||
"print(\"y_train shape:\", y_train.shape) # (n_samples, 14)\n",
|
||||
"\n",
|
||||
"# Масштабирование изображений в диапазон [0, 1]\n",
|
||||
"X_train = X_train.astype(\"float32\") / 255.0\n",
|
||||
"X_test = X_test.astype(\"float32\") / 255.0\n",
|
||||
"\n",
|
||||
"# Определение модели (sigmoid + binary_crossentropy!)\n",
|
||||
"model = tf.keras.Sequential([\n",
|
||||
" layers.Input(shape=(28, 28, 1)),\n",
|
||||
" layers.Conv2D(32, kernel_size=3, activation='relu'),\n",
|
||||
" layers.MaxPooling2D(),\n",
|
||||
" layers.Conv2D(64, kernel_size=3, activation='relu'),\n",
|
||||
" layers.MaxPooling2D(),\n",
|
||||
" layers.Flatten(),\n",
|
||||
" layers.Dense(128, activation='relu'),\n",
|
||||
" layers.Dense(14, activation='sigmoid') # multi-label output\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"# Компиляция модели\n",
|
||||
"model.compile(\n",
|
||||
" optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4), # уменьшенный lr\n",
|
||||
" loss='binary_crossentropy', # Ключевой момент!\n",
|
||||
" metrics=[\n",
|
||||
" tf.keras.metrics.BinaryAccuracy(),\n",
|
||||
" tf.keras.metrics.AUC(name='AUC'),\n",
|
||||
" tf.keras.metrics.Precision(name='Precision'),\n",
|
||||
" tf.keras.metrics.Recall(name='Recall')\n",
|
||||
" ]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Обучение\n",
|
||||
"history = model.fit(\n",
|
||||
" X_train, y_train,\n",
|
||||
" epochs=10,\n",
|
||||
" batch_size=32,\n",
|
||||
" validation_data=(X_test, y_test)\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 141
|
||||
},
|
||||
"id": "KRDuSl4GNf0i",
|
||||
"outputId": "9d4d18c6-08f1-458e-c846-5a373afcc52a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.summary()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/",
|
||||
"height": 506
|
||||
},
|
||||
"id": "9epuUiVJ5z2u",
|
||||
"outputId": "888f592f-b2db-435a-fdf3-d788ba41f7b6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# Визуализация первых n изображений и соответствующих меток\n",
|
||||
"def show_sample_images(dataset, class_names, n=9):\n",
|
||||
" plt.figure(figsize=(5, 5))\n",
|
||||
" for i in range(n):\n",
|
||||
" image, label = dataset[i]\n",
|
||||
" img_np = image.permute(1, 2, 0).numpy().squeeze() # (H, W)\n",
|
||||
"\n",
|
||||
" # Преобразуем one-hot в список меток\n",
|
||||
" active_labels = [class_names[j] for j in range(len(label)) if label[j] == 1]\n",
|
||||
"\n",
|
||||
" plt.subplot(int(n**0.5 + 0.5), int(n**0.5 + 0.5), i + 1)\n",
|
||||
" plt.imshow(img_np, cmap='gray')\n",
|
||||
" plt.axis('off')\n",
|
||||
" plt.title('\\n'.join(active_labels), fontsize=8)\n",
|
||||
"\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
"# Названия меток (по ключу из INFO)\n",
|
||||
"class_names = list(info['label'].values())\n",
|
||||
"\n",
|
||||
"# Вызов визуализации\n",
|
||||
"show_sample_images(train_dataset, class_names, n=9)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "0NDTDphUgpMi",
|
||||
"outputId": "9735171b-9d5a-425d-91d4-1eac2da6de7c"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 2. Предсказания\n",
|
||||
"y_pred_probs = model.predict(X_test)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "mKpWlhC0t45X",
|
||||
"outputId": "ad6e3b10-d232-465d-e6f5-df71eadc51ae"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# Функция для отрисовки метрик\n",
|
||||
"def plot_training_history(history):\n",
|
||||
" metrics = ['loss', 'val_loss', 'AUC', 'val_AUC', 'Precision', 'val_Precision', 'Recall', 'val_Recall']\n",
|
||||
" plt.figure(figsize=(16, 10))\n",
|
||||
"\n",
|
||||
" for i, metric in enumerate(metrics):\n",
|
||||
" plt.subplot(4, 2, i + 1)\n",
|
||||
" plt.plot(history.history[metric])\n",
|
||||
" plt.title(metric)\n",
|
||||
" plt.xlabel('Epoch')\n",
|
||||
" plt.ylabel(metric)\n",
|
||||
" plt.grid(True)\n",
|
||||
"\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
"# Вызов функции\n",
|
||||
"plot_training_history(history)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "wECO7vr_gukD",
|
||||
"outputId": "e704d146-d224-43b3-a2f1-d2d40b2cc492"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score\n",
|
||||
"\n",
|
||||
"def plot_all_roc_pr_curves(y_true, y_probs, class_names):\n",
|
||||
" num_classes = y_true.shape[1]\n",
|
||||
" fig_roc, axs_roc = plt.subplots(4, 4, figsize=(20, 20))\n",
|
||||
" fig_pr, axs_pr = plt.subplots(4, 4, figsize=(20, 20))\n",
|
||||
"\n",
|
||||
" for i in range(num_classes):\n",
|
||||
" fpr, tpr, _ = roc_curve(y_true[:, i], y_probs[:, i])\n",
|
||||
" precision, recall, _ = precision_recall_curve(y_true[:, i], y_probs[:, i])\n",
|
||||
" roc_auc = auc(fpr, tpr)\n",
|
||||
" pr_auc = average_precision_score(y_true[:, i], y_probs[:, i])\n",
|
||||
"\n",
|
||||
" row, col = divmod(i, 4)\n",
|
||||
"\n",
|
||||
" # ROC Curve\n",
|
||||
" axs_roc[row, col].plot(fpr, tpr, label=f'AUC = {roc_auc:.2f}')\n",
|
||||
" axs_roc[row, col].plot([0, 1], [0, 1], 'k--')\n",
|
||||
" axs_roc[row, col].set_title(f'ROC — {class_names[i]}')\n",
|
||||
" axs_roc[row, col].set_xlabel('FPR')\n",
|
||||
" axs_roc[row, col].set_ylabel('TPR')\n",
|
||||
" axs_roc[row, col].legend()\n",
|
||||
"\n",
|
||||
" # PR Curve\n",
|
||||
" axs_pr[row, col].plot(recall, precision, label=f'AP = {pr_auc:.2f}')\n",
|
||||
" axs_pr[row, col].set_title(f'PR — {class_names[i]}')\n",
|
||||
" axs_pr[row, col].set_xlabel('Recall')\n",
|
||||
" axs_pr[row, col].set_ylabel('Precision')\n",
|
||||
" axs_pr[row, col].legend()\n",
|
||||
"\n",
|
||||
" # Отключаем лишние пустые графики, если классов < 16\n",
|
||||
" for j in range(num_classes, 16):\n",
|
||||
" fig_roc.axes[j].axis('off')\n",
|
||||
" fig_pr.axes[j].axis('off')\n",
|
||||
"\n",
|
||||
" fig_roc.tight_layout()\n",
|
||||
" fig_pr.tight_layout()\n",
|
||||
" plt.show()\n",
|
||||
"\n",
|
||||
"# Вызов визуализации ROC и PR кривых\n",
|
||||
"plot_all_roc_pr_curves(y_test, y_pred_probs, list(info['label'].values()))\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "raw",
|
||||
"id": "0",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Установка библиотек"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install pydicom matplotlib nibabel gdown medmnist Pillow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#!/usr/bin/env python3\n",
|
||||
"# Download the 56 zip files in Images_png in batches\n",
|
||||
"import urllib.request\n",
|
||||
"\n",
|
||||
"# URLs for the zip files\n",
|
||||
"links = [\n",
|
||||
" 'https://nihcc.box.com/shared/static/vfk49d74nhbxq3nqjg0900w5nvkorp5c.gz',\n",
|
||||
" 'https://nihcc.box.com/shared/static/i28rlmbvmfjbl8p2n3ril0pptcmcu9d1.gz',\n",
|
||||
" 'https://nihcc.box.com/shared/static/f1t00wrtdk94satdfb9olcolqx20z2jp.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/0aowwzs5lhjrceb3qp67ahp0rd1l1etg.gz',\n",
|
||||
" 'https://nihcc.box.com/shared/static/v5e3goj22zr6h8tzualxfsqlqaygfbsn.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/asi7ikud9jwnkrnkj99jnpfkjdes7l6l.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/jn1b4mw4n6lnh74ovmcjb8y48h8xj07n.gz',\n",
|
||||
" 'https://nihcc.box.com/shared/static/tvpxmn7qyrgl0w8wfh9kqfjskv6nmm1j.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/upyy3ml7qdumlgk2rfcvlb9k6gvqq2pj.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/l6nilvfa9cg3s28tqv1qc1olm3gnz54p.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/hhq8fkdgvcari67vfhs7ppg2w6ni4jze.gz',\n",
|
||||
"\t'https://nihcc.box.com/shared/static/ioqwiy20ihqwyr8pf4c24eazhh281pbu.gz'\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for idx, link in enumerate(links):\n",
|
||||
" fn = 'images_%02d.tar.gz' % (idx+1)\n",
|
||||
" print('downloading'+fn+'...')\n",
|
||||
" urllib.request.urlretrieve(link, fn) # download the zip file\n",
|
||||
"\n",
|
||||
"print(\"Download complete. Please check the checksums\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tarfile\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Распаковка\n",
|
||||
"with tarfile.open('images_02.tar.gz', 'r:gz') as tar:\n",
|
||||
" tar.extractall(path='images_01')\n",
|
||||
"print(\"✅ Распаковано в папку 'images_01'\")\n",
|
||||
"print(\"Количество файлов в папке:\", len(os.listdir('images_01')))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from PIL import Image\n",
|
||||
"import random\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"image_folder = 'images_01/images'\n",
|
||||
"all_images = [f for f in os.listdir(image_folder) if f.endswith('.png')]\n",
|
||||
"print(f\"Всего PNG-файлов: {len(all_images)}\")\n",
|
||||
"\n",
|
||||
"# Покажем 5 случайных\n",
|
||||
"sample = random.sample(all_images, min(5, len(all_images)))\n",
|
||||
"plt.figure(figsize=(15, 10))\n",
|
||||
"for i, img_name in enumerate(sample):\n",
|
||||
" img = Image.open(os.path.join(image_folder, img_name))\n",
|
||||
" plt.subplot(1, 5, i+1)\n",
|
||||
" plt.imshow(img, cmap='gray')\n",
|
||||
" plt.title(img_name[:15])\n",
|
||||
" plt.axis('off')\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!wget -O dataset.zip 'https://storage.yandexcloud.net/aie20.selftest/production/Dataset_for_self-testing_of_AI_services_%28AIE20_SELFTEST%29.zip?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=J0yLHxYnLMK5SrRghejm%2F20260222%2Fru-central1%2Fs3%2Faws4_request&X-Amz-Date=20260222T111520Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=b135a1c05ed66ade15b59a73bc29aef7380f9da179abd74267c4ec235462992a'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install pylibjpeg pylibjpeg-libjpeg"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import random\n",
|
||||
"import pydicom\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"# 1. Настройки\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"# Путь к корневой папке с данными CT (измените, если структура иная)\n",
|
||||
"ct_root = 'Dataset_for_self-testing_of_AI_services_(AIE20_SELFTEST)/AIE20_SELFTEST/FLG'\n",
|
||||
"\n",
|
||||
"# Количество случайных исследований для показа\n",
|
||||
"num_samples = 5\n",
|
||||
"\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"# 2. Функция для рекурсивного поиска всех DICOM-файлов в папке\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"def find_dicom_files(folder):\n",
|
||||
" \"\"\"Возвращает список путей ко всем файлам (не папкам) внутри folder.\"\"\"\n",
|
||||
" all_files = []\n",
|
||||
" for root, dirs, files in os.walk(folder):\n",
|
||||
" for f in files:\n",
|
||||
" if not f.startswith('.'): # игнорируем скрытые файлы\n",
|
||||
" all_files.append(os.path.join(root, f))\n",
|
||||
" return all_files\n",
|
||||
"\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"# 3. Собираем все папки исследований (производитель/исследование)\n",
|
||||
"# ------------------------------------------------------------\n",
|
||||
"if not os.path.exists(ct_root):\n",
|
||||
" print(f\"Ошибка: папка {ct_root} не найдена.\")\n",
|
||||
"else:\n",
|
||||
" # Получаем список производителей\n",
|
||||
" manufacturers = [m for m in os.listdir(ct_root) \n",
|
||||
" if os.path.isdir(os.path.join(ct_root, m))]\n",
|
||||
" if not manufacturers:\n",
|
||||
" print(\"Нет папок производителей.\")\n",
|
||||
" else:\n",
|
||||
" # Список всех полных путей к папкам исследований\n",
|
||||
" study_folders = []\n",
|
||||
" for man in manufacturers:\n",
|
||||
" man_path = os.path.join(ct_root, man)\n",
|
||||
" studies = [s for s in os.listdir(man_path) \n",
|
||||
" if os.path.isdir(os.path.join(man_path, s))]\n",
|
||||
" for study in studies:\n",
|
||||
" study_folders.append(os.path.join(man_path, study))\n",
|
||||
"\n",
|
||||
" print(f\"Всего найдено папок исследований: {len(study_folders)}\")\n",
|
||||
"\n",
|
||||
" if not study_folders:\n",
|
||||
" print(\"Нет доступных исследований.\")\n",
|
||||
" else:\n",
|
||||
" # Выбираем случайные папки (не больше, чем есть)\n",
|
||||
" selected = random.sample(study_folders, min(num_samples, len(study_folders)))\n",
|
||||
"\n",
|
||||
" # ------------------------------------------------------------\n",
|
||||
" # 4. Для каждой выбранной папки показываем случайное изображение\n",
|
||||
" # ------------------------------------------------------------\n",
|
||||
" plt.figure(figsize=(15, 10))\n",
|
||||
" for i, folder in enumerate(selected):\n",
|
||||
" # Рекурсивно ищем все файлы в этой папке\n",
|
||||
" dicom_files = find_dicom_files(folder)\n",
|
||||
" if not dicom_files:\n",
|
||||
" print(f\"В папке {folder} нет файлов, пропускаем.\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" # Выбираем случайный файл\n",
|
||||
" chosen = random.choice(dicom_files)\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" ds = pydicom.dcmread(chosen)\n",
|
||||
" # Проверяем, есть ли массив пикселей\n",
|
||||
" if not hasattr(ds, 'pixel_array'):\n",
|
||||
" print(f\"Файл {chosen} не содержит изображения, пропускаем.\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" # Извлекаем метаданные для подписи\n",
|
||||
" manufacturer = ds.get('Manufacturer', 'Unknown')\n",
|
||||
" modality = ds.get('Modality', 'Unknown')\n",
|
||||
" patient_id = ds.get('PatientID', 'Unknown')\n",
|
||||
" study_desc = ds.get('StudyDescription', 'No description')\n",
|
||||
" series_desc = ds.get('SeriesDescription', '')\n",
|
||||
"\n",
|
||||
" # Формируем подпись\n",
|
||||
" title = (f\"{manufacturer}\\n\"\n",
|
||||
" f\"Modality: {modality}\\n\"\n",
|
||||
" f\"Patient: {patient_id}\\n\"\n",
|
||||
" f\"Study: {study_desc[:30]}\\n\"\n",
|
||||
" f\"Series: {series_desc[:20]}\\n\"\n",
|
||||
" f\"File: {os.path.basename(chosen)}\")\n",
|
||||
"\n",
|
||||
" # Отображаем\n",
|
||||
" plt.subplot(1, num_samples, i+1)\n",
|
||||
" plt.imshow(ds.pixel_array, cmap='gray')\n",
|
||||
" plt.title(title, fontsize=8)\n",
|
||||
" plt.axis('off')\n",
|
||||
"\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Ошибка при обработке файла {chosen}: {e}\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "DataSphere Kernel",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,928 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install nibabel monai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "raw",
|
||||
"id": "1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Task09_Spleen (селезёнка) :\n",
|
||||
"\n",
|
||||
"Размер: всего ~1.5 ГБ (скачается быстро)\n",
|
||||
"\n",
|
||||
"Структура: простая бинарная сегментация (есть селезёнка / нет селезёнки)\n",
|
||||
"\n",
|
||||
"Количество файлов: 41 тренировочное + 20 тестовых изображений"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Task09_Spleen (рекомендую для начала) - 1.5 ГБ\n",
|
||||
"!wget -O Task09_Spleen.tar 'https://msd-for-monai.s3-us-west-2.amazonaws.com/Task09_Spleen.tar'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tarfile\n",
|
||||
"import os\n",
|
||||
"import nibabel as nib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Распаковка\n",
|
||||
"with tarfile.open('Task09_Spleen.tar', 'r') as tar:\n",
|
||||
" tar.extractall()\n",
|
||||
"\n",
|
||||
"# Посмотрим содержимое\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"print(\"Содержимое:\", os.listdir(base_dir))\n",
|
||||
"# Ожидается: imagesTr (тренировочные изображения), labelsTr (маски), imagesTs (тестовые)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"print(\"Папки в датасете:\", os.listdir(base_dir))\n",
|
||||
"# Ожидаем: ['imagesTr', 'labelsTr', 'imagesTs', 'dataset.json']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"train_images = sorted(os.listdir(os.path.join(base_dir, 'imagesTr')))\n",
|
||||
"train_labels = sorted(os.listdir(os.path.join(base_dir, 'labelsTr')))\n",
|
||||
"\n",
|
||||
"print(f\"Тренировочных изображений: {len(train_images)}\")\n",
|
||||
"print(f\"Масок: {len(train_labels)}\")\n",
|
||||
"print(f\"Пример имени файла изображения: {train_images[0]}\")\n",
|
||||
"print(f\"Пример имени файла маски: {train_labels[0]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import nibabel as nib\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"\n",
|
||||
"# Получаем списки файлов, исключая служебные (начинающиеся с ._)\n",
|
||||
"train_images = sorted([\n",
|
||||
" f for f in os.listdir(os.path.join(base_dir, 'imagesTr')) \n",
|
||||
" if not f.startswith('._') and f.endswith('.nii.gz')\n",
|
||||
"])\n",
|
||||
"train_labels = sorted([\n",
|
||||
" f for f in os.listdir(os.path.join(base_dir, 'labelsTr')) \n",
|
||||
" if not f.startswith('._') and f.endswith('.nii.gz')\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"print(f\"Тренировочных изображений (реальных): {len(train_images)}\")\n",
|
||||
"print(f\"Масок (реальных): {len(train_labels)}\")\n",
|
||||
"print(f\"Пример имени файла изображения: {train_images[0]}\")\n",
|
||||
"print(f\"Пример имени файла маски: {train_labels[0]}\")\n",
|
||||
"\n",
|
||||
"# Проверим, совпадают ли имена (в MSD они могут немного отличаться, например, у изображений может быть суффикс '_0000')\n",
|
||||
"# Для простоты возьмём первый файл из изображений и попробуем найти соответствующую маску\n",
|
||||
"# Обычно в Task09_Spleen имена совпадают (без суффиксов)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Выбираем первый реальный файл изображения и соответствующую маску\n",
|
||||
"# Предполагаем, что имена совпадают\n",
|
||||
"img_name = train_images[0]\n",
|
||||
"# Ищем маску с таким же именем (в Task09_Spleen они одинаковые)\n",
|
||||
"mask_name = img_name # если имена совпадают\n",
|
||||
"\n",
|
||||
"img_path = os.path.join(base_dir, 'imagesTr', img_name)\n",
|
||||
"mask_path = os.path.join(base_dir, 'labelsTr', mask_name)\n",
|
||||
"\n",
|
||||
"# Загрузка\n",
|
||||
"img_nii = nib.load(img_path)\n",
|
||||
"mask_nii = nib.load(mask_path)\n",
|
||||
"\n",
|
||||
"img_data = img_nii.get_fdata()\n",
|
||||
"mask_data = mask_nii.get_fdata()\n",
|
||||
"\n",
|
||||
"print(f\"Размер изображения: {img_data.shape}\")\n",
|
||||
"print(f\"Уникальные значения в маске: {np.unique(mask_data)}\") # должно быть [0., 1.] (фон и селезёнка)\n",
|
||||
"\n",
|
||||
"# Покажем несколько срезов\n",
|
||||
"slices = np.linspace(0, img_data.shape[2]-1, 5, dtype=int)\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(15, 10))\n",
|
||||
"for i, idx in enumerate(slices):\n",
|
||||
" plt.subplot(2, 5, i+1)\n",
|
||||
" plt.imshow(img_data[:, :, idx], cmap='gray')\n",
|
||||
" plt.title(f'Срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"\n",
|
||||
" plt.subplot(2, 5, i+6)\n",
|
||||
" plt.imshow(mask_data[:, :, idx], cmap='hot', alpha=0.7)\n",
|
||||
" plt.imshow(img_data[:, :, idx], cmap='gray', alpha=0.3)\n",
|
||||
" plt.title(f'Маска срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import torch\n",
|
||||
"import nibabel as nib\n",
|
||||
"import numpy as np\n",
|
||||
"from monai.networks.nets import UNet\n",
|
||||
"from monai.losses import DiceCELoss\n",
|
||||
"from monai.metrics import DiceMetric\n",
|
||||
"from monai.inferers import sliding_window_inference\n",
|
||||
"from monai.transforms import (\n",
|
||||
" Compose, LoadImaged, EnsureChannelFirstd, ScaleIntensityd,\n",
|
||||
" RandRotate90d, RandFlipd, RandShiftIntensityd, ToTensord,\n",
|
||||
" SpatialPadd\n",
|
||||
")\n",
|
||||
"from monai.data import CacheDataset, DataLoader\n",
|
||||
"\n",
|
||||
"# --- 1. Подготовка данных ---\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"images_dir = os.path.join(base_dir, 'imagesTr')\n",
|
||||
"labels_dir = os.path.join(base_dir, 'labelsTr')\n",
|
||||
"\n",
|
||||
"train_images = sorted([os.path.join(images_dir, f) \n",
|
||||
" for f in os.listdir(images_dir) \n",
|
||||
" if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"train_labels = sorted([os.path.join(labels_dir, f) \n",
|
||||
" for f in os.listdir(labels_dir) \n",
|
||||
" if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"\n",
|
||||
"assert len(train_images) == len(train_labels), \"Количество изображений и масок не совпадает\"\n",
|
||||
"print(f\"Найдено пар изображение-маска: {len(train_images)}\")\n",
|
||||
"\n",
|
||||
"# --- 2. Определение максимального размера и округление до кратного 16 ---\n",
|
||||
"print(\"Определяем максимальный размер по всем осям...\")\n",
|
||||
"max_shape = [0, 0, 0]\n",
|
||||
"for img_path in train_images:\n",
|
||||
" img = nib.load(img_path)\n",
|
||||
" shape = img.shape\n",
|
||||
" max_shape = [max(max_shape[i], shape[i]) for i in range(3)]\n",
|
||||
"print(f\"Максимальный размер (H, W, D): {max_shape}\")\n",
|
||||
"\n",
|
||||
"# Округляем до ближайшего числа, кратного 16\n",
|
||||
"aligned_shape = [((dim + 15) // 16) * 16 for dim in max_shape]\n",
|
||||
"print(f\"Размер после выравнивания (кратно 16): {aligned_shape}\")\n",
|
||||
"\n",
|
||||
"# --- 3. Создание словарей данных и разбиение на train/val ---\n",
|
||||
"data_dicts = [{\"image\": img, \"label\": lbl} for img, lbl in zip(train_images, train_labels)]\n",
|
||||
"split_idx = int(0.8 * len(data_dicts))\n",
|
||||
"train_dicts = data_dicts[:split_idx]\n",
|
||||
"val_dicts = data_dicts[split_idx:]\n",
|
||||
"print(f\"Тренировочных: {len(train_dicts)}, Валидационных: {len(val_dicts)}\")\n",
|
||||
"\n",
|
||||
"# --- 4. Трансформы (используем aligned_shape) ---\n",
|
||||
"train_transforms = Compose([\n",
|
||||
" LoadImaged(keys=[\"image\", \"label\"]),\n",
|
||||
" EnsureChannelFirstd(keys=[\"image\", \"label\"]),\n",
|
||||
" ScaleIntensityd(keys=[\"image\"]),\n",
|
||||
" SpatialPadd(keys=[\"image\", \"label\"], spatial_size=aligned_shape, method=\"symmetric\"),\n",
|
||||
" RandRotate90d(keys=[\"image\", \"label\"], prob=0.5, spatial_axes=(0, 1)),\n",
|
||||
" RandFlipd(keys=[\"image\", \"label\"], prob=0.5, spatial_axis=0),\n",
|
||||
" RandShiftIntensityd(keys=[\"image\"], offsets=0.1, prob=0.5),\n",
|
||||
" ToTensord(keys=[\"image\", \"label\"])\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"val_transforms = Compose([\n",
|
||||
" LoadImaged(keys=[\"image\", \"label\"]),\n",
|
||||
" EnsureChannelFirstd(keys=[\"image\", \"label\"]),\n",
|
||||
" ScaleIntensityd(keys=[\"image\"]),\n",
|
||||
" SpatialPadd(keys=[\"image\", \"label\"], spatial_size=aligned_shape, method=\"symmetric\"),\n",
|
||||
" ToTensord(keys=[\"image\", \"label\"])\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"# --- 5. Датасеты и загрузчики ---\n",
|
||||
"train_ds = CacheDataset(data=train_dicts, transform=train_transforms, cache_rate=0.5)\n",
|
||||
"val_ds = CacheDataset(data=val_dicts, transform=val_transforms, cache_rate=0.5)\n",
|
||||
"\n",
|
||||
"batch_size = 2\n",
|
||||
"train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=2)\n",
|
||||
"val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False, num_workers=2)\n",
|
||||
"\n",
|
||||
"# --- 6. Модель, функция потерь, оптимизатор ---\n",
|
||||
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"print(f\"Используется устройство: {device}\")\n",
|
||||
"\n",
|
||||
"model = UNet(\n",
|
||||
" spatial_dims=3,\n",
|
||||
" in_channels=1,\n",
|
||||
" out_channels=2,\n",
|
||||
" channels=(16, 32, 64, 128, 256),\n",
|
||||
" strides=(2, 2, 2, 2),\n",
|
||||
" num_res_units=2,\n",
|
||||
").to(device)\n",
|
||||
"\n",
|
||||
"loss_function = DiceCELoss(to_onehot_y=True, softmax=True)\n",
|
||||
"optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
|
||||
"dice_metric = DiceMetric(include_background=False, reduction=\"mean\")\n",
|
||||
"\n",
|
||||
"roi_size = (96, 96, 96)\n",
|
||||
"sw_batch_size = 4\n",
|
||||
"\n",
|
||||
"# --- 7. Цикл обучения ---\n",
|
||||
"max_epochs = 50\n",
|
||||
"val_interval = 2\n",
|
||||
"best_metric = -1\n",
|
||||
"best_metric_epoch = -1\n",
|
||||
"\n",
|
||||
"for epoch in range(max_epochs):\n",
|
||||
" print(f\"Эпоха {epoch+1}/{max_epochs}\")\n",
|
||||
" model.train()\n",
|
||||
" epoch_loss = 0\n",
|
||||
" step = 0\n",
|
||||
" for batch_data in train_loader:\n",
|
||||
" inputs, labels = batch_data[\"image\"].to(device), batch_data[\"label\"].to(device)\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" outputs = model(inputs)\n",
|
||||
" loss = loss_function(outputs, labels)\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" epoch_loss += loss.item()\n",
|
||||
" step += 1\n",
|
||||
" \n",
|
||||
" epoch_loss /= step\n",
|
||||
" print(f\" Средняя потеря: {epoch_loss:.4f}\")\n",
|
||||
"\n",
|
||||
" if (epoch + 1) % val_interval == 0:\n",
|
||||
" model.eval()\n",
|
||||
" with torch.no_grad():\n",
|
||||
" for val_data in val_loader:\n",
|
||||
" val_inputs, val_labels = val_data[\"image\"].to(device), val_data[\"label\"].to(device)\n",
|
||||
" val_pred = sliding_window_inference(\n",
|
||||
" val_inputs, roi_size, sw_batch_size, model, overlap=0.5\n",
|
||||
" )\n",
|
||||
" dice_metric(y_pred=val_pred, y=val_labels)\n",
|
||||
" metric = dice_metric.aggregate().item()\n",
|
||||
" dice_metric.reset()\n",
|
||||
" print(f\" Dice на валидации: {metric:.4f}\")\n",
|
||||
" if metric > best_metric:\n",
|
||||
" best_metric = metric\n",
|
||||
" best_metric_epoch = epoch + 1\n",
|
||||
" torch.save(model.state_dict(), \"best_spleen_model.pth\")\n",
|
||||
" print(\" ✅ Модель сохранена\")\n",
|
||||
"\n",
|
||||
"print(f\"🎉 Лучший Dice: {best_metric:.4f} на эпохе {best_metric_epoch}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from monai.networks.nets import UNet\n",
|
||||
"\n",
|
||||
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"\n",
|
||||
"# Создаём архитектуру (те же параметры, что и при обучении)\n",
|
||||
"model = UNet(\n",
|
||||
" spatial_dims=3,\n",
|
||||
" in_channels=1,\n",
|
||||
" out_channels=2,\n",
|
||||
" channels=(16, 32, 64, 128, 256),\n",
|
||||
" strides=(2, 2, 2, 2),\n",
|
||||
" num_res_units=2,\n",
|
||||
").to(device)\n",
|
||||
"\n",
|
||||
"# Загружаем веса\n",
|
||||
"checkpoint = torch.load(\"best_spleen_model.pth\", map_location=device)\n",
|
||||
"model.load_state_dict(checkpoint)\n",
|
||||
"model.eval()\n",
|
||||
"print(\"✅ Модель загружена и готова к инференсу.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "10",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from monai.transforms import Compose, LoadImage, EnsureChannelFirst, ScaleIntensity, SpatialPad, ToTensor\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Параметры (те же, что использовались при обучении)\n",
|
||||
"target_shape = [512, 512, 176] # выровненный размер\n",
|
||||
"\n",
|
||||
"# Трансформы для инференса\n",
|
||||
"infer_transform = Compose([\n",
|
||||
" LoadImage(image_only=True),\n",
|
||||
" EnsureChannelFirst(),\n",
|
||||
" ScaleIntensity(),\n",
|
||||
" SpatialPad(spatial_size=target_shape, method=\"symmetric\"),\n",
|
||||
" ToTensor()\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"# Выберите тестовый файл (например, первый из imagesTs)\n",
|
||||
"import os\n",
|
||||
"test_dir = 'Task09_Spleen/imagesTs'\n",
|
||||
"if os.path.exists(test_dir):\n",
|
||||
" test_files = [f for f in os.listdir(test_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" if test_files:\n",
|
||||
" test_path = os.path.join(test_dir, test_files[0])\n",
|
||||
" else:\n",
|
||||
" # Если нет тестовых, возьмём валидационный из тренировочных (например, 35-й)\n",
|
||||
" train_dir = 'Task09_Spleen/imagesTr'\n",
|
||||
" train_files = [f for f in os.listdir(train_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" test_path = os.path.join(train_dir, train_files[-5]) # последний файл\n",
|
||||
"else:\n",
|
||||
" train_dir = 'Task09_Spleen/imagesTr'\n",
|
||||
" train_files = [f for f in os.listdir(train_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" test_path = os.path.join(train_dir, train_files[-5])\n",
|
||||
"\n",
|
||||
"print(f\"Тестовый файл: {test_path}\")\n",
|
||||
"\n",
|
||||
"# Применяем трансформации и добавляем batch dimension\n",
|
||||
"input_tensor = infer_transform(test_path).unsqueeze(0).to(device) # [1,1,H,W,D]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "11",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from monai.inferers import sliding_window_inference\n",
|
||||
"\n",
|
||||
"roi_size = (96, 96, 96) # размер окна (можно подобрать)\n",
|
||||
"sw_batch_size = 4\n",
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" # Прямой проход (если изображение целиком влезает, можно использовать model(input_tensor))\n",
|
||||
" # Но для 3D лучше sliding window:\n",
|
||||
" pred_logits = sliding_window_inference(\n",
|
||||
" input_tensor, roi_size, sw_batch_size, model, overlap=0.5\n",
|
||||
" )\n",
|
||||
" pred_mask = torch.argmax(pred_logits, dim=1).cpu().numpy()[0] # [H,W,D]\n",
|
||||
"\n",
|
||||
"print(\"Предсказание выполнено.\")\n",
|
||||
"print(f\"Уникальные значения в маске: {np.unique(pred_mask)}\") # должно быть [0,1]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "12",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Загружаем оригинальное изображение (без трансформаций)\n",
|
||||
"import nibabel as nib\n",
|
||||
"orig_img_nii = nib.load(test_path)\n",
|
||||
"orig_img = orig_img_nii.get_fdata() # [H,W,D] исходный размер\n",
|
||||
"\n",
|
||||
"# Размеры\n",
|
||||
"H, W, D = orig_img.shape\n",
|
||||
"print(f\"Исходный размер: {orig_img.shape}\")\n",
|
||||
"\n",
|
||||
"# Обрежем предсказанную маску до исходного размера (если нужно)\n",
|
||||
"pred_mask_cropped = pred_mask[:H, :W, :D]\n",
|
||||
"\n",
|
||||
"# Выберем несколько срезов по оси Z\n",
|
||||
"slices = np.linspace(0, D-1, 5, dtype=int)\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(15, 10))\n",
|
||||
"for i, idx in enumerate(slices):\n",
|
||||
" plt.subplot(2, 5, i+1)\n",
|
||||
" plt.imshow(orig_img[:, :, idx], cmap='gray')\n",
|
||||
" plt.title(f'Срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"\n",
|
||||
" plt.subplot(2, 5, i+6)\n",
|
||||
" plt.imshow(pred_mask_cropped[:, :, idx], cmap='hot', alpha=0.7)\n",
|
||||
" plt.imshow(orig_img[:, :, idx], cmap='gray', alpha=0.3)\n",
|
||||
" plt.title(f'Маска срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "13",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import numpy as np\n",
|
||||
"from monai.networks.nets import UNet\n",
|
||||
"from monai.transforms import Compose, LoadImage, EnsureChannelFirst, ScaleIntensity, SpatialPad, ToTensor\n",
|
||||
"from monai.inferers import sliding_window_inference\n",
|
||||
"from monai.metrics import DiceMetric, HausdorffDistanceMetric\n",
|
||||
"import os\n",
|
||||
"import nibabel as nib\n",
|
||||
"\n",
|
||||
"# 1. Загружаем модель (как вы уже сделали)\n",
|
||||
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"model = UNet(\n",
|
||||
" spatial_dims=3,\n",
|
||||
" in_channels=1,\n",
|
||||
" out_channels=2,\n",
|
||||
" channels=(16, 32, 64, 128, 256),\n",
|
||||
" strides=(2, 2, 2, 2),\n",
|
||||
" num_res_units=2,\n",
|
||||
").to(device)\n",
|
||||
"checkpoint = torch.load(\"best_spleen_model.pth\", map_location=device)\n",
|
||||
"model.load_state_dict(checkpoint)\n",
|
||||
"model.eval()\n",
|
||||
"\n",
|
||||
"# 2. Параметры (те же, что при обучении)\n",
|
||||
"target_shape = [512, 512, 176] # выровненный размер\n",
|
||||
"roi_size = (96, 96, 96)\n",
|
||||
"sw_batch_size = 4\n",
|
||||
"\n",
|
||||
"# Трансформы для инференса\n",
|
||||
"infer_transform = Compose([\n",
|
||||
" LoadImage(image_only=True),\n",
|
||||
" EnsureChannelFirst(),\n",
|
||||
" ScaleIntensity(),\n",
|
||||
" SpatialPad(spatial_size=target_shape, method=\"symmetric\"),\n",
|
||||
" ToTensor()\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"# 3. Собираем все валидационные пары (изображение + маска)\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"val_images = sorted([os.path.join(base_dir, 'imagesTr', f) \n",
|
||||
" for f in os.listdir(os.path.join(base_dir, 'imagesTr')) \n",
|
||||
" if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"val_labels = sorted([os.path.join(base_dir, 'labelsTr', f) \n",
|
||||
" for f in os.listdir(os.path.join(base_dir, 'labelsTr')) \n",
|
||||
" if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"\n",
|
||||
"# Предположим, что последние 9 файлов были валидационными (как в вашем разбиении)\n",
|
||||
"val_images = val_images[-9:]\n",
|
||||
"val_labels = val_labels[-9:]\n",
|
||||
"\n",
|
||||
"print(f\"Валидационных образцов: {len(val_images)}\")\n",
|
||||
"\n",
|
||||
"# 4. Метрики\n",
|
||||
"dice_metric = DiceMetric(include_background=False, reduction=\"mean\")\n",
|
||||
"hausdorff_metric = HausdorffDistanceMetric(include_background=False, percentile=95)\n",
|
||||
"\n",
|
||||
"# 5. Цикл по всем валидационным образцам\n",
|
||||
"all_dice = []\n",
|
||||
"all_hd95 = []\n",
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" for img_path, mask_path in zip(val_images, val_labels):\n",
|
||||
" # Подготовка изображения\n",
|
||||
" input_tensor = infer_transform(img_path).unsqueeze(0).to(device)\n",
|
||||
" \n",
|
||||
" # Предсказание\n",
|
||||
" pred_logits = sliding_window_inference(\n",
|
||||
" input_tensor, roi_size, sw_batch_size, model, overlap=0.5\n",
|
||||
" )\n",
|
||||
" pred_mask = torch.argmax(pred_logits, dim=1).cpu().numpy()[0] # [H,W,D]\n",
|
||||
" \n",
|
||||
" # Загрузка истинной маски (без паддинга)\n",
|
||||
" true_mask_nii = nib.load(mask_path)\n",
|
||||
" true_mask = true_mask_nii.get_fdata()\n",
|
||||
" \n",
|
||||
" # Обрезаем предсказание до исходного размера\n",
|
||||
" H, W, D = true_mask.shape\n",
|
||||
" pred_mask_cropped = pred_mask[:H, :W, :D]\n",
|
||||
" \n",
|
||||
" # Преобразуем в тензоры для MONAI метрик\n",
|
||||
" pred_tensor = torch.from_numpy(pred_mask_cropped).unsqueeze(0).unsqueeze(0).long() # [1,1,H,W,D]\n",
|
||||
" true_tensor = torch.from_numpy(true_mask).unsqueeze(0).unsqueeze(0).long()\n",
|
||||
" \n",
|
||||
" # Вычисляем Dice\n",
|
||||
" dice = dice_metric(pred_tensor, true_tensor)\n",
|
||||
" all_dice.append(dice.item())\n",
|
||||
" \n",
|
||||
" # Вычисляем Hausdorff Distance 95%\n",
|
||||
" hd = hausdorff_metric(pred_tensor, true_tensor)\n",
|
||||
" all_hd95.append(hd.item())\n",
|
||||
" \n",
|
||||
" print(f\"Файл: {os.path.basename(img_path)} -> Dice: {dice.item():.4f}, HD95: {hd.item():.2f} мм\")\n",
|
||||
"\n",
|
||||
"# 6. Итоговая статистика\n",
|
||||
"print(\"\\n\" + \"=\"*40)\n",
|
||||
"print(f\"Средний Dice по валидации: {np.mean(all_dice):.4f} ± {np.std(all_dice):.4f}\")\n",
|
||||
"print(f\"Средний HD95 по валидации: {np.mean(all_hd95):.2f} ± {np.std(all_hd95):.2f} мм\")\n",
|
||||
"print(\"=\"*40)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "14",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from monai.networks.nets import UNet\n",
|
||||
"from monai.transforms import Compose, LoadImage, EnsureChannelFirst, ScaleIntensity, SpatialPad, ToTensor\n",
|
||||
"from monai.inferers import sliding_window_inference\n",
|
||||
"import nibabel as nib\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# --- 1. Загрузка модели ---\n",
|
||||
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"model = UNet(\n",
|
||||
" spatial_dims=3,\n",
|
||||
" in_channels=1,\n",
|
||||
" out_channels=2,\n",
|
||||
" channels=(16, 32, 64, 128, 256),\n",
|
||||
" strides=(2, 2, 2, 2),\n",
|
||||
" num_res_units=2,\n",
|
||||
").to(device)\n",
|
||||
"checkpoint = torch.load(\"best_spleen_model.pth\", map_location=device)\n",
|
||||
"model.load_state_dict(checkpoint)\n",
|
||||
"model.eval()\n",
|
||||
"print(\"✅ Модель загружена\")\n",
|
||||
"\n",
|
||||
"# --- 2. Вывод структуры model.model для выбора целевого слоя ---\n",
|
||||
"print(\"\\nСтруктура model.model:\")\n",
|
||||
"for idx, module in enumerate(model.model):\n",
|
||||
" print(f\"[{idx}]: {module.__class__.__name__}\")\n",
|
||||
"\n",
|
||||
"# --- 3. Выбор целевого слоя ---\n",
|
||||
"# Обычно для Grad‑CAM берут последний сверточный слой энкодера.\n",
|
||||
"# В U‑Net от MONAI энкодер состоит из блоков `Down`.\n",
|
||||
"# Выберите индекс блока, который вы хотите использовать (например, последний Down перед bottleneck).\n",
|
||||
"# Попробуйте индекс 2 или 3, если они есть.\n",
|
||||
"# В вашем случае, возможно, подойдёт индекс 2.\n",
|
||||
"# Если не уверены, начните с последнего Down блока.\n",
|
||||
"target_layer_idx = 2 # Попробуйте изменить на 2 или 3, в зависимости от вывода выше\n",
|
||||
"target_layer = model.model[target_layer_idx]\n",
|
||||
"print(f\"\\nВыбран целевой слой: model.model[{target_layer_idx}] -> {target_layer.__class__.__name__}\")\n",
|
||||
"\n",
|
||||
"# --- 4. Класс Grad-CAM ---\n",
|
||||
"class GradCAM:\n",
|
||||
" def __init__(self, model, target_layer):\n",
|
||||
" self.model = model\n",
|
||||
" self.target_layer = target_layer\n",
|
||||
" self.gradients = None\n",
|
||||
" self.activations = None\n",
|
||||
" \n",
|
||||
" target_layer.register_forward_hook(self.save_activation)\n",
|
||||
" target_layer.register_full_backward_hook(self.save_gradient) # обратите внимание на register_full_backward_hook\n",
|
||||
" \n",
|
||||
" def save_activation(self, module, input, output):\n",
|
||||
" self.activations = output.detach()\n",
|
||||
" \n",
|
||||
" def save_gradient(self, module, grad_input, grad_output):\n",
|
||||
" self.gradients = grad_output[0].detach()\n",
|
||||
" \n",
|
||||
" def generate_cam(self, input_tensor, target_class=1):\n",
|
||||
" self.model.zero_grad()\n",
|
||||
" output = self.model(input_tensor)\n",
|
||||
" \n",
|
||||
" target = output[:, target_class].sum()\n",
|
||||
" target.backward()\n",
|
||||
" \n",
|
||||
" weights = torch.mean(self.gradients, dim=(2, 3, 4), keepdim=True)\n",
|
||||
" cam = torch.sum(weights * self.activations, dim=1, keepdim=True)\n",
|
||||
" cam = torch.relu(cam)\n",
|
||||
" cam = cam / (cam.max() + 1e-8)\n",
|
||||
" return cam.squeeze().cpu().numpy()\n",
|
||||
"\n",
|
||||
"# --- 5. Подготовка тестового изображения ---\n",
|
||||
"test_dir = 'Task09_Spleen/imagesTs'\n",
|
||||
"if os.path.exists(test_dir):\n",
|
||||
" test_files = [f for f in os.listdir(test_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" if test_files:\n",
|
||||
" test_path = os.path.join(test_dir, test_files[0])\n",
|
||||
" else:\n",
|
||||
" train_dir = 'Task09_Spleen/imagesTr'\n",
|
||||
" train_files = [f for f in os.listdir(train_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" test_path = os.path.join(train_dir, train_files[-5])\n",
|
||||
"else:\n",
|
||||
" train_dir = 'Task09_Spleen/imagesTr'\n",
|
||||
" train_files = [f for f in os.listdir(train_dir) if f.endswith('.nii.gz') and not f.startswith('._')]\n",
|
||||
" test_path = os.path.join(train_dir, train_files[-5])\n",
|
||||
"\n",
|
||||
"print(f\"\\nТестовый файл: {test_path}\")\n",
|
||||
"\n",
|
||||
"target_shape = [512, 512, 176]\n",
|
||||
"infer_transform = Compose([\n",
|
||||
" LoadImage(image_only=True),\n",
|
||||
" EnsureChannelFirst(),\n",
|
||||
" ScaleIntensity(),\n",
|
||||
" SpatialPad(spatial_size=target_shape, method=\"symmetric\"),\n",
|
||||
" ToTensor()\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"input_tensor = infer_transform(test_path).unsqueeze(0).to(device)\n",
|
||||
"\n",
|
||||
"# --- 6. Получение Grad-CAM ---\n",
|
||||
"grad_cam = GradCAM(model, target_layer)\n",
|
||||
"cam_3d = grad_cam.generate_cam(input_tensor, target_class=1)\n",
|
||||
"\n",
|
||||
"# --- 7. Визуализация ---\n",
|
||||
"orig_img = nib.load(test_path).get_fdata()\n",
|
||||
"H, W, D = orig_img.shape\n",
|
||||
"\n",
|
||||
"slices = np.linspace(0, D-1, 5, dtype=int)\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(15, 10))\n",
|
||||
"for i, idx in enumerate(slices):\n",
|
||||
" cam_slice = cam_3d[:H, :W, idx]\n",
|
||||
" \n",
|
||||
" plt.subplot(2, 5, i+1)\n",
|
||||
" plt.imshow(orig_img[:, :, idx], cmap='gray')\n",
|
||||
" plt.title(f'Срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
" \n",
|
||||
" plt.subplot(2, 5, i+6)\n",
|
||||
" plt.imshow(orig_img[:, :, idx], cmap='gray', alpha=0.5)\n",
|
||||
" plt.imshow(cam_slice, cmap='jet', alpha=0.5, interpolation='bilinear')\n",
|
||||
" plt.title(f'Grad-CAM срез {idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "15",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from monai.networks.nets import UNet\n",
|
||||
"from monai.transforms import Compose, LoadImage, EnsureChannelFirst, ScaleIntensity, SpatialPad, ToTensor\n",
|
||||
"import nibabel as nib\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# --- 1. Загрузка модели ---\n",
|
||||
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
||||
"model = UNet(\n",
|
||||
" spatial_dims=3,\n",
|
||||
" in_channels=1,\n",
|
||||
" out_channels=2,\n",
|
||||
" channels=(16, 32, 64, 128, 256),\n",
|
||||
" strides=(2, 2, 2, 2),\n",
|
||||
" num_res_units=2,\n",
|
||||
").to(device)\n",
|
||||
"checkpoint = torch.load(\"best_spleen_model.pth\", map_location=device)\n",
|
||||
"model.load_state_dict(checkpoint)\n",
|
||||
"model.eval()\n",
|
||||
"print(\"✅ Модель загружена\")\n",
|
||||
"\n",
|
||||
"# --- 2. Вывод структуры model.model ---\n",
|
||||
"print(\"\\n📋 Доступные слои в model.model:\")\n",
|
||||
"for idx, module in enumerate(model.model):\n",
|
||||
" print(f\" [{idx}]: {module.__class__.__name__}\")\n",
|
||||
"\n",
|
||||
"# --- 3. Класс Grad-CAM ---\n",
|
||||
"class GradCAM:\n",
|
||||
" def __init__(self, model, target_layer):\n",
|
||||
" self.model = model\n",
|
||||
" self.target_layer = target_layer\n",
|
||||
" self.gradients = None\n",
|
||||
" self.activations = None\n",
|
||||
" \n",
|
||||
" target_layer.register_forward_hook(self.save_activation)\n",
|
||||
" target_layer.register_full_backward_hook(self.save_gradient)\n",
|
||||
" \n",
|
||||
" def save_activation(self, module, input, output):\n",
|
||||
" self.activations = output.detach()\n",
|
||||
" \n",
|
||||
" def save_gradient(self, module, grad_input, grad_output):\n",
|
||||
" self.gradients = grad_output[0].detach()\n",
|
||||
" \n",
|
||||
" def generate_cam(self, input_tensor, target_class=1):\n",
|
||||
" self.model.zero_grad()\n",
|
||||
" output = self.model(input_tensor)\n",
|
||||
" \n",
|
||||
" target = output[:, target_class].sum()\n",
|
||||
" target.backward()\n",
|
||||
" \n",
|
||||
" weights = torch.mean(self.gradients, dim=(2, 3, 4), keepdim=True)\n",
|
||||
" cam = torch.sum(weights * self.activations, dim=1, keepdim=True)\n",
|
||||
" cam = torch.relu(cam)\n",
|
||||
" cam = cam / (cam.max() + 1e-8)\n",
|
||||
" return cam.squeeze().cpu().numpy() # [H, W, D]\n",
|
||||
"\n",
|
||||
"# --- 4. Подготовка тестовых данных ---\n",
|
||||
"base_dir = 'Task09_Spleen'\n",
|
||||
"img_dir = os.path.join(base_dir, 'imagesTr') # используем тренировочные, если нет test\n",
|
||||
"label_dir = os.path.join(base_dir, 'labelsTr')\n",
|
||||
"\n",
|
||||
"# Получаем списки файлов\n",
|
||||
"img_files = sorted([f for f in os.listdir(img_dir) if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"label_files = sorted([f for f in os.listdir(label_dir) if f.endswith('.nii.gz') and not f.startswith('._')])\n",
|
||||
"\n",
|
||||
"print(f\"\\n📂 Найдено изображений: {len(img_files)}, масок: {len(label_files)}\")\n",
|
||||
"\n",
|
||||
"# Создаём словарь соответствий: имя маски (без пути) -> полный путь\n",
|
||||
"label_dict = {os.path.basename(f): os.path.join(label_dir, f) for f in label_files}\n",
|
||||
"\n",
|
||||
"# Выбираем файл изображения (например, 10-й)\n",
|
||||
"idx_file = 10 # можно изменить\n",
|
||||
"img_filename = img_files[idx_file]\n",
|
||||
"img_path = os.path.join(img_dir, img_filename)\n",
|
||||
"\n",
|
||||
"# Ищем соответствующую маску: возможно, имя совпадает или отличается суффиксом\n",
|
||||
"mask_path = None\n",
|
||||
"if img_filename in label_dict:\n",
|
||||
" mask_path = label_dict[img_filename]\n",
|
||||
"else:\n",
|
||||
" # Попробуем убрать суффикс _0000, если он есть\n",
|
||||
" base_name = img_filename.replace('_0000.nii.gz', '.nii.gz')\n",
|
||||
" if base_name in label_dict:\n",
|
||||
" mask_path = label_dict[base_name]\n",
|
||||
"\n",
|
||||
"if mask_path is None:\n",
|
||||
" print(f\"❌ Маска для {img_filename} не найдена. Будет показано только КТ.\")\n",
|
||||
" has_mask = False\n",
|
||||
"else:\n",
|
||||
" has_mask = True\n",
|
||||
" print(f\"✅ Найдена маска: {mask_path}\")\n",
|
||||
"\n",
|
||||
"print(f\"📂 Тестовое изображение: {img_path}\")\n",
|
||||
"\n",
|
||||
"# --- 5. Трансформации для инференса ---\n",
|
||||
"target_shape = [512, 512, 176] # выровненный размер\n",
|
||||
"infer_transform = Compose([\n",
|
||||
" LoadImage(image_only=True),\n",
|
||||
" EnsureChannelFirst(),\n",
|
||||
" ScaleIntensity(),\n",
|
||||
" SpatialPad(spatial_size=target_shape, method=\"symmetric\"),\n",
|
||||
" ToTensor()\n",
|
||||
"])\n",
|
||||
"\n",
|
||||
"input_tensor = infer_transform(img_path).unsqueeze(0).to(device)\n",
|
||||
"\n",
|
||||
"# Если есть маска, загрузим её (без паддинга)\n",
|
||||
"if has_mask:\n",
|
||||
" true_nii = nib.load(mask_path)\n",
|
||||
" true_mask = true_nii.get_fdata() # [H, W, D]\n",
|
||||
" H, W, D = true_mask.shape\n",
|
||||
"else:\n",
|
||||
" # Если маски нет, используем размеры изображения (после паддинга мы не знаем исходный размер, поэтому загрузим оригинал)\n",
|
||||
" orig_nii = nib.load(img_path)\n",
|
||||
" orig_img = orig_nii.get_fdata()\n",
|
||||
" H, W, D = orig_img.shape\n",
|
||||
"\n",
|
||||
"# --- 6. Выбор слоёв для Grad-CAM ---\n",
|
||||
"# Запрашиваем у пользователя индексы через ввод (в Jupyter можно использовать input)\n",
|
||||
"print(\"\\n🔍 Введите индексы слоёв для анализа через пробел (например: 2 3 4):\")\n",
|
||||
"user_input = input(\"Индексы: \").strip()\n",
|
||||
"if user_input:\n",
|
||||
" layer_indices = [int(x) for x in user_input.split()]\n",
|
||||
"else:\n",
|
||||
" # Если ничего не введено, используем последний слой\n",
|
||||
" layer_indices = [len(model.model) - 1]\n",
|
||||
" print(f\" Используется последний слой: {layer_indices[0]}\")\n",
|
||||
"\n",
|
||||
"# Проверяем, что индексы в допустимом диапазоне\n",
|
||||
"max_idx = len(model.model) - 1\n",
|
||||
"layer_indices = [idx for idx in layer_indices if 0 <= idx <= max_idx]\n",
|
||||
"if not layer_indices:\n",
|
||||
" raise ValueError(f\"Нет допустимых индексов. Доступны 0..{max_idx}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n🔍 Выбранные слои: {layer_indices}\")\n",
|
||||
"\n",
|
||||
"cam_maps = {}\n",
|
||||
"for idx in layer_indices:\n",
|
||||
" target_layer = model.model[idx]\n",
|
||||
" grad_cam = GradCAM(model, target_layer)\n",
|
||||
" cam = grad_cam.generate_cam(input_tensor, target_class=1)\n",
|
||||
" cam_maps[idx] = cam\n",
|
||||
" print(f\" Слой {idx} ({target_layer.__class__.__name__}) – карта получена\")\n",
|
||||
"\n",
|
||||
"# --- 7. Визуализация ---\n",
|
||||
"# Выбираем несколько равномерно распределённых срезов по оси Z\n",
|
||||
"num_slices = 5\n",
|
||||
"slices = np.linspace(0, D-1, num_slices, dtype=int)\n",
|
||||
"\n",
|
||||
"# Если есть маска, визуализируем её контуром, иначе только КТ + Grad-CAM\n",
|
||||
"plt.figure(figsize=(4*num_slices, 4*len(layer_indices)))\n",
|
||||
"\n",
|
||||
"for i, layer_idx in enumerate(layer_indices):\n",
|
||||
" cam_full = cam_maps[layer_idx] # [H_pad, W_pad, D_pad]\n",
|
||||
" for j, slice_idx in enumerate(slices):\n",
|
||||
" # Обрезаем карту до исходного размера\n",
|
||||
" cam_slice = cam_full[:H, :W, slice_idx]\n",
|
||||
" \n",
|
||||
" # Загружаем срез КТ (можно из оригинального файла, но проще из уже загруженного)\n",
|
||||
" # Для КТ используем orig_img (если нет маски, мы его уже загрузили)\n",
|
||||
" if has_mask:\n",
|
||||
" img_slice = true_nii.dataobj[:, :, slice_idx].T # может потребоваться транспонирование\n",
|
||||
" else:\n",
|
||||
" img_slice = orig_img[:, :, slice_idx].T\n",
|
||||
" \n",
|
||||
" # Позиция в подграфике\n",
|
||||
" plt.subplot(len(layer_indices), num_slices, i*num_slices + j + 1)\n",
|
||||
" \n",
|
||||
" # Показываем КТ подложкой\n",
|
||||
" plt.imshow(img_slice, cmap='gray')\n",
|
||||
" # Накладываем Grad-CAM\n",
|
||||
" plt.imshow(cam_slice, cmap='jet', alpha=0.4, interpolation='bilinear')\n",
|
||||
" \n",
|
||||
" # Если есть маска, добавляем её контур\n",
|
||||
" if has_mask:\n",
|
||||
" mask_slice = true_mask[:, :, slice_idx]\n",
|
||||
" plt.contour(mask_slice, colors='white', linewidths=0.5, alpha=0.8)\n",
|
||||
" \n",
|
||||
" plt.title(f'Слой {layer_idx}, срез {slice_idx}')\n",
|
||||
" plt.axis('off')\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "16",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "DataSphere Kernel",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,23 @@
|
||||
# =============================================================================
|
||||
# Зависимости для 3D радиомики
|
||||
# =============================================================================
|
||||
|
||||
# Основные
|
||||
numpy>=1.23.0
|
||||
pandas>=1.5.0
|
||||
scipy>=1.10.0
|
||||
|
||||
# Работа с медицинскими изображениями
|
||||
nibabel>=5.2.0
|
||||
SimpleITK>=2.3.0
|
||||
pyradiomics>=3.0.0
|
||||
|
||||
# Визуализация и анализ
|
||||
matplotlib>=3.6.0
|
||||
seaborn>=0.12.0
|
||||
scikit-learn>=1.2.0
|
||||
openpyxl>=3.1.0
|
||||
|
||||
# Утилиты
|
||||
tqdm>=4.66.0
|
||||
joblib>=1.4.0
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
Reference in New Issue
Block a user