Files
ai-med-edu/seminars/FeatureExtractionClient.ipynb
T

519 lines
22 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0",
"metadata": {},
"outputs": [],
"source": [
"!pip install pandas"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [],
"source": [
"!pip install pillow"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"!pip install torch==2.6.0 torchaudio==2.6.0 torchvision==0.21.0 --index-url https://download.pytorch.org/whl/cu124"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "4",
"metadata": {},
"outputs": [],
"source": [
"#!/usr/bin/env python3\n",
"# =============================================================================\n",
"# FeatureService Client — Hybrid Version\n",
"# Батчинг + поллинг статуса + пост-обработка CSV\n",
"# =============================================================================\n",
"\n",
"import os\n",
"import csv\n",
"import time\n",
"import requests\n",
"import pandas as pd\n",
"from PIL import Image\n",
"import numpy as np\n",
"import io\n",
"import re\n",
"from http import HTTPStatus\n",
"from pathlib import Path\n",
"\n",
"# =============================================================================\n",
"# CONFIG\n",
"# =============================================================================\n",
"\n",
"# Сервер\n",
"#SERVER_IP = \"85.143.113.40\"\n",
"SERVER_IP = \"192.168.203.91\"\n",
"#SERVER_IP = \"127.0.0.1\"\n",
"SERVER_PORT = 14407\n",
"BASE_URL = f\"http://{SERVER_IP}:{SERVER_PORT}\"\n",
"\n",
"# Пути\n",
"DIR_PATH = \"../data/\"\n",
"RESEARCH_DIR = \"prep/\"\n",
"PATH_IMAGES = DIR_PATH + RESEARCH_DIR + \"images\"\n",
"PATH_MASKS = DIR_PATH + RESEARCH_DIR + \"images_results/masks_sam\"\n",
"PATH_CSV = DIR_PATH + RESEARCH_DIR + \"csv\"\n",
"PATH_CLASSES = DIR_PATH + RESEARCH_DIR + \"predefined_classes.txt\"\n",
"\n",
"# Форматы\n",
"IMAGE_EXT = \".jpg\"\n",
"MASK_EXT = \".png\"\n",
"\n",
"# Параметры обработки\n",
"BATCH_SIZE = 2 # Сколько задач отправлять параллельно\n",
"POLL_INTERVAL = 1 # Интервал опроса статуса (сек)\n",
"POLL_TIMEOUT = 300 # Максимальное время ожидания задачи (сек)\n",
"REQUEST_TIMEOUT = 120 # Таймаут HTTP-запроса (сек)\n",
"DELAY_BETWEEN_TASKS = 0.3 # Пауза между задачами внутри батча\n",
"\n",
"# =============================================================================\n",
"# GLOBAL STATE\n",
"# =============================================================================\n",
"\n",
"in_progress = [] # Очередь задач: [(image, mask, task_id), ...]\n",
"processed = [] # Успешно обработанные: [(image, mask, task_id, output_csv), ...]\n",
"\n",
"# =============================================================================\n",
"# UTILS\n",
"# =============================================================================\n",
"\n",
"def ensure_dirs():\n",
" \"\"\"Создаёт необходимые директории\"\"\"\n",
" for d in [PATH_IMAGES, PATH_MASKS, PATH_CSV]:\n",
" os.makedirs(d, exist_ok=True)\n",
"\n",
"def load_classes():\n",
" \"\"\"Загружает словарь классов из файла\"\"\"\n",
" classes = {}\n",
" try:\n",
" with open(PATH_CLASSES, \"r\", encoding=\"utf-8\") as f:\n",
" for line in f:\n",
" line = line.strip()\n",
" if not line:\n",
" continue\n",
" parts = line.split(maxsplit=1)\n",
" if len(parts) == 2:\n",
" classes[int(parts[0])] = parts[1].strip()\n",
" except FileNotFoundError:\n",
" print(f\"⚠️ {PATH_CLASSES} не найден, метки будут 'Unknown'\")\n",
" except Exception as e:\n",
" print(f\"⚠️ Ошибка загрузки классов: {e}\")\n",
" return classes\n",
"\n",
"def get_label(filename, classes):\n",
" \"\"\"Извлекает метку из имени файла: _class_7_ → класс 7 → название\"\"\"\n",
" match = re.search(r'_class_(\\d+)_', filename)\n",
" if match:\n",
" class_id = int(match.group(1))\n",
" return classes.get(class_id, f\"Class_{class_id}\")\n",
" return \"Unknown\"\n",
"\n",
"def normalize_mask(img):\n",
" \"\"\"Нормализует маску: все ненулевые пиксели → 1 (для PyRadiomics)\"\"\"\n",
" arr = np.array(img)\n",
" arr[arr > 0] = 1\n",
" return Image.fromarray(arr.astype(np.uint8), mode='L')\n",
"\n",
"def match_mask_to_image(image_short_name, masks_list):\n",
" \"\"\"\n",
" Точное сопоставление маски с изображением.\n",
" Ищет маски, начинающиеся с имени изображения.\n",
" \"\"\"\n",
" matched = []\n",
" for mask in masks_list:\n",
" if mask.startswith(image_short_name) and mask.endswith(MASK_EXT):\n",
" matched.append(mask)\n",
" return matched\n",
"\n",
"# =============================================================================\n",
"# CORE: MODEL COMMUNICATION\n",
"# =============================================================================\n",
"\n",
"def launch_model(short_name, image_name, mask_name):\n",
" \"\"\"\n",
" Отправляет изображение и маску на сервер.\n",
" Возвращает task_id или None при ошибке.\n",
" \"\"\"\n",
" image_path = os.path.join(PATH_IMAGES, image_name)\n",
" mask_path = os.path.join(PATH_MASKS, mask_name)\n",
" \n",
" if not os.path.exists(image_path):\n",
" print(f\"✗ Файл не найден: {image_path}\")\n",
" return None\n",
" if not os.path.exists(mask_path):\n",
" print(f\"✗ Файл не найден: {mask_path}\")\n",
" return None\n",
"\n",
" print(f\"🔄 Отправка: {image_name} + {mask_name}\")\n",
" \n",
" try:\n",
" # Читаем изображение\n",
" with open(image_path, 'rb') as img_f:\n",
" image_bytes = img_f.read()\n",
" \n",
" # Нормализуем и читаем маску\n",
" with Image.open(mask_path) as mask_img:\n",
" normalized = normalize_mask(mask_img)\n",
" buf = io.BytesIO()\n",
" normalized.save(buf, format='PNG')\n",
" buf.seek(0)\n",
" mask_bytes = buf.read()\n",
" \n",
" # Формируем multipart/form-data с правильными MIME-типами\n",
" files = [\n",
" ('image', ('image.jpg', image_bytes, 'image/jpeg')),\n",
" ('mask1', ('mask.png', mask_bytes, 'image/png')),\n",
" ]\n",
" \n",
" resp = requests.post(\n",
" f'{BASE_URL}/texture/feature', \n",
" files=files, \n",
" timeout=REQUEST_TIMEOUT\n",
" )\n",
" resp.raise_for_status()\n",
" \n",
" task_id = resp.text.strip().strip('\"').strip(\"'\")\n",
" if not task_id or task_id.lower() == 'error':\n",
" print(f\"✗ Сервер вернул ошибку: {resp.text}\")\n",
" return None\n",
" \n",
" print(f\"✓ Задача создана: {task_id}\")\n",
" return task_id\n",
" \n",
" except requests.exceptions.RequestException as e:\n",
" print(f\"✗ Сетевая ошибка: {e}\")\n",
" return None\n",
" except Exception as e:\n",
" print(f\"✗ Ошибка обработки: {e}\")\n",
" return False\n",
"\n",
"def wait_for_result(task_id, mask_name):\n",
" \"\"\"\n",
" Поллинг статуса задачи до завершения.\n",
" Сохраняет результат в CSV при успехе.\n",
" Возвращает путь к сохранённому файлу или None.\n",
" \"\"\"\n",
" mask_base = os.path.splitext(mask_name)[0]\n",
" output_csv = os.path.join(PATH_CSV, f\"{mask_base}.csv\")\n",
" \n",
" start_time = time.time()\n",
" \n",
" while True:\n",
" # Проверка таймаута\n",
" if time.time() - start_time > POLL_TIMEOUT:\n",
" print(f\"✗ Таймаут ожидания для {task_id}\")\n",
" return None\n",
" \n",
" try:\n",
" status_resp = requests.get(\n",
" f'{BASE_URL}/texture/status?id={task_id}',\n",
" timeout=REQUEST_TIMEOUT\n",
" )\n",
" \n",
" if status_resp.status_code == HTTPStatus.OK:\n",
" print(f\"✓ Задача {task_id} завершена, получаем результат...\")\n",
" \n",
" result_resp = requests.get(\n",
" f'{BASE_URL}/texture/result?id={task_id}',\n",
" timeout=REQUEST_TIMEOUT\n",
" )\n",
" \n",
" if result_resp.status_code == 200 and 'id not found' not in result_resp.text.lower():\n",
" os.makedirs(PATH_CSV, exist_ok=True)\n",
" with open(output_csv, \"w\", encoding=\"utf-8\") as f:\n",
" f.write(result_resp.text)\n",
" print(f\"💾 Сохранено: {output_csv}\")\n",
" return output_csv\n",
" else:\n",
" print(f\"✗ Ошибка получения результата: {result_resp.text[:100]}\")\n",
" return None\n",
" else:\n",
" # Задача ещё в процессе\n",
" time.sleep(POLL_INTERVAL)\n",
" \n",
" except requests.exceptions.RequestException as e:\n",
" print(f\"⚠️ Ошибка опроса статуса: {e}\")\n",
" time.sleep(POLL_INTERVAL)\n",
" except Exception as e:\n",
" print(f\"✗ Неожиданная ошибка: {e}\")\n",
" return None\n",
"\n",
"def process_batch():\n",
" \"\"\"\n",
" Обрабатывает очередь in_progress: ждёт завершения всех задач и сохраняет результаты.\n",
" \"\"\"\n",
" global in_progress, processed\n",
" \n",
" if not in_progress:\n",
" return\n",
" \n",
" print(f\"\\n⏳ Ожидание завершения {len(in_progress)} задач...\")\n",
" \n",
" for image_name, mask_name, task_id in in_progress:\n",
" output_path = wait_for_result(task_id, mask_name)\n",
" if output_path:\n",
" processed.append((image_name, mask_name, task_id, output_path))\n",
" \n",
" in_progress.clear()\n",
" print(\"✅ Батч обработан\\n\")\n",
"\n",
"# =============================================================================\n",
"# POST-PROCESSING: CSV MERGE & ENRICH\n",
"# =============================================================================\n",
"\n",
"def delete_useless_columns(row):\n",
" \"\"\"Удаляет служебные колонки PyRadiomics\"\"\"\n",
" diagnostics_cols = [\n",
" 'diagnostics_Versions_PyRadiomics', 'diagnostics_Versions_Numpy',\n",
" 'diagnostics_Versions_SimpleITK', 'diagnostics_Versions_PyWavelet',\n",
" 'diagnostics_Versions_Python', 'diagnostics_Configuration_Settings',\n",
" 'diagnostics_Configuration_EnabledImageTypes', 'diagnostics_Image-original_Hash',\n",
" 'diagnostics_Image-original_Dimensionality', 'diagnostics_Image-original_Spacing',\n",
" 'diagnostics_Image-original_Size', 'diagnostics_Mask-original_CenterOfMassIndex',\n",
" 'diagnostics_Mask-original_CenterOfMass', 'diagnostics_Mask-original_BoundingBox',\n",
" 'diagnostics_Mask-original_Size', 'diagnostics_Mask-original_Hash',\n",
" 'diagnostics_Mask-original_Spacing'\n",
" ]\n",
" for col in diagnostics_cols:\n",
" row.pop(col, None)\n",
" return row\n",
"\n",
"def merge_csv_files():\n",
" \"\"\"Объединяет все CSV в result.csv\"\"\"\n",
" csv_files = [f for f in os.listdir(PATH_CSV) \n",
" if f.endswith('.csv') and f != 'result.csv']\n",
" if not csv_files:\n",
" print(\"⚠️ Нет CSV-файлов для объединения\")\n",
" return None\n",
" \n",
" print(f\"🔗 Объединяю {len(csv_files)} файлов...\")\n",
" all_rows = []\n",
" \n",
" for csv_file in csv_files:\n",
" try:\n",
" with open(os.path.join(PATH_CSV, csv_file), 'r', newline='', encoding='utf-8') as f:\n",
" for row in csv.DictReader(f):\n",
" row = delete_useless_columns(row)\n",
" row['source_file'] = csv_file\n",
" all_rows.append(row)\n",
" except Exception as e:\n",
" print(f\"⚠️ Ошибка чтения {csv_file}: {e}\")\n",
" \n",
" if not all_rows:\n",
" print(\"⚠️ Нет данных для сохранения\")\n",
" return None\n",
" \n",
" result_path = os.path.join(PATH_CSV, 'result.csv')\n",
" pd.DataFrame(all_rows).to_csv(result_path, index=False, encoding='utf-8')\n",
" print(f\"💾 Объединённый файл: {result_path}\")\n",
" return result_path\n",
"\n",
"def add_label_and_path_to_cell(result_path=None):\n",
" \"\"\"\n",
" Добавляет label и path_to_cell ПЕРВЫМИ двумя столбцами в result.csv\n",
" \"\"\"\n",
" if result_path is None:\n",
" result_path = os.path.join(PATH_CSV, 'result.csv')\n",
" \n",
" if not os.path.exists(result_path):\n",
" print(f\"⚠️ Файл не найден: {result_path}\")\n",
" return\n",
" \n",
" classes = load_classes()\n",
" rows = []\n",
" counter = 1\n",
" \n",
" print(\"🏷️ Добавляю label и path_to_cell...\")\n",
" \n",
" with open(result_path, 'r', newline='', encoding='utf-8') as f:\n",
" reader = csv.DictReader(f)\n",
" \n",
" for row in reader:\n",
" # Ищем имя файла в разных возможных полях\n",
" filename = (row.get('name') or row.get('source_file') or \n",
" row.get('path_to_file') or row.get('filename') or '')\n",
" \n",
" # Извлекаем label\n",
" if filename:\n",
" basename = os.path.splitext(os.path.basename(filename))[0]\n",
" label = get_label(basename, classes)\n",
" else:\n",
" label = \"Unknown\"\n",
" \n",
" # Формируем path_to_cell: \"1_F0000001_class_7_obj_0000.png.csv\"\n",
" source = row.get('source_file', os.path.basename(filename))\n",
" path_to_cell = f\"{counter}_{source}\"\n",
" \n",
" # Создаём новый dict с label и path_to_cell первыми\n",
" new_row = {'label': label, 'path_to_cell': path_to_cell}\n",
" for key, value in row.items():\n",
" if key not in ('label', 'path_to_cell'):\n",
" new_row[key] = value\n",
" \n",
" rows.append(new_row)\n",
" counter += 1\n",
" \n",
" if rows:\n",
" final_columns = ['label', 'path_to_cell'] + [c for c in rows[0].keys() if c not in ['label', 'path_to_cell']]\n",
" pd.DataFrame(rows).to_csv(\n",
" result_path, \n",
" index=False, \n",
" encoding='utf-8',\n",
" columns=final_columns\n",
" )\n",
" print(f\"✅ Готово: {result_path}\")\n",
" return result_path\n",
" else:\n",
" print(\"⚠️ Нет строк для обработки\")\n",
" return None\n",
"\n",
"# =============================================================================\n",
"# MAIN PROCESSING LOOP\n",
"# =============================================================================\n",
"\n",
"def process_all_images():\n",
" \"\"\"\n",
" Основной цикл: сканирует изображения, сопоставляет маски,\n",
" отправляет задачи батчами с поллингом статуса.\n",
" \"\"\"\n",
" global in_progress, counter\n",
" \n",
" ensure_dirs()\n",
" \n",
" # Сканируем файлы\n",
" images = [f for f in os.listdir(PATH_IMAGES) if f.lower().endswith(IMAGE_EXT)]\n",
" masks = [f for f in os.listdir(PATH_MASKS) if f.lower().endswith(MASK_EXT)]\n",
" \n",
" if not images:\n",
" print(f\"⚠️ Нет изображений в {PATH_IMAGES}\")\n",
" return\n",
" if not masks:\n",
" print(f\"⚠️ Нет масок в {PATH_MASKS}\")\n",
" return\n",
" \n",
" print(f\"📁 Найдено: {len(images)} изображений, {len(masks)} масок\")\n",
" \n",
" counter = 0 # Счётчик задач в текущем батче\n",
" \n",
" for img in images:\n",
" short = os.path.splitext(img)[0]\n",
" img_masks = match_mask_to_image(short, masks)\n",
" \n",
" if not img_masks:\n",
" print(f\"⚠️ Нет масок для {img}\")\n",
" continue\n",
" \n",
" for mask in img_masks:\n",
" # Если батч заполнен — обрабатываем его\n",
" if counter >= BATCH_SIZE:\n",
" process_batch()\n",
" counter = 0\n",
" \n",
" # Отправляем задачу\n",
" task_id = launch_model(short, img, mask)\n",
" if task_id:\n",
" in_progress.append((img, mask, task_id))\n",
" counter += 1\n",
" time.sleep(DELAY_BETWEEN_TASKS) # Не перегружаем сервер\n",
" \n",
" # Обрабатываем оставшиеся задачи\n",
" if in_progress:\n",
" process_batch()\n",
" \n",
" print(f\"\\n🎯 Всего обработано: {len(processed)} задач\")\n",
"\n",
"# =============================================================================\n",
"# ENTRY POINT\n",
"# =============================================================================\n",
"\n",
"if __name__ == \"__main__\":\n",
" print(f\"🚀 FeatureService Client (Hybrid)\")\n",
" print(f\"🌐 Сервер: {BASE_URL}\")\n",
" print(f\"📦 BATCH_SIZE: {BATCH_SIZE}, ⏱️ POLL_TIMEOUT: {POLL_TIMEOUT}s\\n\")\n",
" \n",
" try:\n",
" # 1. Обработка изображений\n",
" print(\"=== [1/3] Отправка задач на сервер ===\")\n",
" process_all_images()\n",
" \n",
" if not processed:\n",
" print(\"⚠️ Нет успешно обработанных задач. Пропускаю пост-обработку.\")\n",
" else:\n",
" # 2. Объединение CSV\n",
" print(\"\\n=== [2/3] Объединение результатов ===\")\n",
" result_path = merge_csv_files()\n",
" \n",
" # 3. Добавление меток\n",
" if result_path:\n",
" print(\"\\n=== [3/3] Добавление label и path_to_cell ===\")\n",
" add_label_and_path_to_cell(result_path)\n",
" \n",
" print(f\"\\n✨ Готово. Результат: {PATH_CSV}/result.csv\")\n",
" \n",
" except KeyboardInterrupt:\n",
" print(\"\\n⚠️ Прервано пользователем\")\n",
" except Exception as e:\n",
" print(f\"\\n✗ Критическая ошибка: {e}\")\n",
" import traceback\n",
" traceback.print_exc()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}