From 67fd2bc5acfa9f6d102adfd8e0aa9970038894a0 Mon Sep 17 00:00:00 2001 From: tanexc Date: Fri, 12 Jun 2026 23:53:39 +0300 Subject: [PATCH] Initial commit --- .dockerignore | 13 + .env.example | 1 + .gitignore | 16 ++ Dockerfile | 27 ++ README.md | 265 ++++++++++++++++++ app/__init__.py | 1 + app/domain/Exception.py | 26 ++ app/domain/Model.py | 12 + app/domain/ModelConverter.py | 8 + app/domain/ModelRepository.py | 17 ++ app/domain/ModelStorage.py | 41 +++ app/domain/__init__.py | 1 + app/domain/usecase/GetLatestModelUseCase.py | 21 ++ app/domain/usecase/UploadModelUseCase.py | 74 +++++ app/domain/usecase/__init__.py | 0 app/infrastructure/ModelConverterImpl.py | 119 ++++++++ app/infrastructure/ModelRepositoryImpl.py | 47 ++++ app/infrastructure/ModelStorageImpl.py | 56 ++++ app/infrastructure/__init__.py | 1 + app/infrastructure/bootstrap.py | 40 +++ app/infrastructure/constants.py | 13 + app/infrastructure/database.py | 42 +++ app/main.py | 13 + app/presentation/__init__.py | 1 + app/presentation/dependencies.py | 53 ++++ app/presentation/response/ModelResponse.py | 29 ++ app/presentation/routes.py | 66 +++++ .../systemd/model-update-server.env.example | 4 + deploy/systemd/model-update-server.service | 14 + requirements.txt | 13 + scripts/generate_api_keys.sh | 72 +++++ tmp/.gitkeep | 1 + 32 files changed, 1107 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/__init__.py create mode 100644 app/domain/Exception.py create mode 100644 app/domain/Model.py create mode 100644 app/domain/ModelConverter.py create mode 100644 app/domain/ModelRepository.py create mode 100644 app/domain/ModelStorage.py create mode 100644 app/domain/__init__.py create mode 100644 app/domain/usecase/GetLatestModelUseCase.py create mode 100644 app/domain/usecase/UploadModelUseCase.py create mode 100644 app/domain/usecase/__init__.py create mode 100644 app/infrastructure/ModelConverterImpl.py create mode 100644 app/infrastructure/ModelRepositoryImpl.py create mode 100644 app/infrastructure/ModelStorageImpl.py create mode 100644 app/infrastructure/__init__.py create mode 100644 app/infrastructure/bootstrap.py create mode 100644 app/infrastructure/constants.py create mode 100644 app/infrastructure/database.py create mode 100644 app/main.py create mode 100644 app/presentation/__init__.py create mode 100644 app/presentation/dependencies.py create mode 100644 app/presentation/response/ModelResponse.py create mode 100644 app/presentation/routes.py create mode 100644 deploy/systemd/model-update-server.env.example create mode 100644 deploy/systemd/model-update-server.service create mode 100644 requirements.txt create mode 100755 scripts/generate_api_keys.sh create mode 100644 tmp/.gitkeep diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..613eefb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +__pycache__/ +*.pyc +*.pyo +*.pyd +.pytest_cache/ +.env +.env.* +.idea/ +.venv/ +venv/ +db/ +storage/ +tmp/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5a46441 --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +API_KEYS=replace-with-generated-key-1,replace-with-generated-key-2 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56f9490 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +.env +.env.* +!.env.example + +__pycache__/ +*.py[cod] + +.venv/ +.idea/ + +db/*.db +storage/images/ +storage/models/* +!storage/models/.gitkeep +tmp/* +!tmp/.gitkeep diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7581a37 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + MODELS_DIR=/app/storage/models \ + TMP_DIR=/app/tmp \ + DATABASE_URL=sqlite:////app/db/app.db + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libgl1 \ + libglib2.0-0 \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p /app/storage/models /app/tmp /app/db + +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..c2b28d5 --- /dev/null +++ b/README.md @@ -0,0 +1,265 @@ +# Model Update Server + +Минимальный backend-сервис на FastAPI для загрузки `.pt` моделей, их stub-конвертации в `.ort`, хранения метаданных в SQLite и выдачи последней модели устройствам. + +## Что умеет + +- `POST /models` - загрузка новой `.pt` модели с версией +- `GET /models/latest` - метаданные последней модели +- `GET /models/latest/download` - скачивание последней `.ort` модели + +## API + +Все API-запросы требуют заголовок авторизации: + +```http +X-API-Key: +``` + +Ключи выпускаются вручную на сервере и задаются через переменную окружения `API_KEYS`. +Можно указать один ключ или несколько ключей через запятую. + +### `POST /models` + +Загружает новую модель и регистрирует её как актуальную. + +Принимает: + +- `Content-Type: multipart/form-data` +- поле `file` - файл модели в формате `.pt` +- поле `version` - строковая версия модели, например `1.0.3` + +Возвращает: + +- `201 Created` и JSON: + +```json +{ + "version": "1.0.3", + "created_at": "2026-05-12T10:00:00Z", + "filename": "model_v1.0.3.ort", + "is_latest": true +} +``` + +Ошибки: + +- `400` - пустая версия или файл не `.pt` +- `409` - такая версия уже существует +- `500` - ошибка конвертации + +### `GET /models/latest` + +Возвращает метаданные последней доступной модели. + +Принимает: + +- ничего + +Возвращает: + +- `200 OK` и JSON: + +```json +{ + "version": "1.0.3", + "created_at": "2026-05-12T10:00:00Z", + "filename": "model_v1.0.3.ort" +} +``` + +Ошибки: + +- `404` - если моделей ещё нет + +### `GET /models/latest/download` + +Возвращает файл последней `.ort` модели. + +Принимает: + +- query-параметр `version` +- query-параметр `filename` + +Возвращает: + +- `200 OK` +- бинарный файл `.ort` +- заголовок `Content-Disposition` для скачивания + +Ошибки: + +- `404` - если моделей нет, файл отсутствует на диске или `version`/`filename` не совпадают с актуальной моделью + +## Требования + +- Python 3.11+ + +## Локальный запуск + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +uvicorn app.main:app --reload +``` + +Сервис будет доступен на `http://127.0.0.1:8000`. + +Пример запуска с API-ключом: + +```bash +API_KEYS="local-dev-key" uvicorn app.main:app --reload +``` + +Сгенерировать 10 API-ключей и экспортировать их в текущую shell: + +```bash +source scripts/generate_api_keys.sh +uvicorn app.main:app --reload +``` + +Сгенерировать 10 API-ключей и сохранить их в env-файл: + +```bash +scripts/generate_api_keys.sh --env-file .env +set -a +source .env +set +a +uvicorn app.main:app --reload +``` + +## Переменные окружения + +- `MODELS_DIR` - директория хранения итоговых `.ort` моделей +- `TMP_DIR` - директория для временно загруженных `.pt` +- `DATABASE_URL` - URL подключения к SQLite, например `sqlite:///./db/app.db` +- `API_KEYS` - один или несколько API-ключей через запятую, например `key-1,key-2` + +По умолчанию используются: + +- `storage/models` +- `tmp` +- `sqlite:///db/app.db` +- пустой список API-ключей, при котором API-запросы будут отклоняться с `401` + +## Архитектура + +Проект разложен по слоям clean architecture: + +- `app/domain` - сущности, интерфейсы репозиториев и доменные исключения +- `app/application` - use cases и DTO +- `app/infrastructure` - SQLite, файловое хранилище, stub-конвертер, bootstrap +- `app/presentation` - FastAPI routes, зависимости и response-схемы + +Точка входа приложения: [app/main.py](/Users/tanexc/PyCharmMiscProject/convert-and-update-model-server/app/main.py). + +## Поведение stub-конвертера + +Stub-конвертер уже подключен к `POST /models`, но реальную конвертацию пока не делает. В текущей версии он копирует загруженный `.pt` файл в хранилище с расширением `.ort`, чтобы можно было проверить сценарий загрузки и скачивания end-to-end. + +Точка расширения находится в [app/infrastructure/ModelConverterImpl.py](/Users/tanexc/PyCharmMiscProject/convert-and-update-model-server/app/infrastructure/ModelConverterImpl.py). + +## Примеры запросов + +Загрузка модели: + +```bash +curl -X POST "http://127.0.0.1:8000/models" \ + -H "X-API-Key: local-dev-key" \ + -F "version=1.0.3" \ + -F "file=@./example.pt" +``` + +Получение метаданных: + +```bash +curl -H "X-API-Key: local-dev-key" \ + "http://127.0.0.1:8000/models/latest" +``` + +Скачивание последней модели: + +```bash +curl -OJ -H "X-API-Key: local-dev-key" \ + "http://127.0.0.1:8000/models/latest/download?version=1.0.3&filename=model_v1.0.3.ort" +``` + +## Docker + +Сборка: + +```bash +docker build -t model-update-server . +``` + +Сгенерировать ключи в `.env`: + +```bash +scripts/generate_api_keys.sh --env-file .env +``` + +Запуск через env-файл: + +```bash +docker run --rm -p 8000:8000 \ + --env-file .env \ + -v "$(pwd)/storage/models:/app/storage/models" \ + -v "$(pwd)/db:/app/db" \ + model-update-server +``` + +Файл `.env` содержит реальные ключи и не должен попадать в репозиторий. Шаблон без секретов находится в `.env.example`. + +## systemd + +Пример unit-файла находится в `deploy/systemd/model-update-server.service`. +Он читает переменные окружения из `/etc/model-update-server/model-update-server.env`. + +Установка env-файла с ключами: + +```bash +sudo mkdir -p /etc/model-update-server +scripts/generate_api_keys.sh --env-file /tmp/model-update-server.env +sudo install -m 600 /tmp/model-update-server.env /etc/model-update-server/model-update-server.env +rm /tmp/model-update-server.env +``` + +Установка unit-файла: + +```bash +sudo cp deploy/systemd/model-update-server.service /etc/systemd/system/model-update-server.service +sudo systemctl daemon-reload +sudo systemctl enable --now model-update-server +``` + +## CI + +В CI храните `API_KEYS` как secret/protected variable и передавайте её в окружение сборки или деплоя. +Не генерируйте новые ключи при каждом деплое, если старые клиенты должны продолжать работать. + +Пример для GitHub Actions: + +```yaml +env: + API_KEYS: ${{ secrets.API_KEYS }} +``` + +Пример для GitLab CI: + +```yaml +deploy: + script: + - docker run --env API_KEYS="$API_KEYS" model-update-server +``` + +## Структура данных + +Таблица `models`: + +- `id` +- `version` +- `filename` +- `file_path` +- `created_at` +- `is_latest` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ + diff --git a/app/domain/Exception.py b/app/domain/Exception.py new file mode 100644 index 0000000..62625e1 --- /dev/null +++ b/app/domain/Exception.py @@ -0,0 +1,26 @@ +class DomainError(Exception): + pass + + +class ModelNotFoundError(DomainError): + pass + + +class DuplicateModelVersionError(DomainError): + pass + + +class InvalidModelVersionError(DomainError): + pass + + +class InvalidModelFileError(DomainError): + pass + + +class ModelFileMissingError(DomainError): + pass + + +class ModelConversionError(DomainError): + pass diff --git a/app/domain/Model.py b/app/domain/Model.py new file mode 100644 index 0000000..efdfa95 --- /dev/null +++ b/app/domain/Model.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(slots=True) +class Model: + id: int | None + version: str + filename: str + file_path: str + created_at: datetime + is_latest: bool \ No newline at end of file diff --git a/app/domain/ModelConverter.py b/app/domain/ModelConverter.py new file mode 100644 index 0000000..4352369 --- /dev/null +++ b/app/domain/ModelConverter.py @@ -0,0 +1,8 @@ +from abc import abstractmethod, ABC +from pathlib import Path + + +class ModelConverter(ABC): + @abstractmethod + def convert(self, model_pt_path: Path, results_dir: Path) -> Path: + raise NotImplementedError diff --git a/app/domain/ModelRepository.py b/app/domain/ModelRepository.py new file mode 100644 index 0000000..9aca84b --- /dev/null +++ b/app/domain/ModelRepository.py @@ -0,0 +1,17 @@ +from abc import ABC, abstractmethod + +from app.domain.Model import Model + + +class ModelRepository(ABC): + @abstractmethod + def get_latest(self) -> Model | None: + raise NotImplementedError + + @abstractmethod + def get_by_version(self, version: str) -> Model | None: + raise NotImplementedError + + @abstractmethod + def add(self, model: Model) -> Model: + raise NotImplementedError diff --git a/app/domain/ModelStorage.py b/app/domain/ModelStorage.py new file mode 100644 index 0000000..e550954 --- /dev/null +++ b/app/domain/ModelStorage.py @@ -0,0 +1,41 @@ +from abc import ABC, abstractmethod +from pathlib import Path + + +class ModelStorage(ABC): + @property + @abstractmethod + def models_dir(self) -> Path: + raise NotImplementedError + + @abstractmethod + def ensure_directories(self) -> None: + raise NotImplementedError + + @abstractmethod + def validate_upload_filename(self, filename: str | None) -> str: + raise NotImplementedError + + @abstractmethod + def save_upload_to_tmp(self, source_file, filename: str) -> Path: + raise NotImplementedError + + @abstractmethod + def build_model_filename(self, version: str) -> str: + raise NotImplementedError + + @abstractmethod + def prepare_conversion_input(self, tmp_path: Path, model_filename: str) -> Path: + raise NotImplementedError + + @abstractmethod + def finalize_converted_model(self, converted_path: Path, model_filename: str) -> Path: + raise NotImplementedError + + @abstractmethod + def ensure_model_file_exists(self, file_path: str) -> Path: + raise NotImplementedError + + @abstractmethod + def delete_file_if_exists(self, file_path: Path | None) -> None: + raise NotImplementedError diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/domain/__init__.py @@ -0,0 +1 @@ + diff --git a/app/domain/usecase/GetLatestModelUseCase.py b/app/domain/usecase/GetLatestModelUseCase.py new file mode 100644 index 0000000..50072de --- /dev/null +++ b/app/domain/usecase/GetLatestModelUseCase.py @@ -0,0 +1,21 @@ +from app.domain.ModelStorage import ModelStorage +from app.domain.Model import Model +from app.domain.Exception import ModelNotFoundError +from app.domain.ModelRepository import ModelRepository + + +class GetLatestModelUseCase: + def __init__(self, repository: ModelRepository, storage: ModelStorage) -> None: + self._repository = repository + self._storage = storage + + def execute(self, version: str | None = None, filename: str | None = None) -> Model: + model = self._repository.get_latest() + if model is None: + raise ModelNotFoundError("No models have been uploaded yet.") + if version is not None and model.version != version: + raise ModelNotFoundError("Latest model does not match requested version.") + if filename is not None and model.filename != filename: + raise ModelNotFoundError("Latest model does not match requested filename.") + self._storage.ensure_model_file_exists(model.file_path) + return model diff --git a/app/domain/usecase/UploadModelUseCase.py b/app/domain/usecase/UploadModelUseCase.py new file mode 100644 index 0000000..1a8cac9 --- /dev/null +++ b/app/domain/usecase/UploadModelUseCase.py @@ -0,0 +1,74 @@ +from datetime import datetime, timezone + +from app.domain.Model import Model +from app.domain.Exception import * +from app.domain.ModelRepository import ModelRepository +from app.domain.ModelStorage import ModelStorage +from app.domain.ModelConverter import ModelConverter + + +class UploadModelUseCase: + def __init__( + self, + repository: ModelRepository, + storage: ModelStorage, + converter: ModelConverter, + ) -> None: + self._repository = repository + self._storage = storage + self._converter = converter + + def execute(self, version: str, upload_file) -> Model: + normalized_version = version.strip() + if not normalized_version: + raise InvalidModelVersionError("Version must not be empty.") + if self._repository.get_by_version(normalized_version) is not None: + raise DuplicateModelVersionError(f"Model version '{normalized_version}' already exists.") + + safe_filename = self._storage.validate_upload_filename(upload_file.filename) + tmp_path = self._storage.save_upload_to_tmp(upload_file.file, safe_filename) + conversion_input_path = None + final_model_path = None + + try: + model_filename = self._storage.build_model_filename(normalized_version) + conversion_input_path = self._storage.prepare_conversion_input( + tmp_path=tmp_path, + model_filename=model_filename, + ) + converted_path = self._converter.convert( + model_pt_path=conversion_input_path, + results_dir=self._storage.models_dir, + ) + final_model_path = self._storage.finalize_converted_model( + converted_path=converted_path, + model_filename=model_filename, + ) + + model = self._repository.add( + Model( + id=None, + version=normalized_version, + filename=model_filename, + file_path=str(final_model_path), + created_at=datetime.now(timezone.utc), + is_latest=True, + ) + ) + return model + except Exception as exc: + self._storage.delete_file_if_exists(final_model_path) + if isinstance( + exc, + ( + InvalidModelVersionError, + DuplicateModelVersionError, + ModelNotFoundError, + ), + ): + raise + raise ModelConversionError(f"Model conversion failed: {exc}") from exc + finally: + upload_file.file.close() + self._storage.delete_file_if_exists(tmp_path) + self._storage.delete_file_if_exists(conversion_input_path) \ No newline at end of file diff --git a/app/domain/usecase/__init__.py b/app/domain/usecase/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/infrastructure/ModelConverterImpl.py b/app/infrastructure/ModelConverterImpl.py new file mode 100644 index 0000000..a4426df --- /dev/null +++ b/app/infrastructure/ModelConverterImpl.py @@ -0,0 +1,119 @@ +from pathlib import Path +import shutil +from tempfile import TemporaryDirectory + +import numpy as np +import onnxruntime as ort +from onnxruntime.quantization import ( + CalibrationDataReader, + QuantFormat, + QuantType, + quantize_static, +) +from onnxruntime.tools.convert_onnx_models_to_ort import ( + OptimizationStyle, + convert_onnx_models_to_ort, +) +from PIL import Image +from ultralytics import YOLO + +from app.domain.ModelConverter import ModelConverter +from app.infrastructure.constants import BASE_DIR + + +IMAGES_DIR = BASE_DIR / "storage" / "images" + + +class ModelConverterImpl(ModelConverter): + def convert(self, model_pt_path: Path, results_dir: Path) -> Path: + model_pt_path = Path(model_pt_path) + results_dir = Path(results_dir) + results_dir.mkdir(parents=True, exist_ok=True) + + result_path = results_dir / f"{model_pt_path.stem}.ort" + if result_path.exists(): + result_path.unlink() + + with TemporaryDirectory(dir=results_dir) as temp_dir_name: + temp_dir = Path(temp_dir_name) + fp32_onnx_path = temp_dir / f"{model_pt_path.stem}_fp32.onnx" + int8_onnx_path = temp_dir / f"{model_pt_path.stem}_int8_static.onnx" + + model = YOLO(str(model_pt_path), task="detect") + exported_path = Path( + model.export( + format="onnx", + imgsz=640, + opset=20, + dynamic=False, + simplify=False, + half=False, + ) + ).resolve() + shutil.move(str(exported_path), str(fp32_onnx_path)) + + calibration_reader = ImageCalibrationDataReader(fp32_onnx_path, IMAGES_DIR) + quantize_static( + model_input=str(fp32_onnx_path), + model_output=str(int8_onnx_path), + calibration_data_reader=calibration_reader, + quant_format=QuantFormat.QDQ, + activation_type=QuantType.QUInt8, + weight_type=QuantType.QInt8, + op_types_to_quantize=["Conv"], + ) + + convert_onnx_models_to_ort( + int8_onnx_path, + output_dir=temp_dir, + optimization_styles=[OptimizationStyle.Fixed], + ) + generated_ort_path = temp_dir / f"{int8_onnx_path.stem}.ort" + generated_ort_path.replace(result_path) + + return result_path + + +class ImageCalibrationDataReader(CalibrationDataReader): + def __init__(self, model_path: Path, images_dir: Path, limit: int = 32): + self.session = ort.InferenceSession( + str(model_path), providers=["CPUExecutionProvider"] + ) + self.input_name = self.session.get_inputs()[0].name + self.input_shape = self.session.get_inputs()[0].shape + self.image_paths = [ + path + for path in sorted(images_dir.glob("*")) + if path.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp"} + ][:limit] + self.index = 0 + + if not self.image_paths: + raise FileNotFoundError(f"calibration images not found in {images_dir}") + + def get_next(self) -> dict[str, np.ndarray] | None: + if self.index >= len(self.image_paths): + return None + + image_path = self.image_paths[self.index] + self.index += 1 + image = Image.open(image_path).convert("RGB") + + height = int(self.input_shape[2]) if isinstance(self.input_shape[2], int) else 640 + width = int(self.input_shape[3]) if isinstance(self.input_shape[3], int) else 640 + scale = min(width / image.width, height / image.height) + resized_width = int(round(image.width * scale)) + resized_height = int(round(image.height * scale)) + resized_image = image.resize((resized_width, resized_height), Image.BILINEAR) + + padded_image = Image.new("RGB", (width, height), (114, 114, 114)) + left = (width - resized_width) // 2 + top = (height - resized_height) // 2 + padded_image.paste(resized_image, (left, top)) + + image_array = np.asarray(padded_image, dtype=np.float32) / 255.0 + image_array = np.transpose(image_array, (2, 0, 1)) + return {self.input_name: np.expand_dims(image_array, axis=0)} + + def rewind(self) -> None: + self.index = 0 diff --git a/app/infrastructure/ModelRepositoryImpl.py b/app/infrastructure/ModelRepositoryImpl.py new file mode 100644 index 0000000..389b271 --- /dev/null +++ b/app/infrastructure/ModelRepositoryImpl.py @@ -0,0 +1,47 @@ +from sqlalchemy import select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from app.domain.Exception import DuplicateModelVersionError +from app.domain.Model import Model +from app.domain.ModelRepository import ModelRepository +from app.infrastructure.database import ModelRecord + + +class ModelRepositoryImpl(ModelRepository): + def __init__(self, session: Session) -> None: + self._session = session + + def get_latest(self) -> Model | None: + stmt = ( + select(ModelRecord) + .order_by(ModelRecord.is_latest.desc(), ModelRecord.created_at.desc()) + .limit(1) + ) + record = self._session.scalar(stmt) + return record.to_domain() if record else None + + def get_by_version(self, version: str) -> Model | None: + stmt = select(ModelRecord).where(ModelRecord.version == version) + record = self._session.scalar(stmt) + return record.to_domain() if record else None + + def add(self, model: Model) -> Model: + record = ModelRecord( + version=model.version, + filename=model.filename, + file_path=model.file_path, + created_at=model.created_at, + is_latest=model.is_latest, + ) + try: + self._session.execute(update(ModelRecord).values(is_latest=False)) + self._session.add(record) + self._session.commit() + self._session.refresh(record) + return record.to_domain() + except IntegrityError as exc: + self._session.rollback() + raise DuplicateModelVersionError( + f"Model version '{model.version}' already exists." + ) from exc + diff --git a/app/infrastructure/ModelStorageImpl.py b/app/infrastructure/ModelStorageImpl.py new file mode 100644 index 0000000..9621756 --- /dev/null +++ b/app/infrastructure/ModelStorageImpl.py @@ -0,0 +1,56 @@ +import shutil +from pathlib import Path +from uuid import uuid4 + +from app.domain.Exception import InvalidModelFileError, ModelFileMissingError +from app.domain.ModelStorage import ModelStorage +from app.infrastructure.constants import TMP_DIR, MODEL_DIR + + +class ModelStorageImpl(ModelStorage): + @property + def models_dir(self) -> Path: + return MODEL_DIR + + def ensure_directories(self) -> None: + MODEL_DIR.mkdir(parents=True, exist_ok=True) + TMP_DIR.mkdir(parents=True, exist_ok=True) + + def validate_upload_filename(self, filename: str | None) -> str: + if not filename: + raise InvalidModelFileError("Uploaded file must have a filename.") + + safe_name = Path(filename).name + if Path(safe_name).suffix.lower() != ".pt": + raise InvalidModelFileError("Only .pt model files are supported.") + return safe_name + + def save_upload_to_tmp(self, source_file, filename: str) -> Path: + tmp_path = TMP_DIR / f"{uuid4().hex}_{filename}" + with tmp_path.open("wb") as buffer: + shutil.copyfileobj(source_file, buffer) + return tmp_path + + def build_model_filename(self, version: str) -> str: + return f"model_v{version}.ort" + + def prepare_conversion_input(self, tmp_path: Path, model_filename: str) -> Path: + conversion_input_path = TMP_DIR / f"{uuid4().hex}_{Path(model_filename).stem}.pt" + tmp_path.rename(conversion_input_path) + return conversion_input_path + + def finalize_converted_model(self, converted_path: Path, model_filename: str) -> Path: + target_path = MODEL_DIR / model_filename + if converted_path.resolve() != target_path.resolve(): + converted_path.replace(target_path) + return target_path + + def ensure_model_file_exists(self, file_path: str) -> Path: + resolved_path = Path(file_path) + if not resolved_path.exists(): + raise ModelFileMissingError("Latest model file is missing on disk.") + return resolved_path + + def delete_file_if_exists(self, file_path: Path | None) -> None: + if file_path and file_path.exists(): + file_path.unlink() diff --git a/app/infrastructure/__init__.py b/app/infrastructure/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/infrastructure/__init__.py @@ -0,0 +1 @@ + diff --git a/app/infrastructure/bootstrap.py b/app/infrastructure/bootstrap.py new file mode 100644 index 0000000..d7faf3a --- /dev/null +++ b/app/infrastructure/bootstrap.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass +from pathlib import Path +from app.infrastructure.constants import DATABASE_URL +from app.infrastructure.ModelConverterImpl import ModelConverterImpl +from app.infrastructure.database import Base, create_engine_and_session +from app.infrastructure.ModelStorageImpl import ModelStorageImpl + + +@dataclass(slots=True) +class Container: + storage: ModelStorageImpl + converter: ModelConverterImpl + engine: object + session_factory: object + + +_container: Container | None = None + + +def get_container() -> Container: + global _container + if _container is None: + storage = ModelStorageImpl() + engine, session_factory = create_engine_and_session(DATABASE_URL) + _container = Container( + storage=storage, + converter=ModelConverterImpl(), + engine=engine, + session_factory=session_factory, + ) + return _container + + +def initialize_application() -> None: + container = get_container() + container.storage.ensure_directories() + engine_path = container.engine.url.database + if engine_path: + Path(engine_path).parent.mkdir(parents=True, exist_ok=True) + Base.metadata.create_all(bind=container.engine) diff --git a/app/infrastructure/constants.py b/app/infrastructure/constants.py new file mode 100644 index 0000000..3d5a08a --- /dev/null +++ b/app/infrastructure/constants.py @@ -0,0 +1,13 @@ +import os +from pathlib import Path + + +BASE_DIR = Path(__file__).resolve().parent.parent.parent +MODEL_DIR = Path(os.getenv("MODELS_DIR", str(BASE_DIR / "storage" / "models"))) +TMP_DIR = Path(os.getenv("TMP_DIR", str(BASE_DIR / "tmp"))) +DATABASE_PATH = BASE_DIR / "db" / "app.db" +DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{DATABASE_PATH}") +API_KEY_HEADER_NAME = "X-API-Key" +API_KEYS = tuple( + key.strip() for key in os.getenv("API_KEYS", "").split(",") if key.strip() +) diff --git a/app/infrastructure/database.py b/app/infrastructure/database.py new file mode 100644 index 0000000..a618c79 --- /dev/null +++ b/app/infrastructure/database.py @@ -0,0 +1,42 @@ +from datetime import datetime, timezone +from sqlalchemy import Boolean, DateTime, Integer, String, create_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker +from app.domain.Model import Model + + +class Base(DeclarativeBase): + pass + + +class ModelRecord(Base): + __tablename__ = "models" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True) + version: Mapped[str] = mapped_column(String(64), unique=True, nullable=False, index=True) + filename: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) + file_path: Mapped[str] = mapped_column(String(1024), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(timezone.utc), + ) + is_latest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + + def to_domain(self) -> Model: + return Model( + id=self.id, + version=self.version, + filename=self.filename, + file_path=self.file_path, + created_at=self.created_at, + is_latest=self.is_latest, + ) + + +def create_engine_and_session(database_url: str): + engine = create_engine( + database_url, + connect_args={"check_same_thread": False} if database_url.startswith("sqlite") else {}, + ) + session_factory = sessionmaker(bind=engine, autocommit=False, autoflush=False) + return engine, session_factory diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..a9974d8 --- /dev/null +++ b/app/main.py @@ -0,0 +1,13 @@ +from fastapi import FastAPI + +from app.infrastructure.bootstrap import initialize_application +from app.presentation.routes import router as model_router + + +app = FastAPI() +app.include_router(model_router) + + +@app.on_event("startup") +def on_startup() -> None: + initialize_application() diff --git a/app/presentation/__init__.py b/app/presentation/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/app/presentation/__init__.py @@ -0,0 +1 @@ + diff --git a/app/presentation/dependencies.py b/app/presentation/dependencies.py new file mode 100644 index 0000000..5117022 --- /dev/null +++ b/app/presentation/dependencies.py @@ -0,0 +1,53 @@ +from collections.abc import Generator +from hmac import compare_digest + +from fastapi import Depends, HTTPException, Security, status +from fastapi.security import APIKeyHeader +from sqlalchemy.orm import Session + +from app.domain.usecase.GetLatestModelUseCase import GetLatestModelUseCase +from app.domain.usecase.UploadModelUseCase import UploadModelUseCase +from app.infrastructure.bootstrap import get_container +from app.infrastructure.constants import API_KEY_HEADER_NAME, API_KEYS +from app.infrastructure.ModelRepositoryImpl import ModelRepositoryImpl + +api_key_header = APIKeyHeader(name=API_KEY_HEADER_NAME, auto_error=False) + + +def verify_api_key(api_key: str | None = Security(api_key_header)) -> None: + if api_key and any(compare_digest(api_key, valid_key) for valid_key in API_KEYS): + return + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing API key", + ) + + +def get_db_session() -> Generator[Session, None, None]: + container = get_container() + session = container.session_factory() + try: + yield session + finally: + session.close() + + +def get_upload_model_use_case( + session: Session = Depends(get_db_session), +) -> UploadModelUseCase: + container = get_container() + repository = ModelRepositoryImpl(session) + return UploadModelUseCase( + repository=repository, + storage=container.storage, + converter=container.converter, + ) + + +def get_latest_model_use_case( + session: Session = Depends(get_db_session), +) -> GetLatestModelUseCase: + container = get_container() + repository = ModelRepositoryImpl(session) + return GetLatestModelUseCase(repository=repository, storage=container.storage) diff --git a/app/presentation/response/ModelResponse.py b/app/presentation/response/ModelResponse.py new file mode 100644 index 0000000..54135b5 --- /dev/null +++ b/app/presentation/response/ModelResponse.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from pydantic import BaseModel, field_serializer + +from app.domain.Model import Model + + +class ModelResponse(BaseModel): + version: str + created_at: datetime + filename: str + is_latest: bool + + @field_serializer("created_at") + def serialize_created_at(self, value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + @staticmethod + def from_model(model: Model) -> ModelResponse: + return ModelResponse( + version=model.version, + created_at=model.created_at, + filename=model.filename, + is_latest=model.is_latest, + ) diff --git a/app/presentation/routes.py b/app/presentation/routes.py new file mode 100644 index 0000000..14712bd --- /dev/null +++ b/app/presentation/routes.py @@ -0,0 +1,66 @@ +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, status +from fastapi.responses import FileResponse +from app.domain.Exception import * +from app.domain.usecase.GetLatestModelUseCase import GetLatestModelUseCase +from app.domain.usecase.UploadModelUseCase import UploadModelUseCase +from app.presentation.dependencies import ( + get_latest_model_use_case, + get_upload_model_use_case, + verify_api_key, +) +from app.presentation.response.ModelResponse import ModelResponse + +router = APIRouter(dependencies=[Depends(verify_api_key)]) + + +@router.post("/models", response_model=ModelResponse, status_code=status.HTTP_201_CREATED) +def upload_model( + file: UploadFile = File(...), + version: str = Form(...), + use_case: UploadModelUseCase = Depends(get_upload_model_use_case), +) -> ModelResponse: + try: + result = use_case.execute(version=version, upload_file=file) + except InvalidModelVersionError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except InvalidModelFileError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + except DuplicateModelVersionError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc + except ModelConversionError as exc: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(exc), + ) from exc + return ModelResponse.from_model(result) + + +@router.get("/models/latest", response_model=ModelResponse) +def get_latest_model( + use_case: GetLatestModelUseCase = Depends(get_latest_model_use_case), +) -> ModelResponse: + try: + result = use_case.execute() + except ModelNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + return ModelResponse.from_model(result) + + +@router.get("/models/latest/download") +def download_latest_model( + version: str = Query(...), + filename: str = Query(...), + use_case: GetLatestModelUseCase = Depends(get_latest_model_use_case), +) -> FileResponse: + try: + result = use_case.execute(version=version, filename=filename) + except ModelNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + except ModelFileMissingError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) from exc + + return FileResponse( + path=result.file_path, + filename=result.filename, + media_type="application/octet-stream", + ) diff --git a/deploy/systemd/model-update-server.env.example b/deploy/systemd/model-update-server.env.example new file mode 100644 index 0000000..15b5ff6 --- /dev/null +++ b/deploy/systemd/model-update-server.env.example @@ -0,0 +1,4 @@ +API_KEYS=replace-with-generated-key-1,replace-with-generated-key-2 +MODELS_DIR=/opt/model-update-server/storage/models +TMP_DIR=/opt/model-update-server/tmp +DATABASE_URL=sqlite:////opt/model-update-server/db/app.db diff --git a/deploy/systemd/model-update-server.service b/deploy/systemd/model-update-server.service new file mode 100644 index 0000000..b78a8ef --- /dev/null +++ b/deploy/systemd/model-update-server.service @@ -0,0 +1,14 @@ +[Unit] +Description=Model Update Server +After=network.target + +[Service] +Type=simple +WorkingDirectory=/opt/model-update-server +EnvironmentFile=/etc/model-update-server/model-update-server.env +ExecStart=/opt/model-update-server/.venv/bin/uvicorn app.main:app --host 0.0.0.0 --port 8000 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6251bd7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +sqlalchemy==2.0.40 +python-multipart==0.0.20 + +--extra-index-url https://download.pytorch.org/whl/cpu +torch==2.5.1+cpu +torchvision==0.20.1+cpu +onnxruntime +onnx +numpy +pillow +ultralytics diff --git a/scripts/generate_api_keys.sh b/scripts/generate_api_keys.sh new file mode 100755 index 0000000..923c187 --- /dev/null +++ b/scripts/generate_api_keys.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +KEY_COUNT="${KEY_COUNT:-10}" +KEY_BYTES="${KEY_BYTES:-32}" +ENV_FILE="" + +usage() { + cat <<'EOF' +Usage: + source scripts/generate_api_keys.sh + scripts/generate_api_keys.sh --env-file .env + +Environment: + KEY_COUNT Number of keys to generate. Default: 10 + KEY_BYTES Random bytes per key. Default: 32 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --env-file) + ENV_FILE="${2:-}" + if [[ -z "$ENV_FILE" ]]; then + echo "Missing value for --env-file" >&2 + exit 1 + fi + shift 2 + ;; + -h|--help) + usage + return 0 2>/dev/null || exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +API_KEYS="$( + python3 - "$KEY_COUNT" "$KEY_BYTES" <<'PY' +import secrets +import sys + +count = int(sys.argv[1]) +bytes_count = int(sys.argv[2]) + +keys = [secrets.token_urlsafe(bytes_count) for _ in range(count)] +print(",".join(keys)) +PY +)" + +export API_KEYS + +echo "Generated ${KEY_COUNT} API keys." + +if [[ -n "$ENV_FILE" ]]; then + umask 077 + printf 'API_KEYS=%s\n' "$API_KEYS" > "$ENV_FILE" + echo "Saved API_KEYS to ${ENV_FILE}" +else + echo "API_KEYS=${API_KEYS}" +fi + +if [[ -z "$ENV_FILE" ]] && ! (return 0 2>/dev/null); then + echo + echo "This script was executed in a child shell." + echo "Run it with source to keep API_KEYS in your current shell:" + echo "source scripts/generate_api_keys.sh" +fi diff --git a/tmp/.gitkeep b/tmp/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tmp/.gitkeep @@ -0,0 +1 @@ +