From e208aca53c6b7532324ecb6b5a2e2388edad6ed9 Mon Sep 17 00:00:00 2001 From: Kurt Stolle Date: Mon, 26 Feb 2024 15:10:42 +0100 Subject: [PATCH] Amend: fix video pipe --- notebooks/evaluation.ipynb | 215 -------------------------- sources/unipercept/_api_config.py | 5 + sources/unipercept/_api_data.py | 24 +-- sources/unipercept/api/__init__.py | 0 sources/unipercept/data/_loader.py | 1 - sources/unipercept/render/__init__.py | 1 + sources/unipercept/render/_video.py | 63 ++++++++ 7 files changed, 83 insertions(+), 226 deletions(-) delete mode 100644 notebooks/evaluation.ipynb delete mode 100644 sources/unipercept/api/__init__.py create mode 100644 sources/unipercept/render/_video.py diff --git a/notebooks/evaluation.ipynb b/notebooks/evaluation.ipynb deleted file mode 100644 index e5abe55..0000000 --- a/notebooks/evaluation.ipynb +++ /dev/null @@ -1,215 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Evaluation\n", - "\n", - "In `unipercept`, the evaluation of a trained model is also handled by the `unipercept.engine.Engine` class. This\n", - "notebook shows how metrics can be evaluated using the Python interface." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.integrations.wandb_integration\u001b[0m\u001b[1m:\u001b[0m Reading W&B configuration from wandb-run:\u001b[1m//tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[1m:\u001b[0m Found dataset loader cityscapes-vps/val via key\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (1 frames) with required sources (frozenset({'image', 'panoptic'}),)\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Wrapping dataset:\u001b[1m CityscapesVPSDataset(queue_fn=GroupAdjacentTime(num_frames=1, use_typecheck=False), split='val', root='//datasets/cityscapes-vps', all=False)\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Found 300 sequences with 1 captures!\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Initialized sampler 1 of 1\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Creating dataloader (300 queued; 300 × 1 items):\u001b[1m\n", - "------------------ ------------------------------------------------------------------------\n", - "drop_last False\n", - "pin_memory True\n", - "num_workers 15\n", - "prefetch_factor 2\n", - "persistent_workers False\n", - "sampler \n", - "batch_size 1\n", - "collate_fn >\n", - "worker_init_fn \n", - "------------------ ------------------------------------------------------------------------\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (1 frames) with required sources (frozenset({'image'}),)\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (2 frames) with required sources (frozenset({'image', 'panoptic'}), frozenset({'image', 'panoptic'}))\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (1 frames) with required sources (frozenset({'image'}),)\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (1 frames) with required sources (frozenset({'image', 'panoptic'}),)\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (2 frames) with required sources (frozenset({'image', 'panoptic'}), frozenset({'image', 'panoptic'}))\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:34\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Using adjacent collector (2 frames) with required sources (frozenset({'image', 'panoptic'}), frozenset({'image', 'panoptic'}))\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:35\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Saving configuration to /home/khwstolle/projects/phd-dvps/unipercept/output/multidvps/01HQAVZMT1A50X2Y60M0D4FJAE/config.yaml\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:35\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Starting evaluation procedure...\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:35\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Running inference on loader 'cityscapes-vps/val' for 2 handlers\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:35\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.model\u001b[0m\u001b[1m:\u001b[0m Instantiating model without configuration overrides\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:37\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.model\u001b[0m\u001b[1m:\u001b[0m Loading model weights:\u001b[1m wandb-run://tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:37\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.integrations.wandb_integration\u001b[0m\u001b[1m:\u001b[0m Reading W&B configuration from wandb-run:\u001b[1m//tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:37\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[1m:\u001b[0m Reading W&B model checkpoint from wandb-run:\u001b[1m//tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:38\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[1m:\u001b[0m Downloading model artifact tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2-model:\u001b[1mlatest/model.safetensors\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:38\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.utils.iopath_handlers\u001b[0m\u001b[1m:\u001b[0m Using W&B artifact using the API (without a run)\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:38\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[1m:\u001b[0m Loading checkpoint from path /home/khwstolle/.torch/iopath_cache/tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2-model:\u001b[1mlatest/model.safetensors\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Wrapping dataset:\u001b[1m CityscapesVPSDataset(queue_fn=GroupAdjacentTime(num_frames=1, use_typecheck=False), split='val', root='//datasets/cityscapes-vps', all=False)\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data.collect\u001b[0m\u001b[1m:\u001b[0m Found 300 sequences with 1 captures!\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Initialized sampler 1 of 1\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.data\u001b[0m\u001b[1m:\u001b[0m Creating dataloader (300 queued; 300 × 1 items):\u001b[1m\n", - "------------------ ------------------------------------------------------------------------\n", - "drop_last False\n", - "pin_memory True\n", - "num_workers 15\n", - "prefetch_factor 2\n", - "persistent_workers False\n", - "sampler \n", - "batch_size 1\n", - "collate_fn >\n", - "worker_init_fn \n", - "------------------ ------------------------------------------------------------------------\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Starting inference procedure...\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Setting up experiment trackers\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Loading configuration from /home/khwstolle/projects/phd-dvps/unipercept/output/multidvps/01HQAVZMT1A50X2Y60M0D4FJAE/config.yaml\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:39\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Loading configuration from /home/khwstolle/projects/phd-dvps/unipercept/output/multidvps/01HQAVZMT1A50X2Y60M0D4FJAE/config.yaml\u001b[1m\u001b[0m\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR:wandb.jupyter:Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n", - "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkurt-stolle\u001b[0m (\u001b[33mtue-mps\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" - ] - }, - { - "data": { - "text/html": [ - "wandb version 0.16.3 is available! To upgrade, please run:\n", - " $ pip install wandb --upgrade" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Tracking run with wandb version 0.16.2" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Run data is saved locally in /home/khwstolle/projects/phd-dvps/unipercept/wandb/run-20240223_123540-j6wz32xr" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Syncing run cityscapes/multidvps_resnet50 to Weights & Biases (docs)
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - " View project at https://wandb.ai/tue-mps/multidvps" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - " View run at https://wandb.ai/tue-mps/multidvps/runs/j6wz32xr" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:45\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.integrations.wandb_integration\u001b[0m\u001b[1m:\u001b[0m Tracking current experiment to WandB run cityscapes/multidvps_resnet50\u001b[1m\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:46\u001b[0m 🐛 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Expecting 300 samples (300 batches, offsets [0])\u001b[1m\u001b[0m\n", - "[2024-02-23 12:35:46,982] [INFO] [real_accelerator.py:161:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:47\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Preparing model for evaluation:\u001b[1m GenericFCN\u001b[0m\n", - "\u001b[36m2024-02-23\u001b[0m \u001b[96m12:35:48\u001b[0m 📝 \u001b[1m\u001b[33munipercept\u001b[0m\u001b[33m.engine\u001b[0m\u001b[1m:\u001b[0m Running inference loop on 1 processes.\u001b[1m\u001b[0m\n" - ] - }, - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", - "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", - "\u001b[1;31mClick here for more info. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], - "source": [ - "import unipercept as up\n", - "\n", - "config = up.read_config(\n", - " \"wandb-run://tue-mps/multidvps/01HQ9KH71W0FG092D0C1MMD0AY-train-stage-2\"\n", - ")\n", - "model_factory = up.create_model_factory(config)\n", - "dataset = up.create_dataset(config, \"cityscapes-vps/val\", batch_size=1)\n", - "engine = up.create_engine(config)\n", - "engine.run_evaluation(model_factory, suites=\"cityscapes-vps/val\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "unipercept", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/sources/unipercept/_api_config.py b/sources/unipercept/_api_config.py index cda64c7..51ec5cc 100644 --- a/sources/unipercept/_api_config.py +++ b/sources/unipercept/_api_config.py @@ -51,6 +51,11 @@ "prepare_images", ] + +def __dir__() -> list[str]: + return __all__ + + _logger = get_logger(__name__) diff --git a/sources/unipercept/_api_data.py b/sources/unipercept/_api_data.py index 5f5fe59..d6e8d0c 100644 --- a/sources/unipercept/_api_data.py +++ b/sources/unipercept/_api_data.py @@ -12,67 +12,71 @@ __all__ = ["get_dataset", "get_info", "get_info_at", "list_datasets", "list_info"] +def __dir__() -> list[str]: + return __all__ + + @T.overload def get_dataset( # noqa: D103 - name: T.Literal["cityscapes"], + query: T.Literal["cityscapes"], ) -> type[unisets.cityscapes.CityscapesDataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["cityscapes-vps"], + query: T.Literal["cityscapes-vps"], ) -> type[unisets.cityscapes.CityscapesVPSDataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["kitti-360"], + query: T.Literal["kitti-360"], ) -> type[unisets.kitti_360.KITTI360Dataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["kitti-step"], + query: T.Literal["kitti-step"], ) -> type[unisets.kitti_step.KITTISTEPDataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["kitti-sem"], + query: T.Literal["kitti-sem"], ) -> type[unisets.kitti_sem.SemKITTIDataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["vistas"], + query: T.Literal["vistas"], ) -> type[unisets.vistas.VistasDataset]: ... @T.overload def get_dataset( # noqa: D103 - name: T.Literal["wilddash"], + query: T.Literal["wilddash"], ) -> type[unisets.wilddash.WildDashDataset]: ... @T.overload -def get_dataset(name: str) -> type[unisets.PerceptionDataset]: # noqa: D103 +def get_dataset(query: str) -> type[unisets.PerceptionDataset]: # noqa: D103 ... -def get_dataset(name: str) -> type[unisets.PerceptionDataset]: +def get_dataset(query: str) -> type[unisets.PerceptionDataset]: """ Read a dataset from the catalog, returning the dataset **class** type. """ from unipercept.data.sets import catalog - return catalog.get_dataset(name) + return catalog.get_dataset(query) def get_info(query: str) -> unisets.Metadata: diff --git a/sources/unipercept/api/__init__.py b/sources/unipercept/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sources/unipercept/data/_loader.py b/sources/unipercept/data/_loader.py index 4899a5a..6d100a6 100644 --- a/sources/unipercept/data/_loader.py +++ b/sources/unipercept/data/_loader.py @@ -11,7 +11,6 @@ import math import multiprocessing as M import operator -import os import typing as T import warnings diff --git a/sources/unipercept/render/__init__.py b/sources/unipercept/render/__init__.py index 18da300..04a4b26 100644 --- a/sources/unipercept/render/__init__.py +++ b/sources/unipercept/render/__init__.py @@ -11,3 +11,4 @@ from ._colormap import * from ._plot import * from ._visualizer import * +from ._video import * diff --git a/sources/unipercept/render/_video.py b/sources/unipercept/render/_video.py new file mode 100644 index 0000000..dbf811d --- /dev/null +++ b/sources/unipercept/render/_video.py @@ -0,0 +1,63 @@ +from contextlib import contextmanager +import dataclasses as D +import functools +import os +import sys + +import PIL.Image as pil_image + +import typing as T +import tempfile + +from unipercept.utils.typings import Pathable + +__all__ = ["video_writer"] + + +@contextmanager +def video_writer(out: Pathable, *, fps: int, overwrite: bool = False): + """ + Used for writing a sequence of PIL images to a (temporary) directory, and then + encoding them into a video file using ``ffmpeg`` commands. + """ + + from unipercept.file_io import Path + + def _parse_output_path(out: Pathable) -> str: + out = Path(out) + if out.is_file(): + if not overwrite: + msg = f"File {out!r} already exists, and overwrite is set to False." + raise FileExistsError(msg) + out.unlink() + else: + out.parent.mkdir(parents=True, exist_ok=True) + return str(out) + + def _get_ffmpeg_path() -> str: + return "ffmpeg.exe" if sys.platform == "win32" else "ffmpeg" + + def _get_ffmpeg_cmd(fps: int, dir: str, out: str) -> tuple[str, ...]: + frame_glob = os.path.join(dir, "*.png") + return ( + _get_ffmpeg_path(), + f"-framerate {fps}", + "-pattern_type glob", + f"-i {frame_glob!r}", + "-c:v libx264", + "-pix_fmt yuv420p", + f"{out!r}", + ) + + def _save_image(im: pil_image.Image, *, dir: str): + next_frame = len(os.listdir(dir)) + im.save(os.path.join(dir, f"{next_frame:010d}.png")) + + with tempfile.TemporaryDirectory() as dir: + try: + yield functools.partial(_save_image, dir=dir) + finally: + cmd = " ".join(_get_ffmpeg_cmd(fps, dir, out=_parse_output_path(out))) + + print(cmd, file=sys.stderr) + os.system(cmd)