from copy import copy
import os
import time
from typing import Any, List, Optional, Tuple, Union, Dict
import numpy as np
import pandas as pd
from PIL import Image
import torch
from torch.utils.data import DataLoader, Dataset
from torchvision.transforms import v2 as transforms
try:
from torchvision import tv_tensors
except ImportError:
from torchvision import datapoints as tv_tensors
from tqdm.auto import tqdm
from perceptionmetrics.datasets import detection as detection_dataset
from perceptionmetrics.models import detection as detection_model
from perceptionmetrics.utils import detection_metrics as um
from perceptionmetrics.utils import image as ui
from perceptionmetrics.utils.torch import (
get_device_info,
get_resize_args,
inverse_transform_boxes,
pad_to_closest_divisor,
)
[docs]
def data_to_device(
data: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]],
device: torch.device,
) -> Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]]:
"""Move detection input or target data (dict or list of dicts) to the specified device.
:param data: Detection data (a single dict or list of dicts with tensor values)
:type data: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]]
:param device: Device to move data to
:type device: torch.device
:return: Data with all tensors moved to the target device
:rtype: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]]
"""
if isinstance(data, dict):
return {k: v.to(device) if torch.is_tensor(v) else v for k, v in data.items()}
elif isinstance(data, list):
return [
{k: v.to(device) if torch.is_tensor(v) else v for k, v in item.items()}
for item in data
]
else:
raise TypeError(f"Expected a dict or list of dicts, got {type(data)}")
[docs]
def get_data_shape(data: Union[torch.Tensor, tuple]) -> tuple:
"""Get the shape of the provided data
:param data: Data provided (it can be a single or multiple tensors)
:type data: Union[tuple, list]
:return: Data shape
:rtype: Union[tuple, list]
"""
if isinstance(data, tuple):
return data[0].shape
return data.shape
[docs]
def get_computational_cost(
model: Any,
dummy_input: Union[torch.Tensor, tuple, list],
model_fname: Optional[str] = None,
runs: int = 30,
warm_up_runs: int = 5,
) -> pd.DataFrame:
"""
Get different metrics related to the computational cost of a model.
:param model: TorchScript or PyTorch model (segmentation, detection, etc.)
:type model: Any
:param dummy_input: Dummy input data (Tensor, Tuple, or List of Dicts for detection)
:type dummy_input: Union[torch.Tensor, tuple, list]
:param model_fname: Optional path to model file for size estimation
:type model_fname: Optional[str]
:param runs: Number of timed runs
:type runs: int
:param warm_up_runs: Warm-up iterations before timing
:type warm_up_runs: int
:return: DataFrame with size, inference time, parameter count, etc.
:rtype: pd.DataFrame
"""
# Compute model size if applicable
size_mb = os.path.getsize(model_fname) / 1024**2 if model_fname else None
# Format input consistently
if isinstance(dummy_input, (torch.Tensor, tuple)):
dummy_tuple = dummy_input if isinstance(dummy_input, tuple) else (dummy_input,)
else:
dummy_tuple = dummy_input # e.g., list of dicts for detection
# Warm-up
for _ in range(warm_up_runs):
with torch.no_grad():
if hasattr(model, "inference"):
model.inference(*dummy_tuple)
else:
model(*dummy_tuple)
# Measure inference time
use_cuda = next(model.parameters()).device.type == "cuda"
inference_times = []
for _ in range(runs):
if use_cuda:
torch.cuda.synchronize()
start = time.perf_counter()
with torch.no_grad():
if hasattr(model, "inference"):
model.inference(*dummy_tuple)
else:
model(*dummy_tuple)
if use_cuda:
torch.cuda.synchronize()
inference_times.append(time.perf_counter() - start)
# Get number of parameters
n_params = sum(p.numel() for p in model.parameters())
# Get input shape
input_shape = get_data_shape(dummy_input)
input_shape_str = "x".join(map(str, input_shape))
result = {
"input_shape": [input_shape_str],
"n_params": [n_params],
"size_mb": [size_mb],
"inference_time_s": [np.mean(inference_times)],
}
return pd.DataFrame.from_dict(result)
[docs]
class ImageDetectionTorchDataset(Dataset):
"""Dataset for image detection PyTorch models
:param dataset: Image detection dataset
:type dataset: ImageDetectionDataset
:param transform: Transformation to be applied to images
:type transform: transforms.Compose
:param splits: Splits to be used from the dataset, defaults to ["test"]
:type splits: str, optional
"""
def __init__(
self,
dataset: detection_dataset.ImageDetectionDataset,
transform: transforms.Compose,
splits: List[str] = ["test"],
):
self.dataset = copy(dataset)
# Raise early if any requested split is absent — prevents silent NaN metrics
self.dataset._validate_splits(splits)
# Filter split and make filenames global
self.dataset.dataset = self.dataset.dataset[
self.dataset.dataset["split"].isin(splits)
]
# Use the dataset's make_fname_global method instead of manual path joining
self.dataset.make_fname_global()
self.transform = transform
def __len__(self):
return len(self.dataset.dataset)
def __getitem__(
self, idx: int
) -> Tuple[int, torch.Tensor, Dict[str, torch.Tensor]]:
"""
Load image and annotations, apply transforms.
:param idx: Sample index
:return: Tuple of (sample_id, image_tensor, target_dict)
"""
row = self.dataset.dataset.iloc[idx]
image_path = row["image"]
ann_path = row["annotation"]
image = Image.open(image_path).convert("RGB")
boxes, category_indices = self.dataset.read_annotation(ann_path)
# Convert boxes/labels to tensors
if len(boxes) == 0:
boxes = torch.zeros((0, 4), dtype=torch.float32)
if hasattr(tv_tensors, "BoundingBoxes"):
boxes = tv_tensors.BoundingBoxes(
boxes, format="XYXY", canvas_size=(image.height, image.width)
)
else:
boxes = tv_tensors.BoundingBox(
boxes, format="XYXY", spatial_size=(image.height, image.width)
)
category_indices = torch.as_tensor(category_indices, dtype=torch.int64)
target = {
"boxes": boxes, # [N, 4]
"labels": category_indices, # [N]
"orig_size": torch.tensor([image.width, image.height], dtype=torch.int32),
}
if self.transform:
image, target = self.transform(image, target)
return self.dataset.dataset.index[idx], image, target
[docs]
class TorchImageDetectionModel(detection_model.ImageDetectionModel):
"""TorchImageDetectionModel wrapper.
.. seealso::
`User guide <https://jderobot.github.io/PerceptionMetrics/models/#image-detection-config>`__.
"""
def __init__(
self,
model: Union[str, torch.nn.Module],
model_cfg: str,
ontology_fname: str,
device: torch.device = None,
):
"""Image detection model for PyTorch framework
:param model: Either the filename of a TorchScript model or the model already loaded into a PyTorch module.
:type model: Union[str, torch.nn.Module]
:param model_cfg: JSON file containing model configuration
:type model_cfg: str
:param ontology_fname: JSON file containing model output ontology
:type ontology_fname: str
:param device: torch.device to use (optional). If not provided, best available device is auto-selected using get_device_info() from perceptionmetrics.utils.torch.
:type device: torch.device
"""
# Get device (GPU, MPS, or CPU) if not provided
if device is None:
best_device, _ = get_device_info()
self.device = torch.device(best_device)
else:
self.device = device
# Load model from file or use passed instance
if isinstance(model, str):
assert os.path.isfile(model), "Torch model file not found"
model_fname = model
try:
model = torch.jit.load(model, map_location=self.device)
model_type = "compiled"
except Exception:
try:
loaded = torch.load(model, map_location=self.device, weights_only=False)
# Handle Ultralytics/YOLO-style dict checkpoints
if isinstance(loaded, dict):
candidate = loaded.get("ema") or loaded.get("model")
if candidate is None or not hasattr(candidate, "forward"):
raise ValueError(
"""
The loaded .pt file is a dictionary but doesn't contain a valid model under keys 'model' or 'ema'. Please export to TorchScript for better compatibility.
"""
)
model = candidate
else:
model = loaded
model_type = "native"
# Fallback for missing Ultralytics dependency
except (ModuleNotFoundError, AttributeError) as e:
raise ImportError(
f"Failed to load native .pt model. This often happens if the 'ultralytics' "
f"library is missing or incompatible. \nOriginal error: {e}\n"
f"SUGGESTION: 'pip install ultralytics' or export your model to TorchScript."
) from e
# Init parent class
super().__init__(model, model_type, model_cfg, ontology_fname, model_fname)
# Define the Wrapper with DType Alignment
class DetectionModelWrapper(torch.nn.Module):
def __init__(self, model):
super().__init__()
self.inner_model = model
# Handle input precision, tuple extraction, and output precision
def forward(self, x):
out = self.inner_model(x)
# Only unwrap TUPLES (native YOLO .pt)
# Avoid unwrapping LISTS (torchvision Mask R-CNN)
if isinstance(out, tuple) and len(out) > 0:
out = out[0]
if isinstance(out, torch.Tensor):
return out.float()
return out
self.model = DetectionModelWrapper(self.model).to(self.device).float().eval()
# Load post-processing functions for specific model formats
self.model_format = self.model_cfg.get("model_format", "torchvision")
if self.model_format == "yolo":
from perceptionmetrics.models.utils.yolo import postprocess_detection
elif self.model_format == "torchvision":
from perceptionmetrics.models.utils.torchvision import postprocess_detection
else:
raise ValueError(f"Unsupported model_format: {self.model_format}")
self.postprocess_detection = postprocess_detection
# Load confidence and NMS thresholds from config
self.confidence_threshold = self.model_cfg.get("confidence_threshold", 0.5)
self.nms_threshold = self.model_cfg.get("nms_threshold", 0.3)
self.max_detections_per_image = self.model_cfg.get(
"max_detections_per_image", -1
)
self.postprocess_args = [self.confidence_threshold]
if self.model_format == "yolo":
self.postprocess_args.append(self.nms_threshold)
self.postprocess_args.append(self.max_detections_per_image)
# Add reverse mapping for idx to class_name
self.idx_to_class_name = {v["idx"]: k for k, v in self.ontology.items()}
# Build input transforms (resize, normalize, etc.)
self.transform_input = []
self.closest_divisor = None
resize_cfg = self.model_cfg.get("resize")
if resize_cfg is not None:
self.closest_divisor = resize_cfg.pop("closest_divisor", None)
if resize_cfg:
resize_args = get_resize_args(resize_cfg)
self.transform_input.append(transforms.Resize(**resize_args))
else:
print("'resize_cfg' missing in model config. No resizing will be applied.")
if "crop" in self.model_cfg:
crop_size = (
self.model_cfg["crop"]["height"],
self.model_cfg["crop"]["width"],
)
self.transform_input += [transforms.CenterCrop(crop_size)]
try:
self.transform_input += [
transforms.ToImage(),
transforms.ToDtype(torch.float32, scale=True),
]
except AttributeError:
self.transform_input += [
transforms.ToImageTensor(),
transforms.ConvertDtype(torch.float32),
]
if "normalization" in self.model_cfg:
self.transform_input += [
transforms.Normalize(
mean=self.model_cfg["normalization"]["mean"],
std=self.model_cfg["normalization"]["std"],
)
]
self.transform_input = transforms.Compose(self.transform_input)
[docs]
def predict(
self, image: Image.Image, return_sample: bool = False
) -> Union[Dict[str, torch.Tensor], Tuple[Dict[str, torch.Tensor], torch.Tensor]]:
"""Perform prediction for a single image
:param image: PIL image
:type image: Image.Image
:param return_sample: Whether to return the sample data along with predictions, defaults to False
:type return_sample: bool, optional
:return: Detection result or a tuple with the detection result and the input sample tensor
:rtype: Union[Dict[str, torch.Tensor], Tuple[Dict[str, torch.Tensor], torch.Tensor]]
"""
orig_size = (image.width, image.height)
sample = self.transform_input(image).unsqueeze(0).to(self.device)
result = self.inference(sample)
# Scale bounding boxes back to the original image size
result["boxes"] = inverse_transform_boxes(
result["boxes"], orig_size, self.transform_input.transforms
)
if return_sample:
return result, sample
else:
return result
[docs]
def inference(self, tensor_in: torch.Tensor) -> Dict[str, torch.Tensor]:
"""Perform inference for a tensor
:param tensor_in: Input tensor
:type tensor_in: torch.Tensor
:return: Dictionary with keys 'boxes', 'labels', 'scores'
:rtype: Dict[str, torch.Tensor]
"""
if self.closest_divisor is not None:
tensor_in, (orig_h, orig_w) = pad_to_closest_divisor(
tensor_in, self.closest_divisor
)
with torch.no_grad():
result = self.model(tensor_in.to(self.device))[0] # only first image
# Apply threshold filtering from model config
result = self.postprocess_detection(result, *self.postprocess_args)
# Crop/clamp the bounding boxes to the original shape before padding
if self.closest_divisor is not None:
result["boxes"][:, [0, 2]] = result["boxes"][:, [0, 2]].clamp(0, orig_w)
result["boxes"][:, [1, 3]] = result["boxes"][:, [1, 3]].clamp(0, orig_h)
return result
[docs]
def eval(
self,
dataset: detection_dataset.ImageDetectionDataset,
split: Union[str, List[str]] = "test",
ontology_translation: Optional[str] = None,
predictions_outdir: Optional[str] = None,
results_per_sample: bool = False,
save_visualizations: bool = False,
progress_callback=None,
metrics_callback=None,
) -> pd.DataFrame:
"""Evaluate model over a detection dataset and compute metrics
:param dataset: Image detection dataset
:type dataset: ImageDetectionDataset
:param split: Dataset split(s) to evaluate
:type split: Union[str, List[str]]
:param ontology_translation: Optional translation for class mapping
:type ontology_translation: Optional[str]
:param predictions_outdir: Directory to save predictions, if desired
:type predictions_outdir: Optional[str]
:param results_per_sample: Store per-sample metrics
:type results_per_sample: bool
:param save_visualizations: Save visualized results (GT vs Pred)
:type save_visualizations: bool
:param progress_callback: Optional callback function for progress updates in Streamlit UI
:type progress_callback: Optional[Callable[[int, int], None]]
:param metrics_callback: Optional callback function for intermediate metrics updates in Streamlit UI
:type metrics_callback: Optional[Callable[[pd.DataFrame, int, int], None]]
:return: DataFrame containing evaluation results
:rtype: pd.DataFrame
"""
if (results_per_sample or save_visualizations) and predictions_outdir is None:
raise ValueError(
"predictions_outdir required if results_per_sample or save_visualizations is True"
)
if predictions_outdir is not None:
os.makedirs(predictions_outdir, exist_ok=True)
# Build LUT if ontology translation is provided
lut_ontology = self.get_lut_ontology(dataset.ontology, ontology_translation)
if lut_ontology is not None:
lut_ontology = torch.tensor(lut_ontology, dtype=torch.int64).to(self.device)
# Create DataLoader
torch_dataset = ImageDetectionTorchDataset(
dataset,
transform=self.transform_input,
splits=[split] if isinstance(split, str) else split,
)
# This ensures compatibility with Streamlit and callback functions
if progress_callback is not None and metrics_callback is not None:
num_workers = 0
else:
num_workers = self.model_cfg.get("num_workers", 0)
dataloader = DataLoader(
torch_dataset,
batch_size=self.model_cfg.get("batch_size", 1),
num_workers=num_workers,
collate_fn=lambda batch: tuple(
zip(*batch)
), # handles variable-size targets
)
# Get iou_threshold from model config, default to 0.5 if not present
iou_threshold = self.model_cfg.get("iou_threshold", 0.5)
# Get evaluation_step from model config, default to None (no intermediate updates)
evaluation_step = self.model_cfg.get("evaluation_step", None)
# If evaluation_step is 0, treat as None (disabled)
if evaluation_step == 0:
evaluation_step = None
# Init metrics
metrics_factory = um.DetectionMetricsFactory(
iou_threshold=iou_threshold, num_classes=self.n_classes
)
# Calculate total samples for progress tracking
total_samples = len(dataloader.dataset)
processed_samples = 0
with torch.no_grad():
# Use tqdm if no progress callback provided, otherwise use regular iteration
if progress_callback is None:
pbar = tqdm(dataloader, leave=True)
iterator = pbar
else:
iterator = dataloader
for image_ids, images, targets in iterator:
# Defensive check for empty images
if not images or any(image.numel() == 0 for image in images):
print("Skipping batch: empty image tensor detected.")
continue
images = torch.stack(images).to(self.device)
if self.closest_divisor is not None:
images, (orig_h, orig_w) = pad_to_closest_divisor(
images, self.closest_divisor
)
predictions = self.model(images)
for i in range(len(images)):
gt = targets[i]
pred = predictions[i]
image_tensor = images[i]
sample_id = image_ids[i]
# Post-process predictions
pred = self.postprocess_detection(pred, *self.postprocess_args)
# Crop/clamp the bounding boxes to the original shape before padding
if self.closest_divisor is not None:
pred["boxes"][:, [0, 2]] = pred["boxes"][:, [0, 2]].clamp(
0, orig_w
)
pred["boxes"][:, [1, 3]] = pred["boxes"][:, [1, 3]].clamp(
0, orig_h
)
# Apply ontology translation if needed
if lut_ontology is not None:
gt["labels"] = lut_ontology[gt["labels"]]
# Update metrics
metrics_factory.update(
gt["boxes"],
gt["labels"],
pred["boxes"],
pred["labels"],
pred["scores"],
)
# Store predictions or visualizations if needed
if predictions_outdir is not None:
pred_boxes = pred["boxes"].cpu().numpy()
pred_labels = pred["labels"].cpu().numpy()
pred_scores = pred["scores"].cpu().numpy()
# Save JSON with predictions and csv with metrics per sample
if results_per_sample:
out_data = []
orig_pred_boxes = inverse_transform_boxes(
boxes=pred["boxes"].cpu(),
orig_size=tuple(gt["orig_size"].tolist()),
input_transforms=self.transform_input.transforms,
).numpy()
for box, label, score in zip(
orig_pred_boxes, pred_labels, pred_scores
):
# Convert label index to class name using model ontology
class_name = self.idx_to_class_name.get(
int(label), f"class_{label}"
)
out_data.append(
{
"image_id": sample_id,
"label": class_name,
"score": float(score),
"bbox": box.tolist(),
}
)
df = pd.DataFrame(out_data)
df.to_json(
os.path.join(predictions_outdir, f"{sample_id}.json"),
orient="records",
indent=2,
)
sample_mf = um.DetectionMetricsFactory(
iou_threshold=iou_threshold, num_classes=self.n_classes
)
sample_mf.update(
gt["boxes"],
gt["labels"],
pred["boxes"],
pred["labels"],
pred["scores"],
)
sample_df = sample_mf.get_metrics_dataframe(self.ontology)
sample_df.to_csv(
os.path.join(
predictions_outdir, f"{sample_id}_metrics.csv"
)
)
# Save visualizations per sample
if save_visualizations:
pil_image = transforms.ToPILImage()(image_tensor.cpu())
gt_boxes = gt["boxes"].cpu().numpy()
gt_labels = gt["labels"].cpu().numpy()
gt_class_names = [
self.idx_to_class_name.get(int(label), str(label))
for label in gt_labels
]
pred_class_names = [
self.idx_to_class_name.get(int(label), str(label))
for label in pred_labels
]
image_gt = ui.draw_detections(
pil_image.copy(),
gt_boxes,
gt_labels,
gt_class_names,
scores=None,
)
image_pred = ui.draw_detections(
pil_image.copy(),
pred_boxes,
pred_labels,
pred_class_names,
scores=pred_scores,
)
pil_gt = Image.fromarray(image_gt)
pil_pred = Image.fromarray(image_pred)
combined_width = pil_gt.width + pil_pred.width
combined_height = max(pil_gt.height, pil_pred.height)
combined_image = Image.new(
"RGB", (combined_width, combined_height)
)
combined_image.paste(pil_gt, (0, 0))
combined_image.paste(pil_pred, (pil_gt.width, 0))
out_file = os.path.join(
predictions_outdir, f"{sample_id}.jpg"
)
combined_image.save(out_file)
processed_samples += 1
# Call progress callback if provided
if progress_callback is not None:
progress_callback(processed_samples, total_samples)
# Call metrics callback if provided and evaluation_step is reached
if (
metrics_callback is not None
and evaluation_step is not None
and processed_samples % evaluation_step == 0
):
# Get intermediate metrics
intermediate_metrics = metrics_factory.get_metrics_dataframe(
self.ontology
)
metrics_callback(
intermediate_metrics, processed_samples, total_samples
)
# Return both the DataFrame and the metrics factory for access to precision-recall curves
return {
"metrics_df": metrics_factory.get_metrics_dataframe(self.ontology),
"metrics_factory": metrics_factory,
}
[docs]
def get_computational_cost(
self, image_size: Tuple[int], runs: int = 30, warm_up_runs: int = 5
) -> dict:
"""Get computational cost metrics like inference time
:param image_size: Size of input image (H, W)
:type image_size: Tuple[int]
:param runs: Number of repeated runs to average over
:type runs: int
:param warm_up_runs: Warm-up runs before timing
:type warm_up_runs: int
:return: Dictionary with computational cost details
:rtype: dict
"""
dummy_input = torch.randn(1, 3, *image_size).to(self.device)
return get_computational_cost(
self.model, dummy_input, self.model_fname, runs, warm_up_runs
)