Source code for perceptionmetrics.models.torch_detection

from copy import copy
import os
import time
from typing import Any, List, Optional, Tuple, Union, Dict

import numpy as np
import pandas as pd
from PIL import Image
import torch
from torch.utils.data import DataLoader, Dataset
from torchvision.transforms import v2 as transforms
try:
    from torchvision import tv_tensors
except ImportError:
    from torchvision import datapoints as tv_tensors
from tqdm.auto import tqdm

from perceptionmetrics.datasets import detection as detection_dataset
from perceptionmetrics.models import detection as detection_model
from perceptionmetrics.utils import detection_metrics as um
from perceptionmetrics.utils import image as ui
from perceptionmetrics.utils.torch import (
    get_device_info,
    get_resize_args,
    inverse_transform_boxes,
    pad_to_closest_divisor,
)


[docs] def data_to_device( data: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]], device: torch.device, ) -> Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]]: """Move detection input or target data (dict or list of dicts) to the specified device. :param data: Detection data (a single dict or list of dicts with tensor values) :type data: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]] :param device: Device to move data to :type device: torch.device :return: Data with all tensors moved to the target device :rtype: Union[Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]] """ if isinstance(data, dict): return {k: v.to(device) if torch.is_tensor(v) else v for k, v in data.items()} elif isinstance(data, list): return [ {k: v.to(device) if torch.is_tensor(v) else v for k, v in item.items()} for item in data ] else: raise TypeError(f"Expected a dict or list of dicts, got {type(data)}")
[docs] def get_data_shape(data: Union[torch.Tensor, tuple]) -> tuple: """Get the shape of the provided data :param data: Data provided (it can be a single or multiple tensors) :type data: Union[tuple, list] :return: Data shape :rtype: Union[tuple, list] """ if isinstance(data, tuple): return data[0].shape return data.shape
[docs] def get_computational_cost( model: Any, dummy_input: Union[torch.Tensor, tuple, list], model_fname: Optional[str] = None, runs: int = 30, warm_up_runs: int = 5, ) -> pd.DataFrame: """ Get different metrics related to the computational cost of a model. :param model: TorchScript or PyTorch model (segmentation, detection, etc.) :type model: Any :param dummy_input: Dummy input data (Tensor, Tuple, or List of Dicts for detection) :type dummy_input: Union[torch.Tensor, tuple, list] :param model_fname: Optional path to model file for size estimation :type model_fname: Optional[str] :param runs: Number of timed runs :type runs: int :param warm_up_runs: Warm-up iterations before timing :type warm_up_runs: int :return: DataFrame with size, inference time, parameter count, etc. :rtype: pd.DataFrame """ # Compute model size if applicable size_mb = os.path.getsize(model_fname) / 1024**2 if model_fname else None # Format input consistently if isinstance(dummy_input, (torch.Tensor, tuple)): dummy_tuple = dummy_input if isinstance(dummy_input, tuple) else (dummy_input,) else: dummy_tuple = dummy_input # e.g., list of dicts for detection # Warm-up for _ in range(warm_up_runs): with torch.no_grad(): if hasattr(model, "inference"): model.inference(*dummy_tuple) else: model(*dummy_tuple) # Measure inference time use_cuda = next(model.parameters()).device.type == "cuda" inference_times = [] for _ in range(runs): if use_cuda: torch.cuda.synchronize() start = time.perf_counter() with torch.no_grad(): if hasattr(model, "inference"): model.inference(*dummy_tuple) else: model(*dummy_tuple) if use_cuda: torch.cuda.synchronize() inference_times.append(time.perf_counter() - start) # Get number of parameters n_params = sum(p.numel() for p in model.parameters()) # Get input shape input_shape = get_data_shape(dummy_input) input_shape_str = "x".join(map(str, input_shape)) result = { "input_shape": [input_shape_str], "n_params": [n_params], "size_mb": [size_mb], "inference_time_s": [np.mean(inference_times)], } return pd.DataFrame.from_dict(result)
[docs] class ImageDetectionTorchDataset(Dataset): """Dataset for image detection PyTorch models :param dataset: Image detection dataset :type dataset: ImageDetectionDataset :param transform: Transformation to be applied to images :type transform: transforms.Compose :param splits: Splits to be used from the dataset, defaults to ["test"] :type splits: str, optional """ def __init__( self, dataset: detection_dataset.ImageDetectionDataset, transform: transforms.Compose, splits: List[str] = ["test"], ): self.dataset = copy(dataset) # Raise early if any requested split is absent — prevents silent NaN metrics self.dataset._validate_splits(splits) # Filter split and make filenames global self.dataset.dataset = self.dataset.dataset[ self.dataset.dataset["split"].isin(splits) ] # Use the dataset's make_fname_global method instead of manual path joining self.dataset.make_fname_global() self.transform = transform def __len__(self): return len(self.dataset.dataset) def __getitem__( self, idx: int ) -> Tuple[int, torch.Tensor, Dict[str, torch.Tensor]]: """ Load image and annotations, apply transforms. :param idx: Sample index :return: Tuple of (sample_id, image_tensor, target_dict) """ row = self.dataset.dataset.iloc[idx] image_path = row["image"] ann_path = row["annotation"] image = Image.open(image_path).convert("RGB") boxes, category_indices = self.dataset.read_annotation(ann_path) # Convert boxes/labels to tensors if len(boxes) == 0: boxes = torch.zeros((0, 4), dtype=torch.float32) if hasattr(tv_tensors, "BoundingBoxes"): boxes = tv_tensors.BoundingBoxes( boxes, format="XYXY", canvas_size=(image.height, image.width) ) else: boxes = tv_tensors.BoundingBox( boxes, format="XYXY", spatial_size=(image.height, image.width) ) category_indices = torch.as_tensor(category_indices, dtype=torch.int64) target = { "boxes": boxes, # [N, 4] "labels": category_indices, # [N] "orig_size": torch.tensor([image.width, image.height], dtype=torch.int32), } if self.transform: image, target = self.transform(image, target) return self.dataset.dataset.index[idx], image, target
[docs] class TorchImageDetectionModel(detection_model.ImageDetectionModel): """TorchImageDetectionModel wrapper. .. seealso:: `User guide <https://jderobot.github.io/PerceptionMetrics/models/#image-detection-config>`__. """ def __init__( self, model: Union[str, torch.nn.Module], model_cfg: str, ontology_fname: str, device: torch.device = None, ): """Image detection model for PyTorch framework :param model: Either the filename of a TorchScript model or the model already loaded into a PyTorch module. :type model: Union[str, torch.nn.Module] :param model_cfg: JSON file containing model configuration :type model_cfg: str :param ontology_fname: JSON file containing model output ontology :type ontology_fname: str :param device: torch.device to use (optional). If not provided, best available device is auto-selected using get_device_info() from perceptionmetrics.utils.torch. :type device: torch.device """ # Get device (GPU, MPS, or CPU) if not provided if device is None: best_device, _ = get_device_info() self.device = torch.device(best_device) else: self.device = device # Load model from file or use passed instance if isinstance(model, str): assert os.path.isfile(model), "Torch model file not found" model_fname = model try: model = torch.jit.load(model, map_location=self.device) model_type = "compiled" except Exception: try: loaded = torch.load(model, map_location=self.device, weights_only=False) # Handle Ultralytics/YOLO-style dict checkpoints if isinstance(loaded, dict): candidate = loaded.get("ema") or loaded.get("model") if candidate is None or not hasattr(candidate, "forward"): raise ValueError( """ The loaded .pt file is a dictionary but doesn't contain a valid model under keys 'model' or 'ema'. Please export to TorchScript for better compatibility. """ ) model = candidate else: model = loaded model_type = "native" # Fallback for missing Ultralytics dependency except (ModuleNotFoundError, AttributeError) as e: raise ImportError( f"Failed to load native .pt model. This often happens if the 'ultralytics' " f"library is missing or incompatible. \nOriginal error: {e}\n" f"SUGGESTION: 'pip install ultralytics' or export your model to TorchScript." ) from e # Init parent class super().__init__(model, model_type, model_cfg, ontology_fname, model_fname) # Define the Wrapper with DType Alignment class DetectionModelWrapper(torch.nn.Module): def __init__(self, model): super().__init__() self.inner_model = model # Handle input precision, tuple extraction, and output precision def forward(self, x): out = self.inner_model(x) # Only unwrap TUPLES (native YOLO .pt) # Avoid unwrapping LISTS (torchvision Mask R-CNN) if isinstance(out, tuple) and len(out) > 0: out = out[0] if isinstance(out, torch.Tensor): return out.float() return out self.model = DetectionModelWrapper(self.model).to(self.device).float().eval() # Load post-processing functions for specific model formats self.model_format = self.model_cfg.get("model_format", "torchvision") if self.model_format == "yolo": from perceptionmetrics.models.utils.yolo import postprocess_detection elif self.model_format == "torchvision": from perceptionmetrics.models.utils.torchvision import postprocess_detection else: raise ValueError(f"Unsupported model_format: {self.model_format}") self.postprocess_detection = postprocess_detection # Load confidence and NMS thresholds from config self.confidence_threshold = self.model_cfg.get("confidence_threshold", 0.5) self.nms_threshold = self.model_cfg.get("nms_threshold", 0.3) self.max_detections_per_image = self.model_cfg.get( "max_detections_per_image", -1 ) self.postprocess_args = [self.confidence_threshold] if self.model_format == "yolo": self.postprocess_args.append(self.nms_threshold) self.postprocess_args.append(self.max_detections_per_image) # Add reverse mapping for idx to class_name self.idx_to_class_name = {v["idx"]: k for k, v in self.ontology.items()} # Build input transforms (resize, normalize, etc.) self.transform_input = [] self.closest_divisor = None resize_cfg = self.model_cfg.get("resize") if resize_cfg is not None: self.closest_divisor = resize_cfg.pop("closest_divisor", None) if resize_cfg: resize_args = get_resize_args(resize_cfg) self.transform_input.append(transforms.Resize(**resize_args)) else: print("'resize_cfg' missing in model config. No resizing will be applied.") if "crop" in self.model_cfg: crop_size = ( self.model_cfg["crop"]["height"], self.model_cfg["crop"]["width"], ) self.transform_input += [transforms.CenterCrop(crop_size)] try: self.transform_input += [ transforms.ToImage(), transforms.ToDtype(torch.float32, scale=True), ] except AttributeError: self.transform_input += [ transforms.ToImageTensor(), transforms.ConvertDtype(torch.float32), ] if "normalization" in self.model_cfg: self.transform_input += [ transforms.Normalize( mean=self.model_cfg["normalization"]["mean"], std=self.model_cfg["normalization"]["std"], ) ] self.transform_input = transforms.Compose(self.transform_input)
[docs] def predict( self, image: Image.Image, return_sample: bool = False ) -> Union[Dict[str, torch.Tensor], Tuple[Dict[str, torch.Tensor], torch.Tensor]]: """Perform prediction for a single image :param image: PIL image :type image: Image.Image :param return_sample: Whether to return the sample data along with predictions, defaults to False :type return_sample: bool, optional :return: Detection result or a tuple with the detection result and the input sample tensor :rtype: Union[Dict[str, torch.Tensor], Tuple[Dict[str, torch.Tensor], torch.Tensor]] """ orig_size = (image.width, image.height) sample = self.transform_input(image).unsqueeze(0).to(self.device) result = self.inference(sample) # Scale bounding boxes back to the original image size result["boxes"] = inverse_transform_boxes( result["boxes"], orig_size, self.transform_input.transforms ) if return_sample: return result, sample else: return result
[docs] def inference(self, tensor_in: torch.Tensor) -> Dict[str, torch.Tensor]: """Perform inference for a tensor :param tensor_in: Input tensor :type tensor_in: torch.Tensor :return: Dictionary with keys 'boxes', 'labels', 'scores' :rtype: Dict[str, torch.Tensor] """ if self.closest_divisor is not None: tensor_in, (orig_h, orig_w) = pad_to_closest_divisor( tensor_in, self.closest_divisor ) with torch.no_grad(): result = self.model(tensor_in.to(self.device))[0] # only first image # Apply threshold filtering from model config result = self.postprocess_detection(result, *self.postprocess_args) # Crop/clamp the bounding boxes to the original shape before padding if self.closest_divisor is not None: result["boxes"][:, [0, 2]] = result["boxes"][:, [0, 2]].clamp(0, orig_w) result["boxes"][:, [1, 3]] = result["boxes"][:, [1, 3]].clamp(0, orig_h) return result
[docs] def eval( self, dataset: detection_dataset.ImageDetectionDataset, split: Union[str, List[str]] = "test", ontology_translation: Optional[str] = None, predictions_outdir: Optional[str] = None, results_per_sample: bool = False, save_visualizations: bool = False, progress_callback=None, metrics_callback=None, ) -> pd.DataFrame: """Evaluate model over a detection dataset and compute metrics :param dataset: Image detection dataset :type dataset: ImageDetectionDataset :param split: Dataset split(s) to evaluate :type split: Union[str, List[str]] :param ontology_translation: Optional translation for class mapping :type ontology_translation: Optional[str] :param predictions_outdir: Directory to save predictions, if desired :type predictions_outdir: Optional[str] :param results_per_sample: Store per-sample metrics :type results_per_sample: bool :param save_visualizations: Save visualized results (GT vs Pred) :type save_visualizations: bool :param progress_callback: Optional callback function for progress updates in Streamlit UI :type progress_callback: Optional[Callable[[int, int], None]] :param metrics_callback: Optional callback function for intermediate metrics updates in Streamlit UI :type metrics_callback: Optional[Callable[[pd.DataFrame, int, int], None]] :return: DataFrame containing evaluation results :rtype: pd.DataFrame """ if (results_per_sample or save_visualizations) and predictions_outdir is None: raise ValueError( "predictions_outdir required if results_per_sample or save_visualizations is True" ) if predictions_outdir is not None: os.makedirs(predictions_outdir, exist_ok=True) # Build LUT if ontology translation is provided lut_ontology = self.get_lut_ontology(dataset.ontology, ontology_translation) if lut_ontology is not None: lut_ontology = torch.tensor(lut_ontology, dtype=torch.int64).to(self.device) # Create DataLoader torch_dataset = ImageDetectionTorchDataset( dataset, transform=self.transform_input, splits=[split] if isinstance(split, str) else split, ) # This ensures compatibility with Streamlit and callback functions if progress_callback is not None and metrics_callback is not None: num_workers = 0 else: num_workers = self.model_cfg.get("num_workers", 0) dataloader = DataLoader( torch_dataset, batch_size=self.model_cfg.get("batch_size", 1), num_workers=num_workers, collate_fn=lambda batch: tuple( zip(*batch) ), # handles variable-size targets ) # Get iou_threshold from model config, default to 0.5 if not present iou_threshold = self.model_cfg.get("iou_threshold", 0.5) # Get evaluation_step from model config, default to None (no intermediate updates) evaluation_step = self.model_cfg.get("evaluation_step", None) # If evaluation_step is 0, treat as None (disabled) if evaluation_step == 0: evaluation_step = None # Init metrics metrics_factory = um.DetectionMetricsFactory( iou_threshold=iou_threshold, num_classes=self.n_classes ) # Calculate total samples for progress tracking total_samples = len(dataloader.dataset) processed_samples = 0 with torch.no_grad(): # Use tqdm if no progress callback provided, otherwise use regular iteration if progress_callback is None: pbar = tqdm(dataloader, leave=True) iterator = pbar else: iterator = dataloader for image_ids, images, targets in iterator: # Defensive check for empty images if not images or any(image.numel() == 0 for image in images): print("Skipping batch: empty image tensor detected.") continue images = torch.stack(images).to(self.device) if self.closest_divisor is not None: images, (orig_h, orig_w) = pad_to_closest_divisor( images, self.closest_divisor ) predictions = self.model(images) for i in range(len(images)): gt = targets[i] pred = predictions[i] image_tensor = images[i] sample_id = image_ids[i] # Post-process predictions pred = self.postprocess_detection(pred, *self.postprocess_args) # Crop/clamp the bounding boxes to the original shape before padding if self.closest_divisor is not None: pred["boxes"][:, [0, 2]] = pred["boxes"][:, [0, 2]].clamp( 0, orig_w ) pred["boxes"][:, [1, 3]] = pred["boxes"][:, [1, 3]].clamp( 0, orig_h ) # Apply ontology translation if needed if lut_ontology is not None: gt["labels"] = lut_ontology[gt["labels"]] # Update metrics metrics_factory.update( gt["boxes"], gt["labels"], pred["boxes"], pred["labels"], pred["scores"], ) # Store predictions or visualizations if needed if predictions_outdir is not None: pred_boxes = pred["boxes"].cpu().numpy() pred_labels = pred["labels"].cpu().numpy() pred_scores = pred["scores"].cpu().numpy() # Save JSON with predictions and csv with metrics per sample if results_per_sample: out_data = [] orig_pred_boxes = inverse_transform_boxes( boxes=pred["boxes"].cpu(), orig_size=tuple(gt["orig_size"].tolist()), input_transforms=self.transform_input.transforms, ).numpy() for box, label, score in zip( orig_pred_boxes, pred_labels, pred_scores ): # Convert label index to class name using model ontology class_name = self.idx_to_class_name.get( int(label), f"class_{label}" ) out_data.append( { "image_id": sample_id, "label": class_name, "score": float(score), "bbox": box.tolist(), } ) df = pd.DataFrame(out_data) df.to_json( os.path.join(predictions_outdir, f"{sample_id}.json"), orient="records", indent=2, ) sample_mf = um.DetectionMetricsFactory( iou_threshold=iou_threshold, num_classes=self.n_classes ) sample_mf.update( gt["boxes"], gt["labels"], pred["boxes"], pred["labels"], pred["scores"], ) sample_df = sample_mf.get_metrics_dataframe(self.ontology) sample_df.to_csv( os.path.join( predictions_outdir, f"{sample_id}_metrics.csv" ) ) # Save visualizations per sample if save_visualizations: pil_image = transforms.ToPILImage()(image_tensor.cpu()) gt_boxes = gt["boxes"].cpu().numpy() gt_labels = gt["labels"].cpu().numpy() gt_class_names = [ self.idx_to_class_name.get(int(label), str(label)) for label in gt_labels ] pred_class_names = [ self.idx_to_class_name.get(int(label), str(label)) for label in pred_labels ] image_gt = ui.draw_detections( pil_image.copy(), gt_boxes, gt_labels, gt_class_names, scores=None, ) image_pred = ui.draw_detections( pil_image.copy(), pred_boxes, pred_labels, pred_class_names, scores=pred_scores, ) pil_gt = Image.fromarray(image_gt) pil_pred = Image.fromarray(image_pred) combined_width = pil_gt.width + pil_pred.width combined_height = max(pil_gt.height, pil_pred.height) combined_image = Image.new( "RGB", (combined_width, combined_height) ) combined_image.paste(pil_gt, (0, 0)) combined_image.paste(pil_pred, (pil_gt.width, 0)) out_file = os.path.join( predictions_outdir, f"{sample_id}.jpg" ) combined_image.save(out_file) processed_samples += 1 # Call progress callback if provided if progress_callback is not None: progress_callback(processed_samples, total_samples) # Call metrics callback if provided and evaluation_step is reached if ( metrics_callback is not None and evaluation_step is not None and processed_samples % evaluation_step == 0 ): # Get intermediate metrics intermediate_metrics = metrics_factory.get_metrics_dataframe( self.ontology ) metrics_callback( intermediate_metrics, processed_samples, total_samples ) # Return both the DataFrame and the metrics factory for access to precision-recall curves return { "metrics_df": metrics_factory.get_metrics_dataframe(self.ontology), "metrics_factory": metrics_factory, }
[docs] def get_computational_cost( self, image_size: Tuple[int], runs: int = 30, warm_up_runs: int = 5 ) -> dict: """Get computational cost metrics like inference time :param image_size: Size of input image (H, W) :type image_size: Tuple[int] :param runs: Number of repeated runs to average over :type runs: int :param warm_up_runs: Warm-up runs before timing :type warm_up_runs: int :return: Dictionary with computational cost details :rtype: dict """ dummy_input = torch.randn(1, 3, *image_size).to(self.device) return get_computational_cost( self.model, dummy_input, self.model_fname, runs, warm_up_runs )