Source code for perceptionmetrics.utils.segmentation_metrics

from collections import defaultdict
import logging
import math
from typing import List, Optional, Union

import numpy as np
import pandas as pd


[docs] class SegmentationMetricsFactory: """'Factory' class to accumulate results and compute metrics for segmentation tasks :param n_classes: Number of classes to evaluate :type n_classes: int .. seealso:: `User guide <https://jderobot.github.io/PerceptionMetrics/metrics/#segmentation-metrics>`__. """ METRIC_NAMES = [ "tp", "fp", "fn", "tn", "precision", "recall", "accuracy", "f1_score", "iou", "dice_score", ] def __init__(self, n_classes: int): self.n_classes = n_classes self.confusion_matrix = np.zeros((n_classes, n_classes), dtype=np.int64)
[docs] def reset(self): """Reset all accumulated results to initial state. This allows reusing the same factory instance for multiple independent evaluations without creating a new object. """ self.confusion_matrix = np.zeros( (self.n_classes, self.n_classes), dtype=np.int64 )
[docs] def update( self, pred: np.ndarray, gt: np.ndarray, valid_mask: Optional[np.ndarray] = None ): """Accumulate results for a new batch :param pred: Array containing prediction :type pred: np.ndarray :param gt: Array containing ground truth :type gt: np.ndarray :param valid_mask: Binary mask where False elements will be igonred, defaults to None :type valid_mask: Optional[np.ndarray], optional """ assert pred.shape == gt.shape, "Prediction and GT shapes don't match" assert np.issubdtype(pred.dtype, np.integer), "Prediction should be integer" assert np.issubdtype(gt.dtype, np.integer), "GT should be integer" # Build mask of valid elements mask = (gt >= 0) & (gt < self.n_classes) if valid_mask is not None: mask &= valid_mask # Predictions outside the class range can't be placed in the confusion matrix invalid_pred = mask & ((pred < 0) | (pred >= self.n_classes)) if np.any(invalid_pred): raise ValueError( f"Prediction contains class indices outside [0, {self.n_classes - 1}]: " f"{np.unique(pred[invalid_pred]).tolist()}" ) # Update confusion matrix new_entry = np.bincount( self.n_classes * gt[mask].astype(int) + pred[mask].astype(int), minlength=self.n_classes**2, ) self.confusion_matrix += new_entry.reshape(self.n_classes, self.n_classes)
[docs] def get_metric_names(self) -> List[str]: """Get available metric names :return: List of available metric names :rtype: List[str] """ return self.METRIC_NAMES
[docs] def get_confusion_matrix(self) -> np.ndarray: """Get confusion matrix :return: Confusion matrix :rtype: np.ndarray """ return self.confusion_matrix
[docs] def get_tp(self, per_class: bool = True) -> Union[np.ndarray, int]: """True Positives :param per_class: Return per class TP, defaults to True :type per_class: bool, optional :return: True Positives :rtype: Union[np.ndarray, int] """ tp = np.diag(self.confusion_matrix) return tp if per_class else int(np.nansum(tp))
[docs] def get_fp(self, per_class: bool = True) -> Union[np.ndarray, int]: """False Positives :param per_class: Return per class FP, defaults to True :type per_class: bool, optional :return: False Positives :rtype: Union[np.ndarray, int] """ fp = self.confusion_matrix.sum(axis=0) - np.diag(self.confusion_matrix) return fp if per_class else int(np.nansum(fp))
[docs] def get_fn(self, per_class: bool = True) -> Union[np.ndarray, int]: """False negatives :param per_class: Return per class FN, defaults to True :type per_class: bool, optional :return: False Negatives :rtype: Union[np.ndarray, int] """ fn = self.confusion_matrix.sum(axis=1) - np.diag(self.confusion_matrix) return fn if per_class else int(np.nansum(fn))
[docs] def get_tn(self, per_class: bool = True) -> Union[np.ndarray, int]: """True negatives :param per_class: Return per class TN, defaults to True :type per_class: bool, optional :return: True Negatives :rtype: Union[np.ndarray, int] """ total = self.confusion_matrix.sum() tn = total - (self.get_tp() + self.get_fp() + self.get_fn()) return tn if per_class else int(np.nansum(tn))
[docs] def get_precision(self, per_class: bool = True) -> Union[np.ndarray, float]: """Precision = TP / (TP + FP) :param per_class: Return per class precision, defaults to True :type per_class: bool, optional :return: Precision value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ tp = self.get_tp(per_class) fp = self.get_fp(per_class) denominator = tp + fp if np.isscalar(denominator): return float(tp / denominator) if denominator > 0 else math.nan else: return np.where(denominator > 0, tp / denominator, np.nan)
[docs] def get_recall(self, per_class: bool = True) -> Union[np.ndarray, float]: """Recall = TP / (TP + FN) :param per_class: Return per class recall, defaults to True :type per_class: bool, optional :return: Recall value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ tp = self.get_tp(per_class) fn = self.get_fn(per_class) denominator = tp + fn if np.isscalar(denominator): return float(tp / denominator) if denominator > 0 else math.nan else: return np.where(denominator > 0, tp / denominator, np.nan)
[docs] def get_accuracy(self, per_class: bool = True) -> Union[np.ndarray, float]: """Accuracy = (TP + TN) / (TP + FP + FN + TN) :param per_class: Return per class accuracy, defaults to True :type per_class: bool, optional :return: Accuracy value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ tp = self.get_tp(per_class) fp = self.get_fp(per_class) fn = self.get_fn(per_class) tn = self.get_tn(per_class) total = tp + fp + fn + tn if np.isscalar(total): return float((tp + tn) / total) if total > 0 else math.nan else: return np.where(total > 0, (tp + tn) / total, np.nan)
[docs] def get_f1_score(self, per_class: bool = True) -> Union[np.ndarray, float]: """F1-score = 2 * (Precision * Recall) / (Precision + Recall) :param per_class: Return per class F1 score, defaults to True :type per_class: bool, optional :return: F1-score value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ precision = self.get_precision(per_class) recall = self.get_recall(per_class) denominator = precision + recall if np.isscalar(denominator): return ( 2 * (precision * recall) / denominator if denominator > 0 else math.nan ) else: return np.where( denominator > 0, 2 * (precision * recall) / denominator, np.nan )
[docs] def get_iou(self, per_class: bool = True) -> Union[np.ndarray, float]: """IoU = TP / (TP + FP + FN) :param per_class: Return per class IoU, defaults to True :type per_class: bool, optional :return: IoU value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ tp = self.get_tp(per_class) fp = self.get_fp(per_class) fn = self.get_fn(per_class) union = tp + fp + fn if np.isscalar(union): return float(tp / union) if union > 0 else math.nan else: return np.where(union > 0, tp / union, np.nan)
[docs] def get_dice_score(self, per_class: bool = True) -> Union[np.ndarray, float]: """Dice Score = 2 * TP / (2 * TP + FP + FN) :param per_class: Return per class Dice score, defaults to True :type per_class: bool, optional :return: Dice score value (per class if per_class=True, otherwise global) :rtype: Union[np.ndarray, float] """ tp = self.get_tp(per_class) fp = self.get_fp(per_class) fn = self.get_fn(per_class) denominator = 2 * tp + fp + fn if np.isscalar(denominator): return float(2 * tp / denominator) if denominator > 0 else math.nan else: return np.where(denominator > 0, 2 * tp / denominator, np.nan)
[docs] def get_averaged_metric( self, metric_name: str, method: str, weights: Optional[np.ndarray] = None ) -> float: """Get average metric value :param metric: Name of the metric to compute :type metric: str :param method: Method to use for averaging ('macro', 'micro', 'weighted' or 'normalized_weighted') :type method: str :param weights: Weights for weighted averaging, defaults to None :type weights: Optional[np.ndarray], optional :return: Average metric value :rtype: float """ metric = getattr(self, f"get_{metric_name}") if method == "macro": per_class_values = metric(per_class=True) nan_mask = np.isnan(per_class_values) missing_count = int(np.sum(nan_mask)) if missing_count > 0: # Use implicit string concatenation to keep lines short msg = ( f"Warning: {missing_count} class(es) were missing from the confusion matrix. " f"Their {metric_name.upper()} evaluated to NaN and will be ignored in the macro-average." ) logging.warning(msg) return float(np.nanmean(per_class_values)) if method == "micro": return float(metric(per_class=False)) if method == "weighted": assert ( weights is not None ), "Weights should be provided for weighted averaging" return float(np.nansum(metric(per_class=True) * weights)) if method == "normalized_weighted": assert ( weights is not None ), "Weights should be provided for weighted averaging" weight_sum = np.nansum(weights) if weight_sum == 0: return math.nan return float(np.nansum(metric(per_class=True) * weights) / weight_sum) raise ValueError(f"Unknown method {method}")
[docs] def get_metric_per_name( self, metric_name: str, per_class: bool = True ) -> Union[np.ndarray, float, int]: """Get metric value by name :param metric_name: Name of the metric to compute :type metric_name: str :param per_class: Return per class metric, defaults to True :type per_class: bool, optional :return: Metric value :rtype: Union[np.ndarray, float, int] """ return getattr(self, f"get_{metric_name}")(per_class=per_class)
[docs] def get_metrics_dataframe( metrics_factory: SegmentationMetricsFactory, ontology: dict ) -> pd.DataFrame: """Build a DataFrame with all metrics (global and per class) plus confusion matrix :param metrics_factory: Properly updated SegmentationMetricsFactory object :type metrics_factory: SegmentationMetricsFactory :param ontology: Ontology dictionary :type ontology: dict :return: DataFrame with all metrics :rtype: pd.DataFrame """ # Build results dataframe results = defaultdict(dict) # Add per class and global metrics for metric in metrics_factory.get_metric_names(): per_class = metrics_factory.get_metric_per_name(metric, per_class=True) for class_name, class_data in ontology.items(): results[class_name][metric] = float(per_class[class_data["idx"]]) if metric not in ["tp", "fp", "fn", "tn"]: for avg_method in ["macro", "micro"]: results[avg_method][metric] = metrics_factory.get_averaged_metric( metric, avg_method ) # Add confusion matrix for class_name_a, class_data_a in ontology.items(): for class_name_b, class_data_b in ontology.items(): results[class_name_a][class_name_b] = metrics_factory.confusion_matrix[ class_data_a["idx"], class_data_b["idx"] ] return pd.DataFrame(results)