Source code for perceptionmetrics.models.tf_segmentation

import os
import time
from typing import List, Optional, Tuple, Union

import numpy as np
import pandas as pd
from PIL import Image
import tensorflow as tf
from tensorflow import data as tf_data
from tensorflow import image as tf_image
from tensorflow import io as tf_io
from tqdm import tqdm

from perceptionmetrics.datasets.segmentation import ImageSegmentationDataset
from perceptionmetrics.models.segmentation import ImageSegmentationModel
import perceptionmetrics.utils.segmentation_metrics as um

tf.config.optimizer.set_experimental_options({"layout_optimizer": False})


[docs] def resize_image( image: tf.Tensor, method: str, width: Optional[int] = None, height: Optional[int] = None, closest_divisor: int = 16, ) -> tf.Tensor: """Resize tensorflow image to target size. If only one dimension is provided, the aspect ratio is preserved. :param image: Input image tensor :type image: tf.Tensor :param method: Resizing method (e.g. bilinear, nearest) :type method: str :param width: Target width for resizing :type width: Optional[int], optional :param height: Target height for resizing :type height: Optional[int], optional :param closest_divisor: Closest divisor for the target size, defaults to 16. Only applies to the dimension not provided. :type closest_divisor: int, optional :return: Resized image tensor :rtype: tf.Tensor """ old_size = tf.cast(tf.shape(image)[:2], tf.float32) old_h = old_size[0] old_w = old_size[1] h, w = (old_h, old_w) if width is None: w = int((height / old_h) * old_w) h = height if height is None: h = int((width / old_w) * old_h) w = width h = (h / closest_divisor) * closest_divisor w = (w / closest_divisor) * closest_divisor new_size = [int(h), int(w)] image = tf_image.resize( images=image, size=tf.cast(new_size, tf.int32), method=method ) return image
[docs] def crop_center(image: tf.Tensor, width: int, height: int) -> tf.Tensor: """Crop tensorflow image center to target size :param image: Input image tensor :type image: tf.Tensor :param width: Target width for cropping :type width: int :param height: Target width for cropping :type height: int :return: Cropped image tensor :rtype: tf.Tensor """ old_size = tf.cast(tf.shape(image)[:2], tf.float32) old_h = old_size[0] old_w = old_size[1] offset_height = int((old_h - height) // 2) offset_width = int((old_w - width) // 2) image = tf.image.crop_to_bounding_box( image, offset_height, offset_width, height, width ) return image
[docs] class ImageSegmentationTensorflowDataset: """Dataset for image segmentation Tensorflow models :param dataset: Image segmentation dataset :type dataset: ImageSegmentationDataset :param resize: Target size for resizing images, defaults to None :type resize: Optional[Tuple[int, int]], optional :param crop: Target size for center cropping images, defaults to None :type crop: Optional[Tuple[int, int]], optional :param batch_size: Batch size, defaults to 1 :type batch_size: int, optional :param splits: Splits to be used from the dataset, defaults to ["test"] :type splits: str, optional :param lut_ontology: LUT to transform label classes, defaults to None :type lut_ontology: dict, optional :param normalization: Parameters for normalizing input images, defaults to None :type normalization: dict, optional :param keep_aspect: Whether to keep aspect ratio when resizing images. If true, resize to match smaller sides size and crop center. Defaults to False :type keep_aspect: bool, optional """ def __init__( self, dataset: ImageSegmentationDataset, resize: Optional[Tuple[int, int]] = None, crop: Optional[Tuple[int, int]] = None, batch_size: int = 1, splits: List[str] = ["test"], lut_ontology: Optional[dict] = None, normalization: Optional[dict] = None, keep_aspect: bool = False, ): self.resize = resize self.crop = crop self.normalization = None if normalization is not None: mean = tf.constant(normalization["mean"], dtype=tf.float32) std = tf.constant(normalization["std"], dtype=tf.float32) self.normalization = {"mean": mean, "std": std} self.keep_aspect = keep_aspect # Raise early if any requested split is absent — prevents silent NaN metrics dataset._validate_splits(splits) # Filter split and make filenames global dataset.dataset = dataset.dataset[dataset.dataset["split"].isin(splits)] dataset.make_fname_global() self.lut_ontology = None if lut_ontology is not None: keys = tf.constant(range(len(lut_ontology)), dtype=tf.int32) values = tf.constant(list(lut_ontology), dtype=tf.int32) self.lut_ontology = tf.lookup.StaticHashTable( initializer=tf.lookup.KeyValueTensorInitializer(keys, values), default_value=0, ) # Build tensorflow dataset sample = ( dataset.dataset.index, dataset.dataset["image"], dataset.dataset["label"], ) self.dataset = tf_data.Dataset.from_tensor_slices(sample) self.dataset = self.dataset.map( self.load_data, num_parallel_calls=tf_data.AUTOTUNE ) self.dataset = self.dataset.batch(batch_size, drop_remainder=True)
[docs] def read_image(self, fname: str, label=False) -> tf.Tensor: """Read a single image or label :param fname: Input image or label filename :type fname: str :param label: Whether the input data is a label or not, defaults to False :type label: bool, optional :return: Tensorflow tensor containing read image or label :rtype: tf.Tensor """ # Read image file image = tf_io.read_file(fname) # Decode image n_channels = 1 if label else 3 image = tf_image.decode_png(image, channels=n_channels) image.set_shape([None, None, n_channels]) # Apply LUT for transforming ontology if required if label and self.lut_ontology is not None: image = tf.cast( self.lut_ontology.lookup(tf.cast(image, tf.int32)), tf.uint8 ) # Resize (use NN to avoid interpolation when dealing with labels) if self.resize is not None: method = "nearest" if label else "bilinear" image = resize_image( image, method=method, width=self.resize.get("width", None), height=self.resize.get("height", None), ) if self.crop is not None: image = crop_center( image, width=self.crop.get("width", None), height=self.crop.get("height", None), ) # If label, round values to avoid interpolation artifacts if label: image = tf.round(image) # If normalization parameters are provided, normalize image else: if self.normalization is not None: image = tf.cast(image, tf.float32) / 255.0 image = (image - self.normalization["mean"]) / self.normalization["std"] return image
[docs] def load_data( self, idx: str, images_fnames: List[str], labels_fnames: List[str] ) -> Tuple[tf.Tensor, tf.Tensor]: """Function for loading data for each dataset sample :param idx: Sample index :type idx: str :param images_fnames: List containing all image filenames :type images_fnames: List[str] :param labels_fnames: List containing all corresponding label filenames :type labels_fnames: List[str] :return: Image and label tensor pairs :rtype: Tuple[tf.Tensor, tf.Tensor] """ image = self.read_image(images_fnames) label = self.read_image(labels_fnames, label=True) return idx, image, label
[docs] class TensorflowImageSegmentationModel(ImageSegmentationModel): """Image segmentation model for Tensorflow framework :param model: Either the filename of a Tensorflow model in SavedModel format or the model already loaded into an arbitrary Tensorflow or Keras model. :type model: Union[str, torch.nn.Module] :param model_cfg: JSON file containing model configuration :type model_cfg: str :param ontology_fname: JSON file containing model output ontology :type ontology_fname: str .. seealso:: `User guide <https://jderobot.github.io/PerceptionMetrics/models/#image-segmentation-config>`__. """ def __init__( self, model: Union[str, tf.Module, tf.keras.Model], model_cfg: str, ontology_fname: str, ): # If 'model' contains a string, check that it is a valid filename and load model if isinstance(model, str): assert os.path.isdir(model), "SavedModel directory not found" model_fname = model model = tf.saved_model.load(model) model_type = "compiled" # Otherwise, check that it is a Tensorflow or Keras model elif isinstance(model, tf.Module) or isinstance(model, tf.keras.Model): model_fname = None model_type = "native" else: raise ValueError( "Model must be either a SavedModel directory or a TF/Keras model" ) super().__init__(model, model_type, model_cfg, ontology_fname, model_fname) # Init transformation for input images def t_in(image): tensor = tf.convert_to_tensor(image) if "resize" in self.model_cfg: tensor = resize_image( tensor, method="bilinear", width=self.model_cfg["resize"].get("width", None), height=self.model_cfg["resize"].get("height", None), ) if "crop" in self.model_cfg: tensor = crop_center( tensor, width=self.model_cfg["crop"].get("width", None), height=self.model_cfg["crop"].get("height", None), ) tensor = tf.expand_dims(tensor, axis=0) if "normalization" in self.model_cfg: mean = tf.constant(self.model_cfg["normalization"]["mean"]) std = tf.constant(self.model_cfg["normalization"]["std"]) tensor = tf.cast(tensor, tf.float32) / 255.0 tensor = (tensor - mean) / std return tensor self.t_in = t_in # Init transformation for model output self.t_out = lambda x: Image.fromarray( tf.argmax(tf.squeeze(x), axis=2).numpy().astype(np.uint8) )
[docs] def predict( self, image: Image.Image, return_sample: bool = False ) -> Union[Image.Image, Tuple[Image.Image, tf.Tensor]]: """Perform prediction for a single image :param image: PIL image :type image: Image.Image :param return_sample: Whether to return the sample data along with predictions, defaults to False :type return_sample: bool, optional :return: Segmentation result as a PIL image or a tuple with the segmentation result and the input sample tensor :rtype: Union[Image.Image, Tuple[Image.Image, tf.Tensor]] """ sample = self.t_in(image) result = self.inference(sample) result = self.t_out(result) if return_sample: return result, sample else: return result
[docs] def inference(self, tensor_in: tf.Tensor) -> tf.Tensor: """Perform inference for a tensor :param tensor_in: Input point cloud tensor :type tensor_in: tf.Tensor :return: Segmentation result as tensor :rtype: tf.Tensor """ if self.model_type == "native": tensor_out = self.model(tensor_in, training=False) elif self.model_type == "compiled": tensor_out = self.model.signatures["serving_default"](tensor_in) else: raise ValueError("Model type not recognized") if isinstance(tensor_out, dict): tensor_out = list(tensor_out.values())[0] return tensor_out
[docs] def eval( self, dataset: ImageSegmentationDataset, split: Union[str, List[str]] = "test", ontology_translation: Optional[str] = None, translations_direction: str = "dataset_to_model", predictions_outdir: Optional[str] = None, results_per_sample: bool = False, ) -> pd.DataFrame: """Perform evaluation for an image segmentation dataset :param dataset: Image segmentation dataset for which the evaluation will be performed :type dataset: ImageSegmentationDataset :param split: Split to be used from the dataset, defaults to "test" :type split: Union[str, List[str]], optional :param ontology_translation: JSON file containing translation between dataset and model output ontologies :type ontology_translation: str, optional :param translations_direction: Direction of the ontology translation. Either "dataset_to_model" or "model_to_dataset", defaults to "dataset_to_model" :type translations_direction: str, optional :param predictions_outdir: Directory to save predictions per sample, defaults to None. If None, predictions are not saved. :type predictions_outdir: Optional[str], optional :param results_per_sample: Whether to store results per sample or not, defaults to False. If True, predictions_outdir must be provided. :type results_per_sample: bool, optional :return: DataFrame containing evaluation results :rtype: pd.DataFrame """ # Check that predictions_outdir is provided if results_per_sample is True if results_per_sample and predictions_outdir is None: raise ValueError( "If results_per_sample is True, predictions_outdir must be provided" ) # Create predictions output directory if needed if predictions_outdir is not None: os.makedirs(predictions_outdir, exist_ok=True) # Build a LUT for transforming ontology if needed lut_ontology, eval_ontology = self.get_eval_lut_ontology( dataset.ontology, ontology_translation, translations_direction ) n_classes = len(eval_ontology) # Get Tensorflow dataset dataset = ImageSegmentationTensorflowDataset( dataset, resize=self.model_cfg.get("resize", None), crop=self.model_cfg.get("crop", None), batch_size=self.model_cfg.get("batch_size", 1), splits=[split] if isinstance(split, str) else split, lut_ontology=( lut_ontology if translations_direction == "dataset_to_model" else None ), normalization=self.model_cfg.get("normalization", None), keep_aspect=self.model_cfg.get("keep_aspect", False), ) # Retrieve ignored label indices ignored_label_indices = [] for ignored_class in self.model_cfg.get("ignored_classes", []): ignored_label_indices.append(eval_ontology[ignored_class]["idx"]) # Init metrics metrics_factory = um.SegmentationMetricsFactory(n_classes) # Evaluation loop pbar = tqdm(dataset.dataset) for idx, image, label in pbar: idx = idx.numpy() pred = self.inference(image) # Get valid points masks depending on ignored label indices if ignored_label_indices: valid_mask = tf.ones_like(label, dtype=tf.bool) for ignored_label_idx in ignored_label_indices: valid_mask *= label != ignored_label_idx else: valid_mask = None # Update metrics label = tf.squeeze(label, axis=3).numpy() pred = tf.argmax(pred, axis=3).numpy() if valid_mask is not None: valid_mask = tf.squeeze(valid_mask, axis=3).numpy() # Convert predictions to dataset ontology if needed if ( lut_ontology is not None and translations_direction == "model_to_dataset" ): pred = lut_ontology[pred] metrics_factory.update(pred, label, valid_mask) # Store predictions and results per sample if required if predictions_outdir is not None: for i, (sample_idx, sample_pred, sample_label) in enumerate( zip(idx, pred, label) ): sample_idx = sample_idx.decode("utf-8") if results_per_sample: sample_valid_mask = ( valid_mask[i] if valid_mask is not None else None ) sample_mf = um.SegmentationMetricsFactory(n_classes) sample_mf.update(sample_pred, sample_label, sample_valid_mask) sample_df = um.get_metrics_dataframe(sample_mf, eval_ontology) sample_df.to_csv( os.path.join(predictions_outdir, f"{sample_idx}.csv") ) sample_img = Image.fromarray(np.squeeze(sample_pred).astype(np.uint8)) sample_img.save(os.path.join(predictions_outdir, f"{sample_idx}.png")) return um.get_metrics_dataframe(metrics_factory, eval_ontology)
[docs] def get_computational_cost( self, image_size: Tuple[int] = None, runs: int = 30, warm_up_runs: int = 5, ) -> dict: """Get different metrics related to the computational cost of the model :param image_size: Image size used for inference :type image_size: Tuple[int], optional :param runs: Number of runs to measure inference time, defaults to 30 :type runs: int, optional :param warm_up_runs: Number of warm-up runs, defaults to 5 :type warm_up_runs: int, optional :return: Dictionary containing computational cost information """ # Generate dummy input dummy_input = tf.random.normal([1, *image_size, 3]) # Get model size (if possible) and number of parameters if self.model_fname is not None: size_mb = sum( os.path.getsize(os.path.join(dirpath, f)) for dirpath, _, files in os.walk(self.model_fname) for f in files ) size_mb /= 1024**2 else: size_mb = None n_params = sum(np.prod(var.shape) for var in self.model.variables.variables) # Measure inference time with GPU synchronization for _ in range(warm_up_runs): self.inference(dummy_input) has_gpu = bool(tf.config.list_physical_devices("GPU")) inference_times = [] for _ in range(runs): if has_gpu: tf.config.experimental.set_synchronous_execution(True) start_time = time.perf_counter() self.inference(dummy_input) if has_gpu: tf.config.experimental.set_synchronous_execution(True) inference_times.append(time.perf_counter() - start_time) # Retrieve computational cost information result = { "input_shape": ["x".join(map(str, dummy_input.shape.as_list()))], "n_params": [int(n_params)], "size_mb": [size_mb], "inference_time_s": [np.mean(inference_times)], } return pd.DataFrame.from_dict(result)