import os
from typing import Tuple, List, Optional
import pandas as pd
import numpy as np
import cv2
from perceptionmetrics.datasets.detection import ImageDetectionDataset
from nuimages import NuImages
from nuimages.utils.utils import name_to_index_mapping
from perceptionmetrics.datasets import segmentation as segmentation_dataset
DROP = {}
def _get_rgb_from_idx(class_idx: int) -> List[int]:
"""Generate a deterministic RGB color for a class index."""
rng = np.random.default_rng(seed=class_idx + 17)
rgb = rng.integers(low=40, high=256, size=3)
return [int(rgb[0]), int(rgb[1]), int(rgb[2])]
[docs]
def build_nuimages_detection_dataset(
dataset_dir: str,
version: str = "v1.0-mini",
split: str = "train",
nuim_object: Optional[NuImages] = None,
) -> Tuple[pd.DataFrame, dict]:
"""
Build a nuImages 2D detection dataset index.
Iterates through the nuImages scenes and samples, collects image paths for a given camera, and constructs a dataset index along with a category ontology mapping class names to integer indices.
:param dataset_dir: Path to the nuImages dataset root directory.
:type dataset_dir: str
:param version: nuImages dataset version to load, defaults to "v1.0-mini".
:type version: str
:param split: Dataset split to load ("train" or "val"), defaults to "train".
:type split: str
:param nuim_object: Optional pre-initialized NuImages object to reuse, defaults to None.
:type nuim_object: Optional[NuImages]
:return: Tuple containing:
- A pandas DataFrame with columns ["image", "annotation", "split"] for each sample.
- An ontology dictionary mapping category names to indices.
:rtype: Tuple[pd.DataFrame, dict]
"""
dataset_dir = os.path.abspath(dataset_dir)
assert os.path.isdir(
dataset_dir
), f"Dataset directory {dataset_dir} does not exist."
nuim = (
nuim_object
if nuim_object
else NuImages(version=version, dataroot=dataset_dir, verbose=False)
)
all_categories = [cat["name"] for cat in nuim.category]
categories = [cat for cat in all_categories if cat not in DROP]
ontology = {
name: {"idx": i + 1, "rgb": _get_rgb_from_idx(i + 1)}
for i, name in enumerate(categories)
}
cat_to_idx = {name: ontology[name]["idx"] for name in ontology}
rows = []
for sample in nuim.sample:
key_camera_token = sample["key_camera_token"]
sample_data = nuim.get("sample_data", key_camera_token)
rows.append(
{
"image": os.path.join(dataset_dir, sample_data["filename"]),
"annotation": key_camera_token,
"split": split,
}
)
dataset = pd.DataFrame(rows)
dataset.attrs = {
"ontology": ontology,
"cat_to_idx": cat_to_idx,
}
print(
f"Built nuimages detection dataset with {len(dataset)} samples and "
f"{len(categories)} categories."
)
return dataset, ontology
[docs]
def build_nuimages_segmentation_dataset(
dataset_dir: str,
version: str = "v1.0-mini",
split: str = "train",
labels_rel_dir: str = "generated/nuimages_segmentation_labels",
nuim_object: Optional[NuImages] = None,
) -> Tuple[pd.DataFrame, dict]:
"""
Build a nuImages semantic segmentation dataset index and masks.
:param dataset_dir: Path to the nuImages dataset root directory.
:type dataset_dir: str
:param version: nuImages dataset version to load, defaults to "v1.0-mini".
:type version: str
:param split: Dataset split to load ("train" or "val"), defaults to "train".
:type split: str
:param labels_rel_dir: Relative directory for segmentation labels, defaults to "generated/nuimages_segmentation_labels".
:type labels_rel_dir: str
:param nuim_object: Optional pre-initialized NuImages object to reuse, defaults to None.
:type nuim_object: Optional[NuImages]
:return: Tuple containing:
- A pandas DataFrame with columns ["image", "label", "split"] for each sample.
- An ontology dictionary mapping category names to indices.
:rtype: Tuple[pd.DataFrame, dict]
"""
dataset_dir = os.path.abspath(dataset_dir)
assert os.path.isdir(
dataset_dir
), f"Dataset directory {dataset_dir} does not exist."
nuim = (
nuim_object
if nuim_object
else NuImages(version=version, dataroot=dataset_dir, verbose=False)
)
## For segmentation, we build semantic masks from surface annotations for keyframe images
labels_root = os.path.join(dataset_dir, labels_rel_dir, version)
os.makedirs(labels_root, exist_ok=True)
name_to_global_idx = name_to_index_mapping(nuim.category)
ontology = {"background": {"idx": 0, "rgb": [0, 0, 0]}}
for category_name, category_idx in sorted(
name_to_global_idx.items(), key=lambda item: item[1]
):
ontology[category_name] = {
"idx": int(category_idx),
"rgb": _get_rgb_from_idx(int(category_idx)),
}
rows = []
for sample in nuim.sample:
key_camera_token = sample["key_camera_token"]
sample_data = nuim.get("sample_data", key_camera_token)
image_abs_path = os.path.join(dataset_dir, sample_data["filename"])
semantic_mask, _ = nuim.get_segmentation(key_camera_token)
label = semantic_mask.astype(np.uint8)
label_abs_path = os.path.join(labels_root, f"{key_camera_token}.png")
cv2.imwrite(label_abs_path, label)
rows.append(
{
"image": image_abs_path,
"label": label_abs_path,
"split": split,
}
)
dataset = pd.DataFrame(rows)
dataset.attrs = {"ontology": ontology}
print(
f"Built nuImages segmentation dataset with {len(dataset)} samples and "
f"{len(ontology)} classes."
)
return dataset, ontology
[docs]
class NuImagesDetectionDataset(ImageDetectionDataset):
"""
Dataset class for nuImages 2D object detection.
Inherits from ImageDetectionDataset and parses 2D bounding boxes
from nuImages.
:param dataset_dir: Path to the nuImages dataset root.
:type dataset_dir: str
:param version: nuImages version to load, defaults to "v1.0-mini".
:type version: str
:param split: Dataset split ("train" or "val").
:type split: str
.. seealso::
`User guide and format examples <https://jderobot.github.io/PerceptionMetrics/datasets/#nuimages>`__.
"""
def __init__(
self,
dataset_dir: str,
version: str = "v1.0-mini",
split: str = "train",
):
"""
Initialize the nuImages 2D detection dataset.
:param dataset_dir: Path to the nuImages dataset root directory.
:param version: nuImages dataset version to load, defaults to "v1.0-mini".
:param split: Dataset split to load ("train" or "val"), defaults to "train".
"""
self.dataset_dir = dataset_dir
self.split = split
self.nuim = NuImages(dataroot=dataset_dir, version=version)
dataset, ontology = build_nuimages_detection_dataset(
dataset_dir=dataset_dir,
version=version,
split=split,
nuim_object=self.nuim,
)
self.cat_to_idx = dataset.attrs["cat_to_idx"]
self.ann_by_sample_data_token = {}
for ann in self.nuim.object_ann:
sample_data_token = ann["sample_data_token"]
if sample_data_token not in self.ann_by_sample_data_token:
self.ann_by_sample_data_token[sample_data_token] = []
self.ann_by_sample_data_token[sample_data_token].append(ann)
super().__init__(dataset=dataset, dataset_dir=dataset_dir, ontology=ontology)
[docs]
def read_annotation(self, fname: str) -> Tuple[List[List[float]], List[int]]:
"""
Read annotations for a single nuImages sample and return 2D bounding boxes and class indices.
:param fname: Sample token or filename.
:type fname: str
:return: Tuple containing:
- List of bounding boxes ``[[x1, y1, x2, y2], ...]``.
- List of corresponding class indices.
:rtype: Tuple[List[List[float]], List[int]]
"""
if isinstance(fname, str) and ("/" in fname or "\\" in fname):
fname = os.path.basename(fname)
sample_data = self.nuim.get("sample_data", fname)
image_width = float(sample_data["width"])
image_height = float(sample_data["height"])
boxes_out = []
labels_out = []
for ann in self.ann_by_sample_data_token.get(fname, []):
category_name = self.nuim.get("category", ann["category_token"])["name"]
if category_name in DROP:
continue
if category_name not in self.cat_to_idx:
continue
x1_raw, y1_raw, x2_raw, y2_raw = ann["bbox"]
x1 = max(0.0, min(float(x1_raw), image_width - 1.0))
y1 = max(0.0, min(float(y1_raw), image_height - 1.0))
x2 = max(0.0, min(float(x2_raw), image_width - 1.0))
y2 = max(0.0, min(float(y2_raw), image_height - 1.0))
if x2 <= x1 or y2 <= y1:
continue
boxes_out.append([x1, y1, x2, y2])
labels_out.append(self.cat_to_idx[category_name])
return boxes_out, labels_out
nuimages_detection = NuImagesDetectionDataset
[docs]
class NuImagesSegmentationDataset(segmentation_dataset.ImageSegmentationDataset):
"""Dataset class for nuImages 2D surface segmentation.
Inherits from ImageSegmentationDataset and constructs semantic segmentation masks
param dataset_dir: Path to the nuImages dataset root.
:type dataset_dir: str
:param version: nuImages version to load, defaults to "v1.0-mini".
:type version: str
:param split: Dataset split ("train" or "val").
:type split: str
:param labels_rel_dir: Relative directory within the dataset where generated segmentation label images will be stored, defaults to "generated/nuimages_segmentation_labels".
:type labels_rel_dir: str
.. seealso::
`User guide and format examples <https://jderobot.github.io/PerceptionMetrics/datasets/#nuimages>`__.
"""
def __init__(
self,
dataset_dir: str,
version: str = "v1.0-mini",
split: str = "train",
labels_rel_dir: str = "generated/nuimages_segmentation_labels",
):
dataset_dir = os.path.abspath(dataset_dir)
assert os.path.isdir(
dataset_dir
), f"Dataset directory {dataset_dir} does not exist."
self.nuim = NuImages(dataroot=dataset_dir, version=version)
dataset, ontology = build_nuimages_segmentation_dataset(
dataset_dir=dataset_dir,
version=version,
split=split,
labels_rel_dir=labels_rel_dir,
nuim_object=self.nuim,
)
super().__init__(dataset, dataset_dir, ontology)
import matplotlib.pyplot as plt
if __name__ == "__main__":
dataset_dir = "local/data/nuimages-v1.0-mini"
version = "v1.0-mini"
split = "train"
dataset = NuImagesSegmentationDataset(dataset_dir, version, split)
sample = dataset.dataset.iloc[10]
label_path = sample["label"]
img_path = sample["image"]
# Load images
img = cv2.imread(img_path)
label_img = cv2.imread(label_path, cv2.IMREAD_GRAYSCALE)
bright_colors = [
[255, 0, 0], # red
[0, 255, 0], # green
[0, 0, 255], # blue
[255, 255, 0], # yellow
[255, 0, 255], # magenta
[0, 255, 255], # cyan
[255, 128, 0], # orange
[128, 0, 255], # purple
[0, 128, 255], # sky blue
[128, 255, 0], # lime
[255, 0, 128], # pink
[0, 255, 128], # teal
]
# Create color mask
color_mask = np.zeros_like(img)
for class_name, class_info in dataset.ontology.items():
class_idx = class_info["idx"]
rgb = bright_colors[class_idx % len(bright_colors)]
color_mask[label_img == class_idx] = rgb
# Blend original image with color mask
overlay = cv2.addWeighted(img, 0.5, color_mask, 0.5, 0)
# Draw class names
for class_name, class_info in dataset.ontology.items():
class_idx = class_info["idx"]
positions = np.argwhere(label_img == class_idx)
if positions.shape[0] == 0:
continue
y, x = np.mean(positions, axis=0).astype(int)
cv2.putText(
overlay,
class_name,
(x, y),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(255, 255, 255),
1,
cv2.LINE_AA,
)
# Show overlay
plt.figure(figsize=(12, 8))
plt.imshow(cv2.cvtColor(overlay, cv2.COLOR_BGR2RGB))
plt.axis("off")
plt.title("Image with Segmentation Overlay and Labels")
plt.show()