import time
from typing import Optional, Tuple, Union, Dict
import numpy as np
import torch
try:
from open3d._ml3d.datasets.utils import DataProcessing
except Exception:
print("Open3D-ML3D not available")
from sklearn.neighbors import KDTree
from perceptionmetrics.models.utils.o3d import randlanet, kpconv
from perceptionmetrics.utils import lidar as ul
import perceptionmetrics.utils.torch as ut
[docs]
def inference(
sample: Tuple[np.ndarray, np.ndarray, ul.Sampler],
model: torch.nn.Module,
model_cfg: dict,
measure_processing_time: bool = False,
) -> Union[
Tuple[torch.Tensor, Optional[torch.Tensor], Optional[str]],
Tuple[torch.Tensor, Optional[torch.Tensor], Optional[str], Dict[str, float]],
]:
"""Perform inference on a sample using an Open3D-ML model
:param sample: sample data dictionary
:type sample: dict
:param model: Open3D-ML model
:type model: torch.nn.Module
:param model_cfg: model configuration
:type model_cfg: dict
:param measure_processing_time: whether to measure processing time, defaults to False
:type measure_processing_time: bool, optional
:return: predicted labels, ground truth labels, sample name and optionally processing time
:rtype: Union[ Tuple[torch.Tensor, Optional[torch.Tensor], Optional[str]], Tuple[torch.Tensor, Optional[torch.Tensor], Optional[str], Dict[str, float]] ]
"""
infer_complete = False
points, projected_indices, sampler, label, name, _ = sample
model_format = model_cfg["model_format"]
end_th = model_cfg.get("end_th", 0.5)
processing_time = {"preprocessing": 0, "inference": 0, "postprocessing": 0}
if "kpconv" in model_format:
transform_input = kpconv.transform_input
update_probs = kpconv.update_probs
elif "randlanet" in model_format:
decoder_layers = model.decoder.children()
model_cfg["num_layers"] = sum(1 for _ in decoder_layers)
transform_input = randlanet.transform_input
update_probs = randlanet.update_probs
else:
raise ValueError(f"Unknown model type: {model_format}")
while not infer_complete:
# Get model input data
if measure_processing_time:
start = time.perf_counter()
input_data, selected_indices = transform_input(points, model_cfg, sampler)
if measure_processing_time:
end = time.perf_counter()
processing_time["preprocessing"] += end - start
input_data = ut.data_to_device(input_data, model.device)
if "randlanet" in model_format:
input_data = ut.unsqueeze_data(input_data)
# Perform inference
with torch.no_grad():
if measure_processing_time:
torch.cuda.synchronize()
start = time.perf_counter()
pred = model(*input_data)
if measure_processing_time:
torch.cuda.synchronize()
end = time.perf_counter()
processing_time["inference"] += end - start
# TODO: check if this is consistent across different models
if isinstance(pred, dict):
pred = pred["out"]
# Update probabilities if sampler is used
if measure_processing_time:
start = time.perf_counter()
if sampler is not None:
if "kpconv" in model_format:
sampler.test_probs = update_probs(
pred,
selected_indices,
sampler.test_probs,
lengths=input_data[-1],
)
else:
sampler.test_probs = update_probs(
pred,
selected_indices,
sampler.test_probs,
model_cfg["n_classes"],
)
if sampler.p[sampler.p > end_th].shape[0] == sampler.p.shape[0]:
pred = sampler.test_probs[projected_indices]
infer_complete = True
else:
pred = pred.squeeze().cpu()[projected_indices].cuda()
infer_complete = True
if measure_processing_time:
end = time.perf_counter()
processing_time["postprocessing"] += end - start
if label is not None:
label = torch.from_numpy(label.astype(np.int64)).long().cuda()
result = torch.argmax(pred.squeeze(), axis=-1), label, name
# Return processing time if needed
if measure_processing_time:
return result, processing_time
return result
[docs]
def get_sample(
points_fname: str,
model_cfg: dict,
label_fname: Optional[str] = None,
name: Optional[str] = None,
idx: Optional[int] = None,
has_intensity: bool = True,
measure_processing_time: bool = False,
) -> Tuple[
Union[
Tuple[np.ndarray, np.ndarray, ul.Sampler, np.ndarray, str, int],
Tuple[np.ndarray, np.ndarray, ul.Sampler, np.ndarray, str, int],
Dict[str, float],
]
]:
"""Get sample data for mmdetection3d models
:param points_fname: filename of the point cloud
:type points_fname: str
:param model_cfg: model configuration
:type model_cfg: dict
:param label_fname: filename of the semantic label, defaults to None
:type label_fname: Optional[str], optional
:param name: sample name, defaults to None
:type name: Optional[str], optional
:param idx: sample numerical index, defaults to None
:type idx: Optional[int], optional
:param has_intensity: whether the point cloud has intensity values, defaults to True
:type has_intensity: bool, optional
:param measure_processing_time: whether to measure processing time, defaults to False
:type measure_processing_time: bool, optional
:return: sample data and optionally processing time
:rtype: Union[ Tuple[np.ndarray, np.ndarray, ul.Sampler, np.ndarray, str, int], Tuple[np.ndarray, np.ndarray, ul.Sampler, np.ndarray, str, int], Dict[str, float] ]
"""
points = ul.read_semantickitti_points(points_fname, has_intensity)
label = None
if label_fname is not None:
label, _ = ul.read_semantickitti_label(label_fname)
if measure_processing_time:
start = time.perf_counter()
# Keep only XYZ coordinates
points = np.array(points[:, 0:3], dtype=np.float32)
# Subsample points using a grid of given size
grid_size = model_cfg.get("grid_size", 0.06)
sub_points = DataProcessing.grid_subsampling(points, grid_size=grid_size)
# Create search tree so that we can project points back to the original point cloud
search_tree = KDTree(sub_points)
projected_indices = np.squeeze(search_tree.query(points, return_distance=False))
projected_indices = projected_indices.astype(np.int32)
# Init sampler
sampler = None
if "sampler" in model_cfg:
sampler = ul.Sampler(
sub_points.shape[0],
search_tree,
model_cfg["sampler"],
model_cfg["n_classes"],
)
if measure_processing_time:
end = time.perf_counter()
sample = sub_points, projected_indices, sampler, label, name, idx
# Return processing time if needed
if measure_processing_time:
processing_time = {"preprocessing": end - start}
return sample, processing_time
return sample
[docs]
def reset_sampler(sampler: ul.Sampler, num_points: int, num_classes: int):
"""Reset sampler object probabilities
:param sampler: Sampler object
:type sampler: ul.Sampler
:param num_points: Number of points in the point cloud
:type num_points: int
:param num_classes: Number of semantic classes
:type num_classes: int
"""
sampler.p = np.random.rand(num_points) * 1e-3
sampler.min_p = float(np.min(sampler.p[-1]))
sampler.test_probs = np.zeros((num_points, num_classes), dtype=np.float32)
return sampler