Source code for perceptionmetrics.models.utils.o3d.kpconv
from typing import List, Tuple
import numpy as np
try:
from open3d._ml3d.torch.models.kpconv import batch_grid_subsampling, batch_neighbors
except Exception:
print("Open3D-ML3D not available")
import torch
import perceptionmetrics.utils.lidar as ul
[docs]
def transform_input(points: np.ndarray, cfg: dict, sampler: ul.Sampler) -> Tuple[
Tuple[
torch.Tensor,
List[torch.Tensor],
List[torch.Tensor],
List[torch.Tensor],
List[torch.Tensor],
List[torch.Tensor],
],
List[np.ndarray],
]:
"""Transform point cloud data into input data for the model
:param points: Point cloud data
:type points: np.ndarray
:param cfg: Dictionary containing model configuration file
:type cfg: dict
:param sampler: Object for sampling point cloud
:type sampler: ul.Sampler
:return: Model input data and selected indices
:rtype: Tuple[ Tuple[ torch.Tensor, List[torch.Tensor], List[torch.Tensor], List[torch.Tensor], List[torch.Tensor], List[torch.Tensor], ], List[np.ndarray], ]
"""
p_list = []
p0_list = []
r_mask_list = []
# Divide point cloud into chunks of at most max_in_points
curr_num_points = 0
min_in_points = cfg.get("min_in_points", 10000)
while curr_num_points < min_in_points:
# Sample points if required
new_points, selected_indices, center_point = sampler.sample(
points,
num_points=min_in_points,
radius=cfg.get("in_radius", 4.0),
)
new_points = new_points - center_point
# Recenter point cloud if required
if "recenter" in cfg:
new_points = ul.recenter(new_points, cfg["recenter"]["dims"])
in_pts = new_points
n = in_pts.shape[0]
# Randomly drop some points if max number of points have been exceeded
residual_num_points = cfg.get("max_in_points", 20000) - curr_num_points
if n > residual_num_points:
input_inds = np.random.choice(n, size=residual_num_points, replace=False)
in_pts = in_pts[input_inds, :]
selected_indices = selected_indices[input_inds]
n = input_inds.shape[0]
curr_num_points += n
# Accumulate each chunk data
p_list += [in_pts]
p0_list += [center_point]
r_mask_list += [selected_indices]
# Stack all data
stacked_points = np.concatenate(p_list, axis=0)
stack_lengths = np.array([pp.shape[0] for pp in p_list], dtype=np.int32)
stacked_features = np.ones_like(stacked_points[:, :1], dtype=np.float32)
stacked_features = np.hstack((stacked_features, stacked_points[:, 2:3]))
# From this point until the end of the function, we will get data ready to work as
# inputs for the model
conv_radius = cfg.get("conv_radius", 2.5)
r_normal = cfg.get("first_subsampling_dl", 0.06) * conv_radius
# Lists of inputs
input_points = []
input_neighbors = []
input_pools = []
input_upsamples = []
input_stack_lengths = []
# Generate input data for each block in the model
layer_blocks = []
arch = cfg["architecture"]
for block in arch:
# Get all blocks of the layer
if not (
"pool" in block
or "strided" in block
or "global" in block
or "upsample" in block
):
layer_blocks += [block]
continue
# Convolution neighbors indices
if layer_blocks:
conv_i = batch_neighbors(
stacked_points, stacked_points, stack_lengths, stack_lengths, r_normal
)
else:
# This layer only perform pooling, no neighbors required
conv_i = np.zeros((0, 1), dtype=np.int32)
# Pooling neighbors indices
# If end of layer is a pooling operation
if "pool" in block or "strided" in block:
# New subsampling length
dl = 2 * r_normal / conv_radius
# Subsampled points
pool_p, pool_b = batch_grid_subsampling(
stacked_points, stack_lengths, sampleDl=dl
)
# Subsample indices
pool_i = batch_neighbors(
pool_p, stacked_points, pool_b, stack_lengths, r_normal
)
# Upsample indices (with the radius of the next layer to keep wanted density)
up_i = batch_neighbors(
stacked_points, pool_p, stack_lengths, pool_b, 2 * r_normal
)
else:
# No pooling in the end of this layer, no pooling indices required
pool_i = np.zeros((0, 1), dtype=np.int32)
pool_p = np.zeros((0, 3), dtype=np.float32)
pool_b = np.zeros((0,), dtype=np.int32)
up_i = np.zeros((0, 1), dtype=np.int32)
# Updating input lists
input_points += [stacked_points]
input_neighbors += [conv_i.astype(np.int64)]
input_pools += [pool_i.astype(np.int64)]
input_upsamples += [up_i.astype(np.int64)]
input_stack_lengths += [stack_lengths]
# New points for next layer
stacked_points = pool_p
stack_lengths = pool_b
# Update radius and reset blocks
r_normal *= 2
layer_blocks = []
# Stop when meeting a global pooling or upsampling
if "global" in block or "upsample" in block:
break
# Convert to torch tensors
stacked_features = torch.from_numpy(stacked_features)
input_points = [torch.from_numpy(a) for a in input_points]
input_pools = [torch.from_numpy(a) for a in input_pools]
input_neighbors = [torch.from_numpy(a) for a in input_neighbors]
input_upsamples = [torch.from_numpy(a) for a in input_upsamples]
input_stack_lengths = [torch.from_numpy(a) for a in input_stack_lengths]
return (
stacked_features,
input_points,
input_pools,
input_neighbors,
input_upsamples,
input_stack_lengths,
), r_mask_list
[docs]
def update_probs(
new_probs: torch.Tensor,
indices: torch.Tensor,
test_probs: torch.Tensor,
lengths: int,
weight: float = 0.95,
) -> torch.Tensor:
"""Update test probabilities with new model output using weighted average for a
smooth transition between predictions
:param new_probs: New probabilities to be added to the test probabilities
:type new_probs: torch.Tensor
:param indices: Corresponding indices of the new probabilities
:type indices: torch.Tensor
:param test_probs: Test probabilities to be updated
:type test_probs: torch.Tensor
:param lengths: Length of each subsampled set of points
:type lengths: int
:param weight: Weight used in the weighted average, defaults to 0.95
:type weight: float, optional
:return: Updated test probabilities
:rtype: torch.Tensor
"""
# Format new probabilities
new_probs = torch.nn.functional.softmax(new_probs, dim=-1)
if isinstance(test_probs, np.ndarray):
test_probs = torch.tensor(test_probs, device=new_probs.device)
# Update probabilities using a weighted average for each subsampled set of points
i0 = 0
lengths = lengths[0].cpu().numpy()
for b_i, length in enumerate(lengths):
probs = new_probs[i0 : i0 + length]
proj_mask = indices[b_i]
test_probs[proj_mask] = weight * test_probs[proj_mask] + (1 - weight) * probs
i0 += length
return test_probs