from perceptionmetrics import datasets
from perceptionmetrics import models
from perceptionmetrics.cli.eval_model import eval_model
from perceptionmetrics.cli.eval_preds import eval_preds
from perceptionmetrics.cli.computational_cost import computational_cost
from perceptionmetrics.datasets.coco import find_img_dir_and_ann_file
REGISTRY = {
"eval_model": eval_model,
"eval_preds": eval_preds,
"computational_cost": computational_cost,
}
[docs]
def get_model(task, input_type, model_format, model, ontology, model_cfg):
# Init model from registry
model_name = f"{model_format}_{input_type}_{task}"
if model_name not in models.REGISTRY:
raise ValueError(
f"Model format not supported: {model_format}. "
f"Must be one of {models.REGISTRY.keys()}",
)
return models.REGISTRY[model_name](model, model_cfg, ontology)
[docs]
def get_dataset(
task,
input_type,
dataset_format,
dataset_fname,
dataset_dir,
split_dir,
train_dataset_dir,
val_dataset_dir,
test_dataset_dir,
images_dir,
labels_dir,
data_suffix,
label_suffix,
ontology,
split,
):
# Check if required data is available
if dataset_format == "gaia":
if dataset_fname is None:
raise ValueError("--dataset is required for 'gaia' format")
elif dataset_format in ["rellis3d", "wildscenes"]:
if dataset_dir is None:
raise ValueError(
"--dataset_dir is required for 'rellis3d' and 'wildscenes' formats"
)
if split_dir is None:
raise ValueError(
"--split_dir is required for 'rellis3d' and 'wildscenes' formats"
)
if dataset_format == "rellis3d" and ontology is None:
raise ValueError("--dataset_ontology is required for 'rellis3d' format")
elif dataset_format in ["goose", "generic", "cityscapes"]:
if "train" in split and train_dataset_dir is None:
raise ValueError(
f"--train_dataset_dir is required for 'train' split in '{dataset_format}' format"
)
elif "val" in split and val_dataset_dir is None:
raise ValueError(
f"--val_dataset_dir is required for 'val' split in '{dataset_format}' format"
)
elif "test" in split and test_dataset_dir is None:
raise ValueError(
f"--test_dataset_dir is required for 'test' split in '{dataset_format}' format"
)
if dataset_format == "generic":
if data_suffix is None:
raise ValueError("--data_suffix is required for 'generic' format")
if label_suffix is None:
raise ValueError("--label_suffix is required for 'generic' format")
if ontology is None:
raise ValueError("--dataset_ontology is required for 'generic' format")
elif dataset_format == "rugd":
if images_dir is None:
raise ValueError("--images_dir is required for 'rugd' format")
if labels_dir is None:
raise ValueError("--labels_dir is required for 'rugd' format")
elif dataset_format == "coco":
if dataset_dir is None:
raise ValueError("--dataset_dir is required for 'coco' format")
elif dataset_format == "yolo":
if dataset_fname is None:
raise ValueError("--dataset_fname is required for 'yolo' format")
elif dataset_format == "nuimages":
if dataset_dir is None:
raise ValueError("--dataset_dir is required for 'nuimages' format")
else:
raise ValueError(f"Dataset format not supported: {dataset_format}")
# Get arguments to init dataset
if dataset_format == "gaia":
dataset_args = {"dataset_fname": dataset_fname}
elif dataset_format in ["rellis3d", "wildscenes"]:
dataset_args = {
"dataset_dir": dataset_dir,
"split_dir": split_dir,
}
if dataset_format == "rellis3d":
dataset_args["ontology_fname"] = ontology
elif dataset_format == "goose":
dataset_args = {
"train_dataset_dir": train_dataset_dir,
"val_dataset_dir": val_dataset_dir,
"test_dataset_dir": test_dataset_dir,
}
elif dataset_format == "generic":
dataset_args = {
"data_suffix": data_suffix,
"label_suffix": label_suffix,
"ontology_fname": ontology,
"train_dataset_dir": train_dataset_dir,
"val_dataset_dir": val_dataset_dir,
"test_dataset_dir": test_dataset_dir,
}
elif dataset_format == "cityscapes":
dataset_args = {
"train_dataset_root": train_dataset_dir,
"val_dataset_root": val_dataset_dir,
"test_dataset_root": test_dataset_dir,
}
elif dataset_format == "rugd":
dataset_args = {
"images_dir": images_dir,
"labels_dir": labels_dir,
"ontology_fname": ontology,
}
elif dataset_format == "coco":
# For COCO, we need to construct the annotation file path and image directory
# Assuming standard COCO structure: dataset_dir/annotations/instances_split.json and dataset_dir/images/split/
if len(split) > 1:
raise ValueError("COCO format currently supports only one split at a time")
split_name = split[0]
image_dir, annotation_file = find_img_dir_and_ann_file(
dataset_path=dataset_dir, split=split_name
)
dataset_args = {
"annotation_file": annotation_file,
"image_dir": image_dir,
"split": split_name,
}
elif dataset_format == "yolo":
dataset_args = {
"dataset_fname": dataset_fname,
"dataset_dir": dataset_dir,
}
elif dataset_format == "nuimages":
if len(split) > 1:
raise ValueError("NuImages format currently supports only one split at a time")
dataset_args = {
"dataset_dir": dataset_dir,
"split": split[0],
}
else:
raise ValueError(f"Dataset format not supported: {dataset_format}")
# Init dataset from registry
dataset_name = f"{dataset_format}_{input_type}_{task}"
if dataset_name not in datasets.REGISTRY:
raise ValueError(
f"Dataset format not supported: {dataset_format}. "
f"Must be one of {datasets.REGISTRY.keys()}",
)
return datasets.REGISTRY[dataset_name](**dataset_args)