Source code for perceptionmetrics.cli

from perceptionmetrics import datasets
from perceptionmetrics import models
from perceptionmetrics.cli.eval_model import eval_model
from perceptionmetrics.cli.eval_preds import eval_preds
from perceptionmetrics.cli.computational_cost import computational_cost
from perceptionmetrics.datasets.coco import find_img_dir_and_ann_file

REGISTRY = {
    "eval_model": eval_model,
    "eval_preds": eval_preds,
    "computational_cost": computational_cost,
}


[docs] def get_model(task, input_type, model_format, model, ontology, model_cfg): # Init model from registry model_name = f"{model_format}_{input_type}_{task}" if model_name not in models.REGISTRY: raise ValueError( f"Model format not supported: {model_format}. " f"Must be one of {models.REGISTRY.keys()}", ) return models.REGISTRY[model_name](model, model_cfg, ontology)
[docs] def get_dataset( task, input_type, dataset_format, dataset_fname, dataset_dir, split_dir, train_dataset_dir, val_dataset_dir, test_dataset_dir, images_dir, labels_dir, data_suffix, label_suffix, ontology, split, ): # Check if required data is available if dataset_format == "gaia": if dataset_fname is None: raise ValueError("--dataset is required for 'gaia' format") elif dataset_format in ["rellis3d", "wildscenes"]: if dataset_dir is None: raise ValueError( "--dataset_dir is required for 'rellis3d' and 'wildscenes' formats" ) if split_dir is None: raise ValueError( "--split_dir is required for 'rellis3d' and 'wildscenes' formats" ) if dataset_format == "rellis3d" and ontology is None: raise ValueError("--dataset_ontology is required for 'rellis3d' format") elif dataset_format in ["goose", "generic", "cityscapes"]: if "train" in split and train_dataset_dir is None: raise ValueError( f"--train_dataset_dir is required for 'train' split in '{dataset_format}' format" ) elif "val" in split and val_dataset_dir is None: raise ValueError( f"--val_dataset_dir is required for 'val' split in '{dataset_format}' format" ) elif "test" in split and test_dataset_dir is None: raise ValueError( f"--test_dataset_dir is required for 'test' split in '{dataset_format}' format" ) if dataset_format == "generic": if data_suffix is None: raise ValueError("--data_suffix is required for 'generic' format") if label_suffix is None: raise ValueError("--label_suffix is required for 'generic' format") if ontology is None: raise ValueError("--dataset_ontology is required for 'generic' format") elif dataset_format == "rugd": if images_dir is None: raise ValueError("--images_dir is required for 'rugd' format") if labels_dir is None: raise ValueError("--labels_dir is required for 'rugd' format") elif dataset_format == "coco": if dataset_dir is None: raise ValueError("--dataset_dir is required for 'coco' format") elif dataset_format == "yolo": if dataset_fname is None: raise ValueError("--dataset_fname is required for 'yolo' format") elif dataset_format == "nuimages": if dataset_dir is None: raise ValueError("--dataset_dir is required for 'nuimages' format") else: raise ValueError(f"Dataset format not supported: {dataset_format}") # Get arguments to init dataset if dataset_format == "gaia": dataset_args = {"dataset_fname": dataset_fname} elif dataset_format in ["rellis3d", "wildscenes"]: dataset_args = { "dataset_dir": dataset_dir, "split_dir": split_dir, } if dataset_format == "rellis3d": dataset_args["ontology_fname"] = ontology elif dataset_format == "goose": dataset_args = { "train_dataset_dir": train_dataset_dir, "val_dataset_dir": val_dataset_dir, "test_dataset_dir": test_dataset_dir, } elif dataset_format == "generic": dataset_args = { "data_suffix": data_suffix, "label_suffix": label_suffix, "ontology_fname": ontology, "train_dataset_dir": train_dataset_dir, "val_dataset_dir": val_dataset_dir, "test_dataset_dir": test_dataset_dir, } elif dataset_format == "cityscapes": dataset_args = { "train_dataset_root": train_dataset_dir, "val_dataset_root": val_dataset_dir, "test_dataset_root": test_dataset_dir, } elif dataset_format == "rugd": dataset_args = { "images_dir": images_dir, "labels_dir": labels_dir, "ontology_fname": ontology, } elif dataset_format == "coco": # For COCO, we need to construct the annotation file path and image directory # Assuming standard COCO structure: dataset_dir/annotations/instances_split.json and dataset_dir/images/split/ if len(split) > 1: raise ValueError("COCO format currently supports only one split at a time") split_name = split[0] image_dir, annotation_file = find_img_dir_and_ann_file( dataset_path=dataset_dir, split=split_name ) dataset_args = { "annotation_file": annotation_file, "image_dir": image_dir, "split": split_name, } elif dataset_format == "yolo": dataset_args = { "dataset_fname": dataset_fname, "dataset_dir": dataset_dir, } elif dataset_format == "nuimages": if len(split) > 1: raise ValueError("NuImages format currently supports only one split at a time") dataset_args = { "dataset_dir": dataset_dir, "split": split[0], } else: raise ValueError(f"Dataset format not supported: {dataset_format}") # Init dataset from registry dataset_name = f"{dataset_format}_{input_type}_{task}" if dataset_name not in datasets.REGISTRY: raise ValueError( f"Dataset format not supported: {dataset_format}. " f"Must be one of {datasets.REGISTRY.keys()}", ) return datasets.REGISTRY[dataset_name](**dataset_args)