Core API

This page documents the core APIs in VisDet.

Detection APIs

init_detector(config, checkpoint=None, palette='none', device='cuda:0', cfg_options=None)

Initialize a detector from config file.

Parameters:

Name	Type	Description	Default
config (str,		obj:Path, or :obj:visdet.engine.Config): Config file path, :obj:Path, or the config object.	required
checkpoint	str	Checkpoint path. If left as None, the model will not load any weights.	None
palette	str	Color palette used for visualization. If palette is stored in checkpoint, use checkpoint's palette first, otherwise use externally passed palette. Currently, supports 'coco', 'voc', 'citys' and 'random'. Defaults to none.	'none'
device	str | Sequence[str] | Sequence[int]	The device(s) to run inference on. Single device examples: "cpu", "cuda:0" Multi-GPU single-process example: "cuda:0,1" or ["cuda:0", "cuda:1"] When multiple CUDA devices are provided, the model is wrapped in :class:visdet.engine.model.wrappers.MMDataParallel.	'cuda:0'
cfg_options	dict	Options to override some settings in the used config.	None

Returns:

Type	Description
Module	nn.Module: The constructed detector.

Source code in visdet/apis/inference.py

def init_detector(
    config: str | Path | Config,
    checkpoint: str | None = None,
    palette: str = "none",
    device: str | Sequence[str] | Sequence[int] = "cuda:0",
    cfg_options: dict | None = None,
) -> nn.Module:
    """Initialize a detector from config file.

    Args:
        config (str, :obj:`Path`, or :obj:`visdet.engine.Config`): Config file path,
            :obj:`Path`, or the config object.
        checkpoint (str, optional): Checkpoint path. If left as None, the model
            will not load any weights.
        palette (str): Color palette used for visualization. If palette
            is stored in checkpoint, use checkpoint's palette first, otherwise
            use externally passed palette. Currently, supports 'coco', 'voc',
            'citys' and 'random'. Defaults to none.
        device (str | Sequence[str] | Sequence[int]):
            The device(s) to run inference on.

            - Single device examples: ``"cpu"``, ``"cuda:0"``
            - Multi-GPU single-process example: ``"cuda:0,1"`` or
              ``["cuda:0", "cuda:1"]``

            When multiple CUDA devices are provided, the model is wrapped in
            :class:`visdet.engine.model.wrappers.MMDataParallel`.
        cfg_options (dict, optional): Options to override some settings in
            the used config.

    Returns:
        nn.Module: The constructed detector.
    """
    if isinstance(config, str | Path):
        config = Config.fromfile(config)
    elif not isinstance(config, Config):
        raise TypeError(f"config must be a filename or Config object, but got {type(config)}")
    if cfg_options is not None:
        config.merge_from_dict(cfg_options)
    elif "init_cfg" in config.model.backbone:
        config.model.backbone.init_cfg = None

    scope = config.get("default_scope", "visdet")
    if scope is not None:
        init_default_scope(config.get("default_scope", "visdet"))

    model = MODELS.build(config.model)
    model = revert_sync_batchnorm(model)
    if checkpoint is None:
        warnings.simplefilter("once")
        warnings.warn("checkpoint is None, use COCO classes by default.", stacklevel=2)
        model.dataset_meta = {"classes": get_classes("coco")}
    else:
        checkpoint = load_checkpoint(model, checkpoint, map_location="cpu")
        # Weights converted from elsewhere may not have meta fields.
        checkpoint_meta = checkpoint.get("meta", {})

        # save the dataset_meta in the model for convenience
        if "dataset_meta" in checkpoint_meta:
            # visdet 3.x, all keys should be lowercase
            model.dataset_meta = {k.lower(): v for k, v in checkpoint_meta["dataset_meta"].items()}
        elif "CLASSES" in checkpoint_meta:
            # < visdet 3.x
            classes = checkpoint_meta["CLASSES"]
            model.dataset_meta = {"classes": classes}
        else:
            warnings.simplefilter("once")
            warnings.warn(
                "dataset_meta or class names are not saved in the checkpoint's meta data, use COCO classes by default.",
                stacklevel=2,
            )
            model.dataset_meta = {"classes": get_classes("coco")}

    # Priority:  args.palette -> config -> checkpoint
    if palette != "none":
        model.dataset_meta["palette"] = palette
    else:
        test_dataset_cfg = copy.deepcopy(config.test_dataloader.dataset)
        # lazy init. We only need the metainfo.
        test_dataset_cfg["lazy_init"] = True
        metainfo = DATASETS.build(test_dataset_cfg).metainfo
        cfg_palette = metainfo.get("palette", None)
        if cfg_palette is not None:
            model.dataset_meta["palette"] = cfg_palette
        else:
            if "palette" not in model.dataset_meta:
                warnings.warn(
                    "palette does not exist, random is used by default. You can also set the palette to customize.",
                    stacklevel=2,
                )
                model.dataset_meta["palette"] = "random"

    model.cfg = config  # save the config in the model for convenience

    primary_device, device_ids = _parse_inference_devices(device)
    model.to(primary_device)

    if device_ids is not None and len(device_ids) > 1:
        if not torch.cuda.is_available():
            raise RuntimeError(f"CUDA is not available, cannot use multi-GPU device={device!r}")
        if max(device_ids) >= torch.cuda.device_count():
            raise RuntimeError(
                f"Requested device_ids={device_ids} but only {torch.cuda.device_count()} CUDA device(s) are available"
            )

        from visdet.engine.model import MMDataParallel

        model = MMDataParallel(model, device_ids=device_ids, output_device=device_ids[0])

    model.eval()
    return model

inference_detector(model, imgs, test_pipeline=None, text_prompt=None, custom_entities=False)

Inference image(s) with the detector.

Parameters:

Name	Type	Description	Default
model	Module	The loaded detector.	required
imgs	(str, ndarray, Sequence[str / ndarray])	Either image files or loaded images.	required
test_pipeline (		obj:Compose): Test pipeline.	required

Returns:

Type	Description
DetDataSample | SampleList	obj:DetDataSample or list[:obj:DetDataSample]:
DetDataSample | SampleList	If imgs is a list or tuple, the same length list type results
DetDataSample | SampleList	will be returned, otherwise return the detection results directly.

Source code in visdet/apis/inference.py

def inference_detector(
    model: nn.Module,
    imgs: ImagesType,
    test_pipeline: Compose | None = None,
    text_prompt: str | None = None,
    custom_entities: bool = False,
) -> DetDataSample | SampleList:
    """Inference image(s) with the detector.

    Args:
        model (nn.Module): The loaded detector.
        imgs (str, ndarray, Sequence[str/ndarray]):
           Either image files or loaded images.
        test_pipeline (:obj:`Compose`): Test pipeline.

    Returns:
        :obj:`DetDataSample` or list[:obj:`DetDataSample`]:
        If imgs is a list or tuple, the same length list type results
        will be returned, otherwise return the detection results directly.
    """

    if isinstance(imgs, list | tuple):
        is_batch = True
    else:
        imgs = [imgs]
        is_batch = False

    cfg = model.cfg

    if test_pipeline is None:
        cfg = cfg.copy()
        test_pipeline = get_test_pipeline_cfg(cfg)
        if isinstance(imgs[0], np.ndarray):
            # Calling this method across libraries will result
            # in module unregistered error if not prefixed with visdet.
            test_pipeline[0].type = "visdet.LoadImageFromNDArray"

        test_pipeline = Compose(test_pipeline)

    # RoIPool check removed - eliminating C++ ops

    data_list = []
    for img in imgs:
        # prepare data
        if isinstance(img, np.ndarray):
            # TODO: remove img_id.
            data_ = {"img": img, "img_id": 0}
        else:
            # TODO: remove img_id.
            data_ = {"img_path": img, "img_id": 0}

        if text_prompt:
            data_["text"] = text_prompt
            data_["custom_entities"] = custom_entities

        data_list.append(test_pipeline(data_))

    batch = pseudo_collate(data_list)

    # forward the model
    with torch.inference_mode():
        results = model.test_step(batch)

    if not is_batch:
        return results[0]
    else:
        return results

DetInferencer

Bases: BaseInferencer

Object Detection Inferencer.

Parameters:

Name	Type	Description	Default
model	str	Path to a YAML config file, or a model preset name/alias. For example: "rtmdet-s" or "rtmdet_ins_s". (visdet prefers YAML presets and does not require Python configs.) If model is not specified, user must provide the weights saved by MMEngine which contains the config string. Defaults to None.	None
weights	str	Path to the checkpoint. If it is not specified and model is a model name of metafile, the weights will be loaded from metafile. Defaults to None.	None
device	str	Device to run inference. If None, the available device will be automatically used. Defaults to None.	None
scope	str	The scope of the model. Defaults to visdet.	'visdet'
palette	str	Color palette used for visualization. The order of priority is palette -> config -> checkpoint. Defaults to 'none'.	'none'
show_progress	bool	Control whether to display the progress bar during the inference process. Defaults to True.	True

Source code in visdet/apis/det_inferencer.py

class DetInferencer(BaseInferencer):
    """Object Detection Inferencer.

    Args:
        model (str, optional): Path to a YAML config file, or a model preset
            name/alias. For example: "rtmdet-s" or "rtmdet_ins_s".
            (visdet prefers YAML presets and does not require Python configs.)
            If model is not specified, user must provide the
            `weights` saved by MMEngine which contains the config string.
            Defaults to None.
        weights (str, optional): Path to the checkpoint. If it is not specified
            and model is a model name of metafile, the weights will be loaded
            from metafile. Defaults to None.
        device (str, optional): Device to run inference. If None, the available
            device will be automatically used. Defaults to None.
        scope (str, optional): The scope of the model. Defaults to visdet.
        palette (str): Color palette used for visualization. The order of
            priority is palette -> config -> checkpoint. Defaults to 'none'.
        show_progress (bool): Control whether to display the progress
            bar during the inference process. Defaults to True.
    """

    preprocess_kwargs: set = set()
    forward_kwargs: set = set()
    visualize_kwargs: set = {
        "return_vis",
        "show",
        "wait_time",
        "draw_pred",
        "pred_score_thr",
        "img_out_dir",
        "no_save_vis",
    }
    postprocess_kwargs: set = {
        "print_result",
        "pred_out_dir",
        "return_datasamples",
        "no_save_pred",
    }

    _alias_to_preset: dict[str, str] | None = None

    @classmethod
    def _build_alias_to_preset(cls) -> dict[str, str]:
        if cls._alias_to_preset is None:
            mapping: dict[str, str] = {}
            for preset_name in MODEL_PRESETS.list():
                mapping[preset_name.lower()] = preset_name
                cfg = MODEL_PRESETS.get(preset_name)
                meta = cfg.get("preset_meta", {}) or {}
                for alias in meta.get("aliases", []) or []:
                    mapping[str(alias).lower()] = preset_name
            cls._alias_to_preset = mapping
        return cls._alias_to_preset

    @classmethod
    def _resolve_model_preset(cls, name: str) -> str | None:
        key = name.lower()
        mapping = cls._build_alias_to_preset()
        if key in mapping:
            return mapping[key]
        if key.replace("-", "_") in mapping:
            return mapping[key.replace("-", "_")]
        if key.replace("_", "-") in mapping:
            return mapping[key.replace("_", "-")]
        return None

    @staticmethod
    def _build_infer_cfg_from_preset(preset_cfg: dict) -> tuple[dict, str | None]:
        cfg = copy.deepcopy(preset_cfg)
        meta = cfg.pop("preset_meta", {}) or {}
        weights = meta.get("weights")
        img_scale = meta.get("img_scale", [640, 640])

        infer_cfg = {
            "model": cfg,
            "test_pipeline": [
                {"type": "visdet.InferencerLoader"},
                {"type": "Resize", "scale": img_scale, "keep_ratio": True},
                {"type": "PackDetInputs"},
            ],
            "visualizer": {
                "type": "DetLocalVisualizer",
                "vis_backends": [{"type": "LocalVisBackend"}],
                "name": "visualizer",
            },
        }
        return infer_cfg, weights

    @classmethod
    def list_models(cls, scope: str | None = None, patterns: str = r".*") -> list[str]:
        import re

        matched: set[str] = set()
        for preset_name in MODEL_PRESETS.list():
            cfg = MODEL_PRESETS.get(preset_name)
            meta = cfg.get("preset_meta", {}) or {}
            names = [preset_name]
            names.extend([str(a) for a in meta.get("aliases", []) or []])
            for name in names:
                if re.match(patterns, name) is not None:
                    matched.add(name)
        return sorted(matched)

    def __init__(
        self,
        model: ModelType | str | None = None,
        weights: str | None = None,
        device: str | None = None,
        scope: str | None = "visdet",
        palette: str = "none",
        show_progress: bool = True,
    ) -> None:
        # A global counter tracking the number of images processed, for
        # naming of the output images
        self.num_visualized_imgs = 0
        self.num_predicted_imgs = 0
        self.palette = palette
        init_default_scope(scope)

        if isinstance(model, str) and not osp.isfile(model):
            preset_name = self._resolve_model_preset(model)
            if preset_name is None:
                available = ", ".join(MODEL_PRESETS.list())
                raise ValueError(
                    f"Unknown model preset/alias: {model}. "
                    f"Expected one of: {available}."
                )
            preset_cfg = MODEL_PRESETS.get(preset_name)
            infer_cfg, preset_weights = self._build_infer_cfg_from_preset(preset_cfg)
            if weights is None:
                weights = preset_weights
            model = infer_cfg

        super().__init__(model=model, weights=weights, device=device, scope=scope)
        self.model = revert_sync_batchnorm(self.model)
        self.show_progress = show_progress

    def _load_weights_to_model(
        self, model: nn.Module, checkpoint: dict | None, cfg: ConfigType | None
    ) -> None:
        """Loading model weights and meta information from cfg and checkpoint.

        Args:
            model (nn.Module): Model to load weights and meta information.
            checkpoint (dict, optional): The loaded checkpoint.
            cfg (Config or ConfigDict, optional): The loaded config.
        """

        if checkpoint is not None:
            _load_checkpoint_to_model(model, checkpoint)
            checkpoint_meta = checkpoint.get("meta", {})
            # save the dataset_meta in the model for convenience
            if "dataset_meta" in checkpoint_meta:
                # visdet 3.x, all keys should be lowercase
                model.dataset_meta = {
                    k.lower(): v for k, v in checkpoint_meta["dataset_meta"].items()
                }
            elif "CLASSES" in checkpoint_meta:
                # < visdet 3.x
                classes = checkpoint_meta["CLASSES"]
                model.dataset_meta = {"classes": classes}
            else:
                warnings.warn(
                    "dataset_meta or class names are not saved in the "
                    "checkpoint's meta data, use COCO classes by default.", stacklevel=2
                )
                model.dataset_meta = {"classes": get_classes("coco")}
        else:
            warnings.warn(
                "Checkpoint is not loaded, and the inference "
                "result is calculated by the randomly initialized "
                "model!", stacklevel=2
            )
            warnings.warn("weights is None, use COCO classes by default.", stacklevel=2)
            model.dataset_meta = {"classes": get_classes("coco")}

        # Priority:  args.palette -> config -> checkpoint
        if self.palette != "none":
            model.dataset_meta["palette"] = self.palette
        else:
            cfg_palette = None
            if cfg is not None and hasattr(cfg, "test_dataloader"):
                test_dataset_cfg = copy.deepcopy(cfg.test_dataloader.dataset)
                # lazy init. We only need the metainfo.
                test_dataset_cfg["lazy_init"] = True
                metainfo = DATASETS.build(test_dataset_cfg).metainfo
                cfg_palette = metainfo.get("palette", None)

            if cfg_palette is not None:
                model.dataset_meta["palette"] = cfg_palette
            elif "palette" not in model.dataset_meta:
                warnings.warn(
                    "palette does not exist, random is used by default. "
                    "You can also set the palette to customize.",
                    stacklevel=2,
                )
                model.dataset_meta["palette"] = "random"

    def _init_pipeline(self, cfg: ConfigType) -> Compose:
        """Initialize the test pipeline."""
        if "test_pipeline" in cfg:
            pipeline_cfg = copy.deepcopy(cfg["test_pipeline"])
        else:
            pipeline_cfg = cfg.test_dataloader.dataset.pipeline

        # For inference, the key of ``img_id`` is not used.
        if pipeline_cfg and "meta_keys" in pipeline_cfg[-1]:
            pipeline_cfg[-1]["meta_keys"] = tuple(
                meta_key for meta_key in pipeline_cfg[-1]["meta_keys"] if meta_key != "img_id"
            )

        # Replace the first image loader with InferencerLoader so we can accept
        # both file paths and already-loaded numpy arrays.
        load_img_idx = self._get_transform_idx(pipeline_cfg, ("LoadImageFromFile", LoadImageFromFile))
        if load_img_idx != -1:
            pipeline_cfg[load_img_idx]["type"] = "visdet.InferencerLoader"
        elif pipeline_cfg and pipeline_cfg[0].get("type") not in ("visdet.InferencerLoader", "InferencerLoader"):
            raise ValueError("Cannot find an image loading transform in the test pipeline")

        return Compose(pipeline_cfg)

    def _get_transform_idx(
        self, pipeline_cfg: ConfigType, name: str | tuple[str, type]
    ) -> int:
        """Returns the index of the transform in a pipeline.

        If the transform is not found, returns -1.
        """
        for i, transform in enumerate(pipeline_cfg):
            if transform["type"] in name:
                return i
        return -1

    def _init_visualizer(self, cfg: ConfigType) -> Visualizer | None:
        """Initialize visualizers.

        Args:
            cfg (ConfigType): Config containing the visualizer information.

        Returns:
            Visualizer or None: Visualizer initialized with config.
        """
        visualizer = super()._init_visualizer(cfg)
        visualizer.dataset_meta = self.model.dataset_meta
        return visualizer

    def _inputs_to_list(self, inputs: InputsType) -> list:
        """Preprocess the inputs to a list.

        Preprocess inputs to a list according to its type:

        - list or tuple: return inputs
        - str:
            - Directory path: return all files in the directory
            - other cases: return a list containing the string. The string
              could be a path to file, a url or other types of string according
              to the task.

        Args:
            inputs (InputsType): Inputs for the inferencer.

        Returns:
            list: List of input for the :meth:`preprocess`.
        """
        if isinstance(inputs, str):
            backend = get_file_backend(inputs)
            if hasattr(backend, "isdir") and isdir(inputs):
                # Backends like HttpsBackend do not implement `isdir`, so only
                # those backends that implement `isdir` could accept the inputs
                # as a directory
                filename_list = list_dir_or_file(inputs, list_dir=False, suffix=IMG_EXTENSIONS)
                inputs = [join_path(inputs, filename) for filename in filename_list]

        if not isinstance(inputs, list | tuple):
            inputs = [inputs]

        return list(inputs)

    def preprocess(self, inputs: InputsType, batch_size: int = 1, **kwargs):
        """Process the inputs into a model-feedable format.

        Customize your preprocess by overriding this method. Preprocess should
        return an iterable object, of which each item will be used as the
        input of ``model.test_step``.

        ``BaseInferencer.preprocess`` will return an iterable chunked data,
        which will be used in __call__ like this:

        .. code-block:: python

            def __call__(self, inputs, batch_size=1, **kwargs):
                chunked_data = self.preprocess(inputs, batch_size, **kwargs)
                for batch in chunked_data:
                    preds = self.forward(batch, **kwargs)

        Args:
            inputs (InputsType): Inputs given by user.
            batch_size (int): batch size. Defaults to 1.

        Yields:
            Any: Data processed by the ``pipeline`` and ``collate_fn``.
        """
        chunked_data = self._get_chunk_data(inputs, batch_size)
        yield from map(self.collate_fn, chunked_data)

    def _get_chunk_data(self, inputs: Iterable, chunk_size: int):
        """Get batch data from inputs.

        Args:
            inputs (Iterable): An iterable dataset.
            chunk_size (int): Equivalent to batch size.

        Yields:
            list: batch data.
        """
        inputs_iter = iter(inputs)
        while True:
            try:
                chunk_data = []
                for _ in range(chunk_size):
                    inputs_ = next(inputs_iter)
                    if isinstance(inputs_, dict):
                        if "img" in inputs_:
                            ori_inputs_ = inputs_["img"]
                        else:
                            ori_inputs_ = inputs_["img_path"]
                        chunk_data.append((ori_inputs_, self.pipeline(copy.deepcopy(inputs_))))
                    else:
                        chunk_data.append((inputs_, self.pipeline(inputs_)))
                yield chunk_data
            except StopIteration:
                if chunk_data:
                    yield chunk_data
                break

    # TODO: Video and Webcam are currently not supported and
    #  may consume too much memory if your input folder has a lot of images.
    #  We will be optimized later.
    def __call__(
        self,
        inputs: InputsType,
        batch_size: int = 1,
        return_vis: bool = False,
        show: bool = False,
        wait_time: int = 0,
        no_save_vis: bool = False,
        draw_pred: bool = True,
        pred_score_thr: float = 0.3,
        return_datasamples: bool = False,
        print_result: bool = False,
        no_save_pred: bool = True,
        out_dir: str = "",
        # by open image task
        texts: str | list | None = None,
        # by open panoptic task
        stuff_texts: str | list | None = None,
        # by GLIP and Grounding DINO
        custom_entities: bool = False,
        # by Grounding DINO
        tokens_positive: int | list | None = None,
        **kwargs,
    ) -> dict:
        """Call the inferencer.

        Args:
            inputs (InputsType): Inputs for the inferencer.
            batch_size (int): Inference batch size. Defaults to 1.
            show (bool): Whether to display the visualization results in a
                popup window. Defaults to False.
            wait_time (float): The interval of show (s). Defaults to 0.
            no_save_vis (bool): Whether to force not to save prediction
                vis results. Defaults to False.
            draw_pred (bool): Whether to draw predicted bounding boxes.
                Defaults to True.
            pred_score_thr (float): Minimum score of bboxes to draw.
                Defaults to 0.3.
            return_datasamples (bool): Whether to return results as
                :obj:`DetDataSample`. Defaults to False.
            print_result (bool): Whether to print the inference result w/o
                visualization to the console. Defaults to False.
            no_save_pred (bool): Whether to force not to save prediction
                results. Defaults to True.
            out_dir: Dir to save the inference results or
                visualization. If left as empty, no file will be saved.
                Defaults to ''.
            texts (str | list[str]): Text prompts. Defaults to None.
            stuff_texts (str | list[str]): Stuff text prompts of open
                panoptic task. Defaults to None.
            custom_entities (bool): Whether to use custom entities.
                Defaults to False. Only used in GLIP and Grounding DINO.
            **kwargs: Other keyword arguments passed to :meth:`preprocess`,
                :meth:`forward`, :meth:`visualize` and :meth:`postprocess`.
                Each key in kwargs should be in the corresponding set of
                ``preprocess_kwargs``, ``forward_kwargs``, ``visualize_kwargs``
                and ``postprocess_kwargs``.

        Returns:
            dict: Inference and visualization results.
        """
        (
            preprocess_kwargs,
            forward_kwargs,
            visualize_kwargs,
            postprocess_kwargs,
        ) = self._dispatch_kwargs(**kwargs)

        ori_inputs = self._inputs_to_list(inputs)

        if texts is not None and isinstance(texts, str):
            texts = [texts] * len(ori_inputs)
        if stuff_texts is not None and isinstance(stuff_texts, str):
            stuff_texts = [stuff_texts] * len(ori_inputs)

        # Currently only supports bs=1
        tokens_positive = [tokens_positive] * len(ori_inputs)

        if texts is not None:
            assert len(texts) == len(ori_inputs)
            for i in range(len(texts)):
                if isinstance(ori_inputs[i], str):
                    ori_inputs[i] = {
                        "text": texts[i],
                        "img_path": ori_inputs[i],
                        "custom_entities": custom_entities,
                        "tokens_positive": tokens_positive[i],
                    }
                else:
                    ori_inputs[i] = {
                        "text": texts[i],
                        "img": ori_inputs[i],
                        "custom_entities": custom_entities,
                        "tokens_positive": tokens_positive[i],
                    }
        if stuff_texts is not None:
            assert len(stuff_texts) == len(ori_inputs)
            for i in range(len(stuff_texts)):
                ori_inputs[i]["stuff_text"] = stuff_texts[i]

        inputs = self.preprocess(ori_inputs, batch_size=batch_size, **preprocess_kwargs)

        # Calculate total number of batches for progress bar
        total_images = len(ori_inputs)
        total_batches = (total_images + batch_size - 1) // batch_size

        results_dict = {"predictions": [], "visualization": []}

        if self.show_progress:
            with Progress(
                SpinnerColumn(),
                TextColumn("[progress.description]{task.description}"),
                BarColumn(),
                TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
                TextColumn("•"),
                TextColumn("{task.completed}/{task.total} batches"),
                TextColumn("•"),
                TimeElapsedColumn(),
                TextColumn("•"),
                TimeRemainingColumn(),
            ) as progress:
                task = progress.add_task(
                    f"[cyan]Processing {total_images} images...",
                    total=total_batches
                )
                for ori_imgs, data in inputs:
                    preds = self.forward(data, **forward_kwargs)
                    visualization = self.visualize(
                        ori_imgs,
                        preds,
                        return_vis=return_vis,
                        show=show,
                        wait_time=wait_time,
                        draw_pred=draw_pred,
                        pred_score_thr=pred_score_thr,
                        no_save_vis=no_save_vis,
                        img_out_dir=out_dir,
                        **visualize_kwargs,
                    )
                    results = self.postprocess(
                        preds,
                        visualization,
                        return_datasamples=return_datasamples,
                        print_result=print_result,
                        no_save_pred=no_save_pred,
                        pred_out_dir=out_dir,
                        **postprocess_kwargs,
                    )
                    results_dict["predictions"].extend(results["predictions"])
                    if results["visualization"] is not None:
                        results_dict["visualization"].extend(results["visualization"])
                    progress.update(task, advance=1)
        else:
            for ori_imgs, data in inputs:
                preds = self.forward(data, **forward_kwargs)
                visualization = self.visualize(
                    ori_imgs,
                    preds,
                    return_vis=return_vis,
                    show=show,
                    wait_time=wait_time,
                    draw_pred=draw_pred,
                    pred_score_thr=pred_score_thr,
                    no_save_vis=no_save_vis,
                    img_out_dir=out_dir,
                    **visualize_kwargs,
                )
                results = self.postprocess(
                    preds,
                    visualization,
                    return_datasamples=return_datasamples,
                    print_result=print_result,
                    no_save_pred=no_save_pred,
                    pred_out_dir=out_dir,
                    **postprocess_kwargs,
                )
                results_dict["predictions"].extend(results["predictions"])
                if results["visualization"] is not None:
                    results_dict["visualization"].extend(results["visualization"])
        return results_dict

    def visualize(
        self,
        inputs: InputsType,
        preds: PredType,
        return_vis: bool = False,
        show: bool = False,
        wait_time: int = 0,
        draw_pred: bool = True,
        pred_score_thr: float = 0.3,
        no_save_vis: bool = False,
        img_out_dir: str = "",
        **kwargs,
    ) -> list[np.ndarray] | None:
        """Visualize predictions.

        Args:
            inputs (List[Union[str, np.ndarray]]): Inputs for the inferencer.
            preds (List[:obj:`DetDataSample`]): Predictions of the model.
            return_vis (bool): Whether to return the visualization result.
                Defaults to False.
            show (bool): Whether to display the image in a popup window.
                Defaults to False.
            wait_time (float): The interval of show (s). Defaults to 0.
            draw_pred (bool): Whether to draw predicted bounding boxes.
                Defaults to True.
            pred_score_thr (float): Minimum score of bboxes to draw.
                Defaults to 0.3.
            no_save_vis (bool): Whether to force not to save prediction
                vis results. Defaults to False.
            img_out_dir (str): Output directory of visualization results.
                If left as empty, no file will be saved. Defaults to ''.

        Returns:
            List[np.ndarray] or None: Returns visualization results only if
            applicable.
        """
        if no_save_vis is True:
            img_out_dir = ""

        if not show and img_out_dir == "" and not return_vis:
            return None

        if self.visualizer is None:
            raise ValueError(
                'Visualization needs the "visualizer" term' "defined in the config, but got None."
            )

        results = []

        for single_input, pred in zip(inputs, preds, strict=False):
            if isinstance(single_input, str):
                img_bytes = get(single_input)
                img = imfrombytes(img_bytes)
                img = img[:, :, ::-1]
                img_name = osp.basename(single_input)
            elif isinstance(single_input, np.ndarray):
                img = single_input.copy()
                img_num = str(self.num_visualized_imgs).zfill(8)
                img_name = f"{img_num}.jpg"
            else:
                raise ValueError("Unsupported input type: " f"{type(single_input)}")

            out_file = osp.join(img_out_dir, "vis", img_name) if img_out_dir != "" else None

            self.visualizer.add_datasample(
                img_name,
                img,
                pred,
                show=show,
                wait_time=wait_time,
                draw_gt=False,
                draw_pred=draw_pred,
                pred_score_thr=pred_score_thr,
                out_file=out_file,
            )
            results.append(self.visualizer.get_image())
            self.num_visualized_imgs += 1

        return results

    def postprocess(
        self,
        preds: PredType,
        visualization: list[np.ndarray] | None = None,
        return_datasamples: bool = False,
        print_result: bool = False,
        no_save_pred: bool = False,
        pred_out_dir: str = "",
        **kwargs,
    ) -> dict:
        """Process the predictions and visualization results from ``forward``
        and ``visualize``.

        This method should be responsible for the following tasks:

        1. Convert datasamples into a json-serializable dict if needed.
        2. Pack the predictions and visualization results and return them.
        3. Dump or log the predictions.

        Args:
            preds (List[:obj:`DetDataSample`]): Predictions of the model.
            visualization (Optional[np.ndarray]): Visualized predictions.
            return_datasamples (bool): Whether to use Datasample to store
                inference results. If False, dict will be used.
            print_result (bool): Whether to print the inference result w/o
                visualization to the console. Defaults to False.
            no_save_pred (bool): Whether to force not to save prediction
                results. Defaults to False.
            pred_out_dir: Dir to save the inference results w/o
                visualization. If left as empty, no file will be saved.
                Defaults to ''.

        Returns:
            dict: Inference and visualization results with key ``predictions``
            and ``visualization``.

            - ``visualization`` (Any): Returned by :meth:`visualize`.
            - ``predictions`` (dict or DataSample): Returned by
                :meth:`forward` and processed in :meth:`postprocess`.
                If ``return_datasamples=False``, it usually should be a
                json-serializable dict containing only basic data elements such
                as strings and numbers.
        """
        if no_save_pred is True:
            pred_out_dir = ""

        result_dict = {}
        results = preds
        if not return_datasamples:
            results = []
            for pred in preds:
                result = self.pred2dict(pred, pred_out_dir)
                results.append(result)
        elif pred_out_dir != "":
            warnings.warn(
                "Currently does not support saving datasample "
                "when return_datasamples is set to True. "
                "Prediction results are not saved!", stacklevel=2
            )
        # Add img to the results after printing and dumping
        result_dict["predictions"] = results
        if print_result:
            print(result_dict)
        result_dict["visualization"] = visualization
        return result_dict

    # TODO: The data format and fields saved in json need further discussion.
    #  Maybe should include model name, timestamp, filename, image info etc.
    def pred2dict(self, data_sample: DetDataSample, pred_out_dir: str = "") -> dict:
        """Extract elements necessary to represent a prediction into a
        dictionary.

        It's better to contain only basic data elements such as strings and
        numbers in order to guarantee it's json-serializable.

        Args:
            data_sample (:obj:`DetDataSample`): Predictions of the model.
            pred_out_dir: Dir to save the inference results w/o
                visualization. If left as empty, no file will be saved.
                Defaults to ''.

        Returns:
            dict: Prediction results.
        """
        is_save_pred = True
        if pred_out_dir == "":
            is_save_pred = False

        if is_save_pred and "img_path" in data_sample:
            img_path = osp.basename(data_sample.img_path)
            img_path = osp.splitext(img_path)[0]
            out_img_path = osp.join(pred_out_dir, "preds", img_path + "_panoptic_seg.png")
            out_json_path = osp.join(pred_out_dir, "preds", img_path + ".json")
        elif is_save_pred:
            out_img_path = osp.join(
                pred_out_dir, "preds", f"{self.num_predicted_imgs}_panoptic_seg.png"
            )
            out_json_path = osp.join(pred_out_dir, "preds", f"{self.num_predicted_imgs}.json")
            self.num_predicted_imgs += 1

        result = {}
        if "pred_instances" in data_sample:
            masks = data_sample.pred_instances.get("masks")
            pred_instances = data_sample.pred_instances.numpy()
            result = {
                "labels": pred_instances.labels.tolist(),
                "scores": pred_instances.scores.tolist(),
            }
            if "bboxes" in pred_instances:
                result["bboxes"] = pred_instances.bboxes.tolist()
            if masks is not None:
                if "bboxes" not in pred_instances or pred_instances.bboxes.sum() == 0:
                    # Fake bbox, such as the SOLO.
                    bboxes = mask2bbox(masks.cpu()).numpy().tolist()
                    result["bboxes"] = bboxes
                encode_masks = encode_mask_results(pred_instances.masks)
                for encode_mask in encode_masks:
                    if isinstance(encode_mask["counts"], bytes):
                        encode_mask["counts"] = encode_mask["counts"].decode()
                result["masks"] = encode_masks

        if "pred_panoptic_seg" in data_sample:
            if VOID is None:
                raise RuntimeError(
                    "panopticapi is not installed, please install it by: "
                    "pip install git+https://github.com/cocodataset/"
                    "panopticapi.git."
                )

            pan = data_sample.pred_panoptic_seg.sem_seg.cpu().numpy()[0]
            pan[pan % INSTANCE_OFFSET == len(self.model.dataset_meta["classes"])] = VOID
            pan = id2rgb(pan).astype(np.uint8)

            if is_save_pred:
                imwrite(pan[:, :, ::-1], out_img_path)
                result["panoptic_seg_path"] = out_img_path
            else:
                result["panoptic_seg"] = pan

        if is_save_pred:
            dump(result, out_json_path)

        return result

preprocess(inputs, batch_size=1, **kwargs)

Process the inputs into a model-feedable format.

Customize your preprocess by overriding this method. Preprocess should return an iterable object, of which each item will be used as the input of model.test_step.

BaseInferencer.preprocess will return an iterable chunked data, which will be used in call like this:

.. code-block:: python

def __call__(self, inputs, batch_size=1, **kwargs):
    chunked_data = self.preprocess(inputs, batch_size, **kwargs)
    for batch in chunked_data:
        preds = self.forward(batch, **kwargs)

Parameters:

Name	Type	Description	Default
inputs	InputsType	Inputs given by user.	required
batch_size	int	batch size. Defaults to 1.	1

Yields:

Name	Type	Description
Any		Data processed by the pipeline and collate_fn.

Source code in visdet/apis/det_inferencer.py

def preprocess(self, inputs: InputsType, batch_size: int = 1, **kwargs):
    """Process the inputs into a model-feedable format.

    Customize your preprocess by overriding this method. Preprocess should
    return an iterable object, of which each item will be used as the
    input of ``model.test_step``.

    ``BaseInferencer.preprocess`` will return an iterable chunked data,
    which will be used in __call__ like this:

    .. code-block:: python

        def __call__(self, inputs, batch_size=1, **kwargs):
            chunked_data = self.preprocess(inputs, batch_size, **kwargs)
            for batch in chunked_data:
                preds = self.forward(batch, **kwargs)

    Args:
        inputs (InputsType): Inputs given by user.
        batch_size (int): batch size. Defaults to 1.

    Yields:
        Any: Data processed by the ``pipeline`` and ``collate_fn``.
    """
    chunked_data = self._get_chunk_data(inputs, batch_size)
    yield from map(self.collate_fn, chunked_data)

__call__(inputs, batch_size=1, return_vis=False, show=False, wait_time=0, no_save_vis=False, draw_pred=True, pred_score_thr=0.3, return_datasamples=False, print_result=False, no_save_pred=True, out_dir='', texts=None, stuff_texts=None, custom_entities=False, tokens_positive=None, **kwargs)

Call the inferencer.

Parameters:

Name	Type	Description	Default
inputs	InputsType	Inputs for the inferencer.	required
batch_size	int	Inference batch size. Defaults to 1.	1
show	bool	Whether to display the visualization results in a popup window. Defaults to False.	False
wait_time	float	The interval of show (s). Defaults to 0.	0
no_save_vis	bool	Whether to force not to save prediction vis results. Defaults to False.	False
draw_pred	bool	Whether to draw predicted bounding boxes. Defaults to True.	True
pred_score_thr	float	Minimum score of bboxes to draw. Defaults to 0.3.	0.3
return_datasamples	bool	Whether to return results as :obj:DetDataSample. Defaults to False.	False
print_result	bool	Whether to print the inference result w/o visualization to the console. Defaults to False.	False
no_save_pred	bool	Whether to force not to save prediction results. Defaults to True.	True
out_dir	str	Dir to save the inference results or visualization. If left as empty, no file will be saved. Defaults to ''.	''
texts	str | list[str]	Text prompts. Defaults to None.	None
stuff_texts	str | list[str]	Stuff text prompts of open panoptic task. Defaults to None.	None
custom_entities	bool	Whether to use custom entities. Defaults to False. Only used in GLIP and Grounding DINO.	False
**kwargs		Other keyword arguments passed to :meth:preprocess, :meth:forward, :meth:visualize and :meth:postprocess. Each key in kwargs should be in the corresponding set of preprocess_kwargs, forward_kwargs, visualize_kwargs and postprocess_kwargs.	{}

Returns:

Name	Type	Description
dict	dict	Inference and visualization results.

Source code in visdet/apis/det_inferencer.py

def __call__(
    self,
    inputs: InputsType,
    batch_size: int = 1,
    return_vis: bool = False,
    show: bool = False,
    wait_time: int = 0,
    no_save_vis: bool = False,
    draw_pred: bool = True,
    pred_score_thr: float = 0.3,
    return_datasamples: bool = False,
    print_result: bool = False,
    no_save_pred: bool = True,
    out_dir: str = "",
    # by open image task
    texts: str | list | None = None,
    # by open panoptic task
    stuff_texts: str | list | None = None,
    # by GLIP and Grounding DINO
    custom_entities: bool = False,
    # by Grounding DINO
    tokens_positive: int | list | None = None,
    **kwargs,
) -> dict:
    """Call the inferencer.

    Args:
        inputs (InputsType): Inputs for the inferencer.
        batch_size (int): Inference batch size. Defaults to 1.
        show (bool): Whether to display the visualization results in a
            popup window. Defaults to False.
        wait_time (float): The interval of show (s). Defaults to 0.
        no_save_vis (bool): Whether to force not to save prediction
            vis results. Defaults to False.
        draw_pred (bool): Whether to draw predicted bounding boxes.
            Defaults to True.
        pred_score_thr (float): Minimum score of bboxes to draw.
            Defaults to 0.3.
        return_datasamples (bool): Whether to return results as
            :obj:`DetDataSample`. Defaults to False.
        print_result (bool): Whether to print the inference result w/o
            visualization to the console. Defaults to False.
        no_save_pred (bool): Whether to force not to save prediction
            results. Defaults to True.
        out_dir: Dir to save the inference results or
            visualization. If left as empty, no file will be saved.
            Defaults to ''.
        texts (str | list[str]): Text prompts. Defaults to None.
        stuff_texts (str | list[str]): Stuff text prompts of open
            panoptic task. Defaults to None.
        custom_entities (bool): Whether to use custom entities.
            Defaults to False. Only used in GLIP and Grounding DINO.
        **kwargs: Other keyword arguments passed to :meth:`preprocess`,
            :meth:`forward`, :meth:`visualize` and :meth:`postprocess`.
            Each key in kwargs should be in the corresponding set of
            ``preprocess_kwargs``, ``forward_kwargs``, ``visualize_kwargs``
            and ``postprocess_kwargs``.

    Returns:
        dict: Inference and visualization results.
    """
    (
        preprocess_kwargs,
        forward_kwargs,
        visualize_kwargs,
        postprocess_kwargs,
    ) = self._dispatch_kwargs(**kwargs)

    ori_inputs = self._inputs_to_list(inputs)

    if texts is not None and isinstance(texts, str):
        texts = [texts] * len(ori_inputs)
    if stuff_texts is not None and isinstance(stuff_texts, str):
        stuff_texts = [stuff_texts] * len(ori_inputs)

    # Currently only supports bs=1
    tokens_positive = [tokens_positive] * len(ori_inputs)

    if texts is not None:
        assert len(texts) == len(ori_inputs)
        for i in range(len(texts)):
            if isinstance(ori_inputs[i], str):
                ori_inputs[i] = {
                    "text": texts[i],
                    "img_path": ori_inputs[i],
                    "custom_entities": custom_entities,
                    "tokens_positive": tokens_positive[i],
                }
            else:
                ori_inputs[i] = {
                    "text": texts[i],
                    "img": ori_inputs[i],
                    "custom_entities": custom_entities,
                    "tokens_positive": tokens_positive[i],
                }
    if stuff_texts is not None:
        assert len(stuff_texts) == len(ori_inputs)
        for i in range(len(stuff_texts)):
            ori_inputs[i]["stuff_text"] = stuff_texts[i]

    inputs = self.preprocess(ori_inputs, batch_size=batch_size, **preprocess_kwargs)

    # Calculate total number of batches for progress bar
    total_images = len(ori_inputs)
    total_batches = (total_images + batch_size - 1) // batch_size

    results_dict = {"predictions": [], "visualization": []}

    if self.show_progress:
        with Progress(
            SpinnerColumn(),
            TextColumn("[progress.description]{task.description}"),
            BarColumn(),
            TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
            TextColumn("•"),
            TextColumn("{task.completed}/{task.total} batches"),
            TextColumn("•"),
            TimeElapsedColumn(),
            TextColumn("•"),
            TimeRemainingColumn(),
        ) as progress:
            task = progress.add_task(
                f"[cyan]Processing {total_images} images...",
                total=total_batches
            )
            for ori_imgs, data in inputs:
                preds = self.forward(data, **forward_kwargs)
                visualization = self.visualize(
                    ori_imgs,
                    preds,
                    return_vis=return_vis,
                    show=show,
                    wait_time=wait_time,
                    draw_pred=draw_pred,
                    pred_score_thr=pred_score_thr,
                    no_save_vis=no_save_vis,
                    img_out_dir=out_dir,
                    **visualize_kwargs,
                )
                results = self.postprocess(
                    preds,
                    visualization,
                    return_datasamples=return_datasamples,
                    print_result=print_result,
                    no_save_pred=no_save_pred,
                    pred_out_dir=out_dir,
                    **postprocess_kwargs,
                )
                results_dict["predictions"].extend(results["predictions"])
                if results["visualization"] is not None:
                    results_dict["visualization"].extend(results["visualization"])
                progress.update(task, advance=1)
    else:
        for ori_imgs, data in inputs:
            preds = self.forward(data, **forward_kwargs)
            visualization = self.visualize(
                ori_imgs,
                preds,
                return_vis=return_vis,
                show=show,
                wait_time=wait_time,
                draw_pred=draw_pred,
                pred_score_thr=pred_score_thr,
                no_save_vis=no_save_vis,
                img_out_dir=out_dir,
                **visualize_kwargs,
            )
            results = self.postprocess(
                preds,
                visualization,
                return_datasamples=return_datasamples,
                print_result=print_result,
                no_save_pred=no_save_pred,
                pred_out_dir=out_dir,
                **postprocess_kwargs,
            )
            results_dict["predictions"].extend(results["predictions"])
            if results["visualization"] is not None:
                results_dict["visualization"].extend(results["visualization"])
    return results_dict

visualize(inputs, preds, return_vis=False, show=False, wait_time=0, draw_pred=True, pred_score_thr=0.3, no_save_vis=False, img_out_dir='', **kwargs)

Visualize predictions.

Parameters:

Name	Type	Description	Default
inputs	List[Union[str, ndarray]]	Inputs for the inferencer.	required
preds (List[		obj:DetDataSample]): Predictions of the model.	required
return_vis	bool	Whether to return the visualization result. Defaults to False.	False
show	bool	Whether to display the image in a popup window. Defaults to False.	False
wait_time	float	The interval of show (s). Defaults to 0.	0
draw_pred	bool	Whether to draw predicted bounding boxes. Defaults to True.	True
pred_score_thr	float	Minimum score of bboxes to draw. Defaults to 0.3.	0.3
no_save_vis	bool	Whether to force not to save prediction vis results. Defaults to False.	False
img_out_dir	str	Output directory of visualization results. If left as empty, no file will be saved. Defaults to ''.	''

Returns:

Type	Description
list[ndarray] | None	List[np.ndarray] or None: Returns visualization results only if
list[ndarray] | None	applicable.

Source code in visdet/apis/det_inferencer.py

def visualize(
    self,
    inputs: InputsType,
    preds: PredType,
    return_vis: bool = False,
    show: bool = False,
    wait_time: int = 0,
    draw_pred: bool = True,
    pred_score_thr: float = 0.3,
    no_save_vis: bool = False,
    img_out_dir: str = "",
    **kwargs,
) -> list[np.ndarray] | None:
    """Visualize predictions.

    Args:
        inputs (List[Union[str, np.ndarray]]): Inputs for the inferencer.
        preds (List[:obj:`DetDataSample`]): Predictions of the model.
        return_vis (bool): Whether to return the visualization result.
            Defaults to False.
        show (bool): Whether to display the image in a popup window.
            Defaults to False.
        wait_time (float): The interval of show (s). Defaults to 0.
        draw_pred (bool): Whether to draw predicted bounding boxes.
            Defaults to True.
        pred_score_thr (float): Minimum score of bboxes to draw.
            Defaults to 0.3.
        no_save_vis (bool): Whether to force not to save prediction
            vis results. Defaults to False.
        img_out_dir (str): Output directory of visualization results.
            If left as empty, no file will be saved. Defaults to ''.

    Returns:
        List[np.ndarray] or None: Returns visualization results only if
        applicable.
    """
    if no_save_vis is True:
        img_out_dir = ""

    if not show and img_out_dir == "" and not return_vis:
        return None

    if self.visualizer is None:
        raise ValueError(
            'Visualization needs the "visualizer" term' "defined in the config, but got None."
        )

    results = []

    for single_input, pred in zip(inputs, preds, strict=False):
        if isinstance(single_input, str):
            img_bytes = get(single_input)
            img = imfrombytes(img_bytes)
            img = img[:, :, ::-1]
            img_name = osp.basename(single_input)
        elif isinstance(single_input, np.ndarray):
            img = single_input.copy()
            img_num = str(self.num_visualized_imgs).zfill(8)
            img_name = f"{img_num}.jpg"
        else:
            raise ValueError("Unsupported input type: " f"{type(single_input)}")

        out_file = osp.join(img_out_dir, "vis", img_name) if img_out_dir != "" else None

        self.visualizer.add_datasample(
            img_name,
            img,
            pred,
            show=show,
            wait_time=wait_time,
            draw_gt=False,
            draw_pred=draw_pred,
            pred_score_thr=pred_score_thr,
            out_file=out_file,
        )
        results.append(self.visualizer.get_image())
        self.num_visualized_imgs += 1

    return results

postprocess(preds, visualization=None, return_datasamples=False, print_result=False, no_save_pred=False, pred_out_dir='', **kwargs)

Process the predictions and visualization results from forward and visualize.

This method should be responsible for the following tasks:

1. Convert datasamples into a json-serializable dict if needed.
2. Pack the predictions and visualization results and return them.
3. Dump or log the predictions.

Parameters:

Name	Type	Description	Default
preds (List[		obj:DetDataSample]): Predictions of the model.	required
visualization	Optional[ndarray]	Visualized predictions.	None
return_datasamples	bool	Whether to use Datasample to store inference results. If False, dict will be used.	False
print_result	bool	Whether to print the inference result w/o visualization to the console. Defaults to False.	False
no_save_pred	bool	Whether to force not to save prediction results. Defaults to False.	False
pred_out_dir	str	Dir to save the inference results w/o visualization. If left as empty, no file will be saved. Defaults to ''.	''

Returns:

Name	Type	Description
dict	dict	Inference and visualization results with key predictions
	dict	and visualization.
	dict	visualization (Any): Returned by :meth:visualize.
	dict	predictions (dict or DataSample): Returned by :meth:forward and processed in :meth:postprocess. If return_datasamples=False, it usually should be a json-serializable dict containing only basic data elements such as strings and numbers.

Source code in visdet/apis/det_inferencer.py

def postprocess(
    self,
    preds: PredType,
    visualization: list[np.ndarray] | None = None,
    return_datasamples: bool = False,
    print_result: bool = False,
    no_save_pred: bool = False,
    pred_out_dir: str = "",
    **kwargs,
) -> dict:
    """Process the predictions and visualization results from ``forward``
    and ``visualize``.

    This method should be responsible for the following tasks:

    1. Convert datasamples into a json-serializable dict if needed.
    2. Pack the predictions and visualization results and return them.
    3. Dump or log the predictions.

    Args:
        preds (List[:obj:`DetDataSample`]): Predictions of the model.
        visualization (Optional[np.ndarray]): Visualized predictions.
        return_datasamples (bool): Whether to use Datasample to store
            inference results. If False, dict will be used.
        print_result (bool): Whether to print the inference result w/o
            visualization to the console. Defaults to False.
        no_save_pred (bool): Whether to force not to save prediction
            results. Defaults to False.
        pred_out_dir: Dir to save the inference results w/o
            visualization. If left as empty, no file will be saved.
            Defaults to ''.

    Returns:
        dict: Inference and visualization results with key ``predictions``
        and ``visualization``.

        - ``visualization`` (Any): Returned by :meth:`visualize`.
        - ``predictions`` (dict or DataSample): Returned by
            :meth:`forward` and processed in :meth:`postprocess`.
            If ``return_datasamples=False``, it usually should be a
            json-serializable dict containing only basic data elements such
            as strings and numbers.
    """
    if no_save_pred is True:
        pred_out_dir = ""

    result_dict = {}
    results = preds
    if not return_datasamples:
        results = []
        for pred in preds:
            result = self.pred2dict(pred, pred_out_dir)
            results.append(result)
    elif pred_out_dir != "":
        warnings.warn(
            "Currently does not support saving datasample "
            "when return_datasamples is set to True. "
            "Prediction results are not saved!", stacklevel=2
        )
    # Add img to the results after printing and dumping
    result_dict["predictions"] = results
    if print_result:
        print(result_dict)
    result_dict["visualization"] = visualization
    return result_dict

pred2dict(data_sample, pred_out_dir='')

Extract elements necessary to represent a prediction into a dictionary.

It's better to contain only basic data elements such as strings and numbers in order to guarantee it's json-serializable.

Parameters:

Name	Type	Description	Default
data_sample (		obj:DetDataSample): Predictions of the model.	required
pred_out_dir	str	Dir to save the inference results w/o visualization. If left as empty, no file will be saved. Defaults to ''.	''

Returns:

Name	Type	Description
dict	dict	Prediction results.

Source code in visdet/apis/det_inferencer.py

def pred2dict(self, data_sample: DetDataSample, pred_out_dir: str = "") -> dict:
    """Extract elements necessary to represent a prediction into a
    dictionary.

    It's better to contain only basic data elements such as strings and
    numbers in order to guarantee it's json-serializable.

    Args:
        data_sample (:obj:`DetDataSample`): Predictions of the model.
        pred_out_dir: Dir to save the inference results w/o
            visualization. If left as empty, no file will be saved.
            Defaults to ''.

    Returns:
        dict: Prediction results.
    """
    is_save_pred = True
    if pred_out_dir == "":
        is_save_pred = False

    if is_save_pred and "img_path" in data_sample:
        img_path = osp.basename(data_sample.img_path)
        img_path = osp.splitext(img_path)[0]
        out_img_path = osp.join(pred_out_dir, "preds", img_path + "_panoptic_seg.png")
        out_json_path = osp.join(pred_out_dir, "preds", img_path + ".json")
    elif is_save_pred:
        out_img_path = osp.join(
            pred_out_dir, "preds", f"{self.num_predicted_imgs}_panoptic_seg.png"
        )
        out_json_path = osp.join(pred_out_dir, "preds", f"{self.num_predicted_imgs}.json")
        self.num_predicted_imgs += 1

    result = {}
    if "pred_instances" in data_sample:
        masks = data_sample.pred_instances.get("masks")
        pred_instances = data_sample.pred_instances.numpy()
        result = {
            "labels": pred_instances.labels.tolist(),
            "scores": pred_instances.scores.tolist(),
        }
        if "bboxes" in pred_instances:
            result["bboxes"] = pred_instances.bboxes.tolist()
        if masks is not None:
            if "bboxes" not in pred_instances or pred_instances.bboxes.sum() == 0:
                # Fake bbox, such as the SOLO.
                bboxes = mask2bbox(masks.cpu()).numpy().tolist()
                result["bboxes"] = bboxes
            encode_masks = encode_mask_results(pred_instances.masks)
            for encode_mask in encode_masks:
                if isinstance(encode_mask["counts"], bytes):
                    encode_mask["counts"] = encode_mask["counts"].decode()
            result["masks"] = encode_masks

    if "pred_panoptic_seg" in data_sample:
        if VOID is None:
            raise RuntimeError(
                "panopticapi is not installed, please install it by: "
                "pip install git+https://github.com/cocodataset/"
                "panopticapi.git."
            )

        pan = data_sample.pred_panoptic_seg.sem_seg.cpu().numpy()[0]
        pan[pan % INSTANCE_OFFSET == len(self.model.dataset_meta["classes"])] = VOID
        pan = id2rgb(pan).astype(np.uint8)

        if is_save_pred:
            imwrite(pan[:, :, ::-1], out_img_path)
            result["panoptic_seg_path"] = out_img_path
        else:
            result["panoptic_seg"] = pan

    if is_save_pred:
        dump(result, out_json_path)

    return result

See Also

• Models API
• Datasets API