From d3afbc056371222e3492d84bacab60aa42c3bd1f Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:38:33 +0200 Subject: [PATCH 1/4] Add block for running pt models from local OS --- inference/core/workflows/core_steps/loader.py | 4 + .../models/foundation/ultralytics/__init__.py | 0 .../models/foundation/ultralytics/v1.py | 168 ++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 inference/core/workflows/core_steps/models/foundation/ultralytics/__init__.py create mode 100644 inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py diff --git a/inference/core/workflows/core_steps/loader.py b/inference/core/workflows/core_steps/loader.py index d96b79f18..370e3c405 100644 --- a/inference/core/workflows/core_steps/loader.py +++ b/inference/core/workflows/core_steps/loader.py @@ -144,6 +144,9 @@ from inference.core.workflows.core_steps.models.foundation.stability_ai.inpainting.v1 import ( StabilityAIInpaintingBlockV1, ) +from inference.core.workflows.core_steps.models.foundation.ultralytics.v1 import ( + UltralyticsBlockV1, +) from inference.core.workflows.core_steps.models.foundation.yolo_world.v1 import ( YoloWorldModelBlockV1, ) @@ -426,6 +429,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: TimeInZoneBlockV1, TimeInZoneBlockV2, TriangleVisualizationBlockV1, + UltralyticsBlockV1, VLMAsClassifierBlockV1, VLMAsDetectorBlockV1, YoloWorldModelBlockV1, diff --git a/inference/core/workflows/core_steps/models/foundation/ultralytics/__init__.py b/inference/core/workflows/core_steps/models/foundation/ultralytics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py new file mode 100644 index 000000000..1147ea7e4 --- /dev/null +++ b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py @@ -0,0 +1,168 @@ +import time +from typing import List, Literal, Optional, Type, Union + +import numpy as np +import supervision as sv +from pydantic import ConfigDict, Field, PositiveInt +from ultralytics import YOLO + +from inference.core.logger import logger +from inference.core.workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from inference.core.workflows.execution_engine.entities.types import ( + BOOLEAN_KIND, + FLOAT_ZERO_TO_ONE_KIND, + INTEGER_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + STRING_KIND, + FloatZeroToOne, + ImageInputField, + StepOutputImageSelector, + WorkflowImageSelector, + WorkflowParameterSelector, +) +from inference.core.workflows.prototypes.block import ( + BlockResult, + WorkflowBlock, + WorkflowBlockManifest, +) + +LONG_DESCRIPTION = """ +This block performs inference by executing locally stored ultralytics pth file. +This block expects pth file to be available within local filesystem. +""" + + +class BlockManifest(WorkflowBlockManifest): + model_config = ConfigDict( + json_schema_extra={ + "name": "Ultralytics", + "version": "v1", + "short_description": "Predict the location of objects with bounding boxes by inferring from locally stored pth file.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/ultralytics@v1",] + images: Union[WorkflowImageSelector, StepOutputImageSelector] = ImageInputField + model_path: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + description="Path to locally stored pth file", + examples=["/path/to/model.pth", "$inputs.class_agnostic_nms"], + ) + device: Union[WorkflowParameterSelector(kind=[STRING_KIND]), str] = Field( + default="cpu", + description="Specifies the device for inference (e.g., cpu, cuda:0, mps or 0)", + examples=["cuda:0", "$inputs.device"], + ) + class_agnostic_nms: Union[ + Optional[bool], WorkflowParameterSelector(kind=[BOOLEAN_KIND]) + ] = Field( + default=False, + description="Value to decide if NMS is to be used in class-agnostic mode.", + examples=[True, "$inputs.class_agnostic_nms"], + ) + confidence: Union[ + FloatZeroToOne, + WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.4, + description="Confidence threshold for predictions", + examples=[0.3, "$inputs.confidence_threshold"], + ) + iou_threshold: Union[ + FloatZeroToOne, + WorkflowParameterSelector(kind=[FLOAT_ZERO_TO_ONE_KIND]), + ] = Field( + default=0.3, + description="Parameter of NMS, to decide on minimum box intersection over union to merge boxes", + examples=[0.4, "$inputs.iou_threshold"], + ) + max_detections: Union[ + PositiveInt, WorkflowParameterSelector(kind=[INTEGER_KIND]) + ] = Field( + default=300, + description="Maximum number of detections to return", + examples=[300, "$inputs.max_detections"], + ) + half_precision: Union[ + Optional[bool], WorkflowParameterSelector(kind=[BOOLEAN_KIND]) + ] = Field( + default=False, + description="Enables half-precision (FP16) inference, which can speed up model inference on supported GPUs with minimal impact on accuracy.", + examples=[True, "$inputs.half_precision"], + ) + imgsz: Union[ + int, + WorkflowParameterSelector(kind=[INTEGER_KIND]), + ] = Field( + default=640, + description="Defines the image size for inference.", + examples=[1280, "$inputs.imgsz"], + ) + + @classmethod + def accepts_batch_input(cls) -> bool: + return True + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition(name="inference_id", kind=[STRING_KIND]), + OutputDefinition( + name="predictions", kind=[OBJECT_DETECTION_PREDICTION_KIND] + ), + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.0.0,<2.0.0" + + +class UltralyticsBlockV1(WorkflowBlock): + def __init__(self): + self._model: Optional[YOLO] = None + + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_path: str, + device: str, + class_agnostic_nms: Optional[bool], + confidence: Optional[float], + iou_threshold: Optional[float], + max_detections: Optional[int], + half_precision: bool, + imgsz: int, + ) -> BlockResult: + if not self._model: + self._model = YOLO(model_path) + + predictions = [] + for image in images: + inf = self._model( + image.numpy_image, + imgsz=imgsz, + conf=confidence, + iou=iou_threshold, + half=half_precision, + max_det=max_detections, + agnostic_nms=class_agnostic_nms, + device=device, + verbose=False, + )[0] + detections = sv.Detections.from_ultralytics(inf) + predictions.append(detections) + + return [ + {"inference_id": None, "predictions": prediction} + for prediction in predictions + ] From 3af5080c001767b676ef8f90ed2f7e486a331088 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Thu, 24 Oct 2024 22:48:40 +0200 Subject: [PATCH 2/4] Add ultralytics to extras --- requirements/requirements.ultralytics.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 requirements/requirements.ultralytics.txt diff --git a/requirements/requirements.ultralytics.txt b/requirements/requirements.ultralytics.txt new file mode 100644 index 000000000..6c8857361 --- /dev/null +++ b/requirements/requirements.ultralytics.txt @@ -0,0 +1 @@ +ultralytics>=8.3.0 diff --git a/setup.py b/setup.py index abd150c32..8a7cad2ae 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def read_requirements(path): ), extras_require={ "sam": read_requirements("requirements/requirements.sam.txt"), + "ultralytics": read_requirements("requirements/requirements.ultralytics.txt"), }, classifiers=[ "Programming Language :: Python :: 3", From 74a19af0788448d44baf4214273b6bd6e67e4016 Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Thu, 24 Oct 2024 23:05:26 +0200 Subject: [PATCH 3/4] Conditionally import ultralytics --- .../core_steps/models/foundation/ultralytics/v1.py | 10 +++++++++- requirements/requirements.ultralytics.txt | 1 - 2 files changed, 9 insertions(+), 2 deletions(-) delete mode 100644 requirements/requirements.ultralytics.txt diff --git a/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py index 1147ea7e4..9dc833ffd 100644 --- a/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py @@ -4,7 +4,11 @@ import numpy as np import supervision as sv from pydantic import ConfigDict, Field, PositiveInt -from ultralytics import YOLO + +try: + from ultralytics import YOLO +except ImportError: + pass from inference.core.logger import logger from inference.core.workflows.execution_engine.entities.base import ( @@ -143,6 +147,10 @@ def run( half_precision: bool, imgsz: int, ) -> BlockResult: + if "YOLO" not in locals(): + raise RuntimeError( + "You must install ultralytics in order to use this block." + ) if not self._model: self._model = YOLO(model_path) diff --git a/requirements/requirements.ultralytics.txt b/requirements/requirements.ultralytics.txt deleted file mode 100644 index 6c8857361..000000000 --- a/requirements/requirements.ultralytics.txt +++ /dev/null @@ -1 +0,0 @@ -ultralytics>=8.3.0 From db94afd5bc46653810386cd7a7f90c2f9c6217ba Mon Sep 17 00:00:00 2001 From: Grzegorz Klimaszewski <166530809+grzegorz-roboflow@users.noreply.github.com> Date: Thu, 24 Oct 2024 23:15:33 +0200 Subject: [PATCH 4/4] check for YOLO --- .../workflows/core_steps/models/foundation/ultralytics/v1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py index 9dc833ffd..8adee1083 100644 --- a/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py +++ b/inference/core/workflows/core_steps/models/foundation/ultralytics/v1.py @@ -147,7 +147,7 @@ def run( half_precision: bool, imgsz: int, ) -> BlockResult: - if "YOLO" not in locals(): + if "YOLO" not in globals(): raise RuntimeError( "You must install ultralytics in order to use this block." )