From 41bb348cf7b33215f1592c4af625328c48edbe83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Mal=C3=A9zieux?= Date: Mon, 12 Aug 2024 14:28:32 +0200 Subject: [PATCH 1/4] ENH: adding noise perturbation detector with Gaussian noise --- giskard_vision/core/dataloaders/wrappers.py | 74 +++++++++++++++++++ .../transformation_noise_detector.py | 23 ++++++ 2 files changed, 97 insertions(+) create mode 100644 giskard_vision/core/detectors/transformation_noise_detector.py diff --git a/giskard_vision/core/dataloaders/wrappers.py b/giskard_vision/core/dataloaders/wrappers.py index 0682dbbe..ec4b06cf 100644 --- a/giskard_vision/core/dataloaders/wrappers.py +++ b/giskard_vision/core/dataloaders/wrappers.py @@ -228,6 +228,80 @@ def get_image(self, idx: int) -> np.ndarray: return cv2.GaussianBlur(image, self._kernel_size, *self._sigma) +class NoisyDataLoader(DataLoaderWrapper): + """Wrapper class for a DataIteratorBase, providing noisy images. + + Args: + dataloader (DataIteratorBase): The data loader to be wrapped. + kernel_size (Union[Tuple[int, int], int]): Size of the Gaussian kernel for blurring. Can be a tuple or a single value. + sigma (Union[Tuple[float, float], float]): Standard deviation of the Gaussian kernel for blurring. + Can be a tuple or a single value. + + Returns: + NoisyDataLoader: Noisy data loader instance. + """ + + def __init__( + self, + dataloader: DataIteratorBase, + sigma: float = 0.1, + ) -> None: + """ + Initializes the BlurredDataLoader. + + Args: + dataloader (DataIteratorBase): The data loader to be wrapped. + sigma (float): Standard deviation of the Gaussian noise. + """ + super().__init__(dataloader) + self._sigma = sigma + + @property + def name(self): + """ + Gets the name of the blurred data loader. + + Returns: + str: The name of the blurred data loader. + """ + return "noisy" + + def get_image(self, idx: int) -> np.ndarray: + """ + Gets a blurred image using Gaussian blur. + + Args: + idx (int): Index of the data. + + Returns: + np.ndarray: Blurred image data. + """ + image = super().get_image(idx) + return self.add_gaussian_noise(image, self._sigma * 255) + + def add_gaussian_noise(self, image, std_dev): + """ + Add Gaussian noise to the image + + Args: + image (np.ndarray): Image + std_dev (float): Standard deviation of the Gaussian noise. + + Returns: + np.ndarray: Noisy image + """ + # Generate Gaussian noise + noise = np.random.normal(0, std_dev, image.shape).astype(np.float32) + + # Add the noise to the image + noisy_image = cv2.add(image.astype(np.float32), noise) + + # Clip the values to stay within valid range (0-255 for uint8) + noisy_image = np.clip(noisy_image, 0, 255).astype(np.uint8) + + return noisy_image + + class ColoredDataLoader(DataLoaderWrapper): """Wrapper class for a DataIteratorBase, providing color-altered images using OpenCV color conversion. diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py new file mode 100644 index 00000000..e5c03652 --- /dev/null +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -0,0 +1,23 @@ +from giskard_vision.core.dataloaders.wrappers import NoisyDataLoader + +from ...core.detectors.decorator import maybe_detector +from .perturbation import PerturbationBaseDetector, Robustness + + +@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): + """ + Detector that evaluates models performance on noisy images + """ + + issue_group = Robustness + + def __init__(self, sigma=0.1): + self.sigma = sigma + + def get_dataloaders(self, dataset): + dl = NoisyDataLoader(dataset, self.sigma) + + dls = [dl] + + return dls From afa969a163dab0a24977d3056b20df6183f10798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Mal=C3=A9zieux?= Date: Mon, 12 Aug 2024 14:30:36 +0200 Subject: [PATCH 2/4] FIX: docstring --- giskard_vision/core/dataloaders/wrappers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/giskard_vision/core/dataloaders/wrappers.py b/giskard_vision/core/dataloaders/wrappers.py index ec4b06cf..ff39ed9f 100644 --- a/giskard_vision/core/dataloaders/wrappers.py +++ b/giskard_vision/core/dataloaders/wrappers.py @@ -233,9 +233,7 @@ class NoisyDataLoader(DataLoaderWrapper): Args: dataloader (DataIteratorBase): The data loader to be wrapped. - kernel_size (Union[Tuple[int, int], int]): Size of the Gaussian kernel for blurring. Can be a tuple or a single value. - sigma (Union[Tuple[float, float], float]): Standard deviation of the Gaussian kernel for blurring. - Can be a tuple or a single value. + sigma (float): Standard deviation of the Gaussian noise. Returns: NoisyDataLoader: Noisy data loader instance. From 3fa82427c93c870ad5a60ce850e602cee4ab2ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Mal=C3=A9zieux?= Date: Mon, 12 Aug 2024 15:05:53 +0200 Subject: [PATCH 3/4] FIX: the IoU metric now handles prediction batches --- giskard_vision/core/tests/base.py | 2 +- .../object_detection/tests/performance.py | 57 ++++++++++++------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/giskard_vision/core/tests/base.py b/giskard_vision/core/tests/base.py index e5d5c1b8..25008094 100644 --- a/giskard_vision/core/tests/base.py +++ b/giskard_vision/core/tests/base.py @@ -314,7 +314,7 @@ def run( prediction_time=prediction_time, prediction_fail_rate=prediction_fail_rate, metric_name=self.metric.name, - model_name=model.name, + model_name=model.name if hasattr(model, "name") else model.__class__.__name__, dataloader_name=dataloader.name, dataloader_ref_name=dataloader_ref.name, indexes_examples=indexes, diff --git a/giskard_vision/object_detection/tests/performance.py b/giskard_vision/object_detection/tests/performance.py index 933f6c67..97e5ed20 100644 --- a/giskard_vision/object_detection/tests/performance.py +++ b/giskard_vision/object_detection/tests/performance.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +import numpy as np + from ..types import Types from .base import Metric @@ -17,32 +19,43 @@ def definition(prediction_result: Types.prediction_result, ground_truth: Types.l # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"): # return 0 - gt_box = prediction_result.prediction.item().get("boxes") - pred_box = ground_truth.item().get("boxes") + ious = [] + + for i in range(len(prediction_result.prediction)): + + if isinstance(prediction_result.prediction[i], dict): + gt_box = prediction_result.prediction[i].get("boxes") + else: + ious.append(0) + continue + + pred_box = ground_truth[i].get("boxes") + + x1_min, y1_min, x1_max, y1_max = gt_box + x2_min, y2_min, x2_max, y2_max = pred_box - x1_min, y1_min, x1_max, y1_max = gt_box - x2_min, y2_min, x2_max, y2_max = pred_box + # Calculate the coordinates of the intersection rectangle + x_inter_min = max(x1_min, x2_min) + y_inter_min = max(y1_min, y2_min) + x_inter_max = min(x1_max, x2_max) + y_inter_max = min(y1_max, y2_max) - # Calculate the coordinates of the intersection rectangle - x_inter_min = max(x1_min, x2_min) - y_inter_min = max(y1_min, y2_min) - x_inter_max = min(x1_max, x2_max) - y_inter_max = min(y1_max, y2_max) + # Compute the area of the intersection rectangle + if x_inter_max < x_inter_min or y_inter_max < y_inter_min: + inter_area = 0 + else: + inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min) - # Compute the area of the intersection rectangle - if x_inter_max < x_inter_min or y_inter_max < y_inter_min: - inter_area = 0 - else: - inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min) + # Compute the area of both the prediction and ground-truth rectangles + box1_area = (x1_max - x1_min) * (y1_max - y1_min) + box2_area = (x2_max - x2_min) * (y2_max - y2_min) - # Compute the area of both the prediction and ground-truth rectangles - box1_area = (x1_max - x1_min) * (y1_max - y1_min) - box2_area = (x2_max - x2_min) * (y2_max - y2_min) + # Compute the union area + union_area = box1_area + box2_area - inter_area - # Compute the union area - union_area = box1_area + box2_area - inter_area + # Compute the IoU + iou = inter_area / union_area - # Compute the IoU - iou = inter_area / union_area + ious.append(iou) - return iou + return np.mean(ious) From 53bdcd31c62cc47f10b4dd0a72e8c8b3d6382925 Mon Sep 17 00:00:00 2001 From: Rabah Khalek Date: Mon, 12 Aug 2024 15:18:01 +0200 Subject: [PATCH 4/4] renaming metric IoU to IoUMean --- giskard_vision/core/detectors/metrics.py | 4 ++-- .../object_detection/detectors/metadata_detector.py | 4 ++-- giskard_vision/object_detection/tests/performance.py | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/giskard_vision/core/detectors/metrics.py b/giskard_vision/core/detectors/metrics.py index 206e34d0..744ee1f7 100644 --- a/giskard_vision/core/detectors/metrics.py +++ b/giskard_vision/core/detectors/metrics.py @@ -1,9 +1,9 @@ from giskard_vision.image_classification.tests.performance import Accuracy from giskard_vision.landmark_detection.tests.performance import NMEMean -from giskard_vision.object_detection.tests.performance import IoU +from giskard_vision.object_detection.tests.performance import IoUMean detector_metrics = { "image_classification": Accuracy, "landmark": NMEMean, - "object_detection": IoU, + "object_detection": IoUMean, } diff --git a/giskard_vision/object_detection/detectors/metadata_detector.py b/giskard_vision/object_detection/detectors/metadata_detector.py index 1d4f02eb..74844b98 100644 --- a/giskard_vision/object_detection/detectors/metadata_detector.py +++ b/giskard_vision/object_detection/detectors/metadata_detector.py @@ -15,7 +15,7 @@ SurrogateRelativeTopLeftY, SurrogateStdIntensity, ) -from giskard_vision.object_detection.tests.performance import IoU +from giskard_vision.object_detection.tests.performance import IoUMean from ...core.detectors.decorator import maybe_detector @@ -38,7 +38,7 @@ class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): SurrogateRelativeTopLeftY, SurrogateNormalizedPerimeter, ] - metric = IoU + metric = IoUMean type_task = "regression" metric_type = "absolute" metric_direction = "better_higher" diff --git a/giskard_vision/object_detection/tests/performance.py b/giskard_vision/object_detection/tests/performance.py index 97e5ed20..35c63f60 100644 --- a/giskard_vision/object_detection/tests/performance.py +++ b/giskard_vision/object_detection/tests/performance.py @@ -7,22 +7,20 @@ @dataclass -class IoU(Metric): +class IoUMean(Metric): """Intersection over Union distance between a prediction and a ground truth""" - name = "IoU" + name = "IoUMean" description = "Intersection over Union" @staticmethod def definition(prediction_result: Types.prediction_result, ground_truth: Types.label): - # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"): # return 0 ious = [] for i in range(len(prediction_result.prediction)): - if isinstance(prediction_result.prediction[i], dict): gt_box = prediction_result.prediction[i].get("boxes") else: