scale up based on PreferredMaxReplicas only once 30mins (#348)

* scale up based on PreferredmaxReplicas only once 30mins * modify tests * add the test case
mercari · Feb 23, 2024 · c0d7f0e · c0d7f0e
1 parent 5b748c2
commit c0d7f0e
Show file tree

Hide file tree

Showing 28 changed files with 435 additions and 65 deletions.
diff --git a/api/v1beta3/tortoise_types.go b/api/v1beta3/tortoise_types.go
@@ -344,8 +344,9 @@ type TortoiseConditionType string
 
 const (
 	// TortoiseConditionTypeFailedToReconcile means tortoise failed to reconcile due to some reasons.
-	TortoiseConditionTypeFailedToReconcile           TortoiseConditionType = "FailedToReconcile"
-	TortoiseConditionTypeHPATargetUtilizationUpdated TortoiseConditionType = "HPATargetUtilizationUpdated"
+	TortoiseConditionTypeFailedToReconcile                   TortoiseConditionType = "FailedToReconcile"
+	TortoiseConditionTypeHPATargetUtilizationUpdated         TortoiseConditionType = "HPATargetUtilizationUpdated"
+	TortoiseConditionTypeScaledUpBasedOnPreferredMaxReplicas TortoiseConditionType = "ScaledUpBasedOnPreferredMaxReplicas"
 )
 
 type TortoiseCondition struct {

diff --git a/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: "4"
         memory: 4Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: "3"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/tortoise.yaml
@@ -70,7 +70,11 @@ status:
       resource:
         cpu: "3"
         memory: 3Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/after/tortoise.yaml b/controllers/testdata/mutable-autoscalingpolicy-remove-horizontal/after/tortoise.yaml
@@ -69,7 +69,11 @@ status:
       resource:
         cpu: "3"
         memory: 3Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-istio-enabled-pod-working/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-all-off/after/tortoise.yaml
@@ -60,7 +60,11 @@ status:
       resource:
         cpu: "4"
         memory: 4Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...llers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/after/tortoise.yaml b/...llers/testdata/reconcile-for-the-multiple-containers-pod-all-vertical/after/tortoise.yaml
@@ -60,7 +60,11 @@ status:
       resource:
         cpu: "3"
         memory: 3Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...s/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/tortoise.yaml b/...s/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/tortoise.yaml
@@ -68,6 +68,10 @@ status:
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/.../testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/tortoise.yaml b/.../testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/tortoise.yaml
@@ -68,6 +68,10 @@ status:
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: "4"
         memory: 4Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...estdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml b/...estdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: 100m
         memory: 11Mi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-multiple-containers-pod-working/after/tortoise.yaml
@@ -61,12 +61,23 @@ status:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-backtonormal/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-backtonormal/after/tortoise.yaml
@@ -45,6 +45,10 @@ status:
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-dryrun/after/tortoise.yaml
@@ -38,7 +38,18 @@ status:
       resource:
         cpu: "4"
         memory: 4Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...lers/testdata/reconcile-for-the-single-container-pod-during-emergency/after/tortoise.yaml b/...lers/testdata/reconcile-for-the-single-container-pod-during-emergency/after/tortoise.yaml
@@ -45,6 +45,10 @@ status:
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...ers/testdata/reconcile-for-the-single-container-pod-emergency-started/after/tortoise.yaml b/...ers/testdata/reconcile-for-the-single-container-pod-emergency-started/after/tortoise.yaml
@@ -45,6 +45,10 @@ status:
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...stdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml b/...stdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/tortoise.yaml
@@ -38,12 +38,23 @@ status:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...ollers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml b/...ollers/testdata/reconcile-for-the-single-container-pod-gathering-data/after/tortoise.yaml
@@ -37,7 +37,18 @@ status:
       resource:
         cpu: "4"
         memory: 4Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/tortoise.yaml
@@ -39,12 +39,23 @@ status:
         cpu: "3"
         memory: 4Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml b/controllers/testdata/reconcile-for-the-single-container-pod-initializing/after/tortoise.yaml
@@ -38,7 +38,11 @@ status:
       resource:
         cpu: "4"
         memory: 4Gi
-    tortoiseConditions: null
+    tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases:

diff --git a/...ollers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml b/...ollers/testdata/reconcile-for-the-single-container-pod-partly-working/after/tortoise.yaml
@@ -38,12 +38,23 @@ status:
         cpu: "4"
         memory: 3Gi
     tortoiseConditions:
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      message: the current number of replicas is not bigger than the preferred max
+        replica number
+      reason: ScaledUpBasedOnPreferredMaxReplicas
+      status: "False"
+      type: ScaledUpBasedOnPreferredMaxReplicas
     - lastTransitionTime: "2023-01-01T00:00:00Z"
       lastUpdateTime: "2023-01-01T00:00:00Z"
       message: HPA target utilization is updated
       reason: HPATargetUtilizationUpdated
       status: "True"
       type: HPATargetUtilizationUpdated
+    - lastTransitionTime: "2023-01-01T00:00:00Z"
+      lastUpdateTime: "2023-01-01T00:00:00Z"
+      status: "False"
+      type: FailedToReconcile
   containerResourcePhases:
   - containerName: app
     resourcePhases: