[SAPBTPCFS-7876] Optimize handling of non-transient errors

SAP · Jan 25, 2024 · 4ff906a · 4ff906a
1 parent 01b297d
commit 4ff906a
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 5 deletions.
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -38,7 +38,7 @@ func Get() Config {
 			AllowClusterAccess:        true,
 			RetryBaseDelay:            10 * time.Second,
 			RetryMaxDelay:             time.Hour,
-			IgnoreNonTransientTimeout: 2 * time.Hour,
+			IgnoreNonTransientTimeout: 24 * time.Hour,
 		}
 		envconfig.MustProcess("", &config)
 	})

diff --git a/internal/utils/controller_util.go b/internal/utils/controller_util.go
@@ -71,7 +71,7 @@ func UpdateStatus(ctx context.Context, k8sClient client.Client, object common.SA
 }
 
 func ShouldIgnoreNonTransient(log logr.Logger, serviceInstance *servicesv1.ServiceInstance, timeout time.Duration) bool {
-	firstTrailTimestamp := getFirstErrorTimestamp(serviceInstance)
+	firstTrailTimestamp := getFirstErrorTimestamp(log, serviceInstance)
 	sinceFirstTrail := time.Since(firstTrailTimestamp)
 	if sinceFirstTrail > timeout {
 		log.Info(fmt.Sprintf("timeout of %s reached - error is considered to be non transient. time passed since first trail %s", timeout, sinceFirstTrail))
@@ -81,10 +81,11 @@ func ShouldIgnoreNonTransient(log logr.Logger, serviceInstance *servicesv1.Servi
 	return true
 }
 
-func getFirstErrorTimestamp(serviceInstance *servicesv1.ServiceInstance) time.Time {
-	if serviceInstance.Status.FirstErrorTimestamp.IsZero() {
-		now := metav1.NewTime(time.Now())
+func getFirstErrorTimestamp(log logr.Logger, serviceInstance *servicesv1.ServiceInstance) time.Time {
+	if serviceInstance.Status.FirstErrorTimestamp == nil || serviceInstance.Status.FirstErrorTimestamp.IsZero() {
+		now := metav1.Now()
 		serviceInstance.Status.FirstErrorTimestamp = &now
+		log.Info(fmt.Sprintf("setting new FirstErrorTimestamp: %s , for service instance %s", now, serviceInstance.Name))
 	}
 	return serviceInstance.Status.FirstErrorTimestamp.Time
 }

diff --git a/sapbtp-operator-charts/templates/crd.yml b/sapbtp-operator-charts/templates/crd.yml
@@ -561,6 +561,9 @@ spec:
     - jsonPath: .status.ready
       name: Ready
       type: string
+    - jsonPath: .status.firstErrorTimestamp
+      name: FirstErrorTimestamp
+      type: date
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -763,6 +766,11 @@ spec:
                   - type
                   type: object
                 type: array
+              firstErrorTimestamp:
+                description: The first error timestamp - used to stop retrying after
+                  a while
+                format: date-time
+                type: string
               hashedSpec:
                 description: HashedSpec is the hashed spec without the shared property
                 type: string
@@ -1010,6 +1018,11 @@ spec:
                   - type
                   type: object
                 type: array
+              firstErrorTimestamp:
+                description: The first error timestamp - used to stop retrying after
+                  a while
+                format: date-time
+                type: string
               instanceID:
                 description: The generated ID of the instance, will be automatically
                   filled once the instance is created