[POA-2928] Cover corner case of pod not coming up at all (#87)

I observed some corner cases in the beta environment, where some
services are directly moving to the final state before going to the
running state.
This could happen if there are some config issues like
`CreateContainerConfigError`.

For example, here we got a Pod Delete event where the pod's status is
still Pending.
```
�[31m[ERROR] �[0mPod status is in unknown state, pod name: cronjob-scheduled-work-29008098-8mgxh, status: Pending
�[31m[ERROR] �[0mFailed to change pod state, pod name: cronjob-scheduled-work-29008098-8mgxh, from: PodPending to: PodTerminated, error: Invalid current state for pod cronjob-scheduled-work-29008098-8mgxh: PodPending
```

For such corner cases, I have added an if condition, i.e., if a pod
doesn't exist anymore, then move its state to `PodTerminated`
irrespective of the previous state. This will then stop the apidump
process (if it is still active) and remove it from map. This will remove
the unnecessary garbage data in maps.

Author: mudit-postman

cmd/internal/kube/daemonset/pods_healthcheck_worker.go

@@ -37,13 +37,13 @@ func (d *Daemonset) checkPodsHealth() {
 		podStatus, ok := podStatuses[podUID]
 		if !ok {
 			printer.Infof("Pod status not found for podUID %s, Pod doesn't exists anymore\n", podUID)
-			d.handleTerminatedPod(podUID, errors.Errorf("pod %s doesn't exists anymore", podUID))
+			d.handleTerminatedPod(podUID, errors.Errorf("pod %s doesn't exists anymore", podUID), true)
 		}
 
 		switch podStatus {
 		case coreV1.PodSucceeded, coreV1.PodFailed:
 			printer.Infof("Pod with UID %s has stopped running, status: %s\n", podUID, podStatus)
-			d.handleTerminatedPod(podUID, errors.Errorf("pod %s has stopped running, status: %s", podUID, podStatus))
+			d.handleTerminatedPod(podUID, errors.Errorf("pod %s has stopped running, status: %s", podUID, podStatus), false)
 		case coreV1.PodRunning:
 			printer.Debugf("Pod with UID %s, status:%s\n", podUID, podStatus)
 			d.handleUnmonitoredPod(podUID)
@@ -53,7 +53,7 @@ func (d *Daemonset) checkPodsHealth() {
 
 // handleTerminatedPod handles the terminated pod by changing the pod's traffic monitor state to PodTerminated
 // and stopping the API dump process for that pod.
-func (d *Daemonset) handleTerminatedPod(podUID types.UID, podStatusErr error) {
+func (d *Daemonset) handleTerminatedPod(podUID types.UID, podStatusErr error, podDoesNotExists bool) {
 	podArgs, err := d.getPodArgsFromMap(podUID)
 	if err != nil {
 		printer.Infof("Failed to get podArgs for podUID %s: %v\n", podUID, err)
@@ -65,7 +65,13 @@ func (d *Daemonset) handleTerminatedPod(podUID types.UID, podStatusErr error) {
 		return
 	}
 
-	err = podArgs.changePodTrafficMonitorState(PodTerminated, TrafficMonitoringRunning)
+	// If pod doesn't exists anymore, we don't need to check the pod status
+	// We can directly change the state to PodTerminated
+	if podDoesNotExists {
+		err = podArgs.changePodTrafficMonitorState(PodTerminated)
+	} else {
+		err = podArgs.changePodTrafficMonitorState(PodTerminated, TrafficMonitoringRunning)
+	}
 	if err != nil {
 		printer.Infof("Failed to change pod state, pod name: %s, from: %s to: %s, error: %v\n",
 			podArgs.PodName, podArgs.PodTrafficMonitorState, PodTerminated, err)

cmd/internal/kube/daemonset/telemetry.go

@@ -27,7 +27,7 @@ func (d *Daemonset) sendTelemetry() {
 // dumpPodsApiDumpProcessState logs the current state of active pods.
 // It prints a formatted table with the pod name, project ID, and current state for each pod.
 func (d *Daemonset) dumpPodsApiDumpProcessState() {
-	logf := printer.Debugf
+	logf := printer.Infof
 
 	const hrBr = "================================================================================" +
 		"===========================================================================================\n"