[POA-3109] Handle panic while stopping apidump process while daemonset exit (#90)

This pull request includes changes to prevent the agent from going into
a panic state while closing the apidump process during the daemonset
process shutdown and other changes.

**Problem:**
* Before changing the state of the apidump process to the final state
and then calling the `StopApiDumpProcess`, we are checking if the
process is already in the final state to avoid `StopApiDumpProcess`
being called multiple times.
* But this final state check is not sync, due to which dirty read issue
can happen and `StopApiDumpProcess` can be called multiple times which
will send signal to already stopped channel hence causing program to go
to panic state.

**Changes:**

Improvements to state handling:
* Added a check in `changePodTrafficMonitorState` to prevent state
changes if the pod is already in a final state.
* Ensure the stop channel is closed only after the API dump process is
stopped/exited.
* Removed redundant checks for end states in various methods, as these
are now handled by the new state check in
`changePodTrafficMonitorState`.

Other naming changes:
* Renamed `StopApiDumpProcess` to `SignalApiDumpProcessToStop` for
better clarity on its purpose.
* Renamed `isEndState` to `isTrafficMonitoringInFinalState` for better
readability and consistency.

Author: mudit-postman

cmd/internal/kube/daemonset/apidump_process.go

@@ -37,6 +37,9 @@ func (d *Daemonset) StartApiDumpProcess(podUID types.UID) error {
 			// Decrement the wait group counter
 			d.ApidumpProcessesWG.Done()
 
+			// Close the stop channel once the apidump process is exited
+			close(podArgs.StopChan)
+
 			nextState := TrafficMonitoringEnded
 
 			if err := recover(); err != nil {
@@ -100,18 +103,17 @@ func (d *Daemonset) StartApiDumpProcess(podUID types.UID) error {
 	return nil
 }
 
-// StopApiDumpProcess signals the API dump process to stop for a given pod
+// SignalApiDumpProcessToStop signals the API dump process to stop for a given pod
 // identified by its UID. It retrieves the process's stop channel object from a map
 // and sends a stop signal to trigger apidump shutdown.
-func (d *Daemonset) StopApiDumpProcess(podUID types.UID, stopErr error) error {
+func (d *Daemonset) SignalApiDumpProcessToStop(podUID types.UID, stopErr error) error {
 	podArgs, err := d.getPodArgsFromMap(podUID)
 	if err != nil {
 		return err
 	}
 
 	printer.Infof("Stopping API dump process for pod %s\n", podArgs.PodName)
 	podArgs.StopChan <- stopErr
-	close(podArgs.StopChan)
 
 	return nil
 }

cmd/internal/kube/daemonset/daemonset.go

@@ -343,11 +343,6 @@ func (d *Daemonset) StopAllApiDumpProcesses() {
 		podUID := k.(types.UID)
 		podArgs := v.(*PodArgs)
 
-		if podArgs.isEndState() {
-			printer.Debugf("API dump process for pod %s already stopped, state: %s\n", podArgs.PodName, podArgs.PodTrafficMonitorState)
-			return true
-		}
-
 		// Since this state can happen at any time so no check for allowed current states
 		err := podArgs.changePodTrafficMonitorState(DaemonSetShutdown)
 		if err != nil {
@@ -356,7 +351,7 @@ func (d *Daemonset) StopAllApiDumpProcesses() {
 			return true
 		}
 
-		err = d.StopApiDumpProcess(podUID, errors.Errorf("Daemonset agent is shutting down, stopping pod: %s", podArgs.PodName))
+		err = d.SignalApiDumpProcessToStop(podUID, errors.Errorf("Daemonset agent is shutting down, stopping pod: %s", podArgs.PodName))
 		if err != nil {
 			printer.Errorf("Failed to stop api dump process, pod name: %s, error: %v\n", podArgs.PodName, err)
 		}

cmd/internal/kube/daemonset/kube_events_worker.go

@@ -81,11 +81,6 @@ func (d *Daemonset) handlePodDeleteEvent(pod coreV1.Pod) {
 		return
 	}
 
-	if podArgs.isEndState() {
-		printer.Debugf("Pod %s already stopped monitoring, state: %s\n", podArgs.PodName, podArgs.PodTrafficMonitorState)
-		return
-	}
-
 	var podStatus PodTrafficMonitorState
 	switch pod.Status.Phase {
 	case coreV1.PodSucceeded:
@@ -104,7 +99,7 @@ func (d *Daemonset) handlePodDeleteEvent(pod coreV1.Pod) {
 		return
 	}
 
-	err = d.StopApiDumpProcess(pod.UID, errors.Errorf("got pod delete event, pod: %s", podArgs.PodName))
+	err = d.SignalApiDumpProcessToStop(pod.UID, errors.Errorf("got pod delete event, pod: %s", podArgs.PodName))
 	if err != nil {
 		printer.Errorf("Failed to stop api dump process, pod name: %s, error: %v\n", podArgs.PodName, err)
 	}

cmd/internal/kube/daemonset/pod_args.go

@@ -83,6 +83,10 @@ func (p *PodArgs) changePodTrafficMonitorState(
 	p.StateChangeMutex.Lock()
 	defer p.StateChangeMutex.Unlock()
 
+	if isTrafficMonitoringInFinalState(p) {
+		return errors.New(fmt.Sprintf("API dump process for pod %s already in final state, state: %s\n", p.PodName, p.PodTrafficMonitorState))
+	}
+
 	// Check if the current state is allowed for the transition
 	// If the allowedCurrentStates is empty, then any state is allowed
 	if len(allowedCurrentStates) != 0 && !slices.Contains(allowedCurrentStates, p.PodTrafficMonitorState) {
@@ -97,8 +101,8 @@ func (p *PodArgs) changePodTrafficMonitorState(
 	return nil
 }
 
-// isEndState checks if the pod traffic monitor state is in the final state.
-func (p *PodArgs) isEndState() bool {
+// isTrafficMonitoringInFinalState checks if the pod traffic monitor state is in the final state.
+func isTrafficMonitoringInFinalState(p *PodArgs) bool {
 	switch p.PodTrafficMonitorState {
 	case TrafficMonitoringEnded, TrafficMonitoringFailed, RemovePodFromMap:
 		return true

cmd/internal/kube/daemonset/pods_healthcheck_worker.go

@@ -60,11 +60,6 @@ func (d *Daemonset) handleTerminatedPod(podUID types.UID, podStatusErr error, po
 		return
 	}
 
-	if podArgs.isEndState() {
-		printer.Debugf("Pod %s already stopped monitoring, state: %s\n", podArgs.PodName, podArgs.PodTrafficMonitorState)
-		return
-	}
-
 	// If pod doesn't exists anymore, we don't need to check the pod status
 	// We can directly change the state to PodTerminated
 	if podDoesNotExists {
@@ -78,7 +73,7 @@ func (d *Daemonset) handleTerminatedPod(podUID types.UID, podStatusErr error, po
 		return
 	}
 
-	err = d.StopApiDumpProcess(podUID, podStatusErr)
+	err = d.SignalApiDumpProcessToStop(podUID, podStatusErr)
 	if err != nil {
 		printer.Errorf("Failed to stop api dump process, pod name: %s, error: %v\n", podArgs.PodName, err)
 	}