Handle the case of missing instances when checking for shutdown instances

Author: Pushkar Acharya

Diff for: internal/instances/instances.go

@@ -25,6 +25,7 @@ var ErrAssertTimeTypeFailed = errors.New("failed to assert type time.Time for fi
 
 type Instances struct {
 	nodeFirstSeen sync.Map
+	nodeShutdown  sync.Map
 	apiClient     client.APIClient
 }
 
@@ -100,6 +101,10 @@ func (i *Instances) InstanceShutdownByProviderID(ctx context.Context, providerID
 		defer responseBody.Body.Close()
 	}
 	if err != nil {
+		if errors.Is(err, client.ErrInstanceNotFound) {
+			return i.handleInstanceNotFoundErr(providerID, err)
+		}
+
 		return false, fmt.Errorf("failed to get instance by provider ID %s: %w", providerID, err)
 	}
 	if currInstance == nil || currInstance.State == "STATE_SHUTOFF" || currInstance.State == "STATE_SHUTDOWN" {
@@ -268,3 +273,25 @@ func getNodeAddress(currInstance *crusoeapi.InstanceV1Alpha5) ([]v1.NodeAddress,
 
 	return nodeAddress, nil
 }
+
+func (i *Instances) handleInstanceNotFoundErr(providerID string, orignalErr error) (instanceShutdown bool, err error) {
+	attempt, ok := i.nodeShutdown.Load(providerID)
+	if !ok {
+		i.nodeShutdown.Store(providerID, 1)
+
+		return false, orignalErr
+	}
+	intAttempt, ok := attempt.(int)
+	if !ok {
+		// store correct data to avoid conversion issues in next iteration
+		i.nodeShutdown.Store(providerID, 1)
+
+		return false, orignalErr
+	}
+	if intAttempt >= 3 {
+		return true, nil
+	}
+	i.nodeShutdown.Store(providerID, (intAttempt + 1))
+
+	return false, orignalErr
+}

Diff for: internal/instances/instances_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	v1alpha5 "github.com/crusoecloud/client-go/swagger/v1alpha5"
+	"github.com/crusoecloud/crusoe-cloud-controller-manager/internal/client"
 	mock_client "github.com/crusoecloud/crusoe-cloud-controller-manager/internal/client/mock"
 	"github.com/crusoecloud/crusoe-cloud-controller-manager/internal/instances"
 	"github.com/golang/mock/gomock"
@@ -109,6 +110,38 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 	require.True(t, shutdown)
 }
 
+func TestInstanceShutdownByProviderIDInstanceMissingFromCloudProvider(t *testing.T) {
+	t.Parallel()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mockClient := mock_client.NewMockApiClient(ctrl)
+	mockClient.EXPECT().GetInstanceByID(gomock.Any(), TESTInstanceID).Return(nil,
+		nil, client.ErrInstanceNotFound).AnyTimes()
+	instanceService := instances.NewCrusoeInstances(mockClient)
+
+	// Instance shutdown by ID should return true on third attempt
+	// attempt #1
+	shutdown, err := instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #2
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #3
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #4
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.NoError(t, err)
+	require.True(t, shutdown)
+}
+
 func TestInstanceMetadata(t *testing.T) {
 	t.Parallel()
 	ctrl := gomock.NewController(t)