Handle the case of missing instances when checking for shutdown instances

Author: Pushkar Acharya

Diff for: internal/instances/instances.go

@@ -25,6 +25,7 @@ var ErrAssertTimeTypeFailed = errors.New("failed to assert type time.Time for fi
 
 type Instances struct {
 	nodeFirstSeen sync.Map
+	nodeShutdown  sync.Map
 	apiClient     client.APIClient
 }
 
@@ -100,6 +101,23 @@ func (i *Instances) InstanceShutdownByProviderID(ctx context.Context, providerID
 		defer responseBody.Body.Close()
 	}
 	if err != nil {
+		if err == client.ErrInstanceNotFound {
+			attempt, ok := i.nodeShutdown.Load(providerID)
+			if !ok {
+				attempt = 1
+				i.nodeShutdown.Store(providerID, attempt)
+
+				return false, err
+			}
+			if attempt.(int) >= 3 {
+
+				return true, nil
+			}
+			intAttempt := attempt.(int)
+			i.nodeShutdown.Store(providerID, (intAttempt + 1))
+
+			return false, err
+		}
 		return false, fmt.Errorf("failed to get instance by provider ID %s: %w", providerID, err)
 	}
 	if currInstance == nil || currInstance.State == "STATE_SHUTOFF" || currInstance.State == "STATE_SHUTDOWN" {

Diff for: internal/instances/instances_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	v1alpha5 "github.com/crusoecloud/client-go/swagger/v1alpha5"
+	"github.com/crusoecloud/crusoe-cloud-controller-manager/internal/client"
 	mock_client "github.com/crusoecloud/crusoe-cloud-controller-manager/internal/client/mock"
 	"github.com/crusoecloud/crusoe-cloud-controller-manager/internal/instances"
 	"github.com/golang/mock/gomock"
@@ -109,6 +110,37 @@ func TestInstanceShutdownByProviderID(t *testing.T) {
 	require.True(t, shutdown)
 }
 
+func TestInstanceShutdownByProviderIDInstanceMissingFromCloudProvider(t *testing.T) {
+	t.Parallel()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mockClient := mock_client.NewMockApiClient(ctrl)
+	mockClient.EXPECT().GetInstanceByID(gomock.Any(), TESTInstanceID).Return(nil, nil, client.ErrInstanceNotFound).AnyTimes()
+	instanceService := instances.NewCrusoeInstances(mockClient)
+
+	// Instance shutdown by ID should return true on third attempt
+	// attempt #1
+	shutdown, err := instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #2
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #3
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.ErrorIs(t, err, client.ErrInstanceNotFound)
+	require.False(t, shutdown)
+
+	// attempt #4
+	shutdown, err = instanceService.InstanceShutdownByProviderID(context.Background(), ProviderIDPrefix+TESTInstanceID)
+	require.NoError(t, err)
+	require.True(t, shutdown)
+}
+
 func TestInstanceMetadata(t *testing.T) {
 	t.Parallel()
 	ctrl := gomock.NewController(t)