Skip to content

Commit 1774bc6

Browse files
Merge pull request #2126 from davidfestal/fix-resource-controller
🐛 Fix bugs in the resource controller
2 parents 9f61116 + 98f149d commit 1774bc6

File tree

4 files changed

+32
-6
lines changed

4 files changed

+32
-6
lines changed

pkg/reconciler/workload/resource/resource_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@ func locations(annotations, labels map[string]string, skipPending bool) (locatio
448448
}
449449
for k := range annotations {
450450
if strings.HasPrefix(k, workloadv1alpha1.InternalClusterDeletionTimestampAnnotationPrefix) {
451-
deleting.Insert(strings.TrimPrefix(k, workloadv1alpha1.ClusterResourceStateLabelPrefix))
451+
deleting.Insert(strings.TrimPrefix(k, workloadv1alpha1.InternalClusterDeletionTimestampAnnotationPrefix))
452452
}
453453
}
454454
return

pkg/reconciler/workload/resource/resource_reconcile.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (c *Controller) reconcileResource(ctx context.Context, lclusterName logical
8585
// clean finalizers from removed syncers
8686
filteredFinalizers := make([]string, 0, len(obj.GetFinalizers()))
8787
for _, f := range obj.GetFinalizers() {
88-
logger = logger.WithValues("finalizer", f)
88+
logger := logger.WithValues("finalizer", f)
8989
if !strings.HasPrefix(f, syncershared.SyncerFinalizerNamePrefix) {
9090
filteredFinalizers = append(filteredFinalizers, f)
9191
continue
@@ -176,8 +176,10 @@ func computePlacement(ns *corev1.Namespace, obj metav1.Object) (annotationPatch
176176
// create merge patch
177177
annotationPatch = map[string]interface{}{}
178178
labelPatch = map[string]interface{}{}
179+
180+
// unschedule objects on locations where the namespace is not scheduled
179181
for _, loc := range objLocations.Difference(nsLocations).List() {
180-
// location was removed from namespace, but is still on the object
182+
// That's an inconsistent state, probably due to the namespace deletion reaching its grace period => let's repair it
181183
var hasSyncerFinalizer, hasClusterFinalizer bool
182184
// Check if there's still the syncer or the cluster finalizer.
183185
for _, finalizer := range obj.GetFinalizers() {
@@ -199,16 +201,22 @@ func computePlacement(ns *corev1.Namespace, obj metav1.Object) (annotationPatch
199201
}
200202
}
201203
}
202-
for _, loc := range nsLocations.Intersection(nsLocations).List() {
204+
205+
// sync deletion timestamps if both namespace and object are scheduled
206+
for _, loc := range nsLocations.Intersection(objLocations).List() {
203207
if nsTimestamp, found := ns.Annotations[workloadv1alpha1.InternalClusterDeletionTimestampAnnotationPrefix+loc]; found && validRFC3339(nsTimestamp) {
204208
objTimestamp, found := obj.GetAnnotations()[workloadv1alpha1.InternalClusterDeletionTimestampAnnotationPrefix+loc]
205209
if !found || !validRFC3339(objTimestamp) {
206210
annotationPatch[workloadv1alpha1.InternalClusterDeletionTimestampAnnotationPrefix+loc] = nsTimestamp
207211
}
208212
}
209213
}
214+
215+
// set label on unscheduled objects if namespace is scheduled and not deleting
210216
for _, loc := range nsLocations.Difference(objLocations).List() {
211-
// location was missing on the object
217+
if nsDeleting.Has(loc) {
218+
continue
219+
}
212220
// TODO(sttts): add way to go into pending state first, maybe with a namespace annotation
213221
labelPatch[workloadv1alpha1.ClusterResourceStateLabelPrefix+loc] = string(workloadv1alpha1.ResourceStateSync)
214222
}

pkg/reconciler/workload/resource/resource_reconcile_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,14 @@ func TestComputePlacement(t *testing.T) {
8080
"state.workload.kcp.dev/cluster-1": "Sync",
8181
},
8282
},
83+
{name: "syncing but deleting namespace, unscheduled object, don't schedule the object at all",
84+
ns: namespace(map[string]string{
85+
"deletion.internal.workload.kcp.dev/cluster-1": "2002-10-02T10:00:00-05:00",
86+
}, map[string]string{
87+
"state.workload.kcp.dev/cluster-1": "Sync",
88+
}),
89+
obj: object(nil, nil, nil, nil),
90+
},
8391
{name: "new location on namespace",
8492
ns: namespace(nil, map[string]string{
8593
"state.workload.kcp.dev/cluster-1": "Sync",

test/e2e/reconciler/scheduling/placement_scheduler_test.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
3131
"k8s.io/apimachinery/pkg/api/errors"
3232
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
"k8s.io/apimachinery/pkg/types"
3334
"k8s.io/apimachinery/pkg/util/wait"
3435
"k8s.io/client-go/kubernetes"
3536
"k8s.io/client-go/rest"
@@ -136,13 +137,18 @@ func TestPlacementUpdate(t *testing.T) {
136137
return true
137138
}, wait.ForeverTestTimeout, time.Millisecond*100)
138139

140+
firstSyncTargetKey := workloadv1alpha1.ToSyncTargetKey(syncerFixture.SyncerConfig.SyncTargetWorkspace, firstSyncTargetName)
141+
139142
t.Logf("Create a service in the user workspace")
140143
_, err = kubeClusterClient.CoreV1().Services("default").Create(logicalcluster.WithCluster(ctx, userClusterName), &corev1.Service{
141144
ObjectMeta: metav1.ObjectMeta{
142145
Name: "first",
143146
Labels: map[string]string{
144147
"test.workload.kcp.dev": firstSyncTargetName,
145148
},
149+
Annotations: map[string]string{
150+
"finalizers.workload.kcp.dev/" + firstSyncTargetKey: "wait-a-bit",
151+
},
146152
},
147153
Spec: corev1.ServiceSpec{
148154
Ports: []corev1.ServicePort{
@@ -154,7 +160,6 @@ func TestPlacementUpdate(t *testing.T) {
154160
},
155161
}, metav1.CreateOptions{})
156162
require.NoError(t, err)
157-
firstSyncTargetKey := workloadv1alpha1.ToSyncTargetKey(syncerFixture.SyncerConfig.SyncTargetWorkspace, firstSyncTargetName)
158163

159164
t.Logf("Wait for the service to have the sync label")
160165
framework.Eventually(t, func() (bool, string) {
@@ -233,6 +238,11 @@ func TestPlacementUpdate(t *testing.T) {
233238
return true, ""
234239
}, wait.ForeverTestTimeout, time.Millisecond*100)
235240

241+
t.Logf("Remove the soft finalizer on the service")
242+
_, err = kubeClusterClient.CoreV1().Services("default").Patch(logicalcluster.WithCluster(ctx, userClusterName), "first", types.MergePatchType,
243+
[]byte("{\"metadata\":{\"annotations\":{\"deletion.internal.workload.kcp.dev/"+firstSyncTargetKey+"\":\"\"}}}"), metav1.PatchOptions{})
244+
require.NoError(t, err)
245+
236246
t.Logf("Wait for the service to be removed in the downstream cluster")
237247
require.Eventually(t, func() bool {
238248
downstreamServices, err = syncerFixture.DownstreamKubeClient.CoreV1().Services("").List(ctx, metav1.ListOptions{

0 commit comments

Comments
 (0)