Skip to content

Commit 6367b6f

Browse files
ajanikowajanikow
authored
[Bugfix] Change Plan change discovery (#784)
1 parent ff2eb27 commit 6367b6f

12 files changed

+41
-29
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
- Check if the DB server is cleaned out.
99
- Render Pod Template in ArangoMember Spec and Status
1010
- Add Pod PropagationModes
11+
- Fix MemberUp action for ActiveFailover
1112

1213
## [1.2.1](https://github.com/arangodb/kube-arangodb/tree/1.2.1) (2021-07-28)
1314
- Fix ArangoMember race with multiple ArangoDeployments within single namespace

‎pkg/deployment/reconcile/action.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,12 @@ type ActionPlanAppender interface {
7474
Action
7575

7676
// ActionPlanAppender modify plan after action execution
77-
ActionPlanAppender(current api.Plan) api.Plan
77+
ActionPlanAppender(current api.Plan) (api.Plan, bool)
7878
}
7979

80-
func getActionPlanAppender(a Action, plan api.Plan) api.Plan {
80+
func getActionPlanAppender(a Action, plan api.Plan) (api.Plan, bool) {
8181
if c, ok := a.(ActionPlanAppender); !ok {
82-
return plan
82+
return plan, false
8383
} else {
8484
return c.ActionPlanAppender(plan)
8585
}

‎pkg/deployment/reconcile/action_add_member.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (a *actionAddMember) Start(ctx context.Context) (bool, error) {
7878
}
7979

8080
// ActionPlanAppender appends wait methods to the plan
81-
func (a *actionAddMember) ActionPlanAppender(current api.Plan) api.Plan {
81+
func (a *actionAddMember) ActionPlanAppender(current api.Plan) (api.Plan, bool) {
8282
var app api.Plan
8383

8484
if _, ok := a.action.Params[api.ActionTypeWaitForMemberUp.String()]; ok {
@@ -88,5 +88,10 @@ func (a *actionAddMember) ActionPlanAppender(current api.Plan) api.Plan {
8888
if _, ok := a.action.Params[api.ActionTypeWaitForMemberUp.String()]; ok {
8989
app = append(app, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, a.newMemberID, "Wait for member in sync after creation"))
9090
}
91-
return append(app, current...)
91+
92+
if len(app) > 0 {
93+
return append(app, current...), true
94+
}
95+
96+
return current, false
9297
}

‎pkg/deployment/reconcile/action_cleanout_member.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,23 +128,23 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e
128128
c, err := a.actionCtx.GetDatabaseClient(ctxChild)
129129
if err != nil {
130130
log.Debug().Err(err).Msg("Failed to create database client")
131-
return false, false, errors.WithStack(err)
131+
return false, false, nil
132132
}
133133

134134
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
135135
defer cancel()
136136
cluster, err := c.Cluster(ctxChild)
137137
if err != nil {
138138
log.Debug().Err(err).Msg("Failed to access cluster")
139-
return false, false, errors.WithStack(err)
139+
return false, false, nil
140140
}
141141

142142
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
143143
defer cancel()
144144
cleanedOut, err := cluster.IsCleanedOut(ctxChild, a.action.MemberID)
145145
if err != nil {
146146
log.Debug().Err(err).Msg("IsCleanedOut failed")
147-
return false, false, errors.WithStack(err)
147+
return false, false, nil
148148
}
149149
if !cleanedOut {
150150
// We're not done yet, check job status
@@ -155,30 +155,30 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e
155155
c, err := a.actionCtx.GetDatabaseClient(ctxChild)
156156
if err != nil {
157157
log.Debug().Err(err).Msg("Failed to create database client")
158-
return false, false, errors.WithStack(err)
158+
return false, false, nil
159159
}
160160

161161
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
162162
defer cancel()
163163
agency, err := a.actionCtx.GetAgency(ctxChild)
164164
if err != nil {
165165
log.Debug().Err(err).Msg("Failed to create agency client")
166-
return false, false, errors.WithStack(err)
166+
return false, false, nil
167167
}
168168

169169
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
170170
defer cancel()
171171
jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency)
172172
if err != nil {
173173
log.Debug().Err(err).Msg("Failed to fetch cleanout job status")
174-
return false, false, errors.WithStack(err)
174+
return false, false, nil
175175
}
176176
if jobStatus.IsFailed() {
177177
log.Warn().Str("reason", jobStatus.Reason()).Msg("Cleanout Job failed. Aborting plan")
178178
// Revert cleanout state
179179
m.Phase = api.MemberPhaseCreated
180180
m.CleanoutJobID = ""
181-
if a.actionCtx.UpdateMember(ctx, m); err != nil {
181+
if err := a.actionCtx.UpdateMember(ctx, m); err != nil {
182182
return false, false, errors.WithStack(err)
183183
}
184184
return false, true, nil

‎pkg/deployment/reconcile/action_resign_leadership.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool,
129129

130130
if enabled, err := a.actionCtx.GetAgencyMaintenanceMode(ctx); err != nil {
131131
log.Error().Err(err).Msgf("Unable to get maintenance mode")
132-
return false, false, errors.WithStack(err)
132+
return false, false, nil
133133
} else if enabled {
134134
log.Warn().Msgf("Maintenance is enabled, skipping action")
135135
// We are done, action cannot be handled on maintenance mode
@@ -145,15 +145,15 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool,
145145
agency, err := a.actionCtx.GetAgency(ctxChild)
146146
if err != nil {
147147
log.Debug().Err(err).Msg("Failed to create agency client")
148-
return false, false, errors.WithStack(err)
148+
return false, false, nil
149149
}
150150

151151
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
152152
defer cancel()
153153
c, err := a.actionCtx.GetDatabaseClient(ctxChild)
154154
if err != nil {
155155
log.Debug().Err(err).Msg("Failed to create member client")
156-
return false, false, errors.WithStack(err)
156+
return false, false, nil
157157
}
158158

159159
ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())

‎pkg/deployment/reconcile/action_rotate_member.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,8 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err
9999
// Pod is terminated, we can now remove it
100100
if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil {
101101
if !k8sutil.IsNotFound(err) {
102-
return false, false, errors.WithStack(err)
102+
log.Error().Err(err).Msg("Unable to delete pod")
103+
return false, false, nil
103104
}
104105
}
105106
// Pod is now gone, update the member status

‎pkg/deployment/reconcile/action_rotate_start_member.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,8 @@ func (a *actionRotateStartMember) CheckProgress(ctx context.Context) (bool, bool
9999
// Pod is terminated, we can now remove it
100100
if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil {
101101
if !k8sutil.IsNotFound(err) {
102-
return false, false, errors.WithStack(err)
102+
log.Error().Err(err).Msg("Unable to delete pod")
103+
return false, false, nil
103104
}
104105
}
105106

‎pkg/deployment/reconcile/action_upgrade_current_image.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,8 @@ func (a *setCurrentImageAction) CheckProgress(ctx context.Context) (bool, bool,
7272
return false, false, nil
7373
}
7474
if err := a.actionCtx.SetCurrentImage(ctx, imageInfo); err != nil {
75-
return false, false, errors.WithStack(err)
75+
log.Error().Err(err).Msg("Unable to set current image")
76+
return false, false, nil
7677
}
7778
log.Info().Str("image", a.action.Image).Str("to", imageInfo.Image).Msg("Changed current main image")
7879
return true, false, nil

‎pkg/deployment/reconcile/action_wait_for_member_up.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,11 @@ func (a *actionWaitForMemberUp) checkProgressSingle(ctx context.Context) (bool,
109109
c, err := a.actionCtx.GetDatabaseClient(ctx)
110110
if err != nil {
111111
log.Debug().Err(err).Msg("Failed to create database client")
112-
return false, false, errors.WithStack(err)
112+
return false, false, nil
113113
}
114114
if _, err := c.Version(ctx); err != nil {
115115
log.Debug().Err(err).Msg("Failed to get version")
116-
return false, false, errors.WithStack(err)
116+
return false, false, nil
117117
}
118118
return true, false, nil
119119
}
@@ -125,11 +125,11 @@ func (a *actionWaitForMemberUp) checkProgressSingleInActiveFailover(ctx context.
125125
c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID)
126126
if err != nil {
127127
log.Debug().Err(err).Msg("Failed to create database client")
128-
return false, false, errors.WithStack(err)
128+
return false, false, nil
129129
}
130130
if _, err := c.Version(ctx); err != nil {
131131
log.Debug().Err(err).Msg("Failed to get version")
132-
return false, false, errors.WithStack(err)
132+
return false, false, nil
133133
}
134134
return true, false, nil
135135
}
@@ -141,7 +141,7 @@ func (a *actionWaitForMemberUp) checkProgressAgent(ctx context.Context) (bool, b
141141
clients, err := a.actionCtx.GetAgencyClients(ctx)
142142
if err != nil {
143143
log.Debug().Err(err).Msg("Failed to create agency clients")
144-
return false, false, errors.WithStack(err)
144+
return false, false, nil
145145
}
146146

147147
for _, a := range clients {
@@ -204,11 +204,11 @@ func (a *actionWaitForMemberUp) checkProgressArangoSync(ctx context.Context) (bo
204204
c, err := a.actionCtx.GetSyncServerClient(ctx, a.action.Group, a.action.MemberID)
205205
if err != nil {
206206
log.Debug().Err(err).Msg("Failed to create arangosync client")
207-
return false, false, errors.WithStack(err)
207+
return false, false, nil
208208
}
209209
if err := c.Health(ctx); err != nil {
210210
log.Debug().Err(err).Msg("Health not ok yet")
211-
return false, false, errors.WithStack(err)
211+
return false, false, nil
212212
}
213213
return true, false, nil
214214
}

‎pkg/deployment/reconcile/helper_shutdown.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,8 @@ func (s shutdownHelperDelete) CheckProgress(ctx context.Context) (bool, bool, er
170170
log.Warn().Msgf("Pod still exists")
171171
return false, false, nil
172172
} else if !k8sutil.IsNotFound(err) {
173-
return false, false, errors.WithStack(err)
173+
log.Error().Err(err).Msg("Unable to get pod")
174+
return false, false, nil
174175
}
175176
}
176177

‎pkg/deployment/reconcile/plan_builder_high.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,10 @@ func updateMemberPodTemplateSpec(ctx context.Context,
101101
// Update member specs
102102
status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
103103
for _, m := range members {
104-
if reason, changed := arangoMemberPodTemplateNeedsUpdate(ctx, log, apiObject, spec, group, status, m, cachedStatus, context); changed {
105-
plan = append(plan, api.NewAction(api.ActionTypeArangoMemberUpdatePodSpec, group, m.ID, reason))
104+
if m.Phase != api.MemberPhaseNone {
105+
if reason, changed := arangoMemberPodTemplateNeedsUpdate(ctx, log, apiObject, spec, group, status, m, cachedStatus, context); changed {
106+
plan = append(plan, api.NewAction(api.ActionTypeArangoMemberUpdatePodSpec, group, m.ID, reason))
107+
}
106108
}
107109
}
108110

‎pkg/deployment/reconcile/plan_executor.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ func (d *Reconciler) executePlan(ctx context.Context, cachedStatus inspectorInte
180180
}
181181
}
182182

183-
if newPlan := getActionPlanAppender(action, plan); !newPlan.Equal(plan) {
183+
if newPlan, changed := getActionPlanAppender(action, plan); changed {
184184
// Our actions have been added to the end of plan
185185
log.Info().Msgf("Appending new plan items")
186186
return newPlan, true, nil

0 commit comments

Comments
 (0)