diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index 518920728..604cda7a4 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -427,7 +427,7 @@ const ( // Its condition status can be one of the following: // - "True": The staged update run is making progress. // - "False": The staged update run is waiting/paused/abandoned. - // - "Unknown" means it is unknown. + // - "Unknown": The staged update run is in a transitioning state. StagedUpdateRunConditionProgressing StagedUpdateRunConditionType = "Progressing" // StagedUpdateRunConditionSucceeded indicates whether the staged update run is completed successfully. @@ -489,7 +489,8 @@ const ( // StageUpdatingConditionProgressing indicates whether the stage updating is making progress. // Its condition status can be one of the following: // - "True": The stage updating is making progress. - // - "False": The stage updating is waiting/pausing. + // - "False": The stage updating is waiting. + // - "Unknown": The staged updating is a transitioning state. StageUpdatingConditionProgressing StageUpdatingConditionType = "Progressing" // StageUpdatingConditionSucceeded indicates whether the stage updating is completed successfully. diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 4ceb56ce0..b61bfee01 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -164,7 +164,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim klog.V(2).InfoS("The updateRun is initialized but not executed, waiting to execute", "state", state, "updateRun", runObjRef) case placementv1beta1.StateRun: // Execute the updateRun. - klog.InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) if errors.Is(execErr, errStagedUpdatedAborted) { // errStagedUpdatedAborted cannot be retried. @@ -176,23 +176,23 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) } - // The execution is not finished yet or it encounters a retriable error. - // We need to record the status and requeue. - if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { - return runtime.Result{}, updateErr - } - klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) - if execErr != nil { - return runtime.Result{}, execErr - } - return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil + return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, execErr, state, runObjRef) case placementv1beta1.StateStop: // Stop the updateRun. - klog.InfoS("Stopping the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) - // TODO(britaniar): Implement the stopping logic for in-progress stages. + klog.V(2).InfoS("Stopping the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + finished, waitTime, stopErr := r.stop(updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) + if errors.Is(stopErr, errStagedUpdatedAborted) { + // errStagedUpdatedAborted cannot be retried. + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, stopErr.Error()) + } + + if finished { + klog.V(2).InfoS("The updateRun is stopped", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunStopped(ctx, updateRun) + } + + return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, stopErr, state, runObjRef) - klog.V(2).InfoS("The updateRun is stopped", "updateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunStopped(ctx, updateRun) default: // Initialize, Run, or Stop are the only supported states. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported updateRun state: %s", state)) @@ -202,6 +202,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim return runtime.Result{}, nil } +func (r *Reconciler) handleIncompleteUpdateRun(ctx context.Context, updateRun placementv1beta1.UpdateRunObj, waitTime time.Duration, err error, state placementv1beta1.State, runObjRef klog.ObjectRef) (runtime.Result, error) { + // The execution or stopping is not finished yet or it encounters a retriable error. + // We need to record the status and requeue. + if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { + return runtime.Result{}, updateErr + } + + klog.V(2).InfoS("The updateRun is not finished yet", "state", state, "requeueWaitTime", waitTime, "err", err, "updateRun", runObjRef) + + // Return execution or stopping retriable error if any. + if err != nil { + return runtime.Result{}, err + } + return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil +} + // handleDelete handles the deletion of the updateRun object. // We delete all the dependent resources, including approvalRequest objects, of the updateRun object. func (r *Reconciler) handleDelete(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) (bool, time.Duration, error) { diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index 154d95372..eab39400b 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -333,6 +333,16 @@ func generateFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *p } } +func generateStoppingMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), + string(metav1.ConditionUnknown), condition.UpdateRunStoppingReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateStoppedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), @@ -858,3 +868,12 @@ func generateFalseConditionWithReason(obj client.Object, condType any, reason st falseCond.Reason = reason return falseCond } + +func generateProgressingUnknownConditionWithReason(obj client.Object, reason string) metav1.Condition { + return metav1.Condition{ + Status: metav1.ConditionUnknown, + Type: string(placementv1beta1.StageUpdatingConditionProgressing), + ObservedGeneration: obj.GetGeneration(), + Reason: reason, + } +} diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index fb66e20ad..e8a48d4db 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -18,7 +18,6 @@ package updaterun import ( "context" - "errors" "fmt" "reflect" "strconv" @@ -68,14 +67,7 @@ func (r *Reconciler) execute( // Set up defer function to handle errStagedUpdatedAborted. defer func() { - if errors.Is(err, errStagedUpdatedAborted) { - if updatingStageStatus != nil { - markStageUpdatingFailed(updatingStageStatus, updateRun.GetGeneration(), err.Error()) - } else { - // Handle deletion stage case. - markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), err.Error()) - } - } + checkIfErrorStagedUpdateAborted(err, updateRun, updatingStageStatus) }() // Mark updateRun as progressing if it's not already marked as waiting or stuck. @@ -232,9 +224,7 @@ func (r *Reconciler) executeUpdatingStage( } } markClusterUpdatingStarted(clusterStatus, updateRun.GetGeneration()) - if finishedClusterCount == 0 { - markStageUpdatingStarted(updatingStageStatus, updateRun.GetGeneration()) - } + markStageUpdatingProgressStarted(updatingStageStatus, updateRun.GetGeneration()) // Need to continue as we need to process at most maxConcurrency number of clusters in parallel. continue } @@ -338,7 +328,7 @@ func (r *Reconciler) executeDeleteStage( existingDeleteStageClusterMap[existingDeleteStageStatus.Clusters[i].ClusterName] = &existingDeleteStageStatus.Clusters[i] } // Mark the delete stage as started in case it's not. - markStageUpdatingStarted(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration()) + markStageUpdatingProgressStarted(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration()) for _, binding := range toBeDeletedBindings { bindingSpec := binding.GetBindingSpec() curCluster, exist := existingDeleteStageClusterMap[bindingSpec.TargetCluster] @@ -564,7 +554,7 @@ func calculateMaxConcurrencyValue(status *placementv1beta1.UpdateRunStatus, stag func aggregateUpdateRunStatus(updateRun placementv1beta1.UpdateRunObj, stageName string, stuckClusterNames []string) { if len(stuckClusterNames) > 0 { markUpdateRunStuck(updateRun, stageName, strings.Join(stuckClusterNames, ", ")) - } else { + } else if updateRun.GetUpdateRunSpec().State == placementv1beta1.StateRun { // If there is no stuck cluster but some progress has been made, mark the update run as progressing. markUpdateRunProgressing(updateRun) } @@ -708,8 +698,8 @@ func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, message strin }) } -// markStageUpdatingStarted marks the stage updating status as started in memory. -func markStageUpdatingStarted(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { +// markStageUpdatingProgressStarted marks the stage updating status as started in memory. +func markStageUpdatingProgressStarted(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { if stageUpdatingStatus.StartTime == nil { stageUpdatingStatus.StartTime = &metav1.Time{Time: time.Now()} } diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index f979c7571..759403347 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -706,684 +706,6 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { validateUpdateRunMetricsEmitted(generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) }) }) - - Context("Cluster staged update run should have stopped when state Stop", Ordered, func() { - var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest - var wantMetrics []*promclient.Metric - BeforeAll(func() { - By("Creating a new clusterStagedUpdateRun") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) - - By("Validating the initialization succeeded and the execution has not started") - initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) - wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Validating the first beforeStage approvalRequest has been created") - wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, - Labels: map[string]string{ - placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, - placementv1beta1.TargetUpdateRunLabel: updateRun.Name, - placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, - placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", - }, - }, - Spec: placementv1beta1.ApprovalRequestSpec{ - TargetUpdateRun: updateRun.Name, - TargetStage: updateRun.Status.StagesStatus[0].StageName, - }, - } - validateApprovalRequestCreated(wantApprovalRequest) - - By("Checking update run status metrics are emitted") - wantMetrics = []*promclient.Metric{generateWaitingMetric(updateRun)} - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should accept the approval request and start to rollout 1st stage", func() { - By("Approving the approvalRequest") - approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) - - By("Validating the approvalRequest has ApprovalAccepted status") - Eventually(func() (bool, error) { - var approvalRequest placementv1beta1.ClusterApprovalRequest - if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { - return false, err - } - return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil - }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") - // Approval task has been approved. - wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) - }) - - It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { - By("Validating the 1st clusterResourceBinding is updated to Bound") - binding := resourceBindings[numTargetClusters-1] // cluster-9 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Updating the 1st clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - // 1st stage started. - wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) - - By("Validating the 1st cluster has succeeded and 2nd cluster has started") - wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Validating the 1st stage has startTime set") - Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should stop the in middle of cluster updating when update run state is Stop", func() { - By("Updating updateRun state to Stop") - updateRun.Spec.State = placementv1beta1.StateStop - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the update run is stopped") - // 2nd cluster has started condition but no succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not continue rolling out 1st stage", func() { - By("Validating the 2nd clusterResourceBinding is NOT updated to Bound") - binding := resourceBindings[numTargetClusters-3] // cluster-7 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Validating the 3rd clusterResourceBinding is NOT updated to Bound") - binding = resourceBindings[numTargetClusters-5] // cluster-5 - validateNotBoundBindingState(ctx, binding) - - By("Validating the update run is still stopped") - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should continue executing stage 1 of the update run when state is Run", func() { - By("Updating updateRun state to Run") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating update run is running") - // Mark updateRun progressing condition as true with progressing reason. - meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { - By("Validating the 2nd clusterResourceBinding is updated to Bound") - binding := resourceBindings[numTargetClusters-3] // cluster-7 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Updating the 2nd clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 2nd cluster has succeeded and 3rd cluster has started") - // Mark stage started. - meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - // Mark 2nd cluster succeeded and 3rd cluster started. - wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { - By("Validating the 3rd clusterResourceBinding is updated to Bound") - binding := resourceBindings[numTargetClusters-5] // cluster-5 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Updating the 3rd clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 3rd cluster has succeeded and 4th cluster has started") - wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { - By("Validating the 4th clusterResourceBinding is updated to Bound") - binding := resourceBindings[numTargetClusters-7] // cluster-3 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Updating the 4th clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 4th cluster has succeeded and 5th cluster has started") - wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { - By("Validating the 5th clusterResourceBinding is updated to Bound") - binding := resourceBindings[numTargetClusters-9] // cluster-1 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) - - By("Updating the 5th clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 5th cluster has succeeded and 1st stage has completed and is waiting for AfterStageTasks") - // 5th cluster succeeded. - wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - // Now waiting for after stage tasks of 1st stage. - meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should have approval request created for 1st stage afterStageTask", func() { - By("Validating the approvalRequest has been created") - wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRun.Status.StagesStatus[0].AfterStageTaskStatus[1].ApprovalRequestName, - Labels: map[string]string{ - placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, - placementv1beta1.TargetUpdateRunLabel: updateRun.Name, - placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, - placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", - }, - }, - Spec: placementv1beta1.ApprovalRequestSpec{ - TargetUpdateRun: updateRun.Name, - TargetStage: updateRun.Status.StagesStatus[0].StageName, - }, - } - validateApprovalRequestCreated(wantApprovalRequest) - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should stop the update run in afterStageTasks for 1st stage when state is Stop", func() { - By("Updating updateRun state to Stop") - updateRun.Spec.State = placementv1beta1.StateStop - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the update run is stopped") - // Mark update run stopped. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not complete 1st stage when stopped", func() { - By("Validating the update run is still stopped") - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should approve the approval request for 1st stage afterStageTask while stopped and update run stays the same", func() { - By("Approving the approvalRequest") - approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) - - By("Validating the update run is still stopped") - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should complete the 1st stage once it starts running again when wait time passed and approval request approved then move on to the 2nd stage", func() { - By("Updating updateRun state to Run") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the approvalRequest has ApprovalAccepted status") - Eventually(func() (bool, error) { - var approvalRequest placementv1beta1.ClusterApprovalRequest - if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { - return false, err - } - return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil - }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") - - By("Validating both after stage tasks have completed and 2nd stage has started") - // Timedwait afterStageTask completed. - wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - // Approval afterStageTask completed. - wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) - // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) - wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - // 2nd stage waiting for before stage tasks. - wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Validating the 1st stage has endTime set") - Expect(updateRun.Status.StagesStatus[0].EndTime).ShouldNot(BeNil()) - - By("Validating the waitTime after stage task only completes after the wait time") - waitStartTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)).LastTransitionTime.Time - waitEndTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[0].Conditions, string(placementv1beta1.StageTaskConditionWaitTimeElapsed)).LastTransitionTime.Time - Expect(waitStartTime.Add(updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration).After(waitEndTime)).Should(BeFalse(), - fmt.Sprintf("waitEndTime %v did not pass waitStartTime %v long enough, want at least %v", waitEndTime, waitStartTime, updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration)) - - By("Validating the creation time of the approval request is before the complete time of the timedwait task") - approvalCreateTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[1].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestCreated)).LastTransitionTime.Time - Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should create approval request before 2nd stage", func() { - By("Validating the approvalRequest has been created") - wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRun.Status.StagesStatus[1].BeforeStageTaskStatus[0].ApprovalRequestName, - Labels: map[string]string{ - placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, - placementv1beta1.TargetUpdateRunLabel: updateRun.Name, - placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, - placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", - }, - }, - Spec: placementv1beta1.ApprovalRequestSpec{ - TargetUpdateRun: updateRun.Name, - TargetStage: updateRun.Status.StagesStatus[1].StageName, - }, - } - validateApprovalRequestCreated(wantApprovalRequest) - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not start rolling out 2nd stage while waiting for approval for beforeStageTask", func() { - By("Validating the 1st clusterResourceBinding is not updated to Bound") - binding := resourceBindings[0] // cluster-0 - validateNotBoundBindingState(ctx, binding) - - By("Validating the 1st stage does not have startTime set") - Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should stop the update run when state is Stop while waiting for 2nd stage beforeStageTask approval", func() { - By("Updating updateRun state to Stop") - updateRun.Spec.State = placementv1beta1.StateStop - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the update run is stopped") - // Mark update run stopped. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not start rolling out 2nd stage while in stopped state", func() { - By("Validating the 1st clusterResourceBinding is not updated to Bound") - binding := resourceBindings[0] // cluster-0 - validateNotBoundBindingState(ctx, binding) - - By("Validating the 1st stage does not have startTime set") - Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should start waiting for after Stage approval for 2nd stage in the update run when state is Run", func() { - By("Updating updateRun state to Run") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating update run is running") - // Mark 2nd stage progressing condition as false with waiting reason. - meta.SetStatusCondition(&wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - // Mark updateRun progressing condition as false with waiting reason. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should accept the approval request and start to rollout 2nd stage", func() { - By("Approving the approvalRequest") - approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) - - By("Validating the approvalRequest has ApprovalAccepted status") - Eventually(func() (bool, error) { - var approvalRequest placementv1beta1.ClusterApprovalRequest - if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { - return false, err - } - return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil - }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") - // Approval task has been approved. - wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) - }) - - It("Should mark the 1st cluster in the 2nd stage as succeeded after approving request and marking the binding available", func() { - By("Validating the 1st clusterResourceBinding is updated to Bound") - binding := resourceBindings[0] // cluster-0 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) - - By("Updating the 1st clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - // 2nd stage started. - wantStatus.StagesStatus[1].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) - meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - // 1st cluster started. - wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - - By("Validating the 1st cluster has succeeded and 2nd cluster has started") - wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Validating the 2nd stage has startTime set") - Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 2nd cluster in the 2nd stage as succeeded after marking the binding available", func() { - By("Validating the 2nd clusterResourceBinding is updated to Bound") - binding := resourceBindings[2] // cluster-2 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) - - By("Updating the 2nd clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 2nd cluster has succeeded and 3rd cluster has started") - wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 3rd cluster in the 2nd stage as succeeded after marking the binding available", func() { - By("Validating the 3rd clusterResourceBinding is updated to Bound") - binding := resourceBindings[4] // cluster-4 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) - - By("Updating the 3rd clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 3rd cluster has succeeded and 4th cluster has started") - wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 4th cluster in the 2nd stage as succeeded after marking the binding available", func() { - By("Validating the 4th clusterResourceBinding is updated to Bound") - binding := resourceBindings[6] // cluster-6 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) - - By("Updating the 4th clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 4th cluster has succeeded and 5th cluster has started") - wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should mark the 5th cluster in the 2nd stage as succeeded after marking the binding available", func() { - By("Validating the 5th clusterResourceBinding is updated to Bound") - binding := resourceBindings[8] // cluster-8 - validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) - - By("Updating the 5th clusterResourceBinding to Available") - meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) - Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") - - By("Validating the 5th cluster has succeeded and the stage waiting for AfterStageTask") - wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - // The stage progressing condition now becomes false with waiting reason. - wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) - wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - // Remove any existing waiting metric of the same generation and add to the end. - waitingMetric := generateWaitingMetric(updateRun) - wantMetrics = removeMetricFromMetricList(wantMetrics, waitingMetric) - wantMetrics = append(wantMetrics, waitingMetric) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should create approval request for 2nd stage afterStageTask", func() { - By("Validating the approvalRequest has been created") - wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: updateRun.Status.StagesStatus[1].AfterStageTaskStatus[0].ApprovalRequestName, - Labels: map[string]string{ - placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, - placementv1beta1.TargetUpdateRunLabel: updateRun.Name, - placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, - placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", - }, - }, - Spec: placementv1beta1.ApprovalRequestSpec{ - TargetUpdateRun: updateRun.Name, - TargetStage: updateRun.Status.StagesStatus[1].StageName, - }, - } - validateApprovalRequestCreated(wantApprovalRequest) - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should start stopping the update run when state is Stop", func() { - By("Updating updateRun state to Stop") - updateRun.Spec.State = placementv1beta1.StateStop - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the update run is stopped") - // Mark update run stopped. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not execute 2nd stage afterStageTask when stopped", func() { - By("Validating update run is stopped") - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should not continue to delete stage after approval when still stopped", func() { - By("Approving the approvalRequest") - approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) - - By("Validating the to-be-deleted bindings are NOT deleted") - Eventually(func() error { - for i := numTargetClusters; i < numTargetClusters+numUnscheduledClusters; i++ { - binding := &placementv1beta1.ClusterResourceBinding{} - if err := k8sClient.Get(ctx, types.NamespacedName{Name: resourceBindings[i].Name}, binding); err != nil { - return fmt.Errorf("get binding %s returned a not-found error or another error: %w", binding.Name, err) - } - } - return nil - }, timeout, interval).Should(Succeed(), "failed to validate the to-be-deleted bindings still exist") - - By("Validating update run is stopped") - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should complete the 2nd stage when update run is in Run state and move on to the delete stage", func() { - By("Updating updateRun state to Run") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") - // Update the test's want status to match the new generation. - updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) - - By("Validating the 2nd stage has completed and the delete stage has started") - wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) - wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions, - generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) - wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) - - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) - for i := range wantStatus.DeletionStageStatus.Clusters { - wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) - } - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Validating the 2nd stage has endTime set") - Expect(updateRun.Status.StagesStatus[1].EndTime).ShouldNot(BeNil()) - - By("Validating the waitTime after stage task only completes after the wait time") - waitStartTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)).LastTransitionTime.Time - waitEndTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].AfterStageTaskStatus[1].Conditions, string(placementv1beta1.StageTaskConditionWaitTimeElapsed)).LastTransitionTime.Time - Expect(waitStartTime.Add(updateStrategy.Spec.Stages[1].AfterStageTasks[1].WaitTime.Duration).After(waitEndTime)).Should(BeFalse(), - fmt.Sprintf("waitEndTime %v did not pass waitStartTime %v long enough, want at least %v", waitEndTime, waitStartTime, updateStrategy.Spec.Stages[1].AfterStageTasks[1].WaitTime.Duration)) - - By("Validating the creation time of the approval request is before the complete time of the timedwait task") - approvalCreateTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].AfterStageTaskStatus[0].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestCreated)).LastTransitionTime.Time - Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) - - By("Validating the approvalRequest has ApprovalAccepted status") - Eventually(func() (bool, error) { - var approvalRequest placementv1beta1.ClusterApprovalRequest - if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { - return false, err - } - return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil - }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - - It("Should delete all the clusterResourceBindings in the delete stage and complete the update run", func() { - By("Validating the to-be-deleted bindings are all deleted") - Eventually(func() error { - for i := numTargetClusters; i < numTargetClusters+numUnscheduledClusters; i++ { - binding := &placementv1beta1.ClusterResourceBinding{} - err := k8sClient.Get(ctx, types.NamespacedName{Name: resourceBindings[i].Name}, binding) - if err == nil { - return fmt.Errorf("binding %s is not deleted", binding.Name) - } - if !apierrors.IsNotFound(err) { - return fmt.Errorf("get binding %s does not return a not-found error: %w", binding.Name, err) - } - } - return nil - }, timeout, interval).Should(Succeed(), "failed to validate the deletion of the to-be-deleted bindings") - - By("Validating the delete stage and the clusterStagedUpdateRun has completed") - for i := range wantStatus.DeletionStageStatus.Clusters { - wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - } - // Mark the stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) - wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") - - By("Checking update run status metrics are emitted") - wantMetrics = append(wantMetrics, generateSucceededMetric(updateRun)) - validateUpdateRunMetricsEmitted(wantMetrics...) - }) - }) }) var _ = Describe("UpdateRun execution tests - single stage", func() { @@ -2238,8 +1560,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { It("Should start execution after changing the state to Run", func() { By("Updating the updateRun state to Run") - updateRun.Spec.State = placementv1beta1.StateRun - Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateRun) // Update the test's want status to match the new generation. updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) diff --git a/pkg/controllers/updaterun/stop.go b/pkg/controllers/updaterun/stop.go new file mode 100644 index 000000000..65aab2cbe --- /dev/null +++ b/pkg/controllers/updaterun/stop.go @@ -0,0 +1,237 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package updaterun + +import ( + "errors" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + + placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" + "github.com/kubefleet-dev/kubefleet/pkg/utils/controller" +) + +// stop handles stopping the update run. +func (r *Reconciler) stop( + updateRun placementv1beta1.UpdateRunObj, + updatingStageIndex int, + toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj, +) (finished bool, waitTime time.Duration, stopErr error) { + updateRunStatus := updateRun.GetUpdateRunStatus() + var updatingStageStatus *placementv1beta1.StageUpdatingStatus + + // Set up defer function to handle errStagedUpdatedAborted. + defer func() { + checkIfErrorStagedUpdateAborted(stopErr, updateRun, updatingStageStatus) + }() + + markUpdateRunStopping(updateRun) + + if updatingStageIndex < len(updateRunStatus.StagesStatus) { + return r.stopUpdatingStage(updateRun, updatingStageIndex, toBeUpdatedBindings) + } + // All the stages have finished, stop the delete stage. + finished, stopErr = r.stopDeleteStage(updateRun, toBeDeletedBindings) + return finished, clusterUpdatingWaitTime, stopErr +} + +// stopUpdatingStage stops the updating stage by letting the updating bindings finish and not starting new updates. +func (r *Reconciler) stopUpdatingStage( + updateRun placementv1beta1.UpdateRunObj, + updatingStageIndex int, + toBeUpdatedBindings []placementv1beta1.BindingObj, +) (bool, time.Duration, error) { + updateRunStatus := updateRun.GetUpdateRunStatus() + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] + updateRunRef := klog.KObj(updateRun) + // Create the map of the toBeUpdatedBindings. + toBeUpdatedBindingsMap := make(map[string]placementv1beta1.BindingObj, len(toBeUpdatedBindings)) + for _, binding := range toBeUpdatedBindings { + bindingSpec := binding.GetBindingSpec() + toBeUpdatedBindingsMap[bindingSpec.TargetCluster] = binding + } + // Mark the stage as stopping in case it's not. + markStageUpdatingStopping(updatingStageStatus, updateRun.GetGeneration()) + clusterUpdatingCount := 0 + var stuckClusterNames []string + var clusterUpdateErrors []error + // Go through each cluster in the stage and check if it's updating/succeeded/failed/not started. + for i := 0; i < len(updatingStageStatus.Clusters); i++ { + clusterStatus := &updatingStageStatus.Clusters[i] + clusterStartedCond := meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)) + if !condition.IsConditionStatusTrue(clusterStartedCond, updateRun.GetGeneration()) { + // Cluster has not started updating therefore no need to do anything. + continue + } + + clusterUpdateSucceededCond := meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) + if condition.IsConditionStatusFalse(clusterUpdateSucceededCond, updateRun.GetGeneration()) || condition.IsConditionStatusTrue(clusterUpdateSucceededCond, updateRun.GetGeneration()) { + // The cluster has already been updated or failed to update. + continue + } + + clusterUpdatingCount++ + + binding := toBeUpdatedBindingsMap[clusterStatus.ClusterName] + finished, updateErr := checkClusterUpdateResult(binding, clusterStatus, updatingStageStatus, updateRun) + if updateErr != nil { + clusterUpdateErrors = append(clusterUpdateErrors, updateErr) + } + if finished { + // The cluster has finished successfully, we can process another cluster in this round. + clusterUpdatingCount-- + } else { + // If cluster update has been running for more than "updateRunStuckThreshold", mark the update run as stuck. + timeElapsed := time.Since(clusterStartedCond.LastTransitionTime.Time) + if timeElapsed > updateRunStuckThreshold { + klog.V(2).InfoS("Time waiting for cluster update to finish passes threshold, mark the update run as stuck", "time elapsed", timeElapsed, "threshold", updateRunStuckThreshold, "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + stuckClusterNames = append(stuckClusterNames, clusterStatus.ClusterName) + } + } + } + + // If there are stuck clusters, aggregate them into an error. + aggregateUpdateRunStatus(updateRun, updatingStageStatus.StageName, stuckClusterNames) + + // Aggregate and return errors. + if len(clusterUpdateErrors) > 0 { + // Even though we aggregate errors, we can still check if one of the errors is a staged update aborted error by using errors.Is in the caller. + return false, 0, utilerrors.NewAggregate(clusterUpdateErrors) + } + + if clusterUpdatingCount == 0 { + // All the clusters in the stage have finished updating or not started. + markStageUpdatingStopped(updatingStageStatus, updateRun.GetGeneration()) + klog.InfoS("The stage has finished all clusters updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + return true, 0, nil + } + // Some clusters are still updating. + klog.InfoS("The updating stage is waiting for updating clusters to finish before completely stopping", "numberOfUpdatingClusters", clusterUpdatingCount, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + return false, clusterUpdatingWaitTime, nil +} + +// stopDeleteStage stops the delete stage by letting the deleting bindings finish. +func (r *Reconciler) stopDeleteStage( + updateRun placementv1beta1.UpdateRunObj, + toBeDeletedBindings []placementv1beta1.BindingObj, +) (bool, error) { + updateRunRef := klog.KObj(updateRun) + updateRunStatus := updateRun.GetUpdateRunStatus() + existingDeleteStageStatus := updateRunStatus.DeletionStageStatus + existingDeleteStageClusterMap := make(map[string]*placementv1beta1.ClusterUpdatingStatus, len(existingDeleteStageStatus.Clusters)) + for i := range existingDeleteStageStatus.Clusters { + existingDeleteStageClusterMap[existingDeleteStageStatus.Clusters[i].ClusterName] = &existingDeleteStageStatus.Clusters[i] + } + // Mark the delete stage as stopping in case it's not. + markStageUpdatingStopping(existingDeleteStageStatus, updateRun.GetGeneration()) + + for _, binding := range toBeDeletedBindings { + bindingSpec := binding.GetBindingSpec() + curCluster, exist := existingDeleteStageClusterMap[bindingSpec.TargetCluster] + if !exist { + // This is unexpected because we already checked in validation. + missingErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the to be deleted cluster `%s` is not in the deleting stage during stopping", bindingSpec.TargetCluster)) + klog.ErrorS(missingErr, "The cluster in the deleting stage does not include all the to be deleted binding", "updateRun", updateRunRef) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, missingErr.Error()) + } + // In validation, we already check the binding must exist in the status. + delete(existingDeleteStageClusterMap, bindingSpec.TargetCluster) + if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)), updateRun.GetGeneration()) { + // The cluster status is marked as deleted. + continue + } + if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.GetGeneration()) { + // The cluster status is marked as being deleted. + if binding.GetDeletionTimestamp().IsZero() { + // The cluster is marked as deleting but the binding is not deleting. + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the cluster `%s` in the deleting stage is marked as deleting but its corresponding binding is not deleting", curCluster.ClusterName)) + klog.ErrorS(unexpectedErr, "The binding should be deleting before we mark a cluster deleting", "clusterStatus", curCluster, "updateRun", updateRunRef) + return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) + } + continue + } + } + + // The rest of the clusters in the stage are not in the toBeDeletedBindings so it should be marked as delete succeeded. + for _, clusterStatus := range existingDeleteStageClusterMap { + // Make sure the cluster is marked as deleted. + if !condition.IsConditionStatusTrue(meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.GetGeneration()) { + markClusterUpdatingStarted(clusterStatus, updateRun.GetGeneration()) + } + markClusterUpdatingSucceeded(clusterStatus, updateRun.GetGeneration()) + } + + klog.V(2).InfoS("The delete stage is stopping", "numberOfDeletingClusters", len(toBeDeletedBindings), "updateRun", updateRunRef) + if len(toBeDeletedBindings) == 0 { + markStageUpdatingStopped(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration()) + } + return len(toBeDeletedBindings) == 0, nil +} + +// markUpdateRunStopping marks the update run as stopping in memory. +func markUpdateRunStopping(updateRun placementv1beta1.UpdateRunObj) { + klog.V(2).InfoS("Marking the update run as stopping", "updateRun", klog.KObj(updateRun)) + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionUnknown, + ObservedGeneration: updateRun.GetGeneration(), + Reason: condition.UpdateRunStoppingReason, + Message: "The update run is the process of stopping, waiting for all the updating/deleting clusters to finish updating before completing the stop process", + }) +} + +// markStageUpdatingStopping marks the stage updating status as pausing in memory. +func markStageUpdatingStopping(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { + meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StageUpdatingConditionProgressing), + Status: metav1.ConditionUnknown, + ObservedGeneration: generation, + Reason: condition.StageUpdatingStoppingReason, + Message: "Waiting for all the updating clusters to finish updating before completing the stop process", + }) +} + +// markStageUpdatingStopped marks the stage updating status as stopped in memory. +func markStageUpdatingStopped(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { + meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StageUpdatingConditionProgressing), + Status: metav1.ConditionFalse, + ObservedGeneration: generation, + Reason: condition.StageUpdatingStoppedReason, + Message: "All the updating clusters have finished updating, the stage is now stopped, waiting to be resumed", + }) +} + +func checkIfErrorStagedUpdateAborted(err error, updateRun placementv1beta1.UpdateRunObj, updatingStageStatus *placementv1beta1.StageUpdatingStatus) { + updateRunStatus := updateRun.GetUpdateRunStatus() + if errors.Is(err, errStagedUpdatedAborted) { + if updatingStageStatus != nil { + klog.InfoS("The update run is aborted due to unrecoverable behavior in updating stage, marking the stage as failed", "stage", updatingStageStatus.StageName, "updateRun", klog.KObj(updateRun)) + markStageUpdatingFailed(updatingStageStatus, updateRun.GetGeneration(), err.Error()) + } else { + // Handle deletion stage case. + markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), err.Error()) + } + } +} diff --git a/pkg/controllers/updaterun/stop_integration_test.go b/pkg/controllers/updaterun/stop_integration_test.go new file mode 100644 index 000000000..89b11d0f0 --- /dev/null +++ b/pkg/controllers/updaterun/stop_integration_test.go @@ -0,0 +1,852 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package updaterun + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + promclient "github.com/prometheus/client_model/go" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1" + placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" +) + +var _ = Describe("UpdateRun stop tests", func() { + var updateRun *placementv1beta1.ClusterStagedUpdateRun + var crp *placementv1beta1.ClusterResourcePlacement + var policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot + var updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy + var resourceBindings []*placementv1beta1.ClusterResourceBinding + var targetClusters []*clusterv1beta1.MemberCluster + var unscheduledClusters []*clusterv1beta1.MemberCluster + var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot + var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot + var wantStatus *placementv1beta1.UpdateRunStatus + var numTargetClusters int + var numUnscheduledClusters int + + BeforeEach(OncePerOrdered, func() { + testUpdateRunName = "updaterun-" + utils.RandStr() + testCRPName = "crp-" + utils.RandStr() + testResourceSnapshotName = testCRPName + "-" + testResourceSnapshotIndex + "-snapshot" + testUpdateStrategyName = "updatestrategy-" + utils.RandStr() + testCROName = "cro-" + utils.RandStr() + updateRunNamespacedName = types.NamespacedName{Name: testUpdateRunName} + + updateRun = generateTestClusterStagedUpdateRun() + crp = generateTestClusterResourcePlacement() + updateStrategy = generateTestClusterStagedUpdateStrategy() + clusterResourceOverride = generateTestClusterResourceOverride() + resourceBindings, targetClusters, unscheduledClusters = generateTestClusterResourceBindingsAndClusters(1) + policySnapshot = generateTestClusterSchedulingPolicySnapshot(1, len(targetClusters)) + resourceSnapshot = generateTestClusterResourceSnapshot() + numTargetClusters, numUnscheduledClusters = len(targetClusters), len(unscheduledClusters) + + // Set smaller wait time for testing + stageUpdatingWaitTime = time.Second * 3 + clusterUpdatingWaitTime = time.Second * 2 + + By("Creating a new clusterResourcePlacement") + Expect(k8sClient.Create(ctx, crp)).To(Succeed()) + + By("Creating scheduling policy snapshot") + Expect(k8sClient.Create(ctx, policySnapshot)).To(Succeed()) + + By("Setting the latest policy snapshot condition as fully scheduled") + meta.SetStatusCondition(&policySnapshot.Status.Conditions, metav1.Condition{ + Type: string(placementv1beta1.PolicySnapshotScheduled), + Status: metav1.ConditionTrue, + ObservedGeneration: policySnapshot.Generation, + Reason: "scheduled", + }) + Expect(k8sClient.Status().Update(ctx, policySnapshot)).Should(Succeed(), "failed to update the policy snapshot condition") + + By("Creating the member clusters") + for _, cluster := range targetClusters { + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + } + for _, cluster := range unscheduledClusters { + Expect(k8sClient.Create(ctx, cluster)).To(Succeed()) + } + + By("Creating a bunch of ClusterResourceBindings") + for _, binding := range resourceBindings { + Expect(k8sClient.Create(ctx, binding)).To(Succeed()) + } + + By("Creating a clusterStagedUpdateStrategy") + Expect(k8sClient.Create(ctx, updateStrategy)).To(Succeed()) + + By("Creating a new resource snapshot") + Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) + + By("Creating a new cluster resource override") + Expect(k8sClient.Create(ctx, clusterResourceOverride)).To(Succeed()) + }) + + AfterEach(OncePerOrdered, func() { + By("Deleting the clusterStagedUpdateRun") + Expect(k8sClient.Delete(ctx, updateRun)).Should(Succeed()) + updateRun = nil + + By("Deleting the clusterResourcePlacement") + Expect(k8sClient.Delete(ctx, crp)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + crp = nil + + By("Deleting the clusterSchedulingPolicySnapshot") + Expect(k8sClient.Delete(ctx, policySnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + policySnapshot = nil + + By("Deleting the clusterResourceBindings") + for _, binding := range resourceBindings { + Expect(k8sClient.Delete(ctx, binding)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + } + resourceBindings = nil + + By("Deleting the member clusters") + for _, cluster := range targetClusters { + Expect(k8sClient.Delete(ctx, cluster)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + } + for _, cluster := range unscheduledClusters { + Expect(k8sClient.Delete(ctx, cluster)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + } + targetClusters, unscheduledClusters = nil, nil + + By("Deleting the clusterStagedUpdateStrategy") + Expect(k8sClient.Delete(ctx, updateStrategy)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + updateStrategy = nil + + By("Deleting the clusterResourceSnapshot") + Expect(k8sClient.Delete(ctx, resourceSnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + resourceSnapshot = nil + + By("Deleting the clusterResourceOverride") + Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + clusterResourceOverride = nil + + By("Checking update run status metrics are removed") + // No metrics are emitted as all are removed after updateRun is deleted. + validateUpdateRunMetricsEmitted() + resetUpdateRunMetrics() + }) + + Context("Cluster staged update run should have stopped when state Stop", Ordered, func() { + var wantApprovalRequest *placementv1beta1.ClusterApprovalRequest + var wantMetrics []*promclient.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateRun + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and the execution has not started") + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) + wantStatus = generateExecutionNotStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the first beforeStage approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + wantMetrics = []*promclient.Metric{generateWaitingMetric(updateRun)} + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should accept the approval request and start to rollout 1st stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-1] // cluster-9 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + // 1st stage started. + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should be stopping the in middle of cluster updating when update run state is Stop", func() { + By("Updating updateRun state to Stop") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateStop) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the update run is stopping") + // 2nd cluster has started condition but no succeeded condition. + // Mark stage progressing condition as unknown with stopping reason. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateProgressingUnknownConditionWithReason(updateRun, condition.StageUpdatingStoppingReason)) + // Mark updateRun progressing condition as unknown with stopping reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateProgressingUnknownConditionWithReason(updateRun, condition.UpdateRunStoppingReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateStoppingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should wait for cluster to finish updating before update run is completely stopped", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-3] // cluster-7 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Validating the 2nd cluster has NOT succeeded and the update run is still stopping") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should have completely stopped after the in-progress cluster has finished updating", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-3] // cluster-7 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 2nd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 2nd cluster has succeeded and the update run has completely stopped") + // Mark 2nd cluster succeeded. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + // Mark stage progressing condition as false with stopped reason. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingStoppedReason)) + // Mark updateRun progressing condition as false with stopped reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should continue executing stage 1 of the update run when state is Run", func() { + By("Updating updateRun state to Run") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateRun) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating update run is running") + // Mark 3rd cluster started. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + // Mark stage progressing condition as true with progressing reason. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + // Mark updateRun progressing condition as true with progressing reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 3rd clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-5] // cluster-5 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 3rd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 3rd cluster has succeeded and 4th cluster has started") + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 4th clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-7] // cluster-3 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 4th clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 4th cluster has succeeded and 5th cluster has started") + wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 5th clusterResourceBinding is updated to Bound") + binding := resourceBindings[numTargetClusters-9] // cluster-1 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 5th clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 5th cluster has succeeded and 1st stage has completed and is waiting for AfterStageTasks") + // 5th cluster succeeded. + wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + // Now waiting for after stage tasks of 1st stage. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should have approval request created for 1st stage afterStageTask", func() { + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[0].AfterStageTaskStatus[1].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[0].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[0].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should stop the update run in afterStageTasks for 1st stage when state is Stop", func() { + By("Updating updateRun state to Stop") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateStop) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the update run is stopped") + // Mark stage progressing condition as stopped. + meta.SetStatusCondition(&wantStatus.StagesStatus[0].Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingStoppedReason)) + // Mark update run stopped. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not complete 1st stage when stopped", func() { + By("Validating the update run is still stopped") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should approve the approval request for 1st stage afterStageTask while stopped and update run stays the same", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the update run is still stopped") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should complete the 1st stage once it starts running again when wait time passed and approval request approved then move on to the 2nd stage", func() { + By("Updating updateRun state to Run") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateRun) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + + By("Validating both after stage tasks have completed and 2nd stage has started") + // Timedwait afterStageTask completed. + wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) + // Approval afterStageTask completed. + wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) + wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // 2nd stage waiting for before stage tasks. + wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has endTime set") + Expect(updateRun.Status.StagesStatus[0].EndTime).ShouldNot(BeNil()) + + By("Validating the waitTime after stage task only completes after the wait time") + waitStartTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)).LastTransitionTime.Time + waitEndTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[0].Conditions, string(placementv1beta1.StageTaskConditionWaitTimeElapsed)).LastTransitionTime.Time + Expect(waitStartTime.Add(updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration).After(waitEndTime)).Should(BeFalse(), + fmt.Sprintf("waitEndTime %v did not pass waitStartTime %v long enough, want at least %v", waitEndTime, waitStartTime, updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration)) + + By("Validating the creation time of the approval request is before the complete time of the timedwait task") + approvalCreateTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[1].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestCreated)).LastTransitionTime.Time + Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should create approval request before 2nd stage", func() { + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[1].BeforeStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.BeforeStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[1].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start rolling out 2nd stage while waiting for approval for beforeStageTask", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should stop the update run when state is Stop while waiting for 2nd stage beforeStageTask approval", func() { + By("Updating updateRun state to Stop") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateStop) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the update run is stopped") + // Mark stage progressing condition as stopped. + meta.SetStatusCondition(&wantStatus.StagesStatus[1].Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingStoppedReason)) + // Mark update run stopped. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start rolling out 2nd stage while in stopped state", func() { + By("Validating the 1st clusterResourceBinding is not updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBoundBindingState(ctx, binding) + + By("Validating the 1st stage does not have startTime set") + Expect(updateRun.Status.StagesStatus[1].StartTime).Should(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should start waiting for after Stage approval for 2nd stage in the update run when state is Run", func() { + By("Updating updateRun state to Run") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateRun) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating update run is running") + // Mark 2nd stage progressing condition as false with waiting reason. + meta.SetStatusCondition(&wantStatus.StagesStatus[1].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + // Mark updateRun progressing condition as false with waiting reason. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should accept the approval request and start to rollout 2nd stage", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + // Approval task has been approved. + wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].BeforeStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + }) + + It("Should mark the 1st cluster in the 2nd stage as succeeded after approving request and marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + // 2nd stage started. + wantStatus.StagesStatus[1].Conditions[0] = generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + // 1st cluster started. + wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 2nd stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 2nd cluster in the 2nd stage as succeeded after marking the binding available", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[2] // cluster-2 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Updating the 2nd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 2nd cluster has succeeded and 3rd cluster has started") + wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 3rd cluster in the 2nd stage as succeeded after marking the binding available", func() { + By("Validating the 3rd clusterResourceBinding is updated to Bound") + binding := resourceBindings[4] // cluster-4 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Updating the 3rd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 3rd cluster has succeeded and 4th cluster has started") + wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 4th cluster in the 2nd stage as succeeded after marking the binding available", func() { + By("Validating the 4th clusterResourceBinding is updated to Bound") + binding := resourceBindings[6] // cluster-6 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Updating the 4th clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 4th cluster has succeeded and 5th cluster has started") + wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 5th cluster in the 2nd stage as succeeded after marking the binding available", func() { + By("Validating the 5th clusterResourceBinding is updated to Bound") + binding := resourceBindings[8] // cluster-8 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 1) + + By("Updating the 5th clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 5th cluster has succeeded and the stage waiting for AfterStageTask") + wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + // The stage progressing condition now becomes false with waiting reason. + wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) + wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestCreated)) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + // Remove any existing waiting metric of the same generation and add to the end. + waitingMetric := generateWaitingMetric(updateRun) + wantMetrics = removeMetricFromMetricList(wantMetrics, waitingMetric) + wantMetrics = append(wantMetrics, waitingMetric) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should create approval request for 2nd stage afterStageTask", func() { + By("Validating the approvalRequest has been created") + wantApprovalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRun.Status.StagesStatus[1].AfterStageTaskStatus[0].ApprovalRequestName, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: updateRun.Status.StagesStatus[1].StageName, + placementv1beta1.TargetUpdateRunLabel: updateRun.Name, + placementv1beta1.TaskTypeLabel: placementv1beta1.AfterStageTaskLabelValue, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRun.Name, + TargetStage: updateRun.Status.StagesStatus[1].StageName, + }, + } + validateApprovalRequestCreated(wantApprovalRequest) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should start stopping the update run when state is Stop", func() { + By("Updating updateRun state to Stop") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateStop) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the update run is stopped") + // Mark stage progressing condition as stopped. + meta.SetStatusCondition(&wantStatus.StagesStatus[1].Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingStoppedReason)) + // Mark update run stopped. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseConditionWithReason(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunStoppedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateStoppedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not execute 2nd stage afterStageTask when stopped", func() { + By("Validating update run is stopped") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not continue to delete stage after approval when still stopped", func() { + By("Approving the approvalRequest") + approveClusterApprovalRequest(ctx, wantApprovalRequest.Name) + + By("Validating the to-be-deleted bindings are NOT deleted") + Eventually(func() error { + for i := numTargetClusters; i < numTargetClusters+numUnscheduledClusters; i++ { + binding := &placementv1beta1.ClusterResourceBinding{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: resourceBindings[i].Name}, binding); err != nil { + return fmt.Errorf("get binding %s returned a not-found error or another error: %w", binding.Name, err) + } + } + return nil + }, timeout, interval).Should(Succeed(), "failed to validate the to-be-deleted bindings still exist") + + By("Validating update run is stopped") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should complete the 2nd stage when update run is in Run state and move on to the delete stage", func() { + By("Updating updateRun state to Run") + updateRun = updateClusterStagedUpdateRunState(updateRun.Name, placementv1beta1.StateRun) + // Update the test's want status to match the new generation. + updateAllStatusConditionsGeneration(wantStatus, updateRun.Generation) + + By("Validating the 2nd stage has completed and the delete stage has started") + wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) + wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions, + generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) + wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) + wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + for i := range wantStatus.DeletionStageStatus.Clusters { + wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + } + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 2nd stage has endTime set") + Expect(updateRun.Status.StagesStatus[1].EndTime).ShouldNot(BeNil()) + + By("Validating the waitTime after stage task only completes after the wait time") + waitStartTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)).LastTransitionTime.Time + waitEndTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].AfterStageTaskStatus[1].Conditions, string(placementv1beta1.StageTaskConditionWaitTimeElapsed)).LastTransitionTime.Time + Expect(waitStartTime.Add(updateStrategy.Spec.Stages[1].AfterStageTasks[1].WaitTime.Duration).After(waitEndTime)).Should(BeFalse(), + fmt.Sprintf("waitEndTime %v did not pass waitStartTime %v long enough, want at least %v", waitEndTime, waitStartTime, updateStrategy.Spec.Stages[1].AfterStageTasks[1].WaitTime.Duration)) + + By("Validating the creation time of the approval request is before the complete time of the timedwait task") + approvalCreateTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[1].AfterStageTaskStatus[0].Conditions, string(placementv1beta1.StageTaskConditionApprovalRequestCreated)).LastTransitionTime.Time + Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) + + By("Validating the approvalRequest has ApprovalAccepted status") + Eventually(func() (bool, error) { + var approvalRequest placementv1beta1.ClusterApprovalRequest + if err := k8sClient.Get(ctx, types.NamespacedName{Name: wantApprovalRequest.Name}, &approvalRequest); err != nil { + return false, err + } + return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil + }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should delete all the clusterResourceBindings in the delete stage and complete the update run", func() { + By("Validating the to-be-deleted bindings are all deleted") + Eventually(func() error { + for i := numTargetClusters; i < numTargetClusters+numUnscheduledClusters; i++ { + binding := &placementv1beta1.ClusterResourceBinding{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: resourceBindings[i].Name}, binding) + if err == nil { + return fmt.Errorf("binding %s is not deleted", binding.Name) + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get binding %s does not return a not-found error: %w", binding.Name, err) + } + } + return nil + }, timeout, interval).Should(Succeed(), "failed to validate the deletion of the to-be-deleted bindings") + + By("Validating the delete stage and the clusterStagedUpdateRun has completed") + for i := range wantStatus.DeletionStageStatus.Clusters { + wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + } + // Mark the stage progressing condition as false with succeeded reason and add succeeded condition. + wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason)) + wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) +}) + +func updateClusterStagedUpdateRunState(updateRunName string, state placementv1beta1.State) *placementv1beta1.ClusterStagedUpdateRun { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return fmt.Errorf("failed to get ClusterStagedUpdateRun %s", updateRunName) + } + + updateRun.Spec.State = state + if err := k8sClient.Update(ctx, updateRun); err != nil { + return fmt.Errorf("failed to update ClusterStagedUpdateRun %s", updateRunName) + } + return nil + }, timeout, interval).Should(Succeed(), "Failed to update ClusterStagedUpdateRun %s state to %s", updateRunName, state) + return updateRun +} diff --git a/pkg/controllers/updaterun/stop_test.go b/pkg/controllers/updaterun/stop_test.go new file mode 100644 index 000000000..37ac692cb --- /dev/null +++ b/pkg/controllers/updaterun/stop_test.go @@ -0,0 +1,263 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package updaterun + +import ( + "errors" + "testing" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" +) + +func TestStopDeleteStage(t *testing.T) { + now := metav1.Now() + deletionTime := metav1.NewTime(now.Add(-1 * time.Minute)) + + tests := []struct { + name string + updateRun *placementv1beta1.ClusterStagedUpdateRun + toBeDeletedBindings []placementv1beta1.BindingObj + wantFinished bool + wantError bool + wantAbortError bool + wantStageStatus metav1.ConditionStatus + wantReason string + }{ + { + name: "no bindings to delete - should finish and mark stage as stopped", + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-updaterun", + Generation: 1, + }, + Status: placementv1beta1.UpdateRunStatus{ + DeletionStageStatus: &placementv1beta1.StageUpdatingStatus{ + StageName: "deletion", + Clusters: []placementv1beta1.ClusterUpdatingStatus{}, + }, + }, + }, + toBeDeletedBindings: []placementv1beta1.BindingObj{}, + wantFinished: true, + wantError: false, + wantStageStatus: metav1.ConditionFalse, + wantReason: condition.StageUpdatingStoppedReason, + }, + { + name: "cluster being deleted with proper binding deletion timestamp - should not finish", + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-updaterun", + Generation: 1, + }, + Status: placementv1beta1.UpdateRunStatus{ + DeletionStageStatus: &placementv1beta1.StageUpdatingStatus{ + StageName: "deletion", + Clusters: []placementv1beta1.ClusterUpdatingStatus{ + { + ClusterName: "cluster-1", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterUpdatingConditionStarted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + LastTransitionTime: now, + Reason: condition.ClusterUpdatingStartedReason, + }, + }, + }, + }, + }, + }, + }, + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &deletionTime, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + TargetCluster: "cluster-1", + }, + }, + }, + wantFinished: false, + wantError: false, + wantStageStatus: metav1.ConditionUnknown, + wantReason: condition.StageUpdatingStoppingReason, + }, + { + name: "cluster marked as deleting but binding not deleting - should abort", + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-updaterun", + Generation: 1, + }, + Status: placementv1beta1.UpdateRunStatus{ + DeletionStageStatus: &placementv1beta1.StageUpdatingStatus{ + StageName: "deletion", + Clusters: []placementv1beta1.ClusterUpdatingStatus{ + { + ClusterName: "cluster-1", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterUpdatingConditionStarted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + LastTransitionTime: now, + Reason: condition.ClusterUpdatingStartedReason, + }, + }, + }, + }, + }, + }, + }, + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + // No DeletionTimestamp set + }, + Spec: placementv1beta1.ResourceBindingSpec{ + TargetCluster: "cluster-1", + }, + }, + }, + wantFinished: false, + wantError: true, + wantAbortError: true, + wantStageStatus: metav1.ConditionUnknown, + wantReason: condition.StageUpdatingStoppingReason, + }, + { + name: "multiple clusters with mixed states", + updateRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-updaterun", + Generation: 1, + }, + Status: placementv1beta1.UpdateRunStatus{ + DeletionStageStatus: &placementv1beta1.StageUpdatingStatus{ + StageName: "deletion", + Clusters: []placementv1beta1.ClusterUpdatingStatus{ + { + ClusterName: "cluster-1", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterUpdatingConditionStarted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + LastTransitionTime: now, + Reason: condition.ClusterUpdatingStartedReason, + }, + { + Type: string(placementv1beta1.ClusterUpdatingConditionSucceeded), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + LastTransitionTime: now, + Reason: condition.ClusterUpdatingSucceededReason, + }, + }, + }, + { + ClusterName: "cluster-2", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterUpdatingConditionStarted), + Status: metav1.ConditionTrue, + ObservedGeneration: 1, + LastTransitionTime: now, + Reason: condition.ClusterUpdatingStartedReason, + }, + }, + }, + }, + }, + }, + }, + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &deletionTime, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + TargetCluster: "cluster-1", + }, + }, + &placementv1beta1.ClusterResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &deletionTime, + }, + Spec: placementv1beta1.ResourceBindingSpec{ + TargetCluster: "cluster-2", + }, + }, + }, + wantFinished: false, + wantError: false, + wantStageStatus: metav1.ConditionUnknown, + wantReason: condition.StageUpdatingStoppingReason, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Reconciler{} + + gotFinished, gotErr := r.stopDeleteStage(tt.updateRun, tt.toBeDeletedBindings) + + // Check finished result. + if gotFinished != tt.wantFinished { + t.Errorf("stopDeleteStage() finished = %v, want %v", gotFinished, tt.wantFinished) + } + + // Check error expectations. + if tt.wantError { + if gotErr == nil { + t.Errorf("stopDeleteStage() error = nil, want error") + } else if tt.wantAbortError && !errors.Is(gotErr, errStagedUpdatedAborted) { + t.Errorf("stopDeleteStage() error = %v, want errStagedUpdatedAborted", gotErr) + } + } else if gotErr != nil { + t.Errorf("stopDeleteStage() error = %v, want nil", gotErr) + } + + // Check stage status condition. + progressingCond := meta.FindStatusCondition( + tt.updateRun.Status.DeletionStageStatus.Conditions, + string(placementv1beta1.StageUpdatingConditionProgressing), + ) + if progressingCond == nil { + t.Errorf("stopDeleteStage() missing progressing condition") + } else { + if progressingCond.Status != tt.wantStageStatus { + t.Errorf("stopDeleteStage() progressing condition status = %v, want %v", + progressingCond.Status, tt.wantStageStatus) + } + + if progressingCond.Reason != tt.wantReason { + t.Errorf("stopDeleteStage() progressing condition reason = %v, want %v", + progressingCond.Reason, tt.wantReason) + } + } + }) + } +} diff --git a/pkg/utils/condition/reason.go b/pkg/utils/condition/reason.go index 4ad51bb6f..29d9291a2 100644 --- a/pkg/utils/condition/reason.go +++ b/pkg/utils/condition/reason.go @@ -170,6 +170,9 @@ const ( // UpdateRunWaitingReason is the reason string of condition if the staged update run is waiting for an after-stage task to complete. UpdateRunWaitingReason = "UpdateRunWaiting" + // UpdateRunStoppingReason is the reason string of condition if the staged update run stopping. + UpdateRunStoppingReason = "UpdateRunStopping" + // UpdateRunStoppedReason is the reason string of condition if the staged update run stopped. UpdateRunStoppedReason = "UpdateRunStopped" @@ -182,6 +185,12 @@ const ( // StageUpdatingWaitingReason is the reason string of condition if the stage updating is waiting. StageUpdatingWaitingReason = "StageUpdatingWaiting" + // StageUpdatingStoppingReason is the reason string of condition if the stage updating is stopping. + StageUpdatingStoppingReason = "StageUpdatingStopping" + + // StageUpdatingStoppedReason is the reason string of condition if the stage updating is stopped. + StageUpdatingStoppedReason = "StageUpdatingStopped" + // StageUpdatingFailedReason is the reason string of condition if the stage updating failed. StageUpdatingFailedReason = "StageUpdatingFailed"