From 7045c92086b5f82f0d1d758e830b091e47014bff Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Sat, 22 May 2021 07:46:12 +0200 Subject: [PATCH] Set lokistack condition failed/pending/ready based on pod status map (#53) --- api/v1beta1/lokistack_types.go | 14 +- controllers/lokistack_controller.go | 4 +- .../k8s/k8sfakes/fake_client_extensions.go | 13 + internal/handlers/internal/status/status.go | 38 - .../handlers/internal/status/status_test.go | 53 -- .../handlers/lokistack_create_or_update.go | 6 +- .../status/components.go | 0 internal/status/components_test.go | 162 ++++ internal/status/lokistack.go | 198 +++++ internal/status/lokistack_test.go | 691 ++++++++++++++++++ internal/status/status.go | 61 ++ 11 files changed, 1142 insertions(+), 98 deletions(-) delete mode 100644 internal/handlers/internal/status/status.go delete mode 100644 internal/handlers/internal/status/status_test.go rename {controllers/internal => internal}/status/components.go (100%) create mode 100644 internal/status/components_test.go create mode 100644 internal/status/lokistack.go create mode 100644 internal/status/lokistack_test.go create mode 100644 internal/status/status.go diff --git a/api/v1beta1/lokistack_types.go b/api/v1beta1/lokistack_types.go index 0ba8cebc6c..6e4d1fe0d1 100644 --- a/api/v1beta1/lokistack_types.go +++ b/api/v1beta1/lokistack_types.go @@ -340,6 +340,12 @@ const ( // ConditionReady defines the condition that all components in the Loki deployment are ready. ConditionReady LokiStackConditionType = "Ready" + // ConditionPending defines the conditioin that some or all components are in pending state. + ConditionPending LokiStackConditionType = "Pending" + + // ConditionFailed defines the condition that components in the Loki deployment failed to roll out. + ConditionFailed LokiStackConditionType = "Failed" + // ConditionDegraded defines the condition that some or all components in the Loki deployment // are degraded or the cluster cannot connect to object storage. ConditionDegraded LokiStackConditionType = "Degraded" @@ -349,13 +355,17 @@ const ( type LokiStackConditionReason string const ( + // ReasonFailedComponents when all/some LokiStack components fail to roll out. + ReasonFailedComponents LokiStackConditionReason = "FailedComponents" + // ReasonPendingComponents when all/some LokiStack components pending dependencies + ReasonPendingComponents LokiStackConditionReason = "PendingComponents" + // ReasonReadyComponents when all LokiStack components are ready to serve traffic. + ReasonReadyComponents LokiStackConditionReason = "ReadyComponents" // ReasonMissingObjectStorageSecret when the required secret to store logs to object // storage is missing. ReasonMissingObjectStorageSecret LokiStackConditionReason = "MissingObjectStorageSecret" - // ReasonInvalidObjectStorageSecret when the format of the secret is invalid. ReasonInvalidObjectStorageSecret LokiStackConditionReason = "InvalidObjectStorageSecret" - // ReasonInvalidReplicationConfiguration when the configurated replication factor is not valid // with the select cluster size. ReasonInvalidReplicationConfiguration LokiStackConditionReason = "InvalidReplicationConfiguration" diff --git a/controllers/lokistack_controller.go b/controllers/lokistack_controller.go index 98d654eedf..81858351f2 100644 --- a/controllers/lokistack_controller.go +++ b/controllers/lokistack_controller.go @@ -21,9 +21,9 @@ import ( "time" "github.com/ViaQ/loki-operator/controllers/internal/management/state" - "github.com/ViaQ/loki-operator/controllers/internal/status" "github.com/ViaQ/loki-operator/internal/external/k8s" "github.com/ViaQ/loki-operator/internal/handlers" + "github.com/ViaQ/loki-operator/internal/status" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -117,7 +117,7 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( }, err } - err = status.SetComponentsStatus(ctx, r.Client, req) + err = status.Refresh(ctx, r.Client, req) if err != nil { return ctrl.Result{ Requeue: true, diff --git a/internal/external/k8s/k8sfakes/fake_client_extensions.go b/internal/external/k8s/k8sfakes/fake_client_extensions.go index 13276d9cf0..32b0aa3817 100644 --- a/internal/external/k8s/k8sfakes/fake_client_extensions.go +++ b/internal/external/k8s/k8sfakes/fake_client_extensions.go @@ -18,3 +18,16 @@ import ( func (fake *FakeClient) SetClientObject(out, v client.Object) { reflect.Indirect(reflect.ValueOf(out)).Set(reflect.ValueOf(v).Elem()) } + +// SetClientObjectList sets out list to v. +// This is primarily used within the GetStub to fake the object returned from the API to the vaule of v +// +// Examples: +// +// k.GetStub = func(_ context.Context, _ types.NamespacedName, list client.ObjectList) error { +// k.SetClientObjectList(list, &podList) +// return nil +// } +func (fake *FakeClient) SetClientObjectList(out, v client.ObjectList) { + reflect.Indirect(reflect.ValueOf(out)).Set(reflect.ValueOf(v).Elem()) +} diff --git a/internal/handlers/internal/status/status.go b/internal/handlers/internal/status/status.go deleted file mode 100644 index 4aae7670fe..0000000000 --- a/internal/handlers/internal/status/status.go +++ /dev/null @@ -1,38 +0,0 @@ -package status - -import ( - "context" - - lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" - "github.com/ViaQ/loki-operator/internal/external/k8s" - "sigs.k8s.io/controller-runtime/pkg/client" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// SetDegradedCondition appends the condition Degraded to the lokistack status conditions. -func SetDegradedCondition(ctx context.Context, k k8s.Client, s *lokiv1beta1.LokiStack, msg string, reason lokiv1beta1.LokiStackConditionReason) error { - reasonStr := string(reason) - for _, cond := range s.Status.Conditions { - if cond.Type == string(lokiv1beta1.ConditionDegraded) && cond.Reason == reasonStr { - return nil - } - } - - status := s.Status.DeepCopy() - if status.Conditions == nil { - status.Conditions = []metav1.Condition{} - } - - degraded := metav1.Condition{ - Type: string(lokiv1beta1.ConditionDegraded), - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - Reason: reasonStr, - Message: msg, - } - - status.Conditions = append(status.Conditions, degraded) - s.Status = *status - return k.Status().Update(ctx, s, &client.UpdateOptions{}) -} diff --git a/internal/handlers/internal/status/status_test.go b/internal/handlers/internal/status/status_test.go deleted file mode 100644 index 04d7537b92..0000000000 --- a/internal/handlers/internal/status/status_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package status_test - -import ( - "context" - "testing" - - lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" - "github.com/ViaQ/loki-operator/internal/external/k8s/k8sfakes" - "github.com/ViaQ/loki-operator/internal/handlers/internal/status" - - "github.com/stretchr/testify/require" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func TestSetDegradedCondition_WhenExisting_DoNothing(t *testing.T) { - k := &k8sfakes.FakeClient{} - - msg := "tell me nothing" - reason := lokiv1beta1.ReasonMissingObjectStorageSecret - s := lokiv1beta1.LokiStack{ - Status: lokiv1beta1.LokiStackStatus{ - Conditions: []metav1.Condition{ - { - Type: string(lokiv1beta1.ConditionDegraded), - Reason: string(reason), - }, - }, - }, - } - - err := status.SetDegradedCondition(context.TODO(), k, &s, msg, reason) - require.NoError(t, err) -} - -func TestSetDegradedCondition_WhenNoneExisting_AppendDegradedCondition(t *testing.T) { - sw := &k8sfakes.FakeStatusWriter{} - k := &k8sfakes.FakeClient{} - - k.StatusStub = func() client.StatusWriter { return sw } - - msg := "tell me something" - reason := lokiv1beta1.ReasonMissingObjectStorageSecret - s := lokiv1beta1.LokiStack{} - - err := status.SetDegradedCondition(context.TODO(), k, &s, msg, reason) - require.NoError(t, err) - - require.NotZero(t, k.StatusCallCount()) - require.NotZero(t, sw.UpdateCallCount()) - require.NotEmpty(t, s.Status.Conditions) -} diff --git a/internal/handlers/lokistack_create_or_update.go b/internal/handlers/lokistack_create_or_update.go index e649083ffb..b6e115ee19 100644 --- a/internal/handlers/lokistack_create_or_update.go +++ b/internal/handlers/lokistack_create_or_update.go @@ -11,8 +11,8 @@ import ( lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" "github.com/ViaQ/loki-operator/internal/external/k8s" "github.com/ViaQ/loki-operator/internal/handlers/internal/secrets" - "github.com/ViaQ/loki-operator/internal/handlers/internal/status" "github.com/ViaQ/loki-operator/internal/manifests" + "github.com/ViaQ/loki-operator/internal/status" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -44,7 +44,7 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client key := client.ObjectKey{Name: stack.Spec.Storage.Secret.Name, Namespace: stack.Namespace} if err := k.Get(ctx, key, &s3secret); err != nil { if apierrors.IsNotFound(err) { - return status.SetDegradedCondition(ctx, k, &stack, + return status.SetDegradedCondition(ctx, k, req, "Missing object storage secret", lokiv1beta1.ReasonMissingObjectStorageSecret, ) @@ -54,7 +54,7 @@ func CreateOrUpdateLokiStack(ctx context.Context, req ctrl.Request, k k8s.Client storage, err := secrets.Extract(&s3secret) if err != nil { - return status.SetDegradedCondition(ctx, k, &stack, + return status.SetDegradedCondition(ctx, k, req, "Invalid object storage secret contents", lokiv1beta1.ReasonInvalidObjectStorageSecret, ) diff --git a/controllers/internal/status/components.go b/internal/status/components.go similarity index 100% rename from controllers/internal/status/components.go rename to internal/status/components.go diff --git a/internal/status/components_test.go b/internal/status/components_test.go new file mode 100644 index 0000000000..91328760ff --- /dev/null +++ b/internal/status/components_test.go @@ -0,0 +1,162 @@ +package status_test + +import ( + "context" + "testing" + + lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" + "github.com/ViaQ/loki-operator/internal/external/k8s/k8sfakes" + "github.com/ViaQ/loki-operator/internal/status" + "github.com/stretchr/testify/require" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestSetComponentsStatus_WhenGetLokiStackReturnsError_ReturnError(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewBadRequest("something wasn't found") + } + + err := status.SetComponentsStatus(context.TODO(), k, r) + require.Error(t, err) +} + +func TestSetComponentsStatus_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetComponentsStatus(context.TODO(), k, r) + require.NoError(t, err) +} + +func TestSetComponentsStatus_WhenListReturnError_ReturnError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.ListStub = func(_ context.Context, l client.ObjectList, opts ...client.ListOption) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetComponentsStatus(context.TODO(), k, r) + require.Error(t, err) +} + +func TestSetComponentsStatus_WhenPodListExisting_SetPodStatusMap(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.ListStub = func(_ context.Context, l client.ObjectList, _ ...client.ListOption) error { + pods := v1.PodList{ + Items: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-a", + }, + Status: v1.PodStatus{ + Phase: v1.PodPending, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-b", + }, + Status: v1.PodStatus{ + Phase: v1.PodRunning, + }, + }, + }, + } + k.SetClientObjectList(l, &pods) + return nil + } + + expected := lokiv1beta1.PodStatusMap{ + "Pending": []string{"pod-a"}, + "Running": []string{"pod-b"}, + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + stack := obj.(*lokiv1beta1.LokiStack) + require.Equal(t, expected, stack.Status.Components.Compactor) + return nil + } + + err := status.SetComponentsStatus(context.TODO(), k, r) + require.NoError(t, err) + require.NotZero(t, k.ListCallCount()) + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} diff --git a/internal/status/lokistack.go b/internal/status/lokistack.go new file mode 100644 index 0000000000..cbf8859503 --- /dev/null +++ b/internal/status/lokistack.go @@ -0,0 +1,198 @@ +package status + +import ( + "context" + + "github.com/ViaQ/logerr/kverrors" + lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" + "github.com/ViaQ/loki-operator/internal/external/k8s" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// SetReadyCondition updates or appends the condition Ready to the lokistack status conditions. +// In addition it resets all other Status conditions to false. +func SetReadyCondition(ctx context.Context, k k8s.Client, req ctrl.Request) error { + var s lokiv1beta1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &s); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + } + + for _, cond := range s.Status.Conditions { + if cond.Type == string(lokiv1beta1.ConditionReady) && cond.Status == metav1.ConditionTrue { + return nil + } + } + + ready := metav1.Condition{ + Type: string(lokiv1beta1.ConditionReady), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Message: "All components ready", + Reason: string(lokiv1beta1.ReasonReadyComponents), + } + + index := -1 + for i := range s.Status.Conditions { + // Reset all other conditions first + s.Status.Conditions[i].Status = metav1.ConditionFalse + s.Status.Conditions[i].LastTransitionTime = metav1.Now() + + // Locate existing ready condition if any + if s.Status.Conditions[i].Type == string(lokiv1beta1.ConditionReady) { + index = i + } + } + + if index == -1 { + s.Status.Conditions = append(s.Status.Conditions, ready) + } else { + s.Status.Conditions[index] = ready + } + + return k.Status().Update(ctx, &s, &client.UpdateOptions{}) +} + +// SetFailedCondition updates or appends the condition Failed to the lokistack status conditions. +// In addition it resets all other Status conditions to false. +func SetFailedCondition(ctx context.Context, k k8s.Client, req ctrl.Request) error { + var s lokiv1beta1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &s); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + } + + for _, cond := range s.Status.Conditions { + if cond.Type == string(lokiv1beta1.ConditionFailed) && cond.Status == metav1.ConditionTrue { + return nil + } + } + + failed := metav1.Condition{ + Type: string(lokiv1beta1.ConditionFailed), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Message: "Some LokiStack components failed", + Reason: string(lokiv1beta1.ReasonFailedComponents), + } + + index := -1 + for i := range s.Status.Conditions { + // Reset all other conditions first + s.Status.Conditions[i].Status = metav1.ConditionFalse + s.Status.Conditions[i].LastTransitionTime = metav1.Now() + + // Locate existing failed condition if any + if s.Status.Conditions[i].Type == string(lokiv1beta1.ConditionFailed) { + index = i + } + } + + if index == -1 { + s.Status.Conditions = append(s.Status.Conditions, failed) + } else { + s.Status.Conditions[index] = failed + } + + return k.Status().Update(ctx, &s, &client.UpdateOptions{}) +} + +// SetPendingCondition updates or appends the condition Pending to the lokistack status conditions. +// In addition it resets all other Status conditions to false. +func SetPendingCondition(ctx context.Context, k k8s.Client, req ctrl.Request) error { + var s lokiv1beta1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &s); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + } + + for _, cond := range s.Status.Conditions { + if cond.Type == string(lokiv1beta1.ConditionPending) && cond.Status == metav1.ConditionTrue { + return nil + } + } + + pending := metav1.Condition{ + Type: string(lokiv1beta1.ConditionPending), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Message: "Some LokiStack components pending on dependendies", + Reason: string(lokiv1beta1.ReasonPendingComponents), + } + + index := -1 + for i := range s.Status.Conditions { + // Reset all other conditions first + s.Status.Conditions[i].Status = metav1.ConditionFalse + s.Status.Conditions[i].LastTransitionTime = metav1.Now() + + // Locate existing pending condition if any + if s.Status.Conditions[i].Type == string(lokiv1beta1.ConditionPending) { + index = i + } + } + + if index == -1 { + s.Status.Conditions = append(s.Status.Conditions, pending) + } else { + s.Status.Conditions[index] = pending + } + + return k.Status().Update(ctx, &s, &client.UpdateOptions{}) +} + +// SetDegradedCondition appends the condition Degraded to the lokistack status conditions. +func SetDegradedCondition(ctx context.Context, k k8s.Client, req ctrl.Request, msg string, reason lokiv1beta1.LokiStackConditionReason) error { + var s lokiv1beta1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &s); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + } + + reasonStr := string(reason) + for _, cond := range s.Status.Conditions { + if cond.Type == string(lokiv1beta1.ConditionDegraded) && cond.Reason == reasonStr && cond.Status == metav1.ConditionTrue { + return nil + } + } + + degraded := metav1.Condition{ + Type: string(lokiv1beta1.ConditionDegraded), + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.Now(), + Reason: reasonStr, + Message: msg, + } + + index := -1 + for i := range s.Status.Conditions { + // Reset all other conditions first + s.Status.Conditions[i].Status = metav1.ConditionFalse + s.Status.Conditions[i].LastTransitionTime = metav1.Now() + + // Locate existing pending condition if any + if s.Status.Conditions[i].Type == string(lokiv1beta1.ConditionDegraded) { + index = i + } + } + + if index == -1 { + s.Status.Conditions = append(s.Status.Conditions, degraded) + } else { + s.Status.Conditions[index] = degraded + } + + return k.Status().Update(ctx, &s, &client.UpdateOptions{}) +} diff --git a/internal/status/lokistack_test.go b/internal/status/lokistack_test.go new file mode 100644 index 0000000000..44e926c295 --- /dev/null +++ b/internal/status/lokistack_test.go @@ -0,0 +1,691 @@ +package status_test + +import ( + "context" + "testing" + + lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" + "github.com/ViaQ/loki-operator/internal/external/k8s/k8sfakes" + "github.com/ViaQ/loki-operator/internal/status" + + "github.com/stretchr/testify/require" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestSetReadyCondition_WhenGetLokiStackReturnsError_ReturnError(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewBadRequest("something wasn't found") + } + + err := status.SetReadyCondition(context.TODO(), k, r) + require.Error(t, err) +} + +func TestSetReadyCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetReadyCondition(context.TODO(), k, r) + require.NoError(t, err) +} + +func TestSetReadyCondition_WhenExisting_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionReady), + Status: metav1.ConditionTrue, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetReadyCondition(context.TODO(), k, r) + require.NoError(t, err) + require.Zero(t, k.StatusCallCount()) +} + +func TestSetReadyCondition_WhenExisting_SetReadyConditionTrue(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionReady), + Status: metav1.ConditionFalse, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + require.Equal(t, metav1.ConditionTrue, actual.Status.Conditions[0].Status) + return nil + } + + err := status.SetReadyCondition(context.TODO(), k, r) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetReadyCondition_WhenNoneExisting_AppendReadyCondition(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + return nil + } + + err := status.SetReadyCondition(context.TODO(), k, r) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetFailedCondition_WhenGetLokiStackReturnsError_ReturnError(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewBadRequest("something wasn't found") + } + + err := status.SetFailedCondition(context.TODO(), k, r) + require.Error(t, err) +} + +func TestSetFailedCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetFailedCondition(context.TODO(), k, r) + require.NoError(t, err) +} + +func TestSetFailedCondition_WhenExisting_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionFailed), + Status: metav1.ConditionTrue, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetFailedCondition(context.TODO(), k, r) + require.NoError(t, err) + require.Zero(t, k.StatusCallCount()) +} + +func TestSetFailedCondition_WhenExisting_SetFailedConditionTrue(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionFailed), + Status: metav1.ConditionFalse, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + require.Equal(t, metav1.ConditionTrue, actual.Status.Conditions[0].Status) + return nil + } + + err := status.SetFailedCondition(context.TODO(), k, r) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetFailedCondition_WhenNoneExisting_AppendFailedCondition(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + return nil + } + + err := status.SetFailedCondition(context.TODO(), k, r) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetDegradedCondition_WhenGetLokiStackReturnsError_ReturnError(t *testing.T) { + k := &k8sfakes.FakeClient{} + + msg := "tell me nothing" + reason := lokiv1beta1.ReasonMissingObjectStorageSecret + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewBadRequest("something wasn't found") + } + + err := status.SetDegradedCondition(context.TODO(), k, r, msg, reason) + require.Error(t, err) +} + +func TestSetPendingCondition_WhenGetLokiStackReturnsError_ReturnError(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewBadRequest("something wasn't found") + } + + err := status.SetPendingCondition(context.TODO(), k, r) + require.Error(t, err) +} + +func TestSetPendingCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetPendingCondition(context.TODO(), k, r) + require.NoError(t, err) +} + +func TestSetPendingCondition_WhenExisting_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionPending), + Status: metav1.ConditionTrue, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetPendingCondition(context.TODO(), k, r) + require.NoError(t, err) + require.Zero(t, k.StatusCallCount()) +} + +func TestSetPendingCondition_WhenExisting_SetPendingConditionTrue(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionPending), + Status: metav1.ConditionFalse, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + require.Equal(t, metav1.ConditionTrue, actual.Status.Conditions[0].Status) + return nil + } + + err := status.SetPendingCondition(context.TODO(), k, r) + require.NoError(t, err) + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetPendingCondition_WhenNoneExisting_AppendPendingCondition(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + return nil + } + + err := status.SetPendingCondition(context.TODO(), k, r) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetDegradedCondition_WhenGetLokiStackReturnsNotFound_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + msg := "tell me nothing" + reason := lokiv1beta1.ReasonMissingObjectStorageSecret + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetDegradedCondition(context.TODO(), k, r, msg, reason) + require.NoError(t, err) +} + +func TestSetDegradedCondition_WhenExisting_DoNothing(t *testing.T) { + k := &k8sfakes.FakeClient{} + + msg := "tell me nothing" + reason := lokiv1beta1.ReasonMissingObjectStorageSecret + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionDegraded), + Reason: string(reason), + Status: metav1.ConditionTrue, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + err := status.SetDegradedCondition(context.TODO(), k, r, msg, reason) + require.NoError(t, err) + require.Zero(t, k.StatusCallCount()) +} + +func TestSetDegradedCondition_WhenExisting_SetDegradedConditionTrue(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + msg := "tell me something" + reason := lokiv1beta1.ReasonMissingObjectStorageSecret + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + Status: lokiv1beta1.LokiStackStatus{ + Conditions: []metav1.Condition{ + { + Type: string(lokiv1beta1.ConditionDegraded), + Reason: string(reason), + Status: metav1.ConditionFalse, + }, + }, + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + require.Equal(t, metav1.ConditionTrue, actual.Status.Conditions[0].Status) + return nil + } + + err := status.SetDegradedCondition(context.TODO(), k, r, msg, reason) + require.NoError(t, err) + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} + +func TestSetDegradedCondition_WhenNoneExisting_AppendDegradedCondition(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + + k.StatusStub = func() client.StatusWriter { return sw } + + msg := "tell me something" + reason := lokiv1beta1.ReasonMissingObjectStorageSecret + s := lokiv1beta1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &s) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + sw.UpdateStub = func(_ context.Context, obj client.Object, _ ...client.UpdateOption) error { + actual := obj.(*lokiv1beta1.LokiStack) + require.NotEmpty(t, actual.Status.Conditions) + return nil + } + + err := status.SetDegradedCondition(context.TODO(), k, r, msg, reason) + require.NoError(t, err) + + require.NotZero(t, k.StatusCallCount()) + require.NotZero(t, sw.UpdateCallCount()) +} diff --git a/internal/status/status.go b/internal/status/status.go new file mode 100644 index 0000000000..77c9f829b3 --- /dev/null +++ b/internal/status/status.go @@ -0,0 +1,61 @@ +package status + +import ( + "context" + + "github.com/ViaQ/logerr/kverrors" + lokiv1beta1 "github.com/ViaQ/loki-operator/api/v1beta1" + "github.com/ViaQ/loki-operator/internal/external/k8s" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" +) + +// Refresh executes an aggregate update of the LokiStack Status struct, i.e. +// - It recreates the Status.Components pod status map per component. +// - It sets the appropriate Status.Condition to true that matches the pod status maps. +func Refresh(ctx context.Context, k k8s.Client, req ctrl.Request) error { + if err := SetComponentsStatus(ctx, k, req); err != nil { + return err + } + + var s lokiv1beta1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &s); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.NamespacedName) + } + + cs := s.Status.Components + + // Check for failed pods first + failed := len(cs.Compactor[corev1.PodFailed]) + + len(cs.Distributor[corev1.PodFailed]) + + len(cs.Ingester[corev1.PodFailed]) + + len(cs.Querier[corev1.PodFailed]) + + len(cs.QueryFrontend[corev1.PodFailed]) + + unknown := len(cs.Compactor[corev1.PodUnknown]) + + len(cs.Distributor[corev1.PodUnknown]) + + len(cs.Ingester[corev1.PodUnknown]) + + len(cs.Querier[corev1.PodUnknown]) + + len(cs.QueryFrontend[corev1.PodUnknown]) + + if failed != 0 || unknown != 0 { + return SetFailedCondition(ctx, k, req) + } + + // Check for pending pods + pending := len(cs.Compactor[corev1.PodPending]) + + len(cs.Distributor[corev1.PodPending]) + + len(cs.Ingester[corev1.PodPending]) + + len(cs.Querier[corev1.PodPending]) + + len(cs.QueryFrontend[corev1.PodPending]) + + if pending != 0 { + return SetPendingCondition(ctx, k, req) + } + return SetReadyCondition(ctx, k, req) +}