operator: Use a condition to warn when labels for zone-awareness are empty (#10418)

pull/10503/head
Bayan Taani 3 years ago committed by GitHub
parent 0d542af45f
commit 53c2b2c22a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      operator/CHANGELOG.md
  2. 6
      operator/apis/loki/v1/lokistack_types.go
  3. 9
      operator/docs/operator/api.md
  4. 42
      operator/internal/status/lokistack.go
  5. 13
      operator/internal/status/lokistack_test.go

@ -1,5 +1,6 @@
## Main
- [10418](https://github.com/grafana/loki/pull/10418) **btaani**: Use a condition to warn when labels for zone-awareness are empty
- [9468](https://github.com/grafana/loki/pull/9468) **periklis**: Add support for reconciling loki-mixin dashboards on OpenShift Console
- [9942](https://github.com/grafana/loki/pull/9942) **btaani**: Use a condition to warn when there are no nodes with matching labels for zone-awareness

@ -990,8 +990,10 @@ const (
ReasonFailedCertificateRotation LokiStackConditionReason = "FailedCertificateRotation"
// ReasonQueryTimeoutInvalid when the QueryTimeout can not be parsed.
ReasonQueryTimeoutInvalid LokiStackConditionReason = "ReasonQueryTimeoutInvalid"
// ReasonNoZoneAwareNodes when the cluster does not contain any nodes with the labels needed for zone-awareness.
ReasonNoZoneAwareNodes LokiStackConditionReason = "ReasonNoZoneAwareNodes"
// ReasonZoneAwareNodesMissing when the cluster does not contain any nodes with the labels needed for zone-awareness.
ReasonZoneAwareNodesMissing LokiStackConditionReason = "ReasonZoneAwareNodesMissing"
// ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.
ReasonZoneAwareEmptyLabel LokiStackConditionReason = "ReasonZoneAwareEmptyLabel"
)
// PodStatusMap defines the type for mapping pod status to pod name.

@ -1657,9 +1657,6 @@ storage is missing.</p>
<td><p>ReasonMissingRulerSecret when the required secret to authorization remote write connections
for the ruler is missing.</p>
</td>
</tr><tr><td><p>&#34;ReasonNoZoneAwareNodes&#34;</p></td>
<td><p>ReasonNoZoneAwareNodes when the cluster does not contain any nodes with the labels needed for zone-awareness.</p>
</td>
</tr><tr><td><p>&#34;PendingComponents&#34;</p></td>
<td><p>ReasonPendingComponents when all/some LokiStack components pending dependencies</p>
</td>
@ -1669,6 +1666,12 @@ for the ruler is missing.</p>
</tr><tr><td><p>&#34;ReadyComponents&#34;</p></td>
<td><p>ReasonReadyComponents when all LokiStack components are ready to serve traffic.</p>
</td>
</tr><tr><td><p>&#34;ReasonZoneAwareEmptyLabel&#34;</p></td>
<td><p>ReasonZoneAwareEmptyLabel when the node-label used for zone-awareness has an empty value.</p>
</td>
</tr><tr><td><p>&#34;ReasonZoneAwareNodesMissing&#34;</p></td>
<td><p>ReasonZoneAwareNodesMissing when the cluster does not contain any nodes with the labels needed for zone-awareness.</p>
</td>
</tr></tbody>
</table>

@ -17,10 +17,11 @@ import (
)
const (
messageReady = "All components ready"
messageFailed = "Some LokiStack components failed"
messagePending = "Some LokiStack components pending on dependencies"
messageDegradedNodeLabels = "Cluster contains no nodes matching the labels used for zone-awareness"
messageReady = "All components ready"
messageFailed = "Some LokiStack components failed"
messagePending = "Some LokiStack components pending on dependencies"
messageDegradedMissingNodes = "Cluster contains no nodes matching the labels used for zone-awareness"
messageDegradedEmptyNodeLabel = "No value for the labels used for zone-awareness"
)
var (
@ -41,8 +42,13 @@ var (
}
conditionDegradedNodeLabels = metav1.Condition{
Type: string(lokiv1.ConditionDegraded),
Message: messageDegradedNodeLabels,
Reason: string(lokiv1.ReasonNoZoneAwareNodes),
Message: messageDegradedMissingNodes,
Reason: string(lokiv1.ReasonZoneAwareNodesMissing),
}
conditionDegradedEmptyNodeLabel = metav1.Condition{
Type: string(lokiv1.ConditionDegraded),
Message: messageDegradedEmptyNodeLabel,
Reason: string(lokiv1.ReasonZoneAwareEmptyLabel),
}
)
@ -97,7 +103,7 @@ func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus,
if stack.Spec.Replication != nil && len(stack.Spec.Replication.Zones) > 0 {
// When there are pending pods and zone-awareness is enabled check if there are any nodes
// that can satisfy the constraints and emit a condition if not.
nodesOk, err := checkForZoneawareNodes(ctx, k, stack.Spec.Replication.Zones)
nodesOk, labelsOk, err := checkForZoneawareNodes(ctx, k, stack.Spec.Replication.Zones)
if err != nil {
return metav1.Condition{}, err
}
@ -105,6 +111,10 @@ func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus,
if !nodesOk {
return conditionDegradedNodeLabels, nil
}
if !labelsOk {
return conditionDegradedEmptyNodeLabel, nil
}
}
return conditionPending, nil
@ -113,7 +123,7 @@ func generateCondition(ctx context.Context, cs *lokiv1.LokiStackComponentStatus,
return conditionReady, nil
}
func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1.ZoneSpec) (bool, error) {
func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1.ZoneSpec) (nodesOk bool, labelsOk bool, err error) {
nodeLabels := client.HasLabels{}
for _, z := range zones {
nodeLabels = append(nodeLabels, z.TopologyKey)
@ -121,10 +131,22 @@ func checkForZoneawareNodes(ctx context.Context, k client.Client, zones []lokiv1
nodeList := &corev1.NodeList{}
if err := k.List(ctx, nodeList, nodeLabels); err != nil {
return false, err
return false, false, err
}
if len(nodeList.Items) == 0 {
return false, false, nil
}
for _, node := range nodeList.Items {
for _, nodeLabel := range nodeLabels {
if node.Labels[nodeLabel] == "" {
return true, false, nil
}
}
}
return len(nodeList.Items) > 0, nil
return true, true, nil
}
func updateCondition(ctx context.Context, k k8s.Client, req ctrl.Request, condition metav1.Condition) error {

@ -227,10 +227,21 @@ func TestGenerateCondition_ZoneAwareLokiStack(t *testing.T) {
{
desc: "nodes available",
nodes: []corev1.Node{
{},
{ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"topology-key": "value"},
}},
},
wantCondition: conditionPending,
},
{
desc: "nodes available but empty label value",
nodes: []corev1.Node{
{ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"topology-key": ""},
}},
},
wantCondition: conditionDegradedEmptyNodeLabel,
},
{
desc: "no nodes available",
nodes: []corev1.Node{},

Loading…
Cancel
Save