loki/operator/internal/manifests/node_placement_test.go

package manifests

import (
	"testing"

	configv1 "github.com/grafana/loki/operator/apis/config/v1"
	lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
	"github.com/grafana/loki/operator/internal/manifests/storage"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestTolerationsAreSetForEachComponent(t *testing.T) {
	tolerations := []corev1.Toleration{{
		Key:      "type",
		Operator: corev1.TolerationOpEqual,
		Value:    "storage",
		Effect:   corev1.TaintEffectNoSchedule,
	}}
	optsWithTolerations := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Tolerations: tolerations,
					Replicas:    1,
				},
			},
		},
		ObjectStorage: storage.Options{},
	}

	optsWithoutTolerations := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
			},
		},
		ObjectStorage: storage.Options{},
	}

	t.Run("distributor", func(t *testing.T) {
		assert.Equal(t, tolerations, NewDistributorDeployment(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewDistributorDeployment(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("query_frontend", func(t *testing.T) {
		assert.Equal(t, tolerations, NewQueryFrontendDeployment(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewQueryFrontendDeployment(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("querier", func(t *testing.T) {
		assert.Equal(t, tolerations, NewQuerierDeployment(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewQuerierDeployment(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("ingester", func(t *testing.T) {
		assert.Equal(t, tolerations, NewIngesterStatefulSet(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewIngesterStatefulSet(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("compactor", func(t *testing.T) {
		assert.Equal(t, tolerations, NewCompactorStatefulSet(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewCompactorStatefulSet(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("index_gateway", func(t *testing.T) {
		assert.Equal(t, tolerations, NewIndexGatewayStatefulSet(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewIndexGatewayStatefulSet(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})

	t.Run("ruler", func(t *testing.T) {
		assert.Equal(t, tolerations, NewRulerStatefulSet(optsWithTolerations).Spec.Template.Spec.Tolerations)
		assert.Empty(t, NewRulerStatefulSet(optsWithoutTolerations).Spec.Template.Spec.Tolerations)
	})
}

func TestNodeSelectorsAreSetForEachComponent(t *testing.T) {
	nodeSelectors := map[string]string{"type": "storage"}
	optsWithNodeSelectors := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					NodeSelector: nodeSelectors,
					Replicas:     1,
				},
			},
		},
		ObjectStorage: storage.Options{},
	}

	optsWithoutNodeSelectors := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
			},
		},
		ObjectStorage: storage.Options{},
	}

	t.Run("distributor", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewDistributorDeployment(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewDistributorDeployment(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("query_frontend", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewQueryFrontendDeployment(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewQueryFrontendDeployment(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("querier", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewQuerierDeployment(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewQuerierDeployment(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("ingester", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewIngesterStatefulSet(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewIngesterStatefulSet(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("compactor", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewCompactorStatefulSet(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewCompactorStatefulSet(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("index_gateway", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewIndexGatewayStatefulSet(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewIndexGatewayStatefulSet(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})

	t.Run("ruler", func(t *testing.T) {
		assert.Equal(t, nodeSelectors, NewRulerStatefulSet(optsWithNodeSelectors).Spec.Template.Spec.NodeSelector)
		assert.Empty(t, NewRulerStatefulSet(optsWithoutNodeSelectors).Spec.Template.Spec.NodeSelector)
	})
}

func TestDefaultNodeAffinityForEachComponent(t *testing.T) {
	nodeAffinity := &corev1.NodeAffinity{
		RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
			NodeSelectorTerms: []corev1.NodeSelectorTerm{
				{
					MatchExpressions: []corev1.NodeSelectorRequirement{
						{
							Key:      "kubernetes.io/os",
							Operator: "In",
							Values: []string{
								"linux",
							},
						},
					},
				},
			},
		},
	}
	optsWithNoDefaultAffinity := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
			},
		},
	}
	optsWithDefaultAffinity := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
			},
		},
		Gates: configv1.FeatureGates{
			DefaultNodeAffinity: true,
		},
	}

	t.Run("distributor", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewDistributorDeployment(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewDistributorDeployment(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("query_frontend", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewQueryFrontendDeployment(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewQueryFrontendDeployment(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("querier", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewQuerierDeployment(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewQuerierDeployment(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("ingester", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewIngesterStatefulSet(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewIngesterStatefulSet(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("compactor", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewCompactorStatefulSet(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewCompactorStatefulSet(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("index_gateway", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewIndexGatewayStatefulSet(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewIndexGatewayStatefulSet(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})

	t.Run("ruler", func(t *testing.T) {
		assert.Equal(t, nodeAffinity, NewRulerStatefulSet(optsWithDefaultAffinity).Spec.Template.Spec.Affinity.NodeAffinity)
		affinity := NewRulerStatefulSet(optsWithNoDefaultAffinity).Spec.Template.Spec.Affinity
		if affinity != nil {
			assert.Empty(t, affinity.NodeAffinity)
		}
	})
}

var podAntiAffinityTestTable = []struct {
	component string
	generator func(Options) *corev1.Affinity
}{
	{
		component: "lokistack-gateway",
		generator: func(opts Options) *corev1.Affinity {
			return NewGatewayDeployment(opts, "").Spec.Template.Spec.Affinity
		},
	},
	{
		component: "distributor",
		generator: func(opts Options) *corev1.Affinity {
			return NewDistributorDeployment(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "query-frontend",
		generator: func(opts Options) *corev1.Affinity {
			return NewQueryFrontendDeployment(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "querier",
		generator: func(opts Options) *corev1.Affinity {
			return NewQuerierDeployment(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "ingester",
		generator: func(opts Options) *corev1.Affinity {
			return NewIngesterStatefulSet(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "compactor",
		generator: func(opts Options) *corev1.Affinity {
			return NewCompactorStatefulSet(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "index-gateway",
		generator: func(opts Options) *corev1.Affinity {
			return NewIndexGatewayStatefulSet(opts).Spec.Template.Spec.Affinity
		},
	},
	{
		component: "ruler",
		generator: func(opts Options) *corev1.Affinity {
			return NewRulerStatefulSet(opts).Spec.Template.Spec.Affinity
		},
	},
}

func TestDefaultPodAntiAffinity(t *testing.T) {
	opts := Options{
		// We need to set name here to properly validate default PodAntiAffinity
		Name: "abcd",
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Gateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas: 1,
				},
			},
		},
	}

	for _, tc := range podAntiAffinityTestTable {
		tc := tc
		t.Run(tc.component, func(t *testing.T) {
			t.Parallel()

			wantAffinity := defaultPodAntiAffinity(tc.component, "abcd")

			affinity := tc.generator(opts)
			assert.Equal(t, wantAffinity, affinity.PodAntiAffinity)
		})
	}
}

func TestCustomPodAntiAffinity(t *testing.T) {
	paTerm := []corev1.WeightedPodAffinityTerm{
		{
			Weight: 100,
			PodAffinityTerm: corev1.PodAffinityTerm{
				LabelSelector: &metav1.LabelSelector{
					MatchLabels: map[string]string{
						"foo": "bar",
					},
				},
				TopologyKey: "foo",
			},
		},
	}

	wantAffinity := &corev1.PodAntiAffinity{
		PreferredDuringSchedulingIgnoredDuringExecution: paTerm,
	}

	opts := Options{
		Stack: lokiv1.LokiStackSpec{
			Template: &lokiv1.LokiTemplateSpec{
				Compactor: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				Distributor: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				Gateway: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				Ingester: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				Querier: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				QueryFrontend: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				IndexGateway: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
				Ruler: &lokiv1.LokiComponentSpec{
					Replicas:        1,
					PodAntiAffinity: wantAffinity,
				},
			},
		},
	}

	for _, tc := range podAntiAffinityTestTable {
		tc := tc
		t.Run(tc.component, func(t *testing.T) {
			t.Parallel()

			affinity := tc.generator(opts)
			assert.Equal(t, wantAffinity, affinity.PodAntiAffinity)
		})
	}
}

func TestCustomTopologySpreadConstraints(t *testing.T) {
	template := &corev1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{
			Labels: map[string]string{
				"abc": "edf",
				"gfi": "jkl",
			},
			Annotations: map[string]string{
				"one": "value",
				"two": "values",
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{
				{
					Name:  "a-container",
					Image: "an-image:latest",
					Env: []corev1.EnvVar{
						{
							Name:  "test",
							Value: "other",
						},
					},
				},
				{
					Name:  "b-container",
					Image: "an-image:latest",
					Env: []corev1.EnvVar{
						{
							Name:  "test",
							Value: "other",
						},
					},
				},
			},
		},
	}

	spec := &lokiv1.ReplicationSpec{
		Factor: 2,
		Zones: []lokiv1.ZoneSpec{
			{
				MaxSkew:     1,
				TopologyKey: "datacenter",
			},
			{
				MaxSkew:     1,
				TopologyKey: "rack",
			},
		},
	}

	expectedTemplate := &corev1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{
			Labels: map[string]string{
				"abc":                    "edf",
				"gfi":                    "jkl",
				lokiv1.LabelZoneAwarePod: "enabled",
			},
			Annotations: map[string]string{
				"one":                                   "value",
				"two":                                   "values",
				lokiv1.AnnotationAvailabilityZoneLabels: "datacenter,rack",
			},
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{
				{
					Name:  "a-container",
					Image: "an-image:latest",
					Env: []corev1.EnvVar{
						{
							Name:  "test",
							Value: "other",
						},
						availabilityZoneEnvVar,
					},
				},
				{
					Name:  "b-container",
					Image: "an-image:latest",
					Env: []corev1.EnvVar{
						{
							Name:  "test",
							Value: "other",
						},
						availabilityZoneEnvVar,
					},
				},
			},
			TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
				{
					MaxSkew:           int32(1),
					TopologyKey:       "datacenter",
					WhenUnsatisfiable: corev1.DoNotSchedule,
					LabelSelector: &metav1.LabelSelector{
						MatchLabels: map[string]string{
							kubernetesComponentLabel: "component",
							kubernetesInstanceLabel:  "a-stack",
						},
					},
				},
				{
					MaxSkew:           int32(1),
					TopologyKey:       "rack",
					WhenUnsatisfiable: corev1.DoNotSchedule,
					LabelSelector: &metav1.LabelSelector{
						MatchLabels: map[string]string{
							kubernetesComponentLabel: "component",
							kubernetesInstanceLabel:  "a-stack",
						},
					},
				},
			},
		},
	}

	err := configureReplication(template, spec, "component", "a-stack")
	require.NoError(t, err)
	require.Equal(t, expectedTemplate, template)
}