loki/pkg/validation/limits_test.go

package validation

import (
	"encoding/json"
	"reflect"
	"testing"
	"time"

	"github.com/prometheus/common/model"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gopkg.in/yaml.v2"

	"github.com/grafana/loki/pkg/compactor/deletionmode"
)

func TestLimitsTagsYamlMatchJson(t *testing.T) {
	limits := reflect.TypeOf(Limits{})
	n := limits.NumField()
	var mismatch []string

	for i := 0; i < n; i++ {
		field := limits.Field(i)

		// Note that we aren't requiring YAML and JSON tags to match, just that
		// they either both exist or both don't exist.
		hasYAMLTag := field.Tag.Get("yaml") != ""
		hasJSONTag := field.Tag.Get("json") != ""

		if hasYAMLTag != hasJSONTag {
			mismatch = append(mismatch, field.Name)
		}
	}

	assert.Empty(t, mismatch, "expected no mismatched JSON and YAML tags")
}

func TestLimitsYamlMatchJson(t *testing.T) {
	inputYAML := `
ingestion_rate_strategy: "some-strategy"
ingestion_rate_mb: 34
ingestion_burst_size_mb: 40
max_label_name_length: 10
max_label_value_length: 20
max_label_names_per_series: 30
reject_old_samples: true
reject_old_samples_max_age: 40s
creation_grace_period: 50s
enforce_metric_name: true
max_line_size: 60
max_line_size_truncate: true
max_streams_per_user: 70
max_global_streams_per_user: 80
max_chunks_per_query: 90
max_query_series: 100
max_query_lookback: 110s
max_query_length: 120s
max_query_parallelism: 130
cardinality_limit: 140
max_streams_matchers_per_query: 150
max_concurrent_tail_requests: 160
max_entries_limit_per_query: 170
max_cache_freshness_per_query: 180s
split_queries_by_interval: 190s
ruler_evaluation_delay_duration: 200s
ruler_max_rules_per_rule_group: 210
ruler_max_rule_groups_per_tenant: 220
ruler_remote_write_sigv4_config:
  region: us-east-1
per_tenant_override_config: ""
per_tenant_override_period: 230s
query_timeout: 5m
shard_streams:
  enabled: true
  desired_rate: 4mb
  logging_enabled: true
blocked_queries:
  - pattern: ".*foo.*"
    regex: true
volume_enabled: true
volume_max_series: 10001
`
	inputJSON := `
 {
  "ingestion_rate_strategy": "some-strategy",
  "ingestion_rate_mb": 34,
  "ingestion_burst_size_mb": 40,
  "max_label_name_length": 10,
  "max_label_value_length": 20,
  "max_label_names_per_series": 30,
  "reject_old_samples": true,
  "reject_old_samples_max_age": "40s",
  "creation_grace_period": "50s",
  "enforce_metric_name": true,
  "max_line_size": "60",
  "max_line_size_truncate": true,
  "max_streams_per_user": 70,
  "max_global_streams_per_user": 80,
  "max_chunks_per_query": 90,
  "max_query_series": 100,
  "max_query_lookback": "110s",
  "max_query_length": "120s",
  "max_query_parallelism": 130,
  "cardinality_limit": 140,
  "max_streams_matchers_per_query": 150,
  "max_concurrent_tail_requests": 160,
  "max_entries_limit_per_query": 170,
  "max_cache_freshness_per_query": "180s",
  "split_queries_by_interval": "190s",
  "ruler_evaluation_delay_duration": "200s",
  "ruler_max_rules_per_rule_group": 210,
  "ruler_max_rule_groups_per_tenant":220,
  "ruler_remote_write_sigv4_config": {
    "region": "us-east-1"
  },
  "per_tenant_override_config": "",
  "per_tenant_override_period": "230s",
  "query_timeout": "5m",
  "shard_streams": {
    "desired_rate": "4mb",
    "enabled": true,
    "logging_enabled": true
  },
  "blocked_queries": [
	{
		"pattern": ".*foo.*",
		"regex": true
	}
  ],
  "volume_enabled": true,
  "volume_max_series": 10001
 }
`

	limitsYAML := Limits{}
	err := yaml.Unmarshal([]byte(inputYAML), &limitsYAML)
	require.NoError(t, err, "expected to be able to unmarshal from YAML")

	limitsJSON := Limits{}
	err = json.Unmarshal([]byte(inputJSON), &limitsJSON)
	require.NoError(t, err, "expected to be able to unmarshal from JSON")

	assert.Equal(t, limitsYAML, limitsJSON)
}

func TestOverwriteMarshalingStringMapJSON(t *testing.T) {
	m := NewOverwriteMarshalingStringMap(map[string]string{"foo": "bar"})

	require.Nil(t, json.Unmarshal([]byte(`{"bazz": "buzz"}`), &m))
	require.Equal(t, map[string]string{"bazz": "buzz"}, m.Map())
	out, err := json.Marshal(m)
	require.Nil(t, err)
	var back OverwriteMarshalingStringMap
	require.Nil(t, json.Unmarshal(out, &back))
	require.Equal(t, m, back)
}

func TestOverwriteMarshalingStringMapYAML(t *testing.T) {
	m := NewOverwriteMarshalingStringMap(map[string]string{"foo": "bar"})

	require.Nil(t, yaml.Unmarshal([]byte(`{"bazz": "buzz"}`), &m))
	require.Equal(t, map[string]string{"bazz": "buzz"}, m.Map())
	out, err := yaml.Marshal(m)
	require.Nil(t, err)
	var back OverwriteMarshalingStringMap
	require.Nil(t, yaml.Unmarshal(out, &back))
	require.Equal(t, m, back)
}

func TestLimitsDoesNotMutate(t *testing.T) {
	initialDefault := defaultLimits
	defer func() {
		defaultLimits = initialDefault
	}()

	// Set new defaults with non-nil values for non-scalar types
	newDefaults := Limits{
		RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}},
		StreamRetention: []StreamRetention{
			{
				Period:   model.Duration(24 * time.Hour),
				Selector: `{a="b"}`,
			},
		},
	}
	SetDefaultLimitsForYAMLUnmarshalling(newDefaults)

	for _, tc := range []struct {
		desc string
		yaml string
		exp  Limits
	}{
		{
			desc: "map",
			yaml: `
ruler_remote_write_headers:
  foo: "bar"
`,
			exp: Limits{
				RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"foo": "bar"}},

				// Rest from new defaults
				StreamRetention: []StreamRetention{
					{
						Period:   model.Duration(24 * time.Hour),
						Selector: `{a="b"}`,
					},
				},
			},
		},
		{
			desc: "empty map overrides defaults",
			yaml: `
ruler_remote_write_headers:
`,
			exp: Limits{

				// Rest from new defaults
				StreamRetention: []StreamRetention{
					{
						Period:   model.Duration(24 * time.Hour),
						Selector: `{a="b"}`,
					},
				},
			},
		},
		{
			desc: "slice",
			yaml: `
retention_stream:
  - period: '24h'
    selector: '{foo="bar"}'
`,
			exp: Limits{
				StreamRetention: []StreamRetention{
					{
						Period:   model.Duration(24 * time.Hour),
						Selector: `{foo="bar"}`,
					},
				},

				// Rest from new defaults
				RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}},
			},
		},
		{
			desc: "scalar field",
			yaml: `
reject_old_samples: true
`,
			exp: Limits{
				RejectOldSamples: true,

				// Rest from new defaults
				RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}},
				StreamRetention: []StreamRetention{
					{
						Period:   model.Duration(24 * time.Hour),
						Selector: `{a="b"}`,
					},
				},
			},
		},
		{
			desc: "per tenant query timeout",
			yaml: `
query_timeout: 5m
`,
			exp: Limits{
				QueryTimeout: model.Duration(5 * time.Minute),

				// Rest from new defaults.
				RulerRemoteWriteHeaders: OverwriteMarshalingStringMap{map[string]string{"a": "b"}},
				StreamRetention: []StreamRetention{
					{
						Period:   model.Duration(24 * time.Hour),
						Selector: `{a="b"}`,
					},
				},
			},
		},
	} {

		t.Run(tc.desc, func(t *testing.T) {
			var out Limits
			require.Nil(t, yaml.UnmarshalStrict([]byte(tc.yaml), &out))
			require.Equal(t, tc.exp, out)
		})
	}
}

func TestLimitsValidation_deletionMode(t *testing.T) {
	for _, tc := range []struct {
		mode     string
		expected error
	}{
		{mode: "disabled", expected: nil},
		{mode: "filter-only", expected: nil},
		{mode: "filter-and-delete", expected: nil},
		{mode: "something-else", expected: deletionmode.ErrUnknownMode},
	} {
		t.Run(tc.mode, func(t *testing.T) {
			limits := Limits{DeletionMode: tc.mode}
			require.ErrorIs(t, limits.Validate(), tc.expected)
		})
	}
}