mirror of https://github.com/grafana/loki
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
698 lines
15 KiB
698 lines
15 KiB
package chunk
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/common/model"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
yaml "gopkg.in/yaml.v2"
|
|
)
|
|
|
|
func TestDailyBuckets(t *testing.T) {
|
|
const (
|
|
userID = "0"
|
|
metricName = model.LabelValue("name")
|
|
tableName = "table"
|
|
)
|
|
cfg := PeriodConfig{
|
|
IndexTables: PeriodicTableConfig{Prefix: tableName},
|
|
}
|
|
|
|
type args struct {
|
|
from model.Time
|
|
through model.Time
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
want []Bucket
|
|
}{
|
|
{
|
|
"0 day window",
|
|
args{
|
|
from: model.TimeFromUnix(0),
|
|
through: model.TimeFromUnix(0),
|
|
},
|
|
[]Bucket{{
|
|
from: 0,
|
|
through: 0,
|
|
tableName: "table",
|
|
hashKey: "0:d0",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}},
|
|
},
|
|
{
|
|
"6 hour window",
|
|
args{
|
|
from: model.TimeFromUnix(0),
|
|
through: model.TimeFromUnix(6 * 3600),
|
|
},
|
|
[]Bucket{{
|
|
from: 0,
|
|
through: (6 * 3600) * 1000, // ms
|
|
tableName: "table",
|
|
hashKey: "0:d0",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}},
|
|
},
|
|
{
|
|
"1 day window",
|
|
args{
|
|
from: model.TimeFromUnix(0),
|
|
through: model.TimeFromUnix(24 * 3600),
|
|
},
|
|
[]Bucket{{
|
|
from: 0,
|
|
through: (24 * 3600) * 1000, // ms
|
|
tableName: "table",
|
|
hashKey: "0:d0",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}, {
|
|
from: 0,
|
|
through: 0,
|
|
tableName: "table",
|
|
hashKey: "0:d1",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}},
|
|
},
|
|
{
|
|
"window spanning 3 days with non-zero start",
|
|
args{
|
|
from: model.TimeFromUnix(6 * 3600),
|
|
through: model.TimeFromUnix((2 * 24 * 3600) + (12 * 3600)),
|
|
},
|
|
[]Bucket{{
|
|
from: (6 * 3600) * 1000, // ms
|
|
through: (24 * 3600) * 1000, // ms
|
|
tableName: "table",
|
|
hashKey: "0:d0",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}, {
|
|
from: 0,
|
|
through: (24 * 3600) * 1000, // ms
|
|
tableName: "table",
|
|
hashKey: "0:d1",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}, {
|
|
from: 0,
|
|
through: (12 * 3600) * 1000, // ms
|
|
tableName: "table",
|
|
hashKey: "0:d2",
|
|
bucketSize: uint32(millisecondsInDay),
|
|
}},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := cfg.dailyBuckets(tt.args.from, tt.args.through, userID)
|
|
assert.Equal(t, tt.want, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkTableFor(t *testing.T) {
|
|
tablePeriod, err := time.ParseDuration("168h")
|
|
require.NoError(t, err)
|
|
|
|
periodConfigs := []PeriodConfig{
|
|
{
|
|
From: MustParseDayTime("1970-01-01"),
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "index_1_",
|
|
Period: tablePeriod,
|
|
},
|
|
ChunkTables: PeriodicTableConfig{
|
|
Prefix: "chunks_1_",
|
|
Period: tablePeriod,
|
|
},
|
|
},
|
|
{
|
|
From: MustParseDayTime("2019-01-02"),
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "index_2_",
|
|
Period: tablePeriod,
|
|
},
|
|
ChunkTables: PeriodicTableConfig{
|
|
Prefix: "chunks_2_",
|
|
Period: tablePeriod,
|
|
},
|
|
},
|
|
{
|
|
From: MustParseDayTime("2019-03-06"),
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "index_3_",
|
|
Period: tablePeriod,
|
|
},
|
|
ChunkTables: PeriodicTableConfig{
|
|
Prefix: "chunks_3_",
|
|
Period: tablePeriod,
|
|
},
|
|
},
|
|
}
|
|
|
|
schemaCfg := SchemaConfig{
|
|
Configs: periodConfigs,
|
|
}
|
|
|
|
testCases := []struct {
|
|
timeStr string // RFC3339
|
|
chunkTable string
|
|
}{
|
|
{
|
|
timeStr: "1970-01-01T00:00:00Z",
|
|
chunkTable: "chunks_1_0",
|
|
},
|
|
{
|
|
timeStr: "1970-01-01T00:00:01Z",
|
|
chunkTable: "chunks_1_0",
|
|
},
|
|
{
|
|
timeStr: "2019-01-01T00:00:00Z",
|
|
chunkTable: "chunks_1_2556",
|
|
},
|
|
{
|
|
timeStr: "2019-01-01T23:59:59Z",
|
|
chunkTable: "chunks_1_2556",
|
|
},
|
|
{
|
|
timeStr: "2019-01-02T00:00:00Z",
|
|
chunkTable: "chunks_2_2556",
|
|
},
|
|
{
|
|
timeStr: "2019-03-06T00:00:00Z",
|
|
chunkTable: "chunks_3_2565",
|
|
},
|
|
{
|
|
timeStr: "2020-03-06T00:00:00Z",
|
|
chunkTable: "chunks_3_2618",
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
ts, err := time.Parse(time.RFC3339, tc.timeStr)
|
|
require.NoError(t, err)
|
|
|
|
table, err := schemaCfg.ChunkTableFor(model.TimeFromUnix(ts.Unix()))
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, tc.chunkTable, table)
|
|
}
|
|
}
|
|
|
|
func TestSchemaConfig_Validate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := map[string]struct {
|
|
config *SchemaConfig
|
|
expected *SchemaConfig
|
|
err error
|
|
}{
|
|
"should pass the default config (ie. used cortex runs with a target not requiring the schema config)": {
|
|
config: &SchemaConfig{},
|
|
err: nil,
|
|
},
|
|
"should fail on index table period not multiple of 24h for schema v10": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{Period: 6 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errInvalidTablePeriod,
|
|
},
|
|
"should fail on chunk table period not multiple of 24h for schema v10": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
ChunkTables: PeriodicTableConfig{Period: 6 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errInvalidTablePeriod,
|
|
},
|
|
"should pass on index and chunk table period multiple of 24h for schema v10": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
ChunkTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
expected: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
RowShards: 16,
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
ChunkTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: nil,
|
|
},
|
|
"should pass on index and chunk table period set to zero (no period tables)": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
},
|
|
expected: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
RowShards: 16,
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
},
|
|
err: nil,
|
|
},
|
|
"should set shard factor defaults": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
},
|
|
},
|
|
},
|
|
expected: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
RowShards: 16,
|
|
},
|
|
},
|
|
},
|
|
err: nil,
|
|
},
|
|
"should not override explicit shard factor": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v11",
|
|
RowShards: 6,
|
|
},
|
|
},
|
|
},
|
|
expected: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v11",
|
|
RowShards: 6,
|
|
},
|
|
},
|
|
},
|
|
err: nil,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: aws-dynamo": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "aws-dynamo",
|
|
ObjectType: "aws-dynamo",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: cassandra": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "cassandra",
|
|
ObjectType: "cassandra",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: bigtable-hashed": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "bigtable-hashed",
|
|
ObjectType: "bigtable-hashed",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: gcp": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "gcp",
|
|
ObjectType: "gcp",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: gcp-columnkey": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "gcp-columnkey",
|
|
ObjectType: "gcp-columnkey",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: bigtable": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "bigtable",
|
|
ObjectType: "bigtable",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"should fail if chunks prefix is missing on IndexType: grpc-store": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
Schema: "v10",
|
|
IndexType: "grpc-store",
|
|
ObjectType: "grpc-store",
|
|
IndexTables: PeriodicTableConfig{Period: 24 * time.Hour},
|
|
},
|
|
},
|
|
},
|
|
err: errConfigChunkPrefixNotSet,
|
|
},
|
|
"invalid schema with same from time configs": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: MustParseDayTime("1970-01-01"),
|
|
Schema: "v9",
|
|
},
|
|
{
|
|
From: MustParseDayTime("1970-01-01"),
|
|
Schema: "v10",
|
|
},
|
|
},
|
|
},
|
|
err: errSchemaIncreasingFromTime,
|
|
},
|
|
"invalid schema with from time not in increasing order": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: MustParseDayTime("1970-01-02"),
|
|
Schema: "v9",
|
|
},
|
|
{
|
|
From: MustParseDayTime("1970-01-01"),
|
|
Schema: "v10",
|
|
},
|
|
},
|
|
},
|
|
err: errSchemaIncreasingFromTime,
|
|
},
|
|
"valid schema with different from time configs": {
|
|
config: &SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: MustParseDayTime("1970-01-01"),
|
|
Schema: "v9",
|
|
},
|
|
{
|
|
From: MustParseDayTime("1970-01-02"),
|
|
Schema: "v10",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for testName, testData := range tests {
|
|
testData := testData
|
|
|
|
t.Run(testName, func(t *testing.T) {
|
|
actual := testData.config.Validate()
|
|
assert.Equal(t, testData.err, actual)
|
|
if testData.expected != nil {
|
|
require.Equal(t, testData.expected, testData.config)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPeriodConfig_Validate(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
desc string
|
|
in PeriodConfig
|
|
err string
|
|
}{
|
|
{
|
|
desc: "ignore pre v10 sharding",
|
|
in: PeriodConfig{
|
|
Schema: "v9",
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
{
|
|
desc: "error on invalid schema",
|
|
in: PeriodConfig{
|
|
Schema: "v99",
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
err: "invalid schema version",
|
|
},
|
|
{
|
|
desc: "v10 with shard factor",
|
|
in: PeriodConfig{
|
|
Schema: "v10",
|
|
RowShards: 16,
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
{
|
|
desc: "v11 with shard factor",
|
|
in: PeriodConfig{
|
|
Schema: "v11",
|
|
RowShards: 16,
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
{
|
|
desc: "error v10 no specified shard factor",
|
|
in: PeriodConfig{
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
err: "must have row_shards > 0 (current: 0) for schema (v10)",
|
|
},
|
|
{
|
|
desc: "v12",
|
|
in: PeriodConfig{
|
|
Schema: "v12",
|
|
RowShards: 16,
|
|
IndexTables: PeriodicTableConfig{Period: 0},
|
|
ChunkTables: PeriodicTableConfig{Period: 0},
|
|
},
|
|
},
|
|
} {
|
|
t.Run(tc.desc, func(t *testing.T) {
|
|
if tc.err == "" {
|
|
require.Nil(t, tc.in.validate())
|
|
} else {
|
|
require.Error(t, tc.in.validate(), tc.err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func MustParseDayTime(s string) DayTime {
|
|
t, err := time.Parse("2006-01-02", s)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return DayTime{model.TimeFromUnix(t.Unix())}
|
|
}
|
|
|
|
func TestPeriodicTableConfigCustomUnmarshalling(t *testing.T) {
|
|
yamlFile := `prefix: cortex_
|
|
period: 1w
|
|
tags:
|
|
foo: bar
|
|
`
|
|
|
|
cfg := PeriodicTableConfig{}
|
|
err := yaml.Unmarshal([]byte(yamlFile), &cfg)
|
|
require.NoError(t, err)
|
|
|
|
expectedCfg := PeriodicTableConfig{
|
|
Prefix: "cortex_",
|
|
Period: 7 * 24 * time.Hour,
|
|
Tags: map[string]string{
|
|
"foo": "bar",
|
|
},
|
|
}
|
|
|
|
require.Equal(t, expectedCfg, cfg)
|
|
|
|
yamlGenerated, err := yaml.Marshal(&cfg)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, yamlFile, string(yamlGenerated))
|
|
}
|
|
|
|
func TestSchemaForTime(t *testing.T) {
|
|
schemaCfg := SchemaConfig{Configs: []PeriodConfig{
|
|
{
|
|
From: DayTime{Time: 1564358400000},
|
|
IndexType: "grpc-store",
|
|
ObjectType: "grpc-store",
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "index_",
|
|
Period: 604800000000000,
|
|
Tags: nil,
|
|
},
|
|
RowShards: 16,
|
|
},
|
|
{
|
|
From: DayTime{Time: 1564444800000},
|
|
IndexType: "grpc-store",
|
|
ObjectType: "grpc-store",
|
|
Schema: "v10",
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "index_",
|
|
Period: 604800000000000,
|
|
Tags: nil,
|
|
},
|
|
RowShards: 32,
|
|
},
|
|
}}
|
|
|
|
first, err := schemaCfg.SchemaForTime(model.TimeFromUnix(1564444800 + 100))
|
|
require.NoError(t, err)
|
|
require.Equal(t, schemaCfg.Configs[1], first)
|
|
|
|
second, err := schemaCfg.SchemaForTime(model.TimeFromUnix(1564358400 + 100))
|
|
require.NoError(t, err)
|
|
require.Equal(t, schemaCfg.Configs[0], second)
|
|
}
|
|
|
|
func TestVersionAsInt(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
schemaCfg SchemaConfig
|
|
expected int
|
|
err bool
|
|
}{
|
|
{
|
|
name: "v9",
|
|
schemaCfg: SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: DayTime{Time: 0},
|
|
Schema: "v9",
|
|
},
|
|
},
|
|
},
|
|
expected: int(9),
|
|
},
|
|
{
|
|
name: "malformed",
|
|
schemaCfg: SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: DayTime{Time: 0},
|
|
Schema: "v",
|
|
},
|
|
},
|
|
},
|
|
expected: int(0),
|
|
err: true,
|
|
},
|
|
{
|
|
name: "v12",
|
|
schemaCfg: SchemaConfig{
|
|
Configs: []PeriodConfig{
|
|
{
|
|
From: DayTime{Time: 0},
|
|
Schema: "v12",
|
|
RowShards: 16,
|
|
},
|
|
},
|
|
},
|
|
expected: int(12),
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
version, err := tc.schemaCfg.Configs[0].VersionAsInt()
|
|
require.Equal(t, tc.expected, version)
|
|
if tc.err {
|
|
require.NotNil(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalPeriodConfig(t *testing.T) {
|
|
input := `
|
|
from: "2020-07-31"
|
|
index:
|
|
period: 24h
|
|
prefix: loki_index_
|
|
object_store: gcs
|
|
schema: v11
|
|
store: boltdb-shipper
|
|
`
|
|
|
|
var cfg PeriodConfig
|
|
require.Nil(t, yaml.Unmarshal([]byte(input), &cfg))
|
|
n := 11
|
|
|
|
expected := PeriodConfig{
|
|
From: DayTime{model.Time(1596153600000)},
|
|
IndexType: "boltdb-shipper",
|
|
ObjectType: "gcs",
|
|
Schema: "v11",
|
|
IndexTables: PeriodicTableConfig{
|
|
Prefix: "loki_index_",
|
|
Period: 24 * time.Hour,
|
|
},
|
|
schemaInt: &n,
|
|
}
|
|
|
|
require.Equal(t, expected, cfg)
|
|
}
|
|
|