mirror of https://github.com/grafana/loki
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
340 lines
8.5 KiB
340 lines
8.5 KiB
package aggregation
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-kit/log"
|
|
"github.com/grafana/dskit/backoff"
|
|
"github.com/prometheus/common/config"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/grafana/loki/v3/pkg/logproto"
|
|
"github.com/grafana/loki/v3/pkg/util"
|
|
)
|
|
|
|
const (
|
|
testTenant = "test1"
|
|
testUsername = "user"
|
|
testPassword = "secret"
|
|
LogEntry = "%s %s\n"
|
|
)
|
|
|
|
func Test_Push(t *testing.T) {
|
|
lbls := labels.New(labels.Label{Name: "test", Value: "test"})
|
|
|
|
// create dummy loki server
|
|
responses := make(chan response, 1) // buffered not to block the response handler
|
|
backoff := backoff.Config{
|
|
MinBackoff: 300 * time.Millisecond,
|
|
MaxBackoff: 1 * time.Minute,
|
|
MaxRetries: 1,
|
|
}
|
|
|
|
t.Run("sends log entry to loki server without TLS", func(t *testing.T) {
|
|
// mock loki server
|
|
mock := httptest.NewServer(createServerHandler(responses))
|
|
require.NotNil(t, mock)
|
|
defer mock.Close()
|
|
|
|
// without TLS
|
|
push, err := NewPush(
|
|
mock.Listener.Addr().String(),
|
|
"test1",
|
|
2*time.Second,
|
|
1*time.Second,
|
|
config.DefaultHTTPClientConfig,
|
|
"", "",
|
|
false,
|
|
&backoff,
|
|
log.NewNopLogger(),
|
|
NewMetrics(nil),
|
|
)
|
|
require.NoError(t, err)
|
|
ts, payload := testPayload()
|
|
push.WriteEntry(ts, payload, lbls)
|
|
resp := <-responses
|
|
assertResponse(t, resp, false, labelSet("test", "test"), ts, payload)
|
|
})
|
|
|
|
t.Run("sends log entry to loki server with basic auth", func(t *testing.T) {
|
|
// mock loki server
|
|
mock := httptest.NewServer(createServerHandler(responses))
|
|
require.NotNil(t, mock)
|
|
defer mock.Close()
|
|
|
|
// with basic Auth
|
|
push, err := NewPush(
|
|
mock.Listener.Addr().String(),
|
|
"test1",
|
|
2*time.Second,
|
|
1*time.Second,
|
|
config.DefaultHTTPClientConfig,
|
|
"user", "secret",
|
|
false,
|
|
&backoff,
|
|
log.NewNopLogger(), NewMetrics(nil),
|
|
)
|
|
require.NoError(t, err)
|
|
ts, payload := testPayload()
|
|
push.WriteEntry(ts, payload, lbls)
|
|
resp := <-responses
|
|
assertResponse(t, resp, true, labelSet("test", "test"), ts, payload)
|
|
})
|
|
|
|
t.Run("batches push requests", func(t *testing.T) {
|
|
// mock loki server
|
|
mock := httptest.NewServer(createServerHandler(responses))
|
|
require.NotNil(t, mock)
|
|
defer mock.Close()
|
|
|
|
client, err := config.NewClientFromConfig(
|
|
config.DefaultHTTPClientConfig,
|
|
"pattern-ingester-push-test",
|
|
config.WithHTTP2Disabled(),
|
|
)
|
|
require.NoError(t, err)
|
|
client.Timeout = 2 * time.Second
|
|
|
|
u := url.URL{
|
|
Scheme: "http",
|
|
Host: mock.Listener.Addr().String(),
|
|
Path: pushEndpoint,
|
|
}
|
|
|
|
p := &Push{
|
|
lokiURL: u.String(),
|
|
tenantID: "test1",
|
|
httpClient: client,
|
|
userAgent: defaultUserAgent,
|
|
contentType: defaultContentType,
|
|
username: "user",
|
|
password: "secret",
|
|
logger: log.NewNopLogger(),
|
|
quit: make(chan struct{}),
|
|
backoff: &backoff,
|
|
entries: entries{},
|
|
metrics: NewMetrics(nil),
|
|
}
|
|
|
|
lbls1 := labels.New(labels.Label{Name: "test", Value: "test"})
|
|
lbls2 := labels.New(
|
|
labels.Label{Name: "test", Value: "test"},
|
|
labels.Label{Name: "test2", Value: "test2"},
|
|
)
|
|
|
|
now := time.Now().Truncate(time.Second).UTC()
|
|
then := now.Add(-1 * time.Minute)
|
|
wayBack := now.Add(-5 * time.Minute)
|
|
|
|
p.WriteEntry(
|
|
wayBack,
|
|
AggregatedMetricEntry(model.TimeFromUnix(wayBack.Unix()), 1, 1, "test_service", lbls1),
|
|
lbls1,
|
|
)
|
|
p.WriteEntry(
|
|
then,
|
|
AggregatedMetricEntry(model.TimeFromUnix(then.Unix()), 2, 2, "test_service", lbls1),
|
|
lbls1,
|
|
)
|
|
p.WriteEntry(
|
|
now,
|
|
AggregatedMetricEntry(model.TimeFromUnix(now.Unix()), 3, 3, "test_service", lbls1),
|
|
lbls1,
|
|
)
|
|
|
|
p.WriteEntry(
|
|
wayBack,
|
|
AggregatedMetricEntry(model.TimeFromUnix(wayBack.Unix()), 1, 1, "test2_service", lbls2),
|
|
lbls2,
|
|
)
|
|
p.WriteEntry(
|
|
then,
|
|
AggregatedMetricEntry(model.TimeFromUnix(then.Unix()), 2, 2, "test2_service", lbls2),
|
|
lbls2,
|
|
)
|
|
p.WriteEntry(
|
|
now,
|
|
AggregatedMetricEntry(model.TimeFromUnix(now.Unix()), 3, 3, "test2_service", lbls2),
|
|
lbls2,
|
|
)
|
|
|
|
go p.run(time.Nanosecond)
|
|
|
|
select {
|
|
case resp := <-responses:
|
|
p.Stop()
|
|
req := resp.pushReq
|
|
assert.Len(t, req.Streams, 2)
|
|
|
|
var stream1, stream2 logproto.Stream
|
|
for _, stream := range req.Streams {
|
|
if stream.Labels == lbls1.String() {
|
|
stream1 = stream
|
|
}
|
|
|
|
if stream.Labels == lbls2.String() {
|
|
stream2 = stream
|
|
}
|
|
}
|
|
|
|
require.Len(t, stream1.Entries, 3)
|
|
require.Len(t, stream2.Entries, 3)
|
|
|
|
require.Equal(t, stream1.Entries[0].Timestamp, wayBack)
|
|
require.Equal(t, stream1.Entries[1].Timestamp, then)
|
|
require.Equal(t, stream1.Entries[2].Timestamp, now)
|
|
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(wayBack.Unix()), 1, 1, "test_service", lbls1),
|
|
stream1.Entries[0].Line,
|
|
)
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(then.Unix()), 2, 2, "test_service", lbls1),
|
|
stream1.Entries[1].Line,
|
|
)
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(now.Unix()), 3, 3, "test_service", lbls1),
|
|
stream1.Entries[2].Line,
|
|
)
|
|
|
|
require.Equal(t, stream2.Entries[0].Timestamp, wayBack)
|
|
require.Equal(t, stream2.Entries[1].Timestamp, then)
|
|
require.Equal(t, stream2.Entries[2].Timestamp, now)
|
|
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(wayBack.Unix()), 1, 1, "test2_service", lbls2),
|
|
stream2.Entries[0].Line,
|
|
)
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(then.Unix()), 2, 2, "test2_service", lbls2),
|
|
stream2.Entries[1].Line,
|
|
)
|
|
require.Equal(
|
|
t,
|
|
AggregatedMetricEntry(model.TimeFromUnix(now.Unix()), 3, 3, "test2_service", lbls2),
|
|
stream2.Entries[2].Line,
|
|
)
|
|
|
|
// sanity check that bytes are logged in humanized form without whitespaces
|
|
assert.Contains(t, stream1.Entries[0].Line, "bytes=1B")
|
|
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("timeout")
|
|
}
|
|
})
|
|
}
|
|
|
|
// Test helpers
|
|
|
|
func assertResponse(t *testing.T, resp response, testAuth bool, labels labels.Labels, ts time.Time, payload string) {
|
|
t.Helper()
|
|
|
|
// assert metadata
|
|
assert.Equal(t, testTenant, resp.tenantID)
|
|
|
|
var expUser, expPass string
|
|
|
|
if testAuth {
|
|
expUser = testUsername
|
|
expPass = testPassword
|
|
}
|
|
|
|
assert.Equal(t, expUser, resp.username)
|
|
assert.Equal(t, expPass, resp.password)
|
|
assert.Equal(t, defaultContentType, resp.contentType)
|
|
assert.Equal(t, defaultUserAgent, resp.userAgent)
|
|
|
|
// assert stream labels
|
|
require.Len(t, resp.pushReq.Streams, 1)
|
|
assert.Equal(t, labels.String(), resp.pushReq.Streams[0].Labels)
|
|
assert.Equal(t, labels.Hash(), resp.pushReq.Streams[0].Hash)
|
|
|
|
// assert log entry
|
|
require.Len(t, resp.pushReq.Streams, 1)
|
|
require.Len(t, resp.pushReq.Streams[0].Entries, 1)
|
|
assert.Equal(t, payload, resp.pushReq.Streams[0].Entries[0].Line)
|
|
assert.Equal(t, ts, resp.pushReq.Streams[0].Entries[0].Timestamp)
|
|
}
|
|
|
|
type response struct {
|
|
tenantID string
|
|
pushReq logproto.PushRequest
|
|
contentType string
|
|
userAgent string
|
|
username, password string
|
|
}
|
|
|
|
func createServerHandler(responses chan response) http.HandlerFunc {
|
|
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
|
|
// Parse the request
|
|
var pushReq logproto.PushRequest
|
|
if err := util.ParseProtoReader(req.Context(), req.Body, int(req.ContentLength), math.MaxInt32, &pushReq, util.RawSnappy); err != nil {
|
|
rw.WriteHeader(500)
|
|
return
|
|
}
|
|
|
|
var username, password string
|
|
|
|
basicAuth := req.Header.Get("Authorization")
|
|
if basicAuth != "" {
|
|
encoded := strings.TrimPrefix(basicAuth, "Basic ") // now we have just encoded `username:password`
|
|
decoded, err := base64.StdEncoding.DecodeString(encoded)
|
|
if err != nil {
|
|
rw.WriteHeader(500)
|
|
return
|
|
}
|
|
toks := strings.FieldsFunc(string(decoded), func(r rune) bool {
|
|
return r == ':'
|
|
})
|
|
username, password = toks[0], toks[1]
|
|
}
|
|
|
|
responses <- response{
|
|
tenantID: req.Header.Get("X-Scope-OrgID"),
|
|
contentType: req.Header.Get("Content-Type"),
|
|
userAgent: req.Header.Get("User-Agent"),
|
|
username: username,
|
|
password: password,
|
|
pushReq: pushReq,
|
|
}
|
|
|
|
rw.WriteHeader(http.StatusOK)
|
|
})
|
|
}
|
|
|
|
func labelSet(keyVals ...string) labels.Labels {
|
|
if len(keyVals)%2 != 0 {
|
|
panic("not matching key-value pairs")
|
|
}
|
|
|
|
lbls := labels.Labels{}
|
|
|
|
for i := 0; i < len(keyVals)-1; i += 2 {
|
|
lbls = append(lbls, labels.Label{Name: keyVals[i], Value: keyVals[i+1]})
|
|
}
|
|
|
|
return lbls
|
|
}
|
|
|
|
func testPayload() (time.Time, string) {
|
|
ts := time.Now().UTC()
|
|
payload := fmt.Sprintf(LogEntry, fmt.Sprint(ts.UnixNano()), "pppppp")
|
|
|
|
return ts, payload
|
|
}
|
|
|