- Remove unrelated changes - Refactor code out of the API module - that is already getting pretty crowded. - Don't track reference for AddFast in remote write. This has the potential to consume unlimited server-side memory if a malicious client pushes a different label set for every series. For now, its easier and safer to always use the 'slow' path. - Return 400 on out of order samples. - Use remote.DecodeWriteRequest in the remote write adapters. - Put this behing the 'remote-write-server' feature flag - Add some (very) basic docs. - Used named return & add test for commit error propagation Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>pull/8424/head
Tom Wilkie
4 years ago
parent
72475b8a0c
commit
d479151f1f
@ -0,0 +1,86 @@ |
||||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package remote |
||||
|
||||
import ( |
||||
"context" |
||||
"net/http" |
||||
|
||||
"github.com/go-kit/kit/log" |
||||
"github.com/go-kit/kit/log/level" |
||||
"github.com/prometheus/prometheus/prompb" |
||||
"github.com/prometheus/prometheus/storage" |
||||
) |
||||
|
||||
type handler struct { |
||||
logger log.Logger |
||||
appendable storage.Appendable |
||||
} |
||||
|
||||
// NewWriteHandler creates a http.Handler that accepts remote write requests and
|
||||
// writes them to the provided appendable.
|
||||
func NewWriteHandler(logger log.Logger, appendable storage.Appendable) http.Handler { |
||||
return &handler{ |
||||
logger: logger, |
||||
appendable: appendable, |
||||
} |
||||
} |
||||
|
||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
||||
req, err := DecodeWriteRequest(r.Body) |
||||
if err != nil { |
||||
level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err.Error()) |
||||
http.Error(w, err.Error(), http.StatusBadRequest) |
||||
return |
||||
} |
||||
|
||||
err = h.write(r.Context(), req) |
||||
switch err { |
||||
case nil: |
||||
case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp: |
||||
// Indicated an out of order sample is a bad request to prevent retries.
|
||||
level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error()) |
||||
http.Error(w, err.Error(), http.StatusBadRequest) |
||||
return |
||||
default: |
||||
level.Error(h.logger).Log("msg", "Error appending remote write", "err", err.Error()) |
||||
http.Error(w, err.Error(), http.StatusInternalServerError) |
||||
return |
||||
} |
||||
|
||||
w.WriteHeader(http.StatusNoContent) |
||||
} |
||||
|
||||
func (h *handler) write(ctx context.Context, req *prompb.WriteRequest) (err error) { |
||||
app := h.appendable.Appender(ctx) |
||||
defer func() { |
||||
if err != nil { |
||||
app.Rollback() |
||||
return |
||||
} |
||||
err = app.Commit() |
||||
}() |
||||
|
||||
for _, ts := range req.Timeseries { |
||||
labels := labelProtosToLabels(ts.Labels) |
||||
for _, s := range ts.Samples { |
||||
_, err = app.Add(labels, s.Timestamp, s.Value) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,138 @@ |
||||
// Copyright 2021 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package remote |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"net/http/httptest" |
||||
"testing" |
||||
|
||||
"github.com/go-kit/kit/log" |
||||
"github.com/prometheus/prometheus/pkg/labels" |
||||
"github.com/prometheus/prometheus/prompb" |
||||
"github.com/prometheus/prometheus/storage" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestRemoteWriteHandler(t *testing.T) { |
||||
buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil) |
||||
require.NoError(t, err) |
||||
|
||||
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||
require.NoError(t, err) |
||||
|
||||
appendable := &mockAppendable{} |
||||
handler := NewWriteHandler(nil, appendable) |
||||
|
||||
recorder := httptest.NewRecorder() |
||||
handler.ServeHTTP(recorder, req) |
||||
|
||||
resp := recorder.Result() |
||||
require.Equal(t, http.StatusNoContent, resp.StatusCode) |
||||
|
||||
i := 0 |
||||
for _, ts := range writeRequestFixture.Timeseries { |
||||
labels := labelProtosToLabels(ts.Labels) |
||||
for _, s := range ts.Samples { |
||||
require.Equal(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i]) |
||||
i++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
func TestOutOfOrder(t *testing.T) { |
||||
buf, _, err := buildWriteRequest([]prompb.TimeSeries{{ |
||||
Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}}, |
||||
Samples: []prompb.Sample{{Value: 1, Timestamp: 0}}, |
||||
}}, nil, nil) |
||||
require.NoError(t, err) |
||||
|
||||
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||
require.NoError(t, err) |
||||
|
||||
appendable := &mockAppendable{ |
||||
latest: 100, |
||||
} |
||||
handler := NewWriteHandler(log.NewNopLogger(), appendable) |
||||
|
||||
recorder := httptest.NewRecorder() |
||||
handler.ServeHTTP(recorder, req) |
||||
|
||||
resp := recorder.Result() |
||||
require.Equal(t, http.StatusBadRequest, resp.StatusCode) |
||||
} |
||||
|
||||
func TestCommitErr(t *testing.T) { |
||||
buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil) |
||||
require.NoError(t, err) |
||||
|
||||
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||
require.NoError(t, err) |
||||
|
||||
appendable := &mockAppendable{ |
||||
commitErr: fmt.Errorf("commit error"), |
||||
} |
||||
handler := NewWriteHandler(log.NewNopLogger(), appendable) |
||||
|
||||
recorder := httptest.NewRecorder() |
||||
handler.ServeHTTP(recorder, req) |
||||
|
||||
resp := recorder.Result() |
||||
body, err := ioutil.ReadAll(resp.Body) |
||||
require.NoError(t, err) |
||||
require.Equal(t, http.StatusInternalServerError, resp.StatusCode) |
||||
require.Equal(t, "commit error\n", string(body)) |
||||
} |
||||
|
||||
type mockAppendable struct { |
||||
latest int64 |
||||
samples []mockSample |
||||
commitErr error |
||||
} |
||||
|
||||
type mockSample struct { |
||||
l labels.Labels |
||||
t int64 |
||||
v float64 |
||||
} |
||||
|
||||
func (m *mockAppendable) Appender(_ context.Context) storage.Appender { |
||||
return m |
||||
} |
||||
|
||||
func (m *mockAppendable) Add(l labels.Labels, t int64, v float64) (uint64, error) { |
||||
if t < m.latest { |
||||
return 0, storage.ErrOutOfOrderSample |
||||
} |
||||
|
||||
m.latest = t |
||||
m.samples = append(m.samples, mockSample{l, t, v}) |
||||
return 0, nil |
||||
} |
||||
|
||||
func (m *mockAppendable) Commit() error { |
||||
return m.commitErr |
||||
} |
||||
|
||||
func (*mockAppendable) AddFast(uint64, int64, float64) error { |
||||
return fmt.Errorf("not implemented") |
||||
} |
||||
|
||||
func (*mockAppendable) Rollback() error { |
||||
return fmt.Errorf("not implemented") |
||||
} |
||||
Loading…
Reference in new issue