- Remove unrelated changes - Refactor code out of the API module - that is already getting pretty crowded. - Don't track reference for AddFast in remote write. This has the potential to consume unlimited server-side memory if a malicious client pushes a different label set for every series. For now, its easier and safer to always use the 'slow' path. - Return 400 on out of order samples. - Use remote.DecodeWriteRequest in the remote write adapters. - Put this behing the 'remote-write-server' feature flag - Add some (very) basic docs. - Used named return & add test for commit error propagation Signed-off-by: Tom Wilkie <tom.wilkie@gmail.com>pull/8424/head
Tom Wilkie
5 years ago
parent
72475b8a0c
commit
d479151f1f
@ -0,0 +1,86 @@ |
|||||||
|
// Copyright 2021 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package remote |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"net/http" |
||||||
|
|
||||||
|
"github.com/go-kit/kit/log" |
||||||
|
"github.com/go-kit/kit/log/level" |
||||||
|
"github.com/prometheus/prometheus/prompb" |
||||||
|
"github.com/prometheus/prometheus/storage" |
||||||
|
) |
||||||
|
|
||||||
|
type handler struct { |
||||||
|
logger log.Logger |
||||||
|
appendable storage.Appendable |
||||||
|
} |
||||||
|
|
||||||
|
// NewWriteHandler creates a http.Handler that accepts remote write requests and
|
||||||
|
// writes them to the provided appendable.
|
||||||
|
func NewWriteHandler(logger log.Logger, appendable storage.Appendable) http.Handler { |
||||||
|
return &handler{ |
||||||
|
logger: logger, |
||||||
|
appendable: appendable, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
||||||
|
req, err := DecodeWriteRequest(r.Body) |
||||||
|
if err != nil { |
||||||
|
level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err.Error()) |
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
err = h.write(r.Context(), req) |
||||||
|
switch err { |
||||||
|
case nil: |
||||||
|
case storage.ErrOutOfOrderSample, storage.ErrOutOfBounds, storage.ErrDuplicateSampleForTimestamp: |
||||||
|
// Indicated an out of order sample is a bad request to prevent retries.
|
||||||
|
level.Error(h.logger).Log("msg", "Out of order sample from remote write", "err", err.Error()) |
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest) |
||||||
|
return |
||||||
|
default: |
||||||
|
level.Error(h.logger).Log("msg", "Error appending remote write", "err", err.Error()) |
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError) |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
w.WriteHeader(http.StatusNoContent) |
||||||
|
} |
||||||
|
|
||||||
|
func (h *handler) write(ctx context.Context, req *prompb.WriteRequest) (err error) { |
||||||
|
app := h.appendable.Appender(ctx) |
||||||
|
defer func() { |
||||||
|
if err != nil { |
||||||
|
app.Rollback() |
||||||
|
return |
||||||
|
} |
||||||
|
err = app.Commit() |
||||||
|
}() |
||||||
|
|
||||||
|
for _, ts := range req.Timeseries { |
||||||
|
labels := labelProtosToLabels(ts.Labels) |
||||||
|
for _, s := range ts.Samples { |
||||||
|
_, err = app.Add(labels, s.Timestamp, s.Value) |
||||||
|
if err != nil { |
||||||
|
return err |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return nil |
||||||
|
} |
||||||
@ -0,0 +1,138 @@ |
|||||||
|
// Copyright 2021 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package remote |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"io/ioutil" |
||||||
|
"net/http" |
||||||
|
"net/http/httptest" |
||||||
|
"testing" |
||||||
|
|
||||||
|
"github.com/go-kit/kit/log" |
||||||
|
"github.com/prometheus/prometheus/pkg/labels" |
||||||
|
"github.com/prometheus/prometheus/prompb" |
||||||
|
"github.com/prometheus/prometheus/storage" |
||||||
|
"github.com/stretchr/testify/require" |
||||||
|
) |
||||||
|
|
||||||
|
func TestRemoteWriteHandler(t *testing.T) { |
||||||
|
buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
appendable := &mockAppendable{} |
||||||
|
handler := NewWriteHandler(nil, appendable) |
||||||
|
|
||||||
|
recorder := httptest.NewRecorder() |
||||||
|
handler.ServeHTTP(recorder, req) |
||||||
|
|
||||||
|
resp := recorder.Result() |
||||||
|
require.Equal(t, http.StatusNoContent, resp.StatusCode) |
||||||
|
|
||||||
|
i := 0 |
||||||
|
for _, ts := range writeRequestFixture.Timeseries { |
||||||
|
labels := labelProtosToLabels(ts.Labels) |
||||||
|
for _, s := range ts.Samples { |
||||||
|
require.Equal(t, mockSample{labels, s.Timestamp, s.Value}, appendable.samples[i]) |
||||||
|
i++ |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func TestOutOfOrder(t *testing.T) { |
||||||
|
buf, _, err := buildWriteRequest([]prompb.TimeSeries{{ |
||||||
|
Labels: []prompb.Label{{Name: "__name__", Value: "test_metric"}}, |
||||||
|
Samples: []prompb.Sample{{Value: 1, Timestamp: 0}}, |
||||||
|
}}, nil, nil) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
appendable := &mockAppendable{ |
||||||
|
latest: 100, |
||||||
|
} |
||||||
|
handler := NewWriteHandler(log.NewNopLogger(), appendable) |
||||||
|
|
||||||
|
recorder := httptest.NewRecorder() |
||||||
|
handler.ServeHTTP(recorder, req) |
||||||
|
|
||||||
|
resp := recorder.Result() |
||||||
|
require.Equal(t, http.StatusBadRequest, resp.StatusCode) |
||||||
|
} |
||||||
|
|
||||||
|
func TestCommitErr(t *testing.T) { |
||||||
|
buf, _, err := buildWriteRequest(writeRequestFixture.Timeseries, nil, nil) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
req, err := http.NewRequest("", "", bytes.NewReader(buf)) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
appendable := &mockAppendable{ |
||||||
|
commitErr: fmt.Errorf("commit error"), |
||||||
|
} |
||||||
|
handler := NewWriteHandler(log.NewNopLogger(), appendable) |
||||||
|
|
||||||
|
recorder := httptest.NewRecorder() |
||||||
|
handler.ServeHTTP(recorder, req) |
||||||
|
|
||||||
|
resp := recorder.Result() |
||||||
|
body, err := ioutil.ReadAll(resp.Body) |
||||||
|
require.NoError(t, err) |
||||||
|
require.Equal(t, http.StatusInternalServerError, resp.StatusCode) |
||||||
|
require.Equal(t, "commit error\n", string(body)) |
||||||
|
} |
||||||
|
|
||||||
|
type mockAppendable struct { |
||||||
|
latest int64 |
||||||
|
samples []mockSample |
||||||
|
commitErr error |
||||||
|
} |
||||||
|
|
||||||
|
type mockSample struct { |
||||||
|
l labels.Labels |
||||||
|
t int64 |
||||||
|
v float64 |
||||||
|
} |
||||||
|
|
||||||
|
func (m *mockAppendable) Appender(_ context.Context) storage.Appender { |
||||||
|
return m |
||||||
|
} |
||||||
|
|
||||||
|
func (m *mockAppendable) Add(l labels.Labels, t int64, v float64) (uint64, error) { |
||||||
|
if t < m.latest { |
||||||
|
return 0, storage.ErrOutOfOrderSample |
||||||
|
} |
||||||
|
|
||||||
|
m.latest = t |
||||||
|
m.samples = append(m.samples, mockSample{l, t, v}) |
||||||
|
return 0, nil |
||||||
|
} |
||||||
|
|
||||||
|
func (m *mockAppendable) Commit() error { |
||||||
|
return m.commitErr |
||||||
|
} |
||||||
|
|
||||||
|
func (*mockAppendable) AddFast(uint64, int64, float64) error { |
||||||
|
return fmt.Errorf("not implemented") |
||||||
|
} |
||||||
|
|
||||||
|
func (*mockAppendable) Rollback() error { |
||||||
|
return fmt.Errorf("not implemented") |
||||||
|
} |
||||||
Loading…
Reference in new issue