mirror of https://github.com/grafana/grafana
Alerting: Introduce a Mimir client as part of the Remote Alertmanager (#78357)
* Alerting: Introduce a Mimir client as part of the Remote Alertmanager This is our first attempt at making Grafana communicate use Mimir as a backend - it uses a new set of APIs that we've developed on the Mimir side to upload the grafana configuration and alertmanager state so that it can then be ported over. Codewise, we've introduced a couple of things: A client to isolate in its own package all the communication that happens with Mimir A few changes to the remote/alertmanager to include uploading the configuration and state when it starts A few refactors that align a bit better with the design approach that we're thinking An integration tests again these newly developed APIs using a custom image --------- Signed-off-by: gotjosh <josue.abreu@gmail.com> Co-authored-by: Santiago <santiagohernandez.1997@gmail.com>pull/78611/head
gotjosh
2 years ago
committed by
GitHub
parent
eedc19f9f0
commit
23fe8f4e9c
@ -0,0 +1,59 @@ |
||||
package client |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
"net/http" |
||||
) |
||||
|
||||
const ( |
||||
grafanaAlertmanagerConfigPath = "/api/v1/grafana/config" |
||||
) |
||||
|
||||
type UserGrafanaConfig struct { |
||||
ID int64 `json:"id"` |
||||
GrafanaAlertmanagerConfig string `json:"configuration"` |
||||
Hash string `json:"configuration_hash"` |
||||
CreatedAt int64 `json:"created"` |
||||
Default bool `json:"default"` |
||||
} |
||||
|
||||
func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) { |
||||
gc := &UserGrafanaConfig{} |
||||
response := successResponse{ |
||||
Data: gc, |
||||
} |
||||
// nolint:bodyclose
|
||||
// closed within `do`
|
||||
_, err := mc.do(ctx, grafanaAlertmanagerConfigPath, http.MethodGet, nil, &response) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if response.Status != "success" { |
||||
return nil, fmt.Errorf("returned non-success `status` from the MimirAPI: %s", response.Status) |
||||
} |
||||
|
||||
return gc, nil |
||||
} |
||||
|
||||
func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, c, hash string, id, created int64, d bool) error { |
||||
payload, err := json.Marshal(&UserGrafanaConfig{ |
||||
ID: id, |
||||
GrafanaAlertmanagerConfig: c, |
||||
Hash: hash, |
||||
CreatedAt: created, |
||||
Default: d, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return mc.doOK(ctx, grafanaAlertmanagerConfigPath, http.MethodPost, bytes.NewBuffer(payload)) |
||||
} |
||||
|
||||
func (mc *Mimir) DeleteGrafanaAlertmanagerConfig(ctx context.Context) error { |
||||
return mc.doOK(ctx, grafanaAlertmanagerConfigPath, http.MethodDelete, nil) |
||||
} |
||||
@ -0,0 +1,51 @@ |
||||
package client |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
"net/http" |
||||
) |
||||
|
||||
const ( |
||||
grafanaAlertmanagerStatePath = "/grafana/state" |
||||
) |
||||
|
||||
type UserGrafanaState struct { |
||||
State string `json:"state"` |
||||
} |
||||
|
||||
func (mc *Mimir) GetGrafanaAlertmanagerState(ctx context.Context) (*UserGrafanaState, error) { |
||||
gs := &UserGrafanaState{} |
||||
response := successResponse{ |
||||
Data: gs, |
||||
} |
||||
// nolint:bodyclose
|
||||
// closed within `do`
|
||||
_, err := mc.do(ctx, grafanaAlertmanagerStatePath, http.MethodGet, nil, &response) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if response.Status != "success" { |
||||
return nil, fmt.Errorf("returned non-success `status` from the MimirAPI: %s", response.Status) |
||||
} |
||||
|
||||
return gs, nil |
||||
} |
||||
|
||||
func (mc *Mimir) CreateGrafanaAlertmanagerState(ctx context.Context, state string) error { |
||||
payload, err := json.Marshal(&UserGrafanaState{ |
||||
State: state, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return mc.doOK(ctx, grafanaAlertmanagerStatePath, http.MethodPost, bytes.NewBuffer(payload)) |
||||
} |
||||
|
||||
func (mc *Mimir) DeleteGrafanaAlertmanagerState(ctx context.Context) error { |
||||
return mc.doOK(ctx, grafanaAlertmanagerStatePath, http.MethodDelete, nil) |
||||
} |
||||
@ -0,0 +1,179 @@ |
||||
package client |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"io" |
||||
"net/http" |
||||
"net/url" |
||||
"path" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
) |
||||
|
||||
// MimirClient contains all the methods to query the migration critical endpoints of Mimir instance, it's an interface to allow multiple implementations.
|
||||
type MimirClient interface { |
||||
GetGrafanaAlertmanagerState(ctx context.Context) (*UserGrafanaState, error) |
||||
CreateGrafanaAlertmanagerState(ctx context.Context, s string) error |
||||
DeleteGrafanaAlertmanagerState(ctx context.Context) error |
||||
|
||||
GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) |
||||
CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration string, hash string, id int64, at int64, d bool) error |
||||
DeleteGrafanaAlertmanagerConfig(ctx context.Context) error |
||||
} |
||||
|
||||
type Mimir struct { |
||||
endpoint *url.URL |
||||
client http.Client |
||||
logger log.Logger |
||||
} |
||||
|
||||
type Config struct { |
||||
Address string |
||||
TenantID string |
||||
Password string |
||||
|
||||
Logger log.Logger |
||||
} |
||||
|
||||
// successResponse represents a successful response from the Mimir API.
|
||||
type successResponse struct { |
||||
Status string `json:"status"` |
||||
Data any `json:"data"` |
||||
} |
||||
|
||||
// errorResponse represents an error from the Mimir API.
|
||||
type errorResponse struct { |
||||
Status string `json:"status"` |
||||
Error1 string `json:"error"` |
||||
Error2 string `json:"Error"` |
||||
} |
||||
|
||||
func (e *errorResponse) Error() string { |
||||
if e.Error1 != "" { |
||||
return e.Error1 |
||||
} |
||||
|
||||
return e.Error2 |
||||
} |
||||
|
||||
func New(cfg *Config) (*Mimir, error) { |
||||
endpoint, err := url.Parse(cfg.Address) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
rt := &MimirAuthRoundTripper{ |
||||
TenantID: cfg.TenantID, |
||||
Password: cfg.Password, |
||||
Next: http.DefaultTransport, |
||||
} |
||||
|
||||
c := http.Client{ |
||||
Transport: rt, |
||||
} |
||||
|
||||
return &Mimir{ |
||||
endpoint: endpoint, |
||||
client: c, |
||||
logger: cfg.Logger, |
||||
}, nil |
||||
} |
||||
|
||||
// do execute an HTTP requests against the specified path and method using the specified payload.
|
||||
// It returns the HTTP response.
|
||||
func (mc *Mimir) do(ctx context.Context, p, method string, payload io.Reader, out any) (*http.Response, error) { |
||||
pathURL, err := url.Parse(p) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
endpoint := *mc.endpoint |
||||
endpoint.Path = path.Join(endpoint.Path, pathURL.Path) |
||||
|
||||
r, err := http.NewRequestWithContext(ctx, method, endpoint.String(), payload) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
r.Header.Set("Accept", "application/json") |
||||
r.Header.Set("Content-Type", "application/json") |
||||
|
||||
resp, err := mc.client.Do(r) |
||||
if err != nil { |
||||
msg := "Unable to fulfill request to the Mimir API" |
||||
mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method) |
||||
return nil, fmt.Errorf("%s: %w", msg, err) |
||||
} |
||||
defer func() { |
||||
if err := resp.Body.Close(); err != nil { |
||||
mc.logger.Error("Error closing HTTP body", "err", err, "url", r.URL.String(), "method", r.Method) |
||||
} |
||||
}() |
||||
|
||||
ct := resp.Header.Get("Content-Type") |
||||
if !strings.HasPrefix(ct, "application/json") { |
||||
msg := "Response content-type is not application/json" |
||||
mc.logger.Error(msg, "content-type", "url", r.URL.String(), "method", r.Method, ct, "status", resp.StatusCode) |
||||
return nil, fmt.Errorf("%s: %s", msg, ct) |
||||
} |
||||
|
||||
if out == nil { |
||||
return resp, nil |
||||
} |
||||
|
||||
body, err := io.ReadAll(resp.Body) |
||||
if err != nil { |
||||
msg := "Failed to read the request body" |
||||
mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode) |
||||
return nil, fmt.Errorf("%s: %w", msg, err) |
||||
} |
||||
|
||||
if resp.StatusCode/100 != 2 { |
||||
errResponse := &errorResponse{} |
||||
err = json.Unmarshal(body, errResponse) |
||||
|
||||
if err == nil && errResponse.Error() != "" { |
||||
msg := "Error response from the Mimir API" |
||||
mc.logger.Error(msg, "err", errResponse, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode) |
||||
return nil, fmt.Errorf("%s: %w", msg, errResponse) |
||||
} |
||||
|
||||
msg := "Failed to decode non-2xx JSON response" |
||||
mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode) |
||||
return nil, fmt.Errorf("%s: %w", msg, err) |
||||
} |
||||
|
||||
if err = json.Unmarshal(body, out); err != nil { |
||||
msg := "Failed to decode 2xx JSON response" |
||||
mc.logger.Error(msg, "err", err, "url", r.URL.String(), "method", r.Method, "status", resp.StatusCode) |
||||
return nil, fmt.Errorf("%s: %w", msg, err) |
||||
} |
||||
|
||||
return resp, nil |
||||
} |
||||
|
||||
func (mc *Mimir) doOK(ctx context.Context, p, method string, payload io.Reader) error { |
||||
var sr successResponse |
||||
resp, err := mc.do(ctx, p, method, payload, &sr) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
defer func() { |
||||
if err := resp.Body.Close(); err != nil { |
||||
mc.logger.Error("Error closing HTTP body", "err", err) |
||||
} |
||||
}() |
||||
|
||||
switch sr.Status { |
||||
case "success": |
||||
return nil |
||||
case "error": |
||||
return errors.New("received an 2xx status code but the request body reflected an error") |
||||
default: |
||||
return fmt.Errorf("received an unknown status from the request body: %s", sr.Status) |
||||
} |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
package client |
||||
|
||||
import ( |
||||
"net/http" |
||||
) |
||||
|
||||
const mimirTenantHeader = "X-Scope-OrgID" |
||||
|
||||
type MimirAuthRoundTripper struct { |
||||
TenantID string |
||||
Password string |
||||
Next http.RoundTripper |
||||
} |
||||
|
||||
// RoundTrip implements the http.RoundTripper interface
|
||||
// It adds an `X-Scope-OrgID` header with the TenantID if only provided with a tenantID or sets HTTP Basic Authentication if both
|
||||
// a tenantID and a password are provided.
|
||||
func (r *MimirAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { |
||||
if r.TenantID != "" && r.Password == "" { |
||||
req.Header.Set(mimirTenantHeader, r.TenantID) |
||||
} |
||||
|
||||
if r.TenantID != "" && r.Password != "" { |
||||
req.SetBasicAuth(r.TenantID, r.Password) |
||||
} |
||||
|
||||
return r.Next.RoundTrip(req) |
||||
} |
||||
Loading…
Reference in new issue