apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

api/v1: Coninue work exposing rules and alerts

Browse Source 
Signed-off-by: Max Leonard Inden <IndenML@gmail.com>
pull/4318/head
Max Leonard Inden 8 years ago
parent 31f8ca0dfb
commit 71fafad099
No known key found for this signature in database
GPG Key ID: 5403C5464810BC26
6 changed files with 304 additions and 153 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 97
      docs/querying/api.md
  2. 25
      rules/alerting.go
  3. 3
      rules/manager.go
  4. 10
      rules/recording.go
  5. 192
      web/api/v1/api.go
  6. 130
      web/api/v1/api_test.go

97
docs/querying/api.md
Unescape View File

@ -363,6 +363,103 @@ $ curl http://localhost:9090/api/v1/targets
}
```
## Rules
The `/rules` API endpoint returns a list of alerting and recording rules that
are currently loaded. In addition it returns the currently active alerts fired
by the Prometheus instance of each alerting rule.
As the `/rules` endpoint is fairly new, it does not have the same stability
guarantees as the overarching API v1.
```
GET /api/v1/rules
```
```json
$ curl http://localhost:9090/api/v1/rules
{
"data": {
"groups": [
{
"rules": [
{
"alerts": [
{
"activeAt": "2018-07-04T20:27:12.60602144+02:00",
"annotations": {
"summary": "High request latency"
},
"labels": {
"alertname": "HighRequestLatency",
"severity": "page"
},
"state": "firing",
"value": 1
}
],
"annotations": {
"summary": "High request latency"
},
"duration": 600,
"labels": {
"severity": "page"
},
"name": "HighRequestLatency",
"query": "job:request_latency_seconds:mean5m{job=\"myjob\"} > 0.5",
"type": "alerting"
},
{
"name": "job:http_inprogress_requests:sum",
"query": "sum(http_inprogress_requests) by (job)",
"type": "recording"
}
],
"file": "/rules.yaml",
"interval": 60,
"name": "example"
}
]
},
"status": "success"
}
```
## Alerts
The `/alerts` endpoint returns a list of all active alerts.
As the `/alerts` endpoint is fairly new, it does not have the same stability
guarantees as the overarching API v1.
```
GET /api/v1/alerts
```
```json
$ curl http://localhost:9090/api/v1/alerts
{
"data": {
"alerts": [
{
"activeAt": "2018-07-04T20:27:12.60602144+02:00",
"annotations": {},
"labels": {
"alertname": "my-alert"
},
"state": "firing",
"value": 1
}
]
},
"status": "success"
}
```
## Querying target metadata
The following endpoint returns metadata about metrics currently scraped by targets.

25
rules/alerting.go
Unescape View File

@ -126,46 +126,31 @@ func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, ann
}
}
// Name returns the name of the alert.
// Name returns the name of the alerting rule.
func (r *AlertingRule) Name() string {
return r.name
}
// Query returns the query expression of the alert.
// Query returns the query expression of the alerting rule.
func (r *AlertingRule) Query() promql.Expr {
return r.vector
}
// Duration returns the hold duration of the alert.
// Duration returns the hold duration of the alerting rule.
func (r *AlertingRule) Duration() time.Duration {
return r.holdDuration
}
// Labels returns the labels of the alert.
// Labels returns the labels of the alerting rule.
func (r *AlertingRule) Labels() labels.Labels {
return r.labels
}
// Annotations returns the annotations of the alert.
// Annotations returns the annotations of the alerting rule.
func (r *AlertingRule) Annotations() labels.Labels {
return r.annotations
}
// Alertinfo return an array of alerts
func (r *AlertingRule) Alertinfo() []*Alert {
activealerts := &r.active
alertsarr := make([]*Alert, 0)
if len(*activealerts) > 0 {
for _, a := range *activealerts {
if a.ResolvedAt.IsZero() {
alertsarr = append(alertsarr, a)
}
}
return alertsarr
}
return nil
}
func (r *AlertingRule) equal(o *AlertingRule) bool {
return r.name == o.name && labels.Equal(r.labels, o.labels)
}

3
rules/manager.go
Unescape View File

@ -188,6 +188,9 @@ func (g *Group) File() string { return g.file }
// Rules returns the group's rules.
func (g *Group) Rules() []Rule { return g.rules }
// Interval returns the group's interval.
func (g *Group) Interval() time.Duration { return g.interval }
func (g *Group) run(ctx context.Context) {
defer close(g.terminated)

10
rules/recording.go
Unescape View File

@ -52,6 +52,16 @@ func (rule *RecordingRule) Name() string {
return rule.name
}
// Query returns the rule query expression.
func (rule *RecordingRule) Query() promql.Expr {
return rule.vector
}
// Labels returns the rule labels.
func (rule *RecordingRule) Labels() labels.Labels {
return rule.labels
}
// Eval evaluates the rule and then overrides the metric names and labels accordingly.
func (rule *RecordingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, _ *url.URL) (promql.Vector, error) {
vector, err := query(ctx, rule.vector.String(), ts)

192
web/api/v1/api.go
Unescape View File

@ -96,17 +96,9 @@ type alertmanagerRetriever interface {
DroppedAlertmanagers() []*url.URL
}
type alertRetreiver interface {
AlertingRules() []*rules.AlertingRule
}
type rulesRetreiver interface {
type rulesRetriever interface {
RuleGroups() []*rules.Group
}
type alertsrulesRetreiver interface {
alertRetreiver
rulesRetreiver
AlertingRules() []*rules.AlertingRule
}
type response struct {
@ -133,7 +125,7 @@ type API struct {
targetRetriever targetRetriever
alertmanagerRetriever alertmanagerRetriever
alertsrulesRetreiver alertsrulesRetreiver
rulesRetriever rulesRetriever
now func() time.Time
config func() config.Config
flagsMap map[string]string
@ -156,20 +148,20 @@ func NewAPI(
db func() *tsdb.DB,
enableAdmin bool,
logger log.Logger,
al alertsrulesRetreiver,
rr rulesRetriever,
) *API {
return &API{
QueryEngine: qe,
Queryable: q,
targetRetriever: tr,
alertmanagerRetriever: ar,
now: time.Now,
config: configFunc,
flagsMap: flagsMap,
ready: readyFunc,
db: db,
enableAdmin: enableAdmin,
alertsrulesRetreiver: al,
now: time.Now,
config: configFunc,
flagsMap: flagsMap,
ready: readyFunc,
db: db,
enableAdmin: enableAdmin,
rulesRetriever: rr,
}
}
@ -597,92 +589,130 @@ func (api *API) alertmanagers(r *http.Request) (interface{}, *apiError, func())
return ams, nil, nil
}
// AlertDiscovery has info for all alerts
// AlertDiscovery has info for all active alerts.
type AlertDiscovery struct {
Alertgrps []*Alertgrp `json:"alertgrp"`
Alerts []*Alert `json:"alerts"`
}
// Alert has info for a alert
// Alert has info for an alert.
type Alert struct {
Labels labels.Labels `json:"labels"`
Status string `json:"status"`
Activesince *time.Time `json:"activesince,omitempty"`
Annotations labels.Labels `json:"annotations"`
State string `json:"state"`
ActiveAt *time.Time `json:"activeAt,omitempty"`
Value float64 `json:"value"`
}
// Alertgrp has info for alerts part of a group
type Alertgrp struct {
Name string `json:"name"`
Query string `json:"query"`
Duration string `json:"duration"`
Annotations labels.Labels `json:"annotations,omitempty"`
Alerts []*Alert `json:"alerts"`
}
func (api *API) alerts(r *http.Request) (interface{}, *apiError, func()) {
alertingRules := api.rulesRetriever.AlertingRules()
alerts := []*Alert{}
func (api *API) alerts(r *http.Request) (interface{}, *apiError) {
alertingrules := api.alertsrulesRetreiver.AlertingRules()
var alertgrps []*Alertgrp
res := &AlertDiscovery{Alertgrps: alertgrps}
for _, activerule := range alertingrules {
t := &Alertgrp{
Name: activerule.Name(),
Query: fmt.Sprintf("%v", activerule.Query()),
Duration: activerule.Duration().String(),
Annotations: activerule.Annotations(),
}
alerts := activerule.Alertinfo()
var activealerts []*Alert
for _, alert := range alerts {
q := &Alert{
Labels: alert.Labels,
Status: alert.State.String(),
Activesince: &alert.ActiveAt,
}
for _, alertingRule := range alertingRules {
alerts = append(
alerts,
rulesAlertsToAPIAlerts(alertingRule.ActiveAlerts())...,
)
}
res := &AlertDiscovery{Alerts: alerts}
activealerts = append(activealerts, q)
return res, nil, nil
}
func rulesAlertsToAPIAlerts(rulesAlerts []*rules.Alert) []*Alert {
apiAlerts := make([]*Alert, len(rulesAlerts))
for i, ruleAlert := range rulesAlerts {
apiAlerts[i] = &Alert{
Labels: ruleAlert.Labels,
Annotations: ruleAlert.Annotations,
State: ruleAlert.State.String(),
ActiveAt: &ruleAlert.ActiveAt,
Value: ruleAlert.Value,
}
t.Alerts = activealerts
res.Alertgrps = append(res.Alertgrps, t)
}
return res, nil
return apiAlerts
}
// RuleDiscovery has info for all rules
type RuleDiscovery struct {
RuleGroups []*RuleGroup `json:"groups"`
}
// GroupDiscovery has info for all rules
type GroupDiscovery struct {
Rulegrps []*Rulegrp `json:"groups"`
// RuleGroup has info for rules which are part of a group
type RuleGroup struct {
Name string `json:"name"`
File string `json:"file"`
// In order to preserve rule ordering, while exposing type (alerting or recording)
// specific properties, both alerting and recording rules are exposed in the
// same array.
Rules []rule `json:"rules"`
Interval float64 `json:"interval"`
}
// Rulegrp has info for rules which are part of a group
type Rulegrp struct {
Name string `json:"name"`
File string `json:"file"`
Rules []*Ruleinfo `json:"rules"`
type rule interface{}
type alertingRule struct {
Name string `json:"name"`
Query string `json:"query"`
Duration float64 `json:"duration"`
Labels labels.Labels `json:"labels"`
Annotations labels.Labels `json:"annotations"`
Alerts []*Alert `json:"alerts"`
// Type of an alertingRule is always "alerting".
Type string `json:"type"`
}
// Ruleinfo has rule in human readable format using \n as line separators
type Ruleinfo struct {
Rule string `json:"rule"`
type recordingRule struct {
Name string `json:"name"`
Query string `json:"query"`
Labels labels.Labels `json:"labels,omitempty"`
// Type of a recordingRule is always "recording".
Type string `json:"type"`
}
func (api *API) rules(r *http.Request) (interface{}, *apiError) {
grps := api.alertsrulesRetreiver.RuleGroups()
res := &GroupDiscovery{Rulegrps: make([]*Rulegrp, len(grps))}
for i, grp := range grps {
t := &Rulegrp{
Name: grp.Name(),
File: grp.File(),
func (api *API) rules(r *http.Request) (interface{}, *apiError, func()) {
ruleGroups := api.rulesRetriever.RuleGroups()
res := &RuleDiscovery{RuleGroups: make([]*RuleGroup, len(ruleGroups))}
for i, grp := range ruleGroups {
apiRuleGroup := &RuleGroup{
Name: grp.Name(),
File: grp.File(),
Interval: grp.Interval().Seconds(),
Rules: []rule{},
}
var rulearr []*Ruleinfo
for _, rule := range grp.Rules() {
q := &Ruleinfo{
Rule: rule.String(),
for _, r := range grp.Rules() {
var enrichedRule rule
switch rule := r.(type) {
case *rules.AlertingRule:
enrichedRule = alertingRule{
Name: rule.Name(),
Query: rule.Query().String(),
Duration: rule.Duration().Seconds(),
Labels: rule.Labels(),
Annotations: rule.Annotations(),
Alerts: rulesAlertsToAPIAlerts(rule.ActiveAlerts()),
Type: "alerting",
}
case *rules.RecordingRule:
enrichedRule = recordingRule{
Name: rule.Name(),
Query: rule.Query().String(),
Labels: rule.Labels(),
Type: "recording",
}
default:
err := fmt.Errorf("failed to assert type of rule '%v'", rule.Name())
return nil, &apiError{errorInternal, err}, nil
}
rulearr = append(rulearr, q)
apiRuleGroup.Rules = append(apiRuleGroup.Rules, enrichedRule)
}
t.Rules = rulearr
res.Rulegrps[i] = t
res.RuleGroups[i] = apiRuleGroup
}
return res, nil
return res, nil, nil
}
type prometheusConfig struct {

130
web/api/v1/api_test.go
Unescape View File

@ -21,7 +21,6 @@ import (
"fmt"
"github.com/go-kit/kit/log"
"io/ioutil"
stdlog "log"
"math"
"net/http"
"net/http/httptest"
@ -102,18 +101,18 @@ func (t testAlertmanagerRetriever) DroppedAlertmanagers() []*url.URL {
}
}
type testalertsrulesfunc struct {
test *testing.T
type rulesRetrieverMock struct {
testing *testing.T
}
func (t testalertsrulesfunc) AlertingRules() []*rules.AlertingRule {
func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule {
expr1, err := promql.ParseExpr(`absent(test_metric3) != 1`)
if err != nil {
stdlog.Fatalf("Unable to parse alert expression: %s", err)
m.testing.Fatalf("unable to parse alert expression: %s", err)
}
expr2, err := promql.ParseExpr(`up == 1`)
if err != nil {
stdlog.Fatalf("Unable to parse alert expression: %s", err)
m.testing.Fatalf("Unable to parse alert expression: %s", err)
}
rule1 := rules.NewAlertingRule(
@ -138,10 +137,10 @@ func (t testalertsrulesfunc) AlertingRules() []*rules.AlertingRule {
return r
}
func (t testalertsrulesfunc) RuleGroups() []*rules.Group {
var ar testalertsrulesfunc
func (m rulesRetrieverMock) RuleGroups() []*rules.Group {
var ar rulesRetrieverMock
arules := ar.AlertingRules()
storage := testutil.NewStorage(t.test)
storage := testutil.NewStorage(m.testing)
defer storage.Close()
engine := promql.NewEngine(nil, nil, 10, 10*time.Second)
@ -158,10 +157,15 @@ func (t testalertsrulesfunc) RuleGroups() []*rules.Group {
r = append(r, alertrule)
}
recordingExpr, err := promql.ParseExpr(`vector(1)`)
if err != nil {
m.testing.Fatalf("unable to parse alert expression: %s", err)
}
recordingRule := rules.NewRecordingRule("recording-rule-1", recordingExpr, labels.Labels{})
r = append(r, recordingRule)
group := rules.NewGroup("grp", "/path/to/file", time.Second, r, opts)
fmt.Println(group)
return []*rules.Group{group}
}
var samplePrometheusCfg = config.Config{
@ -196,10 +200,14 @@ func TestEndpoints(t *testing.T) {
now := time.Now()
t.Run("local", func(t *testing.T) {
var algr rulesRetrieverMock
algr.testing = t
algr.AlertingRules()
algr.RuleGroups()
var algr testalertsrulesfunc
algr.test = t
t.Run("local", func(t *testing.T) {
var algr rulesRetrieverMock
algr.testing = t
algr.AlertingRules()
@ -210,11 +218,11 @@ func TestEndpoints(t *testing.T) {
QueryEngine: suite.QueryEngine(),
targetRetriever: testTargetRetriever{},
alertmanagerRetriever: testAlertmanagerRetriever{},
now: func() time.Time { return now },
config: func() config.Config { return samplePrometheusCfg },
flagsMap: sampleFlagMap,
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
alertsrulesRetreiver: algr,
now: func() time.Time { return now },
config: func() config.Config { return samplePrometheusCfg },
flagsMap: sampleFlagMap,
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
rulesRetriever: algr,
}
testEndpoints(t, api, true)
@ -251,8 +259,8 @@ func TestEndpoints(t *testing.T) {
t.Fatal(err)
}
var algr testalertsrulesfunc
algr.test = t
var algr rulesRetrieverMock
algr.testing = t
algr.AlertingRules()
@ -263,11 +271,11 @@ func TestEndpoints(t *testing.T) {
QueryEngine: suite.QueryEngine(),
targetRetriever: testTargetRetriever{},
alertmanagerRetriever: testAlertmanagerRetriever{},
now: func() time.Time { return now },
config: func() config.Config { return samplePrometheusCfg },
flagsMap: sampleFlagMap,
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
alertsrulesRetreiver: algr,
now: func() time.Time { return now },
config: func() config.Config { return samplePrometheusCfg },
flagsMap: sampleFlagMap,
ready: func(f http.HandlerFunc) http.HandlerFunc { return f },
rulesRetriever: algr,
}
testEndpoints(t, api, false)
@ -652,37 +660,41 @@ func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
{
endpoint: api.alerts,
response: &AlertDiscovery{
Alertgrps: []*Alertgrp{
{
Name: "test_metric3",
Query: "absent(test_metric3) != 1",
Duration: "1s",
Alerts: nil,
Annotations: labels.Labels{},
},
{
Name: "test_metric4",
Query: "up == 1",
Duration: "1s",
Alerts: nil,
Annotations: labels.Labels{},
},
},
Alerts: []*Alert{},
},
},
{
endpoint: api.rules,
response: &GroupDiscovery{
Rulegrps: []*Rulegrp{
response: &RuleDiscovery{
RuleGroups: []*RuleGroup{
{
Name: "grp",
File: "/path/to/file",
Rules: []*Ruleinfo{
{
Rule: "alert: test_metric3\nexpr: absent(test_metric3) != 1\nfor: 1s\n",
Name: "grp",
File: "/path/to/file",
Interval: 1,
Rules: []rule{
alertingRule{
Name: "test_metric3",
Query: "absent(test_metric3) != 1",
Duration: 1,
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Type: "alerting",
},
alertingRule{
Name: "test_metric4",
Query: "up == 1",
Duration: 1,
Labels: labels.Labels{},
Annotations: labels.Labels{},
Alerts: []*Alert{},
Type: "alerting",
},
{
Rule: "alert: test_metric4\nexpr: up == 1\nfor: 1s\n",
recordingRule{
Name: "recording-rule-1",
Query: "vector(1)",
Labels: labels.Labels{},
Type: "recording",
},
},
},
@ -768,7 +780,21 @@ func testEndpoints(t *testing.T, api *API, testLabelAPI bool) {
t.Fatalf("Expected error of type %q but got none", test.errType)
}
if !reflect.DeepEqual(resp, test.response) {
t.Fatalf("Response does not match, expected:\n%+v\ngot:\n%+v", test.response, resp)
respJSON, err := json.Marshal(resp)
if err != nil {
t.Fatalf("failed to marshal response as JSON: %v", err.Error())
}
expectedRespJSON, err := json.Marshal(test.response)
if err != nil {
t.Fatalf("failed to marshal expected response as JSON: %v", err.Error())
}
t.Fatalf(
"Response does not match, expected:\n%+v\ngot:\n%+v",
string(expectedRespJSON),
string(respJSON),
)
}
}
}

Write Preview
Loading…
Cancel
Save