apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make scraping offset consistent.

Browse Source 
To evenly distribute scraping load we currently rely on random
jittering. This commit hashes over the target's identity and calculates
a consistent offset. This also ensures that scrape intervals
are constantly spaced between config/target changes.
pull/1393/head
Fabian Reinartz 10 years ago
parent 938ebe78c2
commit fe7e91e2eb
3 changed files with 123 additions and 56 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 31
      retrieval/target.go
  2. 46
      retrieval/target_test.go
  3. 102
      web/ui/bindata.go

31
retrieval/target.go
Unescape View File

@ -18,7 +18,6 @@ import (
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
"strings"
@ -267,6 +266,30 @@ func (t *Target) String() string {
return t.host()
}
// fingerprint returns an identifying hash for the target.
func (t *Target) fingerprint() model.Fingerprint {
t.RLock()
defer t.RUnlock()
return t.labels.Fingerprint()
}
// offset returns the time until the next scrape cycle for the target.
func (t *Target) offset(interval time.Duration) time.Duration {
now := time.Now().UnixNano()
var (
base = now % int64(interval)
offset = uint64(t.fingerprint()) % uint64(interval)
next = base + int64(offset)
)
if next > int64(interval) {
next -= int64(interval)
}
return time.Duration(next)
}
func (t *Target) client() (*http.Client, error) {
t.RLock()
defer t.RUnlock()
@ -366,14 +389,12 @@ func (t *Target) RunScraper(sampleAppender storage.SampleAppender) {
log.Debugf("Starting scraper for target %v...", t)
jitterTimer := time.NewTimer(time.Duration(float64(lastScrapeInterval) * rand.Float64()))
select {
case <-jitterTimer.C:
case <-time.After(t.offset(lastScrapeInterval)):
// Continue after scraping offset.
case <-t.scraperStopping:
jitterTimer.Stop()
return
}
jitterTimer.Stop()
ticker := time.NewTicker(lastScrapeInterval)
defer ticker.Stop()

46
retrieval/target_test.go
Unescape View File

@ -45,6 +45,52 @@ func TestTargetLabels(t *testing.T) {
}
}
func TestTargetOffset(t *testing.T) {
interval := 10 * time.Second
offsets := make([]time.Duration, 10000)
// Calculate offsets for 10000 different targets.
for i := range offsets {
target := newTestTarget("example.com:80", 0, model.LabelSet{
"label": model.LabelValue(fmt.Sprintf("%d", i)),
})
offsets[i] = target.offset(interval)
}
// Put the offsets into buckets and validate that they are all
// within bounds.
bucketSize := 1 * time.Second
buckets := make([]int, interval/bucketSize)
for _, offset := range offsets {
if offset < 0 || offset >= interval {
t.Fatalf("Offset %v out of bounds", offset)
}
bucket := offset / bucketSize
buckets[bucket]++
}
t.Log(buckets)
// Calculate whether the the number of targets per bucket
// does not differ more than a given tolerance.
avg := len(offsets) / len(buckets)
tolerance := 0.15
for _, bucket := range buckets {
diff := bucket - avg
if diff < 0 {
diff = -diff
}
if float64(diff)/float64(avg) > tolerance {
t.Fatalf("Bucket out of tolerance bounds")
}
}
}
func TestOverwriteLabels(t *testing.T) {
type test struct {
metric string

102
web/ui/bindata.go
Unescape View File

@ -791,38 +791,38 @@ func AssetNames() []string {
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
"web/ui/templates/_base.html": webUiTemplates_baseHtml,
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
"web/ui/templates/graph.html": webUiTemplatesGraphHtml,
"web/ui/templates/status.html": webUiTemplatesStatusHtml,
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
"web/ui/static/css/graph.css": webUiStaticCssGraphCss,
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
"web/ui/static/js/graph.js": webUiStaticJsGraphJs,
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss,
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
}
// AssetDir returns the file names below a certain
@ -864,69 +864,70 @@ type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"web": &bintree{nil, map[string]*bintree{
"ui": &bintree{nil, map[string]*bintree{
"static": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
"prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}},
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
}},
"img": &bintree{nil, map[string]*bintree{
"ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
"graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}},
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
}},
"vendor": &bintree{nil, map[string]*bintree{
"bootstrap-3.3.1": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{
"bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}},
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
}},
"fonts": &bintree{nil, map[string]*bintree{
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
"glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}},
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
}},
}},
"bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
"bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}},
}},
"bootstrap3-typeahead": &bintree{nil, map[string]*bintree{
"bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}},
}},
"js": &bintree{nil, map[string]*bintree{
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
"jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}},
}},
"rickshaw": &bintree{nil, map[string]*bintree{
"rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}},
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
"vendor": &bintree{nil, map[string]*bintree{
"d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}},
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
}},
}},
}},
}},
"templates": &bintree{nil, map[string]*bintree{
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
"alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}},
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
"status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}},
}},
}},
@ -979,4 +980,3 @@ func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

Write Preview
Loading…
Cancel
Save