@ -37,30 +37,31 @@ const (
storageStopping
)
type memorySeriesStorage struct {
fpLocker * fingerprintLocker
persistDone chan bool
stopServing chan chan <- bool
type persistRequest struct {
fingerprint clientmodel . Fingerprint
chunkDesc * chunkDesc
}
type memorySeriesStorage struct {
fpLocker * fingerprintLocker
fingerprintToSeries * seriesMap
memoryEvictionInterval time . Duration
memoryRetentionPeriod time . Duration
persistencePurgeInterval time . Duration
persistenceRetentionPeriod time . Duration
persistQueue chan * persistRequest
persistence * persistence
persistLatency prometheus . Summary
persistErrors * prometheus . CounterVec
persistQueueLength prometheus . Gauge
numSeries prometheus . Gauge
seriesOps * prometheus . CounterVec
ingestedSamplesCount prometheus . Counter
purgeDuration , eviction Duration prometheus . Gauge
loopStopping , loopStopped chan struct { }
evictInterval , evictAfter time . Duration
purgeInterval , purgeAfter time . Duration
checkpointInterval time . Duration
persistQueue chan * persistRequest
persistStopped chan struct { }
persistence * persistence
persistLatency prometheus . Summary
persistErrors * prometheus . CounterVec
persistQueueLength prometheus . Gauge
numSeries prometheus . Gauge
seriesOps * prometheus . CounterVec
ingestedSamplesCount prometheus . Counter
purgeDuration , evictDuration prometheus . Gauge
}
// MemorySeriesStorageOptions contains options needed by
@ -72,6 +73,7 @@ type MemorySeriesStorageOptions struct {
PersistenceStoragePath string // Location of persistence files.
PersistencePurgeInterval time . Duration // How often to check for purging.
PersistenceRetentionPeriod time . Duration // Chunks at least that old are purged.
CheckpointInterval time . Duration // How often to checkpoint the series map and head chunks.
}
// NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still
@ -96,20 +98,20 @@ func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) (Storage, error) {
numSeries . Set ( float64 ( fingerprintToSeries . length ( ) ) )
return & memorySeriesStorage {
fpLocker : newFingerprintLocker ( 100 ) , // TODO: Tweak value.
fpLocker : newFingerprintLocker ( 100 ) , // TODO: Tweak value.
fingerprintToSeries : fingerprintToSeries ,
persistDone : make ( chan bool ) ,
stopServing : make ( chan chan <- bool ) ,
memoryEvictionInterval : o . MemoryEvictionInterval ,
memoryRetentionPeriod : o . MemoryRetentionPeriod ,
loopStopping : make ( chan struct { } ) ,
loopStopped : make ( chan struct { } ) ,
evictInterval : o . MemoryEvictionInterval ,
evictAfter : o . MemoryRetentionPeriod ,
purgeInterval : o . PersistencePurgeInterval ,
purgeAfter : o . PersistenceRetentionPeriod ,
checkpointInterval : o . CheckpointInterval ,
persistencePurgeInterval : o . PersistencePurgeInterval ,
persistenceRetentionPeriod : o . PersistenceRetentionPeriod ,
persistQueue : make ( chan * persistRequest , persistQueueCap ) ,
persistence : p ,
persistQueue : make ( chan * persistRequest , persistQueueCap ) ,
persistStopped : make ( chan struct { } ) ,
persistence : p ,
persistLatency : prometheus . NewSummary ( prometheus . SummaryOpts {
Namespace : namespace ,
@ -154,18 +156,166 @@ func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) (Storage, error) {
Name : "purge_duration_milliseconds" ,
Help : "The duration of the last storage purge iteration in milliseconds." ,
} ) ,
eviction Duration : prometheus . NewGauge ( prometheus . GaugeOpts {
evictDuration : prometheus . NewGauge ( prometheus . GaugeOpts {
Namespace : namespace ,
Subsystem : subsystem ,
Name : "eviction _duration_milliseconds" ,
Name : "evict_duration_milliseconds" ,
Help : "The duration of the last memory eviction iteration in milliseconds." ,
} ) ,
} , nil
}
type persistRequest struct {
fingerprint clientmodel . Fingerprint
chunkDesc * chunkDesc
// Start implements Storage.
func ( s * memorySeriesStorage ) Start ( ) {
go s . handlePersistQueue ( )
go s . loop ( )
}
// Stop implements Storage.
func ( s * memorySeriesStorage ) Stop ( ) error {
glog . Info ( "Stopping maintenance loop..." )
close ( s . loopStopping )
<- s . loopStopped
glog . Info ( "Stopping persist loop..." )
close ( s . persistQueue )
<- s . persistStopped
// One final checkpoint of the series map and the head chunks.
if err := s . persistence . checkpointSeriesMapAndHeads ( s . fingerprintToSeries , s . fpLocker ) ; err != nil {
return err
}
if err := s . persistence . close ( ) ; err != nil {
return err
}
return nil
}
// WaitForIndexing implements Storage.
func ( s * memorySeriesStorage ) WaitForIndexing ( ) {
s . persistence . waitForIndexing ( )
}
// NewIterator implements storage.
func ( s * memorySeriesStorage ) NewIterator ( fp clientmodel . Fingerprint ) SeriesIterator {
s . fpLocker . Lock ( fp )
defer s . fpLocker . Unlock ( fp )
series , ok := s . fingerprintToSeries . get ( fp )
if ! ok {
// Oops, no series for fp found. That happens if, after
// preloading is done, the whole series is identified as old
// enough for purging and hence purged for good. As there is no
// data left to iterate over, return an iterator that will never
// return any values.
return nopSeriesIterator { }
}
return series . newIterator (
func ( ) { s . fpLocker . Lock ( fp ) } ,
func ( ) { s . fpLocker . Unlock ( fp ) } ,
)
}
// NewPreloader implements Storage.
func ( s * memorySeriesStorage ) NewPreloader ( ) Preloader {
return & memorySeriesPreloader {
storage : s ,
}
}
// GetFingerprintsForLabelMatchers implements Storage.
func ( s * memorySeriesStorage ) GetFingerprintsForLabelMatchers ( labelMatchers metric . LabelMatchers ) clientmodel . Fingerprints {
var result map [ clientmodel . Fingerprint ] struct { }
for _ , matcher := range labelMatchers {
intersection := map [ clientmodel . Fingerprint ] struct { } { }
switch matcher . Type {
case metric . Equal :
fps , err := s . persistence . getFingerprintsForLabelPair (
metric . LabelPair {
Name : matcher . Name ,
Value : matcher . Value ,
} ,
)
if err != nil {
glog . Error ( "Error getting fingerprints for label pair: " , err )
}
if len ( fps ) == 0 {
return nil
}
for _ , fp := range fps {
if _ , ok := result [ fp ] ; ok || result == nil {
intersection [ fp ] = struct { } { }
}
}
default :
values , err := s . persistence . getLabelValuesForLabelName ( matcher . Name )
if err != nil {
glog . Errorf ( "Error getting label values for label name %q: %v" , matcher . Name , err )
}
matches := matcher . Filter ( values )
if len ( matches ) == 0 {
return nil
}
for _ , v := range matches {
fps , err := s . persistence . getFingerprintsForLabelPair (
metric . LabelPair {
Name : matcher . Name ,
Value : v ,
} ,
)
if err != nil {
glog . Error ( "Error getting fingerprints for label pair: " , err )
}
for _ , fp := range fps {
if _ , ok := result [ fp ] ; ok || result == nil {
intersection [ fp ] = struct { } { }
}
}
}
}
if len ( intersection ) == 0 {
return nil
}
result = intersection
}
fps := make ( clientmodel . Fingerprints , 0 , len ( result ) )
for fp := range result {
fps = append ( fps , fp )
}
return fps
}
// GetLabelValuesForLabelName implements Storage.
func ( s * memorySeriesStorage ) GetLabelValuesForLabelName ( labelName clientmodel . LabelName ) clientmodel . LabelValues {
lvs , err := s . persistence . getLabelValuesForLabelName ( labelName )
if err != nil {
glog . Errorf ( "Error getting label values for label name %q: %v" , labelName , err )
}
return lvs
}
// GetMetricForFingerprint implements Storage.
func ( s * memorySeriesStorage ) GetMetricForFingerprint ( fp clientmodel . Fingerprint ) clientmodel . Metric {
s . fpLocker . Lock ( fp )
defer s . fpLocker . Unlock ( fp )
series , ok := s . fingerprintToSeries . get ( fp )
if ok {
// Copy required here because caller might mutate the returned
// metric.
m := make ( clientmodel . Metric , len ( series . metric ) )
for ln , lv := range series . metric {
m [ ln ] = lv
}
return m
}
metric , err := s . persistence . getArchivedMetric ( fp )
if err != nil {
glog . Errorf ( "Error retrieving archived metric for fingerprint %v: %v" , fp , err )
}
return metric
}
// AppendSamples implements Storage.
@ -250,51 +400,6 @@ func (s *memorySeriesStorage) preloadChunksForRange(
return series . preloadChunksForRange ( from , through , fp , s . persistence )
}
// NewIterator implements storage.
func ( s * memorySeriesStorage ) NewIterator ( fp clientmodel . Fingerprint ) SeriesIterator {
s . fpLocker . Lock ( fp )
defer s . fpLocker . Unlock ( fp )
series , ok := s . fingerprintToSeries . get ( fp )
if ! ok {
// Oops, no series for fp found. That happens if, after
// preloading is done, the whole series is identified as old
// enough for purging and hence purged for good. As there is no
// data left to iterate over, return an iterator that will never
// return any values.
return nopSeriesIterator { }
}
return series . newIterator (
func ( ) { s . fpLocker . Lock ( fp ) } ,
func ( ) { s . fpLocker . Unlock ( fp ) } ,
)
}
func ( s * memorySeriesStorage ) evictMemoryChunks ( ttl time . Duration ) {
defer func ( begin time . Time ) {
s . evictionDuration . Set ( float64 ( time . Since ( begin ) ) / float64 ( time . Millisecond ) )
} ( time . Now ( ) )
for m := range s . fingerprintToSeries . iter ( ) {
s . fpLocker . Lock ( m . fp )
if m . series . evictOlderThan (
clientmodel . TimestampFromTime ( time . Now ( ) ) . Add ( - 1 * ttl ) ,
m . fp , s . persistQueue ,
) {
s . fingerprintToSeries . del ( m . fp )
s . numSeries . Dec ( )
if err := s . persistence . archiveMetric (
m . fp , m . series . metric , m . series . firstTime ( ) , m . series . lastTime ( ) ,
) ; err != nil {
glog . Errorf ( "Error archiving metric %v: %v" , m . series . metric , err )
} else {
s . seriesOps . WithLabelValues ( archive ) . Inc ( )
}
}
s . fpLocker . Unlock ( m . fp )
}
}
func ( s * memorySeriesStorage ) handlePersistQueue ( ) {
for req := range s . persistQueue {
s . persistQueueLength . Set ( float64 ( len ( s . persistQueue ) ) )
@ -310,58 +415,71 @@ func (s *memorySeriesStorage) handlePersistQueue() {
req . chunkDesc . unpin ( )
chunkOps . WithLabelValues ( persistAndUnpin ) . Inc ( )
}
s . persistDone <- true
}
// WaitForIndexing implements Storage.
func ( s * memorySeriesStorage ) WaitForIndexing ( ) {
s . persistence . waitForIndexing ( )
}
// Close stops serving, flushes all pending operations, and frees all
// resources. It implements Storage.
func ( s * memorySeriesStorage ) Close ( ) error {
stopped := make ( chan bool )
glog . Info ( "Waiting for storage to stop serving..." )
s . stopServing <- stopped
<- stopped
glog . Info ( "Serving stopped." )
glog . Info ( "Stopping persist loop..." )
close ( s . persistQueue )
<- s . persistDone
glog . Info ( "Persist loop stopped." )
glog . Info ( "Persisting head chunks..." )
if err := s . persistence . persistSeriesMapAndHeads ( s . fingerprintToSeries ) ; err != nil {
return err
}
glog . Info ( "Done persisting head chunks." )
if err := s . persistence . close ( ) ; err != nil {
return err
}
return nil
close ( s . persistStopped )
}
func ( s * memorySeriesStorage ) purgePeriodically ( stop <- chan bool ) {
purgeTicker := time . NewTicker ( s . persistencePurgeInterval )
defer purgeTicker . Stop ( )
func ( s * memorySeriesStorage ) loop ( ) {
evictTicker := time . NewTicker ( s . evictInterval )
purgeTicker := time . NewTicker ( s . purgeInterval )
checkpointTicker := time . NewTicker ( s . checkpointInterval )
defer func ( ) {
evictTicker . Stop ( )
purgeTicker . Stop ( )
checkpointTicker . Stop ( )
glog . Info ( "Maintenance loop stopped." )
close ( s . loopStopped )
} ( )
for {
select {
case <- stop :
glog . Info ( "Purging loop stopped." )
case <- s . loopStopping :
return
case <- checkpointTicker . C :
s . persistence . checkpointSeriesMapAndHeads ( s . fingerprintToSeries , s . fpLocker )
case <- evictTicker . C :
// TODO: Change this to be based on number of chunks in memory.
glog . Info ( "Evicting chunks..." )
begin := time . Now ( )
for m := range s . fingerprintToSeries . iter ( ) {
select {
case <- s . loopStopping :
glog . Info ( "Interrupted evicting chunks." )
return
default :
// Keep going.
}
s . fpLocker . Lock ( m . fp )
if m . series . evictOlderThan (
clientmodel . TimestampFromTime ( time . Now ( ) ) . Add ( - 1 * s . evictAfter ) ,
m . fp , s . persistQueue ,
) {
s . fingerprintToSeries . del ( m . fp )
s . numSeries . Dec ( )
if err := s . persistence . archiveMetric (
m . fp , m . series . metric , m . series . firstTime ( ) , m . series . lastTime ( ) ,
) ; err != nil {
glog . Errorf ( "Error archiving metric %v: %v" , m . series . metric , err )
} else {
s . seriesOps . WithLabelValues ( archive ) . Inc ( )
}
}
s . fpLocker . Unlock ( m . fp )
}
duration := time . Since ( begin )
s . evictDuration . Set ( float64 ( duration ) / float64 ( time . Millisecond ) )
glog . Infof ( "Done evicting chunks in %v." , duration )
case <- purgeTicker . C :
glog . Info ( "Purging old series data..." )
ts := clientmodel . TimestampFromTime ( time . Now ( ) ) . Add ( - 1 * s . persistenceRetentionPeriod )
ts := clientmodel . TimestampFromTime ( time . Now ( ) ) . Add ( - 1 * s . purgeAfter )
begin := time . Now ( )
for fp := range s . fingerprintToSeries . fpIter ( ) {
select {
case <- stop :
glog . Info ( "Interrupted running series purge." )
case <- s . loopS topping :
glog . Info ( "Interrupted purging series ." )
return
default :
s . purgeSeries ( fp , ts )
@ -375,15 +493,16 @@ func (s *memorySeriesStorage) purgePeriodically(stop <-chan bool) {
}
for _ , fp := range persistedFPs {
select {
case <- stop :
glog . Info ( "Interrupted running series purge ." )
case <- s . loopS topping :
glog . Info ( "Interrupted purnging series ." )
return
default :
s . purgeSeries ( fp , ts )
}
}
s . purgeDuration . Set ( float64 ( time . Since ( begin ) ) / float64 ( time . Millisecond ) )
glog . Info ( "Done purging old series data." )
duration := time . Since ( begin )
s . purgeDuration . Set ( float64 ( duration ) / float64 ( time . Millisecond ) )
glog . Infof ( "Done purging old series data in %v." , duration )
}
}
}
@ -427,130 +546,6 @@ func (s *memorySeriesStorage) purgeSeries(fp clientmodel.Fingerprint, beforeTime
}
}
// Serve implements Storage.
func ( s * memorySeriesStorage ) Serve ( started chan struct { } ) {
evictMemoryTicker := time . NewTicker ( s . memoryEvictionInterval )
defer evictMemoryTicker . Stop ( )
go s . handlePersistQueue ( )
stopPurge := make ( chan bool )
go s . purgePeriodically ( stopPurge )
close ( started )
for {
select {
case <- evictMemoryTicker . C :
s . evictMemoryChunks ( s . memoryRetentionPeriod )
case stopped := <- s . stopServing :
stopPurge <- true
stopped <- true
return
}
}
}
// NewPreloader implements Storage.
func ( s * memorySeriesStorage ) NewPreloader ( ) Preloader {
return & memorySeriesPreloader {
storage : s ,
}
}
// GetFingerprintsForLabelMatchers implements Storage.
func ( s * memorySeriesStorage ) GetFingerprintsForLabelMatchers ( labelMatchers metric . LabelMatchers ) clientmodel . Fingerprints {
var result map [ clientmodel . Fingerprint ] struct { }
for _ , matcher := range labelMatchers {
intersection := map [ clientmodel . Fingerprint ] struct { } { }
switch matcher . Type {
case metric . Equal :
fps , err := s . persistence . getFingerprintsForLabelPair (
metric . LabelPair {
Name : matcher . Name ,
Value : matcher . Value ,
} ,
)
if err != nil {
glog . Error ( "Error getting fingerprints for label pair: " , err )
}
if len ( fps ) == 0 {
return nil
}
for _ , fp := range fps {
if _ , ok := result [ fp ] ; ok || result == nil {
intersection [ fp ] = struct { } { }
}
}
default :
values , err := s . persistence . getLabelValuesForLabelName ( matcher . Name )
if err != nil {
glog . Errorf ( "Error getting label values for label name %q: %v" , matcher . Name , err )
}
matches := matcher . Filter ( values )
if len ( matches ) == 0 {
return nil
}
for _ , v := range matches {
fps , err := s . persistence . getFingerprintsForLabelPair (
metric . LabelPair {
Name : matcher . Name ,
Value : v ,
} ,
)
if err != nil {
glog . Error ( "Error getting fingerprints for label pair: " , err )
}
for _ , fp := range fps {
if _ , ok := result [ fp ] ; ok || result == nil {
intersection [ fp ] = struct { } { }
}
}
}
}
if len ( intersection ) == 0 {
return nil
}
result = intersection
}
fps := make ( clientmodel . Fingerprints , 0 , len ( result ) )
for fp := range result {
fps = append ( fps , fp )
}
return fps
}
// GetLabelValuesForLabelName implements Storage.
func ( s * memorySeriesStorage ) GetLabelValuesForLabelName ( labelName clientmodel . LabelName ) clientmodel . LabelValues {
lvs , err := s . persistence . getLabelValuesForLabelName ( labelName )
if err != nil {
glog . Errorf ( "Error getting label values for label name %q: %v" , labelName , err )
}
return lvs
}
// GetMetricForFingerprint implements Storage.
func ( s * memorySeriesStorage ) GetMetricForFingerprint ( fp clientmodel . Fingerprint ) clientmodel . Metric {
s . fpLocker . Lock ( fp )
defer s . fpLocker . Unlock ( fp )
series , ok := s . fingerprintToSeries . get ( fp )
if ok {
// Copy required here because caller might mutate the returned
// metric.
m := make ( clientmodel . Metric , len ( series . metric ) )
for ln , lv := range series . metric {
m [ ln ] = lv
}
return m
}
metric , err := s . persistence . getArchivedMetric ( fp )
if err != nil {
glog . Errorf ( "Error retrieving archived metric for fingerprint %v: %v" , fp , err )
}
return metric
}
// To expose persistQueueCap as metric:
var (
persistQueueCapDesc = prometheus . NewDesc (
@ -574,7 +569,7 @@ func (s *memorySeriesStorage) Describe(ch chan<- *prometheus.Desc) {
s . seriesOps . Describe ( ch )
ch <- s . ingestedSamplesCount . Desc ( )
ch <- s . purgeDuration . Desc ( )
ch <- s . eviction Duration . Desc ( )
ch <- s . evictDuration . Desc ( )
ch <- persistQueueCapDesc
@ -593,7 +588,7 @@ func (s *memorySeriesStorage) Collect(ch chan<- prometheus.Metric) {
s . seriesOps . Collect ( ch )
ch <- s . ingestedSamplesCount
ch <- s . purgeDuration
ch <- s . eviction Duration
ch <- s . evictDuration
ch <- persistQueueCapGauge
Bjoern Rabenstein
12 years ago