@ -34,12 +34,16 @@ import (
)
const (
readPeriod = 10 * time . Millisecond
checkpointPeriod = 5 * time . Second
segmentCheckPeriod = 100 * time . Millisecond
consumer = "consumer"
)
var (
ErrIgnorable = errors . New ( "ignore me" )
readTimeout = 15 * time . Second
)
// WriteTo is an interface used by the Watcher to send the samples it's read
// from the WAL on to somewhere else. Functions will be called concurrently
// and it is left to the implementer to make sure they are safe.
@ -61,11 +65,17 @@ type WriteTo interface {
SeriesReset ( int )
}
// Used to notifier the watcher that data has been written so that it can read.
type WriteNotified interface {
Notify ( )
}
type WatcherMetrics struct {
recordsRead * prometheus . CounterVec
recordDecodeFails * prometheus . CounterVec
samplesSentPreTailing * prometheus . CounterVec
currentSegment * prometheus . GaugeVec
notificationsSkipped * prometheus . CounterVec
}
// Watcher watches the TSDB WAL for a given WriteTo.
@ -88,9 +98,11 @@ type Watcher struct {
recordDecodeFailsMetric prometheus . Counter
samplesSentPreTailing prometheus . Counter
currentSegmentMetric prometheus . Gauge
notificationsSkipped prometheus . Counter
quit chan struct { }
done chan struct { }
readNotify chan struct { }
quit chan struct { }
done chan struct { }
// For testing, stop when we hit this segment.
MaxSegment int
@ -134,6 +146,15 @@ func NewWatcherMetrics(reg prometheus.Registerer) *WatcherMetrics {
} ,
[ ] string { consumer } ,
) ,
notificationsSkipped : prometheus . NewCounterVec (
prometheus . CounterOpts {
Namespace : "prometheus" ,
Subsystem : "wal_watcher" ,
Name : "notifications_skipped_total" ,
Help : "The number of WAL write notifications that the Watcher has skipped due to already being in a WAL read routine." ,
} ,
[ ] string { consumer } ,
) ,
}
if reg != nil {
@ -141,6 +162,7 @@ func NewWatcherMetrics(reg prometheus.Registerer) *WatcherMetrics {
reg . MustRegister ( m . recordDecodeFails )
reg . MustRegister ( m . samplesSentPreTailing )
reg . MustRegister ( m . currentSegment )
reg . MustRegister ( m . notificationsSkipped )
}
return m
@ -161,13 +183,25 @@ func NewWatcher(metrics *WatcherMetrics, readerMetrics *LiveReaderMetrics, logge
sendExemplars : sendExemplars ,
sendHistograms : sendHistograms ,
quit : make ( chan struct { } ) ,
done : make ( chan struct { } ) ,
readNotify : make ( chan struct { } ) ,
quit : make ( chan struct { } ) ,
done : make ( chan struct { } ) ,
MaxSegment : - 1 ,
}
}
func ( w * Watcher ) Notify ( ) {
select {
case w . readNotify <- struct { } { } :
return
default : // default so we can exit
// we don't need a buffered channel or any buffering since
// for each notification it recv's the watcher will read until EOF
w . notificationsSkipped . Inc ( )
}
}
func ( w * Watcher ) setMetrics ( ) {
// Setup the WAL Watchers metrics. We do this here rather than in the
// constructor because of the ordering of creating Queue Managers's,
@ -177,6 +211,8 @@ func (w *Watcher) setMetrics() {
w . recordDecodeFailsMetric = w . metrics . recordDecodeFails . WithLabelValues ( w . name )
w . samplesSentPreTailing = w . metrics . samplesSentPreTailing . WithLabelValues ( w . name )
w . currentSegmentMetric = w . metrics . currentSegment . WithLabelValues ( w . name )
w . notificationsSkipped = w . metrics . notificationsSkipped . WithLabelValues ( w . name )
}
}
@ -262,7 +298,7 @@ func (w *Watcher) Run() error {
// On start, after reading the existing WAL for series records, we have a pointer to what is the latest segment.
// On subsequent calls to this function, currentSegment will have been incremented and we should open that segment.
if err := w . watch ( currentSegment , currentSegment >= lastSegment ) ; err != nil {
if err := w . watch ( currentSegment , currentSegment >= lastSegment ) ; err != nil && ! errors . Is ( err , ErrIgnorable ) {
return err
}
@ -330,6 +366,26 @@ func (w *Watcher) segments(dir string) ([]int, error) {
return refs , nil
}
func ( w * Watcher ) readAndHandleError ( r * LiveReader , segmentNum int , tail bool , size int64 ) error {
err := w . readSegment ( r , segmentNum , tail )
// Ignore all errors reading to end of segment whilst replaying the WAL.
if ! tail {
if err != nil && errors . Cause ( err ) != io . EOF {
level . Warn ( w . logger ) . Log ( "msg" , "Ignoring error reading to end of segment, may have dropped data" , "segment" , segmentNum , "err" , err )
} else if r . Offset ( ) != size {
level . Warn ( w . logger ) . Log ( "msg" , "Expected to have read whole segment, may have dropped data" , "segment" , segmentNum , "read" , r . Offset ( ) , "size" , size )
}
return ErrIgnorable
}
// Otherwise, when we are tailing, non-EOFs are fatal.
if errors . Cause ( err ) != io . EOF {
return err
}
return nil
}
// Use tail true to indicate that the reader is currently on a segment that is
// actively being written to. If false, assume it's a full segment and we're
// replaying it on start to cache the series records.
@ -342,7 +398,7 @@ func (w *Watcher) watch(segmentNum int, tail bool) error {
reader := NewLiveReader ( w . logger , w . readerMetrics , segment )
readTicker := time . NewTicker ( readPeriod )
readTicker := time . NewTicker ( readTimeout )
defer readTicker . Stop ( )
checkpointTicker := time . NewTicker ( checkpointPeriod )
@ -400,7 +456,6 @@ func (w *Watcher) watch(segmentNum int, tail bool) error {
if last <= segmentNum {
continue
}
err = w . readSegment ( reader , segmentNum , tail )
// Ignore errors reading to end of segment whilst replaying the WAL.
@ -421,24 +476,23 @@ func (w *Watcher) watch(segmentNum int, tail bool) error {
return nil
// we haven't read due to a notification in quite some time, try reading anyways
case <- readTicker . C :
err = w . readSegment ( reader , segmentNum , tail )
// Ignore all errors reading to end of segment whilst replaying the WAL.
if ! tail {
switch {
case err != nil && errors . Cause ( err ) != io . EOF :
level . Warn ( w . logger ) . Log ( "msg" , "Ignoring error reading to end of segment, may have dropped data" , "segment" , segmentNum , "err" , err )
case reader . Offset ( ) != size :
level . Warn ( w . logger ) . Log ( "msg" , "Expected to have read whole segment, may have dropped data" , "segment" , segmentNum , "read" , reader . Offset ( ) , "size" , size )
}
return nil
level . Debug ( w . logger ) . Log ( "msg" , "Watcher is reading the WAL due to timeout, haven't received any write notifications recently" , "timeout" , readTimeout )
err := w . readAndHandleError ( reader , segmentNum , tail , size )
if err != nil {
return err
}
// still want to reset the ticker so we don't read too often
readTicker . Reset ( readTimeout )
// Otherwise, when we are tailing, non-EOFs are fatal.
if errors . Cause ( err ) != io . EOF {
case <- w . readNotify :
err := w . readAndHandleError ( reader , segmentNum , tail , size )
if err != nil {
return err
}
// still want to reset the ticker so we don't read too often
readTicker . Reset ( readTimeout )
}
}
}
Callum Styan
3 years ago
committed by