Adding a check to make sure we don't set tailer offset past the end of the file.

Fixing lagging alert to also catch when the read_bytes is past the end of the file.
Adding an alert to let us know if we are missing tailing of any files.

pkg/promtail/targets/tailer.go

@@ -46,6 +46,16 @@ func newTailer(logger log.Logger, handler api.EntryHandler, positions *positions
 		reOpen = true
 	}
 
+	// Simple check to make sure the file we are tailing doesn't
+	// have a position already saved which is past the end of the file.
+	fi, err = os.Stat(filename)
+	if err != nil {
+		return nil, err
+	}
+	if fi.Size() < positions.Get(filename) {
+		positions.Remove(filename)
+	}
+
 	tail, err := tail.TailFile(filename, tail.Config{
 		Follow: true,
 		Poll:   true,

production/loki-mixin/alerts.libsonnet

@@ -116,7 +116,7 @@
           {
             alert: 'PromtailFileLagging',
             expr: |||
-              promtail_file_bytes_total - promtail_read_bytes_total > 100000
+              abs(promtail_file_bytes_total - promtail_read_bytes_total) > 100000
             |||,
             'for': '15m',
             labels: {
@@ -128,6 +128,21 @@
               |||,
             },
           },
+          {
+            alert: 'PromtailFileMissing',
+            expr: |||
+              count by (path,instance,job) (promtail_file_bytes_total) unless count by (path,instance,job) (promtail_read_bytes_total)
+            |||,
+            'for': '15m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              message: |||
+                {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} matches the glob but is not being tailed.
+              |||,
+            },
+          },
         ],
       },
     ],