apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

adding ability to supply timezone to timestamp pipeline stage

Browse Source
pull/796/head
Edward Welch 6 years ago committed by Ed
parent 8feadf7924
commit 3d5319e72a
8 changed files with 116 additions and 18 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      cmd/promtail/Dockerfile
  2. 2
      cmd/promtail/Dockerfile.debug
  3. 2
      docs/logentry/processing-log-lines.md
  4. 2
      pkg/logentry/stages/extensions.go
  5. 13
      pkg/logentry/stages/timestamp.go
  6. 42
      pkg/logentry/stages/timestamp_test.go
  7. 17
      pkg/logentry/stages/util.go
  8. 38
      pkg/logentry/stages/util_test.go

2
cmd/promtail/Dockerfile
Unescape View File

@ -1,5 +1,5 @@
FROM alpine:3.9
RUN apk add --update --no-cache ca-certificates
RUN apk add --update --no-cache ca-certificates tzdata
ADD promtail /usr/bin
COPY promtail-local-config.yaml /etc/promtail/local-config.yaml
COPY promtail-docker-config.yaml /etc/promtail/docker-config.yaml

2
cmd/promtail/Dockerfile.debug
Unescape View File

@ -1,5 +1,5 @@
FROM alpine:3.9
RUN apk add --update --no-cache ca-certificates
RUN apk add --update --no-cache ca-certificates tzdata
ADD promtail-debug /usr/bin
ADD dlv /usr/bin
COPY promtail-local-config.yaml /etc/promtail/local-config.yaml

2
docs/logentry/processing-log-lines.md
Unescape View File

@ -366,10 +366,12 @@ A timestamp stage will parse data from the `extracted` map and set the `time` va
- timestamp:
source: ①
format: ②
location: ③
```
① `source` is **required** and is the key name to data in the `extracted` map.
② `format` is **required** and is the input to Go's [time.parse](https://golang.org/pkg/time/#Parse) function.
③ `location` is **optional** and is an IANA Timezone Database string, see the [go docs](https://golang.org/pkg/time/#LoadLocation) for more info
Several of Go's pre-defined format's can be used by their name:

2
pkg/logentry/stages/extensions.go
Unescape View File

@ -26,6 +26,7 @@ func NewDocker(logger log.Logger, registerer prometheus.Registerer) (Stage, erro
StageTypeTimestamp: TimestampConfig{
"timestamp",
RFC3339Nano,
nil,
}},
PipelineStage{
StageTypeOutput: OutputConfig{
@ -52,6 +53,7 @@ func NewCRI(logger log.Logger, registerer prometheus.Registerer) (Stage, error)
StageTypeTimestamp: TimestampConfig{
"time",
RFC3339Nano,
nil,
},
},
PipelineStage{

13
pkg/logentry/stages/timestamp.go
Unescape View File

@ -2,6 +2,7 @@ package stages
import (
"errors"
"fmt"
"reflect"
"time"
@ -15,6 +16,7 @@ const (
ErrEmptyTimestampStageConfig = "timestamp stage config cannot be empty"
ErrTimestampSourceRequired = "timestamp source value is required if timestamp is specified"
ErrTimestampFormatRequired = "timestamp format is required"
ErrInvalidLocation = "invalid location specified: %v"
Unix = "Unix"
UnixMs = "UnixMs"
@ -25,6 +27,7 @@ const (
type TimestampConfig struct {
Source string `mapstructure:"source"`
Format string `mapstructure:"format"`
Location *string `mapstructure:"location"`
}
// parser can convert the time string into a time.Time value
@ -41,7 +44,15 @@ func validateTimestampConfig(cfg *TimestampConfig) (parser, error) {
if cfg.Format == "" {
return nil, errors.New(ErrTimestampFormatRequired)
}
return convertDateLayout(cfg.Format), nil
var loc *time.Location
var err error
if cfg.Location != nil {
loc, err = time.LoadLocation(*cfg.Location)
if err != nil {
return nil, fmt.Errorf(ErrInvalidLocation, err)
}
}
return convertDateLayout(cfg.Format, loc), nil
}

42
pkg/logentry/stages/timestamp_test.go
Unescape View File

@ -1,6 +1,8 @@
package stages
import (
"fmt"
"strings"
"testing"
"time"
@ -43,9 +45,17 @@ func TestTimestampPipeline(t *testing.T) {
assert.Equal(t, time.Date(2012, 11, 01, 22, 8, 41, 0, time.FixedZone("", -4*60*60)).Unix(), ts.Unix())
}
var (
invalidLocationString = "America/Canada"
validLocationString = "America/New_York"
validLocation, _ = time.LoadLocation(validLocationString)
)
func TestTimestampValidation(t *testing.T) {
tests := map[string]struct {
config *TimestampConfig
// Note the error text validation is a little loosey as it only validates with strings.HasPrefix
// this is to work around different errors related to timezone loading on different systems
err error
testString string
expectedTime time.Time
@ -64,6 +74,14 @@ func TestTimestampValidation(t *testing.T) {
},
err: errors.New(ErrTimestampFormatRequired),
},
"invalid location": {
config: &TimestampConfig{
Source: "source1",
Format: "2006-01-02",
Location: &invalidLocationString,
},
err: fmt.Errorf(ErrInvalidLocation, ""),
},
"standard format": {
config: &TimestampConfig{
Source: "source1",
@ -91,6 +109,16 @@ func TestTimestampValidation(t *testing.T) {
testString: "Jul 15 01:02:03",
expectedTime: time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC),
},
"custom format with location": {
config: &TimestampConfig{
Source: "source1",
Format: "2006-01-02 15:04:05",
Location: &validLocationString,
},
err: nil,
testString: "2009-07-01 03:30:20",
expectedTime: time.Date(2009, 7, 1, 3, 30, 20, 0, validLocation),
},
"unix_ms": {
config: &TimestampConfig{
Source: "source1",
@ -110,7 +138,7 @@ func TestTimestampValidation(t *testing.T) {
t.Errorf("validateOutputConfig() expected error = %v, actual error = %v", test.err, err)
return
}
if (err != nil) && (err.Error() != test.err.Error()) {
if (err != nil) && !strings.HasPrefix(err.Error(), test.err.Error()) {
t.Errorf("validateOutputConfig() expected error = %v, actual error = %v", test.err, err)
return
}
@ -176,6 +204,18 @@ func TestTimestampStage_Process(t *testing.T) {
},
time.Date(2019, 7, 9, 21, 48, 36, 123, time.UTC),
},
"with location success": {
TimestampConfig{
Source: "ts",
Format: "2006-01-02 15:04:05",
Location: &validLocationString,
},
map[string]interface{}{
"somethigelse": "notimportant",
"ts": "2019-07-22 20:29:32",
},
time.Date(2019, 7, 22, 20, 29, 32, 0, validLocation),
},
}
for name, test := range tests {
test := test

17
pkg/logentry/stages/util.go
Unescape View File

@ -12,7 +12,7 @@ const (
)
// convertDateLayout converts pre-defined date format layout into date format
func convertDateLayout(predef string) parser {
func convertDateLayout(predef string, location *time.Location) parser {
switch predef {
case "ANSIC":
return func(t string) (time.Time, error) {
@ -81,10 +81,13 @@ func convertDateLayout(predef string) parser {
default:
if !strings.Contains(predef, "2006") {
return func(t string) (time.Time, error) {
return parseTimestampWithoutYear(predef, t, time.Now())
return parseTimestampWithoutYear(predef, location, t, time.Now())
}
}
return func(t string) (time.Time, error) {
if location != nil {
return time.ParseInLocation(predef, t, location)
}
return time.Parse(predef, t)
}
}
@ -93,8 +96,14 @@ func convertDateLayout(predef string) parser {
// parseTimestampWithoutYear parses the input timestamp without the year component,
// assuming the timestamp is related to a point in time close to "now", and correctly
// handling the edge cases around new year's eve
func parseTimestampWithoutYear(layout string, timestamp string, now time.Time) (time.Time, error) {
parsedTime, err := time.Parse(layout, timestamp)
func parseTimestampWithoutYear(layout string, location *time.Location, timestamp string, now time.Time) (time.Time, error) {
var parsedTime time.Time
var err error
if location != nil {
parsedTime, err = time.ParseInLocation(layout, timestamp, location)
} else {
parsedTime, err = time.Parse(layout, timestamp)
}
if err != nil {
return parsedTime, err
}

38
pkg/logentry/stages/util_test.go
Unescape View File

@ -67,24 +67,37 @@ func TestGetString(t *testing.T) {
assert.Equal(t, "1562723913000", s64_1)
}
var (
location, _ = time.LoadLocation("America/New_York")
)
func TestConvertDateLayout(t *testing.T) {
t.Parallel()
tests := map[string]struct {
layout string
location *time.Location
timestamp string
expected time.Time
}{
"custom layout with year": {
"2006 Jan 02 15:04:05",
nil,
"2019 Jul 15 01:02:03",
time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC),
},
"custom layout without year": {
"Jan 02 15:04:05",
nil,
"Jul 15 01:02:03",
time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, time.UTC),
},
"custom layout with year and location": {
"Jan 02 15:04:05",
location,
"Jul 15 01:02:03",
time.Date(time.Now().Year(), 7, 15, 1, 2, 3, 0, location),
},
}
for testName, testData := range tests {
@ -93,7 +106,7 @@ func TestConvertDateLayout(t *testing.T) {
t.Run(testName, func(t *testing.T) {
t.Parallel()
parser := convertDateLayout(testData.layout)
parser := convertDateLayout(testData.layout, testData.location)
parsed, err := parser(testData.timestamp)
if err != nil {
t.Errorf("convertDateLayout() parser returned an unexpected error = %v", err)
@ -110,6 +123,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
tests := map[string]struct {
layout string
location *time.Location
timestamp string
now time.Time
expected time.Time
@ -117,13 +131,31 @@ func TestParseTimestampWithoutYear(t *testing.T) {
}{
"parse timestamp within current year": {
"Jan 02 15:04:05",
nil,
"Jul 15 01:02:03",
time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
time.Date(2019, 7, 15, 1, 2, 3, 0, time.UTC),
nil,
},
"parse timestamp with location DST": {
"Jan 02 15:04:05",
location,
"Jul 15 01:02:03",
time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
time.Date(2019, 7, 15, 1, 2, 3, 0, location),
nil,
},
"parse timestamp with location non DST": {
"Jan 02 15:04:05",
location,
"Jan 15 01:02:03",
time.Date(2019, 7, 14, 0, 0, 0, 0, time.UTC),
time.Date(2019, 1, 15, 1, 2, 3, 0, location),
nil,
},
"parse timestamp on 31th Dec and today is 1st Jan": {
"Jan 02 15:04:05",
nil,
"Dec 31 23:59:59",
time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC),
@ -131,6 +163,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
},
"parse timestamp on 1st Jan and today is 31st Dec": {
"Jan 02 15:04:05",
nil,
"Jan 01 01:02:03",
time.Date(2018, 12, 31, 23, 59, 59, 0, time.UTC),
time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
@ -138,6 +171,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
},
"error if the input layout actually includes the year component": {
"2006 Jan 02 15:04:05",
nil,
"2019 Jan 01 01:02:03",
time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
time.Date(2019, 1, 1, 1, 2, 3, 0, time.UTC),
@ -151,7 +185,7 @@ func TestParseTimestampWithoutYear(t *testing.T) {
t.Run(testName, func(t *testing.T) {
t.Parallel()
parsed, err := parseTimestampWithoutYear(testData.layout, testData.timestamp, testData.now)
parsed, err := parseTimestampWithoutYear(testData.layout, testData.location, testData.timestamp, testData.now)
if ((err != nil) != (testData.err != nil)) || (err != nil && testData.err != nil && err.Error() != testData.err.Error()) {
t.Errorf("parseTimestampWithoutYear() expected error = %v, actual error = %v", testData.err, err)
return

Write Preview
Loading…
Cancel
Save