Promtail: adding pipeline stage inspector (#4011)

* Adding promtail pipeline stage inspector

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Adding library

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Adding docs

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Appeasing the linter

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Further appeasing the linter

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Removing old version of go-cmp to pass check-mod step

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Making screenshot narrower and correcting link

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Apply suggestions from code review

Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com>

* Copy changes

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

* Copy changes

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com>
pull/4025/head
Danny Kopping 4 years ago committed by GitHub
parent 2fd633cded
commit ac63ea8500
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 12
      clients/cmd/promtail/main.go
  2. 129
      clients/pkg/logentry/stages/inspector.go
  3. 36
      clients/pkg/logentry/stages/stage.go
  4. 3
      clients/pkg/logentry/stages/util.go
  5. BIN
      docs/sources/clients/promtail/inspect.png
  6. 24
      docs/sources/clients/promtail/troubleshooting.md
  7. 1
      go.mod
  8. 3
      go.sum
  9. 2
      vendor/github.com/google/go-cmp/cmp/path.go
  10. 202
      vendor/github.com/google/go-cmp/cmp/report_slices.go
  11. 3
      vendor/modules.txt

@ -6,6 +6,9 @@ import (
"os"
"reflect"
// embed time zone data
_ "time/tzdata"
"k8s.io/klog"
"github.com/cortexproject/cortex/pkg/util/flagext"
@ -15,9 +18,6 @@ import (
"github.com/prometheus/common/version"
"github.com/weaveworks/common/logging"
// embed time zone data
_ "time/tzdata"
"github.com/grafana/loki/clients/pkg/logentry/stages"
"github.com/grafana/loki/clients/pkg/promtail"
"github.com/grafana/loki/clients/pkg/promtail/config"
@ -39,6 +39,7 @@ type Config struct {
dryRun bool
configFile string
configExpandEnv bool
inspect bool
}
func (c *Config) RegisterFlags(f *flag.FlagSet) {
@ -47,6 +48,7 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
f.BoolVar(&c.logConfig, "log-config-reverse-order", false, "Dump the entire Loki config object at Info log "+
"level with the order reversed, reversing the order makes viewing the entries easier in Grafana.")
f.BoolVar(&c.dryRun, "dry-run", false, "Start Promtail but print entries instead of sending them to Loki.")
f.BoolVar(&c.inspect, "inspect", false, "Allows for detailed inspection of pipeline stages")
f.StringVar(&c.configFile, "config.file", "", "yaml file to load")
f.BoolVar(&c.configExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.")
c.Config.RegisterFlags(f)
@ -84,6 +86,10 @@ func main() {
// Use Stderr instead of files for the klog.
klog.SetOutput(os.Stderr)
if config.inspect {
stages.Inspect = true
}
// Set the global debug variable in the stages package which is used to conditionally log
// debug messages which otherwise cause huge allocations processing log lines for log messages never printed
if config.ServerConfig.Config.LogLevel.String() == "debug" {

@ -0,0 +1,129 @@
package stages
import (
"fmt"
"io"
"strings"
"github.com/fatih/color"
"github.com/google/go-cmp/cmp"
)
type inspector struct {
writer io.Writer
formatter *formatter
}
func newInspector(writer io.Writer, disableFormatting bool) *inspector {
f := &formatter{
red: color.New(color.FgRed),
yellow: color.New(color.FgYellow),
green: color.New(color.FgGreen),
bold: color.New(color.Bold),
}
if disableFormatting {
f.disable()
}
return &inspector{
writer: writer,
formatter: f,
}
}
type formatter struct {
red *color.Color
yellow *color.Color
green *color.Color
bold *color.Color
}
func (f *formatter) disable() {
f.red.DisableColor()
f.yellow.DisableColor()
f.green.DisableColor()
f.bold.DisableColor()
}
func (i inspector) inspect(stageName string, before *Entry, after Entry) {
if before == nil {
fmt.Fprintln(i.writer, i.formatter.red.Sprintf("could not copy entry in '%s' stage; inspect aborted", stageName))
return
}
r := diffReporter{
formatter: i.formatter,
}
cmp.Equal(*before, after, cmp.Reporter(&r))
diff := r.String()
if strings.TrimSpace(diff) == "" {
diff = i.formatter.red.Sprintf("none")
}
fmt.Fprintf(i.writer, "[inspect: %s stage]: %s\n", i.formatter.bold.Sprintf("%s", stageName), diff)
}
// diffReporter is a simple custom reporter that only records differences
// detected during comparison.
type diffReporter struct {
path cmp.Path
formatter *formatter
diffs []string
}
func (r *diffReporter) PushStep(ps cmp.PathStep) {
r.path = append(r.path, ps)
}
func (r *diffReporter) Report(rs cmp.Result) {
if rs.Equal() {
return
}
vx, vy := r.path.Last().Values()
// TODO(dannyk): try using go-cmp to filter this condition out with Equal(), but for now this just makes it work
if fmt.Sprintf("%v", vx) == fmt.Sprintf("%v", vy) {
return
}
change := vx.IsValid()
addition := vy.IsValid()
removal := change && !addition
mod := addition && change
var titleColor *color.Color
switch {
case mod:
titleColor = r.formatter.yellow
case removal:
titleColor = r.formatter.red
default:
titleColor = r.formatter.green
}
r.diffs = append(r.diffs, titleColor.Sprintf("%#v:", r.path))
if removal {
r.diffs = append(r.diffs, r.formatter.red.Sprintf("\t-: %v", vx))
}
if mod {
r.diffs = append(r.diffs, r.formatter.yellow.Sprintf("\t-: %v", vx))
}
if addition {
r.diffs = append(r.diffs, r.formatter.green.Sprintf("\t+: %v", vy))
}
}
func (r *diffReporter) PopStep() {
r.path = r.path[:len(r.path)-1]
}
func (r *diffReporter) String() string {
return fmt.Sprintf("\n%s", strings.Join(r.diffs, "\n"))
}

@ -1,12 +1,15 @@
package stages
import (
"os"
"runtime"
"time"
"github.com/go-kit/kit/log"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"gopkg.in/yaml.v2"
"github.com/grafana/loki/clients/pkg/promtail/api"
)
@ -50,20 +53,51 @@ type Stage interface {
Run(chan Entry) chan Entry
}
func (entry *Entry) copy() *Entry {
out, err := yaml.Marshal(entry)
if err != nil {
return nil
}
var n *Entry
err = yaml.Unmarshal(out, &n)
if err != nil {
return nil
}
return n
}
// stageProcessor Allow to transform a Processor (old synchronous pipeline stage) into an async Stage
type stageProcessor struct {
Processor
inspector *inspector
}
func (s stageProcessor) Run(in chan Entry) chan Entry {
return RunWith(in, func(e Entry) Entry {
var before *Entry
if Inspect {
before = e.copy()
}
s.Process(e.Labels, e.Extracted, &e.Timestamp, &e.Line)
if Inspect {
s.inspector.inspect(s.Processor.Name(), before, e)
}
return e
})
}
func toStage(p Processor) Stage {
return &stageProcessor{Processor: p}
return &stageProcessor{
Processor: p,
inspector: newInspector(os.Stderr, runtime.GOOS == "windows"),
}
}
// New creates a new stage for the given type and configuration.

@ -14,6 +14,9 @@ var (
// debug level when debug level logging is not enabled. Log level allocations can become very expensive
// as we log numerous log entries per log line at debug level.
Debug = false
// Inspect is used to debug promtail pipelines by showing diffs between pipeline stages
Inspect = false
)
const (

Binary file not shown.

After

Width:  |  Height:  |  Size: 119 KiB

@ -19,6 +19,30 @@ To start Promtail in dry run mode use the flag `--dry-run` as shown in the examp
cat my.log | promtail --stdin --dry-run --client.url http://127.0.0.1:3100/loki/api/v1/push
```
## Inspecting pipeline stages
Promtail can output all changes to log entries as each pipeline stage is executed.
Each log entry contains four fields:
- line
- timestamp
- labels
- extracted fields
Enable the inspection output using the `--inspect` command-line option. The `--inspect` option can be used in combination with `--stdin` and `--dry-run`.
```bash
cat my.log | promtail --stdin --dry-run --inspect --client.url http://127.0.0.1:3100/loki/api/v1/push
```
![screenshot](../inspect.png)
The output uses color to highlight changes. Additions are in green, modifications in yellow, and removals in red.
If no changes are applied during a stage, that is usually an indication of a misconfiguration or undesired behavior.
The `--inspect` flag should not be used in production, as the calculation of changes between pipeline stages negatively
impacts Promtail's performance.
## Pipe data to Promtail
Promtail supports piping data for sending logs to Loki (via the flag `--stdin`). This is a very useful way to troubleshooting your configuration.

@ -29,6 +29,7 @@ require (
github.com/gofrs/uuid v4.0.0+incompatible
github.com/gogo/protobuf v1.3.2 // remember to update loki-build-image/Dockerfile too
github.com/golang/snappy v0.0.3
github.com/google/go-cmp v0.5.6
github.com/gorilla/mux v1.7.3
github.com/gorilla/websocket v1.4.2
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2

@ -771,8 +771,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=

@ -315,7 +315,7 @@ func (tf Transform) Option() Option { return tf.trans }
// pops the address from the stack. Thus, when traversing into a pointer from
// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
// by checking whether the pointer has already been visited. The cycle detection
// uses a seperate stack for the x and y values.
// uses a separate stack for the x and y values.
//
// If a cycle is detected we need to determine whether the two pointers
// should be considered equal. The definition of equality chosen by Equal

@ -7,6 +7,7 @@ package cmp
import (
"bytes"
"fmt"
"math"
"reflect"
"strconv"
"strings"
@ -96,15 +97,16 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
}
// Auto-detect the type of the data.
var isLinedText, isText, isBinary bool
var sx, sy string
var ssx, ssy []string
var isString, isMostlyText, isPureLinedText, isBinary bool
switch {
case t.Kind() == reflect.String:
sx, sy = vx.String(), vy.String()
isText = true // Initial estimate, verify later
isString = true
case t.Kind() == reflect.Slice && t.Elem() == reflect.TypeOf(byte(0)):
sx, sy = string(vx.Bytes()), string(vy.Bytes())
isBinary = true // Initial estimate, verify later
isString = true
case t.Kind() == reflect.Array:
// Arrays need to be addressable for slice operations to work.
vx2, vy2 := reflect.New(t).Elem(), reflect.New(t).Elem()
@ -112,13 +114,12 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
vy2.Set(vy)
vx, vy = vx2, vy2
}
if isText || isBinary {
var numLines, lastLineIdx, maxLineLen int
isBinary = !utf8.ValidString(sx) || !utf8.ValidString(sy)
if isString {
var numTotalRunes, numValidRunes, numLines, lastLineIdx, maxLineLen int
for i, r := range sx + sy {
if !(unicode.IsPrint(r) || unicode.IsSpace(r)) || r == utf8.RuneError {
isBinary = true
break
numTotalRunes++
if (unicode.IsPrint(r) || unicode.IsSpace(r)) && r != utf8.RuneError {
numValidRunes++
}
if r == '\n' {
if maxLineLen < i-lastLineIdx {
@ -128,8 +129,26 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
numLines++
}
}
isText = !isBinary
isLinedText = isText && numLines >= 4 && maxLineLen <= 1024
isPureText := numValidRunes == numTotalRunes
isMostlyText = float64(numValidRunes) > math.Floor(0.90*float64(numTotalRunes))
isPureLinedText = isPureText && numLines >= 4 && maxLineLen <= 1024
isBinary = !isMostlyText
// Avoid diffing by lines if it produces a significantly more complex
// edit script than diffing by bytes.
if isPureLinedText {
ssx = strings.Split(sx, "\n")
ssy = strings.Split(sy, "\n")
esLines := diff.Difference(len(ssx), len(ssy), func(ix, iy int) diff.Result {
return diff.BoolResult(ssx[ix] == ssy[iy])
})
esBytes := diff.Difference(len(sx), len(sy), func(ix, iy int) diff.Result {
return diff.BoolResult(sx[ix] == sy[iy])
})
efficiencyLines := float64(esLines.Dist()) / float64(len(esLines))
efficiencyBytes := float64(esBytes.Dist()) / float64(len(esBytes))
isPureLinedText = efficiencyLines < 4*efficiencyBytes
}
}
// Format the string into printable records.
@ -138,9 +157,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
switch {
// If the text appears to be multi-lined text,
// then perform differencing across individual lines.
case isLinedText:
ssx := strings.Split(sx, "\n")
ssy := strings.Split(sy, "\n")
case isPureLinedText:
list = opts.formatDiffSlice(
reflect.ValueOf(ssx), reflect.ValueOf(ssy), 1, "line",
func(v reflect.Value, d diffMode) textRecord {
@ -229,7 +246,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
// If the text appears to be single-lined text,
// then perform differencing in approximately fixed-sized chunks.
// The output is printed as quoted strings.
case isText:
case isMostlyText:
list = opts.formatDiffSlice(
reflect.ValueOf(sx), reflect.ValueOf(sy), 64, "byte",
func(v reflect.Value, d diffMode) textRecord {
@ -237,7 +254,6 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
return textRecord{Diff: d, Value: textLine(s)}
},
)
delim = ""
// If the text appears to be binary data,
// then perform differencing in approximately fixed-sized chunks.
@ -299,7 +315,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
// Wrap the output with appropriate type information.
var out textNode = &textWrap{Prefix: "{", Value: list, Suffix: "}"}
if !isText {
if !isMostlyText {
// The "{...}" byte-sequence literal is not valid Go syntax for strings.
// Emit the type for extra clarity (e.g. "string{...}").
if t.Kind() == reflect.String {
@ -338,8 +354,11 @@ func (opts formatOptions) formatDiffSlice(
vx, vy reflect.Value, chunkSize int, name string,
makeRec func(reflect.Value, diffMode) textRecord,
) (list textList) {
es := diff.Difference(vx.Len(), vy.Len(), func(ix int, iy int) diff.Result {
return diff.BoolResult(vx.Index(ix).Interface() == vy.Index(iy).Interface())
eq := func(ix, iy int) bool {
return vx.Index(ix).Interface() == vy.Index(iy).Interface()
}
es := diff.Difference(vx.Len(), vy.Len(), func(ix, iy int) diff.Result {
return diff.BoolResult(eq(ix, iy))
})
appendChunks := func(v reflect.Value, d diffMode) int {
@ -364,6 +383,7 @@ func (opts formatOptions) formatDiffSlice(
groups := coalesceAdjacentEdits(name, es)
groups = coalesceInterveningIdentical(groups, chunkSize/4)
groups = cleanupSurroundingIdentical(groups, eq)
maxGroup := diffStats{Name: name}
for i, ds := range groups {
if maxLen >= 0 && numDiffs >= maxLen {
@ -416,25 +436,36 @@ func (opts formatOptions) formatDiffSlice(
// coalesceAdjacentEdits coalesces the list of edits into groups of adjacent
// equal or unequal counts.
//
// Example:
//
// Input: "..XXY...Y"
// Output: [
// {NumIdentical: 2},
// {NumRemoved: 2, NumInserted 1},
// {NumIdentical: 3},
// {NumInserted: 1},
// ]
//
func coalesceAdjacentEdits(name string, es diff.EditScript) (groups []diffStats) {
var prevCase int // Arbitrary index into which case last occurred
lastStats := func(i int) *diffStats {
if prevCase != i {
var prevMode byte
lastStats := func(mode byte) *diffStats {
if prevMode != mode {
groups = append(groups, diffStats{Name: name})
prevCase = i
prevMode = mode
}
return &groups[len(groups)-1]
}
for _, e := range es {
switch e {
case diff.Identity:
lastStats(1).NumIdentical++
lastStats('=').NumIdentical++
case diff.UniqueX:
lastStats(2).NumRemoved++
lastStats('!').NumRemoved++
case diff.UniqueY:
lastStats(2).NumInserted++
lastStats('!').NumInserted++
case diff.Modified:
lastStats(2).NumModified++
lastStats('!').NumModified++
}
}
return groups
@ -444,6 +475,35 @@ func coalesceAdjacentEdits(name string, es diff.EditScript) (groups []diffStats)
// equal groups into adjacent unequal groups that currently result in a
// dual inserted/removed printout. This acts as a high-pass filter to smooth
// out high-frequency changes within the windowSize.
//
// Example:
//
// WindowSize: 16,
// Input: [
// {NumIdentical: 61}, // group 0
// {NumRemoved: 3, NumInserted: 1}, // group 1
// {NumIdentical: 6}, // ├── coalesce
// {NumInserted: 2}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 9}, // └── coalesce
// {NumIdentical: 64}, // group 2
// {NumRemoved: 3, NumInserted: 1}, // group 3
// {NumIdentical: 6}, // ├── coalesce
// {NumInserted: 2}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 7}, // ├── coalesce
// {NumIdentical: 1}, // ├── coalesce
// {NumRemoved: 2}, // └── coalesce
// {NumIdentical: 63}, // group 4
// ]
// Output: [
// {NumIdentical: 61},
// {NumIdentical: 7, NumRemoved: 12, NumInserted: 3},
// {NumIdentical: 64},
// {NumIdentical: 8, NumRemoved: 12, NumInserted: 3},
// {NumIdentical: 63},
// ]
//
func coalesceInterveningIdentical(groups []diffStats, windowSize int) []diffStats {
groups, groupsOrig := groups[:0], groups
for i, ds := range groupsOrig {
@ -463,3 +523,91 @@ func coalesceInterveningIdentical(groups []diffStats, windowSize int) []diffStat
}
return groups
}
// cleanupSurroundingIdentical scans through all unequal groups, and
// moves any leading sequence of equal elements to the preceding equal group and
// moves and trailing sequence of equal elements to the succeeding equal group.
//
// This is necessary since coalesceInterveningIdentical may coalesce edit groups
// together such that leading/trailing spans of equal elements becomes possible.
// Note that this can occur even with an optimal diffing algorithm.
//
// Example:
//
// Input: [
// {NumIdentical: 61},
// {NumIdentical: 1 , NumRemoved: 11, NumInserted: 2}, // assume 3 leading identical elements
// {NumIdentical: 67},
// {NumIdentical: 7, NumRemoved: 12, NumInserted: 3}, // assume 10 trailing identical elements
// {NumIdentical: 54},
// ]
// Output: [
// {NumIdentical: 64}, // incremented by 3
// {NumRemoved: 9},
// {NumIdentical: 67},
// {NumRemoved: 9},
// {NumIdentical: 64}, // incremented by 10
// ]
//
func cleanupSurroundingIdentical(groups []diffStats, eq func(i, j int) bool) []diffStats {
var ix, iy int // indexes into sequence x and y
for i, ds := range groups {
// Handle equal group.
if ds.NumDiff() == 0 {
ix += ds.NumIdentical
iy += ds.NumIdentical
continue
}
// Handle unequal group.
nx := ds.NumIdentical + ds.NumRemoved + ds.NumModified
ny := ds.NumIdentical + ds.NumInserted + ds.NumModified
var numLeadingIdentical, numTrailingIdentical int
for i := 0; i < nx && i < ny && eq(ix+i, iy+i); i++ {
numLeadingIdentical++
}
for i := 0; i < nx && i < ny && eq(ix+nx-1-i, iy+ny-1-i); i++ {
numTrailingIdentical++
}
if numIdentical := numLeadingIdentical + numTrailingIdentical; numIdentical > 0 {
if numLeadingIdentical > 0 {
// Remove leading identical span from this group and
// insert it into the preceding group.
if i-1 >= 0 {
groups[i-1].NumIdentical += numLeadingIdentical
} else {
// No preceding group exists, so prepend a new group,
// but do so after we finish iterating over all groups.
defer func() {
groups = append([]diffStats{{Name: groups[0].Name, NumIdentical: numLeadingIdentical}}, groups...)
}()
}
// Increment indexes since the preceding group would have handled this.
ix += numLeadingIdentical
iy += numLeadingIdentical
}
if numTrailingIdentical > 0 {
// Remove trailing identical span from this group and
// insert it into the succeeding group.
if i+1 < len(groups) {
groups[i+1].NumIdentical += numTrailingIdentical
} else {
// No succeeding group exists, so append a new group,
// but do so after we finish iterating over all groups.
defer func() {
groups = append(groups, diffStats{Name: groups[len(groups)-1].Name, NumIdentical: numTrailingIdentical})
}()
}
// Do not increment indexes since the succeeding group will handle this.
}
// Update this group since some identical elements were removed.
nx -= numIdentical
ny -= numIdentical
groups[i] = diffStats{Name: ds.Name, NumRemoved: nx, NumInserted: ny}
}
ix += nx
iy += ny
}
return groups
}

@ -491,7 +491,8 @@ github.com/golang/protobuf/ptypes/wrappers
github.com/golang/snappy
# github.com/google/btree v1.0.0
github.com/google/btree
# github.com/google/go-cmp v0.5.5
# github.com/google/go-cmp v0.5.6
## explicit
github.com/google/go-cmp/cmp
github.com/google/go-cmp/cmp/internal/diff
github.com/google/go-cmp/cmp/internal/flags

Loading…
Cancel
Save