feat: add detected-fields command to logcli (#12739)

cmd/logcli/main.go

@@ -16,6 +16,7 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/grafana/loki/v3/pkg/logcli/client"
+	"github.com/grafana/loki/v3/pkg/logcli/detected"
 	"github.com/grafana/loki/v3/pkg/logcli/index"
 	"github.com/grafana/loki/v3/pkg/logcli/labelquery"
 	"github.com/grafana/loki/v3/pkg/logcli/output"
@@ -253,6 +254,39 @@ Example:
 	   'my-query'
   `)
 	volumeRangeQuery = newVolumeQuery(true, volumeRangeCmd)
+
+	detectedFieldsCmd = app.Command("detected-fields", `Run a query for detected fields..
+
+The "detected-fields" command will return information about fields detected using either
+the "logfmt" or "json" parser against the log lines returned by the provided query for the
+provided time range.
+
+The "detected-fields" command will output extra information about the query
+and its results, such as the API URL, set of common labels, and set
+of excluded labels. This extra information can be suppressed with the
+--quiet flag.
+
+By default we look over the last hour of data; use --since to modify
+or provide specific start and end times with --from and --to respectively.
+
+Notice that when using --from and --to then ensure to use RFC3339Nano
+time format, but without timezone at the end. The local timezone will be added
+automatically or if using  --timezone flag.
+
+Example:
+
+	logcli detected-fields
+	   --timezone=UTC
+	   --from="2021-01-19T10:00:00Z"
+	   --to="2021-01-19T20:00:00Z"
+	   --output=jsonl
+	   'my-query'
+
+The output is limited to 100 fields by default; use --field-limit to increase.
+The query is limited to processing 1000 lines per subquery; use --line-limit to increase.
+`)
+
+	detectedFieldsQuery = newDetectedFieldsQuery(detectedFieldsCmd)
 )
 
 func main() {
@@ -388,6 +422,8 @@ func main() {
 		} else {
 			index.GetVolume(volumeQuery, queryClient, out, *statistics)
 		}
+	case detectedFieldsCmd.FullCommand():
+		detectedFieldsQuery.Do(queryClient, *outputMode)
 	}
 }
 
@@ -652,3 +688,41 @@ func newVolumeQuery(rangeQuery bool, cmd *kingpin.CmdClause) *volume.Query {
 
 	return q
 }
+
+func newDetectedFieldsQuery(cmd *kingpin.CmdClause) *detected.FieldsQuery {
+	// calculate query range from cli params
+	var from, to string
+	var since time.Duration
+
+	q := &detected.FieldsQuery{}
+
+	// executed after all command flags are parsed
+	cmd.Action(func(c *kingpin.ParseContext) error {
+		defaultEnd := time.Now()
+		defaultStart := defaultEnd.Add(-since)
+
+		q.Start = mustParse(from, defaultStart)
+		q.End = mustParse(to, defaultEnd)
+
+		q.Quiet = *quiet
+
+		return nil
+	})
+
+	cmd.Flag("field-limit", "Limit on number of fields to return.").
+		Default("100").
+		IntVar(&q.FieldLimit)
+	cmd.Flag("line-limit", "Limit the number of lines each subquery is allowed to process.").
+		Default("1000").
+		IntVar(&q.LineLimit)
+	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").
+		Required().
+		StringVar(&q.QueryString)
+	cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
+	cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
+	cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
+	cmd.Flag("step", "Query resolution step width, for metric queries. Evaluate the query at the specified step over the time range.").
+		DurationVar(&q.Step)
+
+	return q
+}

pkg/logcli/client/client.go

@@ -28,16 +28,17 @@ import (
 )
 
 const (
-	queryPath         = "/loki/api/v1/query"
-	queryRangePath    = "/loki/api/v1/query_range"
-	labelsPath        = "/loki/api/v1/labels"
-	labelValuesPath   = "/loki/api/v1/label/%s/values"
-	seriesPath        = "/loki/api/v1/series"
-	tailPath          = "/loki/api/v1/tail"
-	statsPath         = "/loki/api/v1/index/stats"
-	volumePath        = "/loki/api/v1/index/volume"
-	volumeRangePath   = "/loki/api/v1/index/volume_range"
-	defaultAuthHeader = "Authorization"
+	queryPath          = "/loki/api/v1/query"
+	queryRangePath     = "/loki/api/v1/query_range"
+	labelsPath         = "/loki/api/v1/labels"
+	labelValuesPath    = "/loki/api/v1/label/%s/values"
+	seriesPath         = "/loki/api/v1/series"
+	tailPath           = "/loki/api/v1/tail"
+	statsPath          = "/loki/api/v1/index/stats"
+	volumePath         = "/loki/api/v1/index/volume"
+	volumeRangePath    = "/loki/api/v1/index/volume_range"
+	detectedFieldsPath = "/loki/api/v1/detected_fields"
+	defaultAuthHeader  = "Authorization"
 )
 
 var userAgent = fmt.Sprintf("loki-logcli/%s", build.Version)
@@ -54,6 +55,7 @@ type Client interface {
 	GetStats(queryStr string, start, end time.Time, quiet bool) (*logproto.IndexStatsResponse, error)
 	GetVolume(query *volume.Query) (*loghttp.QueryResponse, error)
 	GetVolumeRange(query *volume.Query) (*loghttp.QueryResponse, error)
+	GetDetectedFields(queryStr string, fieldLimit, lineLimit int, start, end time.Time, step time.Duration, quiet bool) (*loghttp.DetectedFieldsResponse, error)
 }
 
 // Tripperware can wrap a roundtripper.
@@ -224,7 +226,36 @@ func (c *DefaultClient) getVolume(path string, query *volume.Query) (*loghttp.Qu
 	return &resp, nil
 }
 
-func (c *DefaultClient) doQuery(path string, query string, quiet bool) (*loghttp.QueryResponse, error) {
+func (c *DefaultClient) GetDetectedFields(
+	queryStr string,
+	fieldLimit, lineLimit int,
+	start, end time.Time,
+	step time.Duration,
+	quiet bool,
+) (*loghttp.DetectedFieldsResponse, error) {
+	qsb := util.NewQueryStringBuilder()
+	qsb.SetString("query", queryStr)
+	qsb.SetInt("field_limit", int64(fieldLimit))
+	qsb.SetInt("line_limit", int64(lineLimit))
+	qsb.SetInt("start", start.UnixNano())
+	qsb.SetInt("end", end.UnixNano())
+	qsb.SetString("step", step.String())
+
+	var err error
+	var r loghttp.DetectedFieldsResponse
+
+	if err = c.doRequest(detectedFieldsPath, qsb.Encode(), quiet, &r); err != nil {
+		return nil, err
+	}
+
+	return &r, nil
+}
+
+func (c *DefaultClient) doQuery(
+	path string,
+	query string,
+	quiet bool,
+) (*loghttp.QueryResponse, error) {
 	var err error
 	var r loghttp.QueryResponse
 

pkg/logcli/client/file.go

@@ -190,17 +190,28 @@ func (f *FileClient) GetOrgID() string {
 }
 
 func (f *FileClient) GetStats(_ string, _, _ time.Time, _ bool) (*logproto.IndexStatsResponse, error) {
-	// TODO(trevorwhitney): could we teach logcli to read from an actual index file?
+	// TODO(twhitney): could we teach logcli to read from an actual index file?
 	return nil, ErrNotSupported
 }
 
 func (f *FileClient) GetVolume(_ *volume.Query) (*loghttp.QueryResponse, error) {
-	// TODO(trevorwhitney): could we teach logcli to read from an actual index file?
+	// TODO(twhitney): could we teach logcli to read from an actual index file?
 	return nil, ErrNotSupported
 }
 
 func (f *FileClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryResponse, error) {
-	// TODO(trevorwhitney): could we teach logcli to read from an actual index file?
+	// TODO(twhitney): could we teach logcli to read from an actual index file?
+	return nil, ErrNotSupported
+}
+
+func (f *FileClient) GetDetectedFields(
+	_ string,
+	_, _ int,
+	_, _ time.Time,
+	_ time.Duration,
+	_ bool,
+) (*loghttp.DetectedFieldsResponse, error) {
+	// TODO(twhitney): could we teach logcli to do this?
 	return nil, ErrNotSupported
 }
 

pkg/logcli/detected/fields.go

@@ -0,0 +1,57 @@
+package detected
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+
+	"github.com/grafana/loki/v3/pkg/logcli/client"
+	"github.com/grafana/loki/v3/pkg/loghttp"
+)
+
+type FieldsQuery struct {
+	QueryString   string
+	Start         time.Time
+	End           time.Time
+	FieldLimit    int
+	LineLimit     int
+	Step          time.Duration
+	Quiet         bool
+	ColoredOutput bool
+}
+
+// DoQuery executes the query and prints out the results
+func (q *FieldsQuery) Do(c client.Client, outputMode string) {
+	var resp *loghttp.DetectedFieldsResponse
+	var err error
+
+	resp, err = c.GetDetectedFields(q.QueryString, q.FieldLimit, q.LineLimit, q.Start, q.End, q.Step, q.Quiet)
+	if err != nil {
+		log.Fatalf("Error doing request: %+v", err)
+	}
+
+	switch outputMode {
+	case "raw":
+		out, err := json.Marshal(resp)
+		if err != nil {
+			log.Fatalf("Error marshalling response: %+v", err)
+		}
+		fmt.Println(string(out))
+	default:
+		output := make([]string, len(resp.Fields))
+		for i, field := range resp.Fields {
+			bold := color.New(color.Bold)
+			output[i] = fmt.Sprintf("label: %s\t\t", bold.Sprintf("%s", field.Label)) +
+				fmt.Sprintf("type: %s\t\t", bold.Sprintf("%s", field.Type)) +
+				fmt.Sprintf("cardinality: %s", bold.Sprintf("%d", field.Cardinality))
+		}
+
+		slices.Sort(output)
+		fmt.Println(strings.Join(output, "\n"))
+	}
+}

pkg/logcli/query/query_test.go

@@ -485,6 +485,16 @@ func (t *testQueryClient) GetVolumeRange(_ *volume.Query) (*loghttp.QueryRespons
 	panic("not implemented")
 }
 
+func (t *testQueryClient) GetDetectedFields(
+	_ string,
+	_, _ int,
+	_, _ time.Time,
+	_ time.Duration,
+	_ bool,
+) (*loghttp.DetectedFieldsResponse, error) {
+	panic("not implemented")
+}
+
 var legacySchemaConfigContents = `schema_config:
   configs:
   - from: 2020-05-15

pkg/loghttp/detected.go

@@ -0,0 +1,14 @@
+package loghttp
+
+import "github.com/grafana/loki/v3/pkg/logproto"
+
+// LabelResponse represents the http json response to a label query
+type DetectedFieldsResponse struct {
+	Fields []DetectedField `json:"fields,omitempty"`
+}
+
+type DetectedField struct {
+	Label       string                     `json:"label,omitempty"`
+	Type        logproto.DetectedFieldType `json:"type,omitempty"`
+	Cardinality uint64                     `json:"cardinality,omitempty"`
+}