Explore Metrics: Support prometheus utf8 metrics names and labels (#98285)

* utf8 metrics for prometheus devenv

* introduce utf8 support

* completions and suggestions

* don't wrap the utf8 label in quotes

* linting

* support utf8 labels and metrics on visual query builder

* lint

* update raw view for utf8 metric syntax

* betterer

* support utf8 metric names in explore metrics

* utf8 support in grouop by

* utf8 support in label break down

* support series endpoint

* support series endpoint

* support series endpoint

* Explore metrics: Utf8 support in Explore metrics with OTel experience enabled (#98707)

* betterer

---------

Co-authored-by: Brendan O'Handley <brendan.ohandley@grafana.com>

devenv/docker/blocks/prometheus_utf8/main.go

@@ -60,6 +60,14 @@ func main() {
 			label:        "label.with.spaß",
 			getNextValue: staticList([]string{"this_is_fun"}),
 		},
+		{
+			label:        "instance",
+			getNextValue: staticList([]string{"instance"}),
+		},
+		{
+			label:        "job",
+			getNextValue: staticList([]string{"job"}),
+		},
 		{
 			label:        "site",
 			getNextValue: staticList([]string{"LA-EPI"}),
@@ -85,6 +93,12 @@ func main() {
 		Help: "a metric with utf8 labels",
 	}, dimensions)
 
+	target_info := promauto.NewGauge(prometheus.GaugeOpts{
+		Name:        "target_info",
+		Help:        "an info metric model for otel",
+		ConstLabels: map[string]string{"job": "job", "instance": "instance", "resource 1": "1", "resource 2": "2", "resource ę": "e", "deployment_environment": "prod"},
+	})
+
 	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
@@ -101,6 +115,7 @@ func main() {
 
 			utf8Metric.WithLabelValues(labels...).Inc()
 			opsProcessed.WithLabelValues(labels...).Inc()
+			target_info.Set(1)
 
 			time.Sleep(time.Second * 5)
 		}

packages/grafana-prometheus/src/datasource.ts

@@ -71,6 +71,7 @@ import {
   RawRecordingRules,
   RuleQueryMapping,
 } from './types';
+import { utf8Support, wrapUtf8Filters } from './utf8_support';
 import { PrometheusVariableSupport } from './variables';
 
 const ANNOTATION_QUERY_STEP_DEFAULT = '60s';
@@ -925,7 +926,21 @@ export class PrometheusDatasource
 
     // We need a first replace to evaluate variables before applying adhoc filters
     // This is required for an expression like `metric > $VAR` where $VAR is a float to which we must not add adhoc filters
-    const expr = this.templateSrv.replace(target.expr, variables, this.interpolateQueryExpr);
+    const expr = this.templateSrv.replace(
+      target.expr,
+      variables,
+      (value: string | string[] = [], variable: QueryVariableModel | CustomVariableModel) => {
+        if (typeof value === 'string' && target.fromExploreMetrics) {
+          if (variable.name === 'filters') {
+            return wrapUtf8Filters(value);
+          }
+          if (variable.name === 'groupby') {
+            return utf8Support(value);
+          }
+        }
+        return this.interpolateQueryExpr(value, variable);
+      }
+    );
 
     // Apply ad-hoc filters
     // When ad-hoc filters are applied, we replace again the variables in case the ad-hoc filters also reference a variable

packages/grafana-prometheus/src/types.ts

@@ -20,6 +20,7 @@ export interface PromQuery extends GenPromQuery, DataQuery {
   disableTextWrap?: boolean;
   fullMetaSearch?: boolean;
   includeNullMetadata?: boolean;
+  fromExploreMetrics?: boolean;
 }
 
 export enum PrometheusCacheLevel {

packages/grafana-prometheus/src/utf8_support.test.ts

@@ -1,4 +1,4 @@
-import { escapeForUtf8Support, utf8Support } from './utf8_support';
+import { escapeForUtf8Support, utf8Support, wrapUtf8Filters } from './utf8_support';
 
 describe('utf8 support', () => {
   it('should return utf8 labels wrapped in quotes', () => {
@@ -33,3 +33,95 @@ describe('applyValueEncodingEscaping', () => {
     expect(excapedLabels).toEqual(expected);
   });
 });
+
+describe('wrapUtf8Filters', () => {
+  it('should correctly wrap UTF-8 labels and values for multiple key-value pairs', () => {
+    const result = wrapUtf8Filters('label.with.spaß="this_is_fun",instance="localhost:9112"');
+    const expected = '"label.with.spaß"="this_is_fun",instance="localhost:9112"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly wrap UTF-8 labels and values for a single key-value pair', () => {
+    const result = wrapUtf8Filters('label.with.spaß="this_is_fun"');
+    const expected = '"label.with.spaß"="this_is_fun"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly handle commas within values', () => {
+    const result = wrapUtf8Filters('label.with.spaß="this,is,fun",instance="localhost:9112"');
+    const expected = '"label.with.spaß"="this,is,fun",instance="localhost:9112"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly handle escaped quotes within values', () => {
+    const result = wrapUtf8Filters(`label.with.spaß="this_is_\\"fun\\"",instance="localhost:9112"`);
+    const expected = `"label.with.spaß"="this_is_\\"fun\\"",instance="localhost:9112"`;
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly handle spaces within keys', () => {
+    const result = wrapUtf8Filters('label with space="value with space",instance="localhost:9112"');
+    const expected = '"label with space"="value with space",instance="localhost:9112"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly process mixed inputs with various formats', () => {
+    const result = wrapUtf8Filters('key1="value1",key2="value,with,comma",key3="val3"');
+    const expected = 'key1="value1",key2="value,with,comma",key3="val3"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly handle empty values', () => {
+    const result = wrapUtf8Filters('key1="",key2="value2"');
+    const expected = 'key1="",key2="value2"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle an empty input string', () => {
+    const result = wrapUtf8Filters('');
+    const expected = '';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle a single key with an empty value', () => {
+    const result = wrapUtf8Filters('key1=""');
+    const expected = 'key1=""';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle multiple consecutive commas in a value', () => {
+    const result = wrapUtf8Filters('key1="value1,,value2",key2="value3"');
+    const expected = 'key1="value1,,value2",key2="value3"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle a key-value pair with special characters in the key', () => {
+    const result = wrapUtf8Filters('special@key#="value1",key2="value2"');
+    const expected = '"special@key#"="value1",key2="value2"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle a key-value pair with special characters in the value', () => {
+    const result = wrapUtf8Filters('key1="value@#&*",key2="value2"');
+    const expected = 'key1="value@#&*",key2="value2"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should correctly process keys without special characters', () => {
+    const result = wrapUtf8Filters('key1="value1",key2="value2"');
+    const expected = 'key1="value1",key2="value2"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle nested escaped quotes correctly', () => {
+    const result = wrapUtf8Filters('key1="nested \\"escaped\\" quotes",key2="value2"');
+    const expected = 'key1="nested \\"escaped\\" quotes",key2="value2"';
+    expect(result).toEqual(expected);
+  });
+
+  it('should handle escaped quotes correctly', () => {
+    const result = wrapUtf8Filters('key1="nested \\"escaped\\" quotes",key2="value with \\"escaped\\" quotes"');
+    const expected = 'key1="nested \\"escaped\\" quotes",key2="value with \\"escaped\\" quotes"';
+    expect(result).toEqual(expected);
+  });
+});

packages/grafana-prometheus/src/utf8_support.ts

@@ -79,3 +79,35 @@ const isValidCodePoint = (codePoint: number): boolean => {
   // Validate the code point for UTF-8 compliance if needed.
   return codePoint >= 0 && codePoint <= 0x10ffff;
 };
+
+export const wrapUtf8Filters = (filterStr: string): string => {
+  const resultArray: string[] = [];
+  let currentKey = '';
+  let currentValue = '';
+  let inQuotes = false;
+  let temp = '';
+
+  for (const char of filterStr) {
+    if (char === '"' && temp[temp.length - 1] !== '\\') {
+      // Toggle inQuotes when an unescaped quote is found
+      inQuotes = !inQuotes;
+      temp += char;
+    } else if (char === ',' && !inQuotes) {
+      // When outside quotes and encountering ',', finalize the current pair
+      [currentKey, currentValue] = temp.split('=');
+      resultArray.push(`${utf8Support(currentKey.trim())}="${currentValue.slice(1, -1)}"`);
+      temp = ''; // Reset for the next pair
+    } else {
+      // Collect characters
+      temp += char;
+    }
+  }
+
+  // Handle the last key-value pair
+  if (temp) {
+    [currentKey, currentValue] = temp.split('=');
+    resultArray.push(`${utf8Support(currentKey.trim())}="${currentValue.slice(1, -1)}"`);
+  }
+
+  return resultArray.join(',');
+};

public/app/features/trails/ActionTabs/MetricOverviewScene.tsx

@@ -1,6 +1,7 @@
 import { useEffect } from 'react';
 
 import { PromMetricsMetadataItem } from '@grafana/prometheus';
+import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support';
 import {
   QueryVariable,
   SceneComponentProps,
@@ -90,9 +91,14 @@ export class MetricOverviewScene extends SceneObjectBase<MetricOverviewSceneStat
       // when the group left variable is changed we should get all the resource attributes + labels
       const resourceAttributes = sceneGraph.lookupVariable(VAR_OTEL_GROUP_LEFT, trail)?.getValue();
       if (typeof resourceAttributes === 'string') {
-        const attributeArray: VariableValueOption[] = resourceAttributes
+        const attributeArray: VariableValueOption[] = resourceAttributes.split(',').map((el) => {
-          .split(',')
+          let label = el;
-          .map((el) => ({ label: el, value: el }));
+          if (!isValidLegacyName(el)) {
+            // remove '' from label
+            label = el.slice(1, -1);
+          }
+          return { label, value: el };
+        });
         allLabelOptions = attributeArray.concat(allLabelOptions);
       }
     }

public/app/features/trails/Breakdown/LabelBreakdownScene.tsx

@@ -4,6 +4,7 @@ import { isNumber, max, min, throttle } from 'lodash';
 import { useEffect, useState } from 'react';
 
 import { DataFrame, FieldType, GrafanaTheme2, PanelData, SelectableValue } from '@grafana/data';
+import { isValidLegacyName, utf8Support } from '@grafana/prometheus/src/utf8_support';
 import { config } from '@grafana/runtime';
 import {
   ConstantVariable,
@@ -313,7 +314,14 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
       return [];
     }
 
-    const attributeArray: SelectableValue[] = resourceAttributes.split(',').map((el) => ({ label: el, value: el }));
+    const attributeArray: SelectableValue[] = resourceAttributes.split(',').map((el) => {
+      let label = el;
+      if (!isValidLegacyName(el)) {
+        // remove '' from label
+        label = el.slice(1, -1);
+      }
+      return { label, value: el };
+    });
     // shift ALL value to the front
     const all: SelectableValue = [{ label: 'All', value: ALL_VARIABLE_VALUE }];
     const firstGroup = all.concat(attributeArray);
@@ -449,7 +457,7 @@ export function buildAllLayout(
       break;
     }
 
-    const expr = queryDef.queries[0].expr.replaceAll(VAR_GROUP_BY_EXP, String(option.value));
+    const expr = queryDef.queries[0].expr.replaceAll(VAR_GROUP_BY_EXP, utf8Support(String(option.value)));
     const unit = queryDef.unit;
 
     const vizPanel = PanelBuilders.timeseries()
@@ -465,6 +473,7 @@ export function buildAllLayout(
               refId: `A-${option.label}`,
               expr,
               legendFormat: `{{${option.label}}}`,
+              fromExploreMetrics: true,
             },
           ],
         })

public/app/features/trails/MetricSelect/api.ts

@@ -1,6 +1,7 @@
 import { AdHocVariableFilter, RawTimeRange, Scope } from '@grafana/data';
 import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
 import { PromQueryModeller } from '@grafana/prometheus/src/querybuilder/PromQueryModeller';
+import { utf8Support } from '@grafana/prometheus/src/utf8_support';
 import { config, getBackendSrv } from '@grafana/runtime';
 
 import { limitOtelMatchTerms } from '../otel/util';
@@ -38,7 +39,7 @@ export async function getMetricNamesWithoutScopes(
     ? adhocFilters.map((filter) =>
         removeBrackets(queryModeller.renderLabels([{ label: filter.key, op: filter.operator, value: filter.value }]))
       )
-    : adhocFilters.map((filter) => `${filter.key}${filter.operator}"${filter.value}"`);
+    : adhocFilters.map((filter) => `${utf8Support(filter.key)}${filter.operator}"${filter.value}"`);
   let missingOtelTargets = false;
 
   if (jobs.length > 0 && instances.length > 0) {

public/app/features/trails/MetricSelect/previewPanel.test.ts

@@ -20,7 +20,7 @@ describe('getPreviewPanelFor', () => {
     });
 
     test('When there are 1 or more filters, append to the ${filters} variable', () => {
-      const expected = 'avg(${metric}{${filters},__ignore_usage__=""} ${otel_join_query})';
+      const expected = 'avg(${metric}{__ignore_usage__="",${filters}} ${otel_join_query})';
 
       for (let i = 1; i < 10; ++i) {
         const expr = callAndGetExpr(1);

public/app/features/trails/MetricSelect/previewPanel.ts

@@ -54,7 +54,7 @@ export function getPreviewPanelFor(
 
 function convertPreviewQueriesToIgnoreUsage(query: PromQuery, currentFilterCount: number) {
   // If there are filters, we append to the list. Otherwise, we replace the empty list.
-  const replacement = currentFilterCount > 0 ? '${filters},__ignore_usage__=""' : '__ignore_usage__=""';
+  const replacement = currentFilterCount > 0 ? '__ignore_usage__="",${filters}' : '__ignore_usage__=""';
 
   const expr = query.expr?.replace('${filters}', replacement);
 

public/app/features/trails/autoQuery/getAutoQueriesForMetric.ts

@@ -1,3 +1,5 @@
+import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support';
+
 import { createDefaultMetricQueryDefs } from './queryGenerators/default';
 import { createHistogramMetricQueryDefs } from './queryGenerators/histogram';
 import { createSummaryMetricQueryDefs } from './queryGenerators/summary';
@@ -5,7 +7,7 @@ import { AutoQueryContext, AutoQueryInfo } from './types';
 import { getUnit } from './units';
 
 export function getAutoQueriesForMetric(metric: string, nativeHistogram?: boolean): AutoQueryInfo {
-  const isUtf8Metric = false;
+  const isUtf8Metric = !isValidLegacyName(metric);
   const metricParts = metric.split('_');
   const suffix = metricParts.at(-1);
 

public/app/features/trails/autoQuery/queryGenerators/common.ts

@@ -24,6 +24,7 @@ export function generateCommonAutoQueryInfo({
     refId: 'A',
     expr: mainQueryExpr,
     legendFormat: description,
+    fromExploreMetrics: true,
   };
 
   const main = {
@@ -48,6 +49,7 @@ export function generateCommonAutoQueryInfo({
         refId: 'A',
         expr: breakdownQueryExpr,
         legendFormat: `{{${VAR_GROUP_BY_EXP}}}`,
+        fromExploreMetrics: true,
       },
     ],
     vizBuilder: () => simpleGraphBuilder(breakdown),

public/app/features/trails/autoQuery/queryGenerators/histogram.ts

@@ -44,6 +44,7 @@ export function createHistogramMetricQueryDefs(context: AutoQueryContext) {
           isUtf8Metric: context.isUtf8Metric,
           groupings: ['le'],
         }),
+        fromExploreMetrics: true,
         format: 'heatmap',
       },
     ],
@@ -73,5 +74,6 @@ function percentileQuery(context: AutoQueryContext, percentile: number, grouping
     refId: `Percentile${percentile}`,
     expr: `histogram_quantile(${percent}, ${query})`,
     legendFormat,
+    fromExploreMetrics: true,
   };
 }

public/app/features/trails/helpers/MetricDatasourceHelper.ts

@@ -149,6 +149,7 @@ export class MetricDatasourceHelper {
     const ds = await this.getDatasource();
 
     if (ds instanceof PrometheusDatasource) {
+      options.key = unwrapQuotes(options.key);
       const keys = await ds.getTagValues(options);
       return keys;
     }
@@ -172,3 +173,15 @@ export function getMetricDescription(metadata?: PromMetricsMetadataItem) {
 
   return lines.join('\n\n');
 }
+
+function unwrapQuotes(value: string): string {
+  if (value === '' || !isWrappedInQuotes(value)) {
+    return value;
+  }
+  return value.slice(1, -1);
+}
+
+function isWrappedInQuotes(value: string): boolean {
+  const wrappedInQuotes = /^".*"$/;
+  return wrappedInQuotes.test(value);
+}

public/app/features/trails/otel/api.ts

@@ -1,5 +1,6 @@
 import { RawTimeRange, Scope } from '@grafana/data';
 import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
+import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support';
 import { config, getBackendSrv } from '@grafana/runtime';
 
 import { callSuggestionsApi } from '../utils';
@@ -40,7 +41,10 @@ export async function totalOtelResources(
 ): Promise<OtelTargetType> {
   const start = getPrometheusTime(timeRange.from, false);
   const end = getPrometheusTime(timeRange.to, true);
-
+  // check that the metric is utf8 before doing a resource query
+  if (metric && !isValidLegacyName(metric)) {
+    metric = `{"${metric}"}`;
+  }
   const query = metric ? metricOtelJobInstanceQuery(metric) : otelTargetInfoQuery(filters);
 
   const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/query`;
@@ -204,7 +208,13 @@ export async function getFilteredResourceAttributes(
   // The match param for the metric to get all possible labels for this metric
   const metricMatchTerms = limitOtelMatchTerms([], metricResources.jobs, metricResources.instances);
 
-  let metricMatchParam = `${metric}{${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`;
+  let metricMatchParam = '';
+  // check metric is utf8 to give corrrect syntax
+  if (!isValidLegacyName(metric)) {
+    metricMatchParam = `{'${metric}',${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`;
+  } else {
+    metricMatchParam = `${metric}{${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`;
+  }
 
   const start = getPrometheusTime(timeRange.from, false);
   const end = getPrometheusTime(timeRange.to, true);

public/app/features/trails/otel/util.ts

@@ -1,4 +1,5 @@
 import { AdHocVariableFilter, MetricFindValue, RawTimeRange, VariableHide } from '@grafana/data';
+import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support';
 import { config } from '@grafana/runtime';
 import { AdHocFiltersVariable, ConstantVariable, sceneGraph, SceneObject } from '@grafana/scenes';
 
@@ -110,7 +111,13 @@ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: strin
 
     // add the other OTEL resource filters
     for (let i = 0; i < otelFilters?.length; i++) {
-      const labelName = otelFilters[i].key;
+      let labelName = otelFilters[i].key;
+
+      // when adding an otel resource filter with utfb
+      if (!isValidLegacyName(labelName)) {
+        labelName = `'${labelName}'`;
+      }
+
       const op = otelFilters[i].operator;
       const labelValue = otelFilters[i].value;
 
@@ -281,8 +288,15 @@ export async function updateOtelJoinWithGroupLeft(trail: DataTrail, metric: stri
   );
   // here we start to add the attributes to the group left
   if (attributes.length > 0) {
+    // loop through attributes to check for utf8
+    const utf8Attributes = attributes.map((a) => {
+      if (!isValidLegacyName(a)) {
+        return `'${a}'`;
+      }
+      return a;
+    });
     // update the group left variable that contains all the filtered resource attributes
-    otelGroupLeft.setState({ value: attributes.join(',') });
+    otelGroupLeft.setState({ value: utf8Attributes.join(',') });
     // get the new otel join query that includes the group left attributes
     const resourceObject = getOtelResourcesObject(trail);
     const otelJoinQuery = getOtelJoinQuery(resourceObject, trail);