diff --git a/devenv/docker/blocks/prometheus_utf8/main.go b/devenv/docker/blocks/prometheus_utf8/main.go index 1aaf3f49b1f..8a61eb97bba 100644 --- a/devenv/docker/blocks/prometheus_utf8/main.go +++ b/devenv/docker/blocks/prometheus_utf8/main.go @@ -60,6 +60,14 @@ func main() { label: "label.with.spaß", getNextValue: staticList([]string{"this_is_fun"}), }, + { + label: "instance", + getNextValue: staticList([]string{"instance"}), + }, + { + label: "job", + getNextValue: staticList([]string{"job"}), + }, { label: "site", getNextValue: staticList([]string{"LA-EPI"}), @@ -85,6 +93,12 @@ func main() { Help: "a metric with utf8 labels", }, dimensions) + target_info := promauto.NewGauge(prometheus.GaugeOpts{ + Name: "target_info", + Help: "an info metric model for otel", + ConstLabels: map[string]string{"job": "job", "instance": "instance", "resource 1": "1", "resource 2": "2", "resource ę": "e", "deployment_environment": "prod"}, + }) + http.Handle("/metrics", promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -101,6 +115,7 @@ func main() { utf8Metric.WithLabelValues(labels...).Inc() opsProcessed.WithLabelValues(labels...).Inc() + target_info.Set(1) time.Sleep(time.Second * 5) } diff --git a/packages/grafana-prometheus/src/datasource.ts b/packages/grafana-prometheus/src/datasource.ts index 8ece500ac12..a6c0daffd37 100644 --- a/packages/grafana-prometheus/src/datasource.ts +++ b/packages/grafana-prometheus/src/datasource.ts @@ -71,6 +71,7 @@ import { RawRecordingRules, RuleQueryMapping, } from './types'; +import { utf8Support, wrapUtf8Filters } from './utf8_support'; import { PrometheusVariableSupport } from './variables'; const ANNOTATION_QUERY_STEP_DEFAULT = '60s'; @@ -925,7 +926,21 @@ export class PrometheusDatasource // We need a first replace to evaluate variables before applying adhoc filters // This is required for an expression like `metric > $VAR` where $VAR is a float to which we must not add adhoc filters - const expr = this.templateSrv.replace(target.expr, variables, this.interpolateQueryExpr); + const expr = this.templateSrv.replace( + target.expr, + variables, + (value: string | string[] = [], variable: QueryVariableModel | CustomVariableModel) => { + if (typeof value === 'string' && target.fromExploreMetrics) { + if (variable.name === 'filters') { + return wrapUtf8Filters(value); + } + if (variable.name === 'groupby') { + return utf8Support(value); + } + } + return this.interpolateQueryExpr(value, variable); + } + ); // Apply ad-hoc filters // When ad-hoc filters are applied, we replace again the variables in case the ad-hoc filters also reference a variable diff --git a/packages/grafana-prometheus/src/types.ts b/packages/grafana-prometheus/src/types.ts index 57903231250..25887e7d8eb 100644 --- a/packages/grafana-prometheus/src/types.ts +++ b/packages/grafana-prometheus/src/types.ts @@ -20,6 +20,7 @@ export interface PromQuery extends GenPromQuery, DataQuery { disableTextWrap?: boolean; fullMetaSearch?: boolean; includeNullMetadata?: boolean; + fromExploreMetrics?: boolean; } export enum PrometheusCacheLevel { diff --git a/packages/grafana-prometheus/src/utf8_support.test.ts b/packages/grafana-prometheus/src/utf8_support.test.ts index 0c62733bddf..5b02d3920e3 100644 --- a/packages/grafana-prometheus/src/utf8_support.test.ts +++ b/packages/grafana-prometheus/src/utf8_support.test.ts @@ -1,4 +1,4 @@ -import { escapeForUtf8Support, utf8Support } from './utf8_support'; +import { escapeForUtf8Support, utf8Support, wrapUtf8Filters } from './utf8_support'; describe('utf8 support', () => { it('should return utf8 labels wrapped in quotes', () => { @@ -33,3 +33,95 @@ describe('applyValueEncodingEscaping', () => { expect(excapedLabels).toEqual(expected); }); }); + +describe('wrapUtf8Filters', () => { + it('should correctly wrap UTF-8 labels and values for multiple key-value pairs', () => { + const result = wrapUtf8Filters('label.with.spaß="this_is_fun",instance="localhost:9112"'); + const expected = '"label.with.spaß"="this_is_fun",instance="localhost:9112"'; + expect(result).toEqual(expected); + }); + + it('should correctly wrap UTF-8 labels and values for a single key-value pair', () => { + const result = wrapUtf8Filters('label.with.spaß="this_is_fun"'); + const expected = '"label.with.spaß"="this_is_fun"'; + expect(result).toEqual(expected); + }); + + it('should correctly handle commas within values', () => { + const result = wrapUtf8Filters('label.with.spaß="this,is,fun",instance="localhost:9112"'); + const expected = '"label.with.spaß"="this,is,fun",instance="localhost:9112"'; + expect(result).toEqual(expected); + }); + + it('should correctly handle escaped quotes within values', () => { + const result = wrapUtf8Filters(`label.with.spaß="this_is_\\"fun\\"",instance="localhost:9112"`); + const expected = `"label.with.spaß"="this_is_\\"fun\\"",instance="localhost:9112"`; + expect(result).toEqual(expected); + }); + + it('should correctly handle spaces within keys', () => { + const result = wrapUtf8Filters('label with space="value with space",instance="localhost:9112"'); + const expected = '"label with space"="value with space",instance="localhost:9112"'; + expect(result).toEqual(expected); + }); + + it('should correctly process mixed inputs with various formats', () => { + const result = wrapUtf8Filters('key1="value1",key2="value,with,comma",key3="val3"'); + const expected = 'key1="value1",key2="value,with,comma",key3="val3"'; + expect(result).toEqual(expected); + }); + + it('should correctly handle empty values', () => { + const result = wrapUtf8Filters('key1="",key2="value2"'); + const expected = 'key1="",key2="value2"'; + expect(result).toEqual(expected); + }); + + it('should handle an empty input string', () => { + const result = wrapUtf8Filters(''); + const expected = ''; + expect(result).toEqual(expected); + }); + + it('should handle a single key with an empty value', () => { + const result = wrapUtf8Filters('key1=""'); + const expected = 'key1=""'; + expect(result).toEqual(expected); + }); + + it('should handle multiple consecutive commas in a value', () => { + const result = wrapUtf8Filters('key1="value1,,value2",key2="value3"'); + const expected = 'key1="value1,,value2",key2="value3"'; + expect(result).toEqual(expected); + }); + + it('should handle a key-value pair with special characters in the key', () => { + const result = wrapUtf8Filters('special@key#="value1",key2="value2"'); + const expected = '"special@key#"="value1",key2="value2"'; + expect(result).toEqual(expected); + }); + + it('should handle a key-value pair with special characters in the value', () => { + const result = wrapUtf8Filters('key1="value@#&*",key2="value2"'); + const expected = 'key1="value@#&*",key2="value2"'; + expect(result).toEqual(expected); + }); + + it('should correctly process keys without special characters', () => { + const result = wrapUtf8Filters('key1="value1",key2="value2"'); + const expected = 'key1="value1",key2="value2"'; + expect(result).toEqual(expected); + }); + + it('should handle nested escaped quotes correctly', () => { + const result = wrapUtf8Filters('key1="nested \\"escaped\\" quotes",key2="value2"'); + const expected = 'key1="nested \\"escaped\\" quotes",key2="value2"'; + expect(result).toEqual(expected); + }); + + it('should handle escaped quotes correctly', () => { + const result = wrapUtf8Filters('key1="nested \\"escaped\\" quotes",key2="value with \\"escaped\\" quotes"'); + const expected = 'key1="nested \\"escaped\\" quotes",key2="value with \\"escaped\\" quotes"'; + expect(result).toEqual(expected); + }); +}); diff --git a/packages/grafana-prometheus/src/utf8_support.ts b/packages/grafana-prometheus/src/utf8_support.ts index 386d41ba987..9f757c0bc73 100644 --- a/packages/grafana-prometheus/src/utf8_support.ts +++ b/packages/grafana-prometheus/src/utf8_support.ts @@ -79,3 +79,35 @@ const isValidCodePoint = (codePoint: number): boolean => { // Validate the code point for UTF-8 compliance if needed. return codePoint >= 0 && codePoint <= 0x10ffff; }; + +export const wrapUtf8Filters = (filterStr: string): string => { + const resultArray: string[] = []; + let currentKey = ''; + let currentValue = ''; + let inQuotes = false; + let temp = ''; + + for (const char of filterStr) { + if (char === '"' && temp[temp.length - 1] !== '\\') { + // Toggle inQuotes when an unescaped quote is found + inQuotes = !inQuotes; + temp += char; + } else if (char === ',' && !inQuotes) { + // When outside quotes and encountering ',', finalize the current pair + [currentKey, currentValue] = temp.split('='); + resultArray.push(`${utf8Support(currentKey.trim())}="${currentValue.slice(1, -1)}"`); + temp = ''; // Reset for the next pair + } else { + // Collect characters + temp += char; + } + } + + // Handle the last key-value pair + if (temp) { + [currentKey, currentValue] = temp.split('='); + resultArray.push(`${utf8Support(currentKey.trim())}="${currentValue.slice(1, -1)}"`); + } + + return resultArray.join(','); +}; diff --git a/public/app/features/trails/ActionTabs/MetricOverviewScene.tsx b/public/app/features/trails/ActionTabs/MetricOverviewScene.tsx index c82226fc763..4f3807c6034 100644 --- a/public/app/features/trails/ActionTabs/MetricOverviewScene.tsx +++ b/public/app/features/trails/ActionTabs/MetricOverviewScene.tsx @@ -1,6 +1,7 @@ import { useEffect } from 'react'; import { PromMetricsMetadataItem } from '@grafana/prometheus'; +import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support'; import { QueryVariable, SceneComponentProps, @@ -90,9 +91,14 @@ export class MetricOverviewScene extends SceneObjectBase ({ label: el, value: el })); + const attributeArray: VariableValueOption[] = resourceAttributes.split(',').map((el) => { + let label = el; + if (!isValidLegacyName(el)) { + // remove '' from label + label = el.slice(1, -1); + } + return { label, value: el }; + }); allLabelOptions = attributeArray.concat(allLabelOptions); } } diff --git a/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx b/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx index 1ab14b84aac..c650e3876ae 100644 --- a/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx +++ b/public/app/features/trails/Breakdown/LabelBreakdownScene.tsx @@ -4,6 +4,7 @@ import { isNumber, max, min, throttle } from 'lodash'; import { useEffect, useState } from 'react'; import { DataFrame, FieldType, GrafanaTheme2, PanelData, SelectableValue } from '@grafana/data'; +import { isValidLegacyName, utf8Support } from '@grafana/prometheus/src/utf8_support'; import { config } from '@grafana/runtime'; import { ConstantVariable, @@ -313,7 +314,14 @@ export class LabelBreakdownScene extends SceneObjectBase ({ label: el, value: el })); + const attributeArray: SelectableValue[] = resourceAttributes.split(',').map((el) => { + let label = el; + if (!isValidLegacyName(el)) { + // remove '' from label + label = el.slice(1, -1); + } + return { label, value: el }; + }); // shift ALL value to the front const all: SelectableValue = [{ label: 'All', value: ALL_VARIABLE_VALUE }]; const firstGroup = all.concat(attributeArray); @@ -449,7 +457,7 @@ export function buildAllLayout( break; } - const expr = queryDef.queries[0].expr.replaceAll(VAR_GROUP_BY_EXP, String(option.value)); + const expr = queryDef.queries[0].expr.replaceAll(VAR_GROUP_BY_EXP, utf8Support(String(option.value))); const unit = queryDef.unit; const vizPanel = PanelBuilders.timeseries() @@ -465,6 +473,7 @@ export function buildAllLayout( refId: `A-${option.label}`, expr, legendFormat: `{{${option.label}}}`, + fromExploreMetrics: true, }, ], }) diff --git a/public/app/features/trails/MetricSelect/api.ts b/public/app/features/trails/MetricSelect/api.ts index 2d499efc3c0..cd61a3782d5 100644 --- a/public/app/features/trails/MetricSelect/api.ts +++ b/public/app/features/trails/MetricSelect/api.ts @@ -1,6 +1,7 @@ import { AdHocVariableFilter, RawTimeRange, Scope } from '@grafana/data'; import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; import { PromQueryModeller } from '@grafana/prometheus/src/querybuilder/PromQueryModeller'; +import { utf8Support } from '@grafana/prometheus/src/utf8_support'; import { config, getBackendSrv } from '@grafana/runtime'; import { limitOtelMatchTerms } from '../otel/util'; @@ -38,7 +39,7 @@ export async function getMetricNamesWithoutScopes( ? adhocFilters.map((filter) => removeBrackets(queryModeller.renderLabels([{ label: filter.key, op: filter.operator, value: filter.value }])) ) - : adhocFilters.map((filter) => `${filter.key}${filter.operator}"${filter.value}"`); + : adhocFilters.map((filter) => `${utf8Support(filter.key)}${filter.operator}"${filter.value}"`); let missingOtelTargets = false; if (jobs.length > 0 && instances.length > 0) { diff --git a/public/app/features/trails/MetricSelect/previewPanel.test.ts b/public/app/features/trails/MetricSelect/previewPanel.test.ts index 59daafa3297..6175af65793 100644 --- a/public/app/features/trails/MetricSelect/previewPanel.test.ts +++ b/public/app/features/trails/MetricSelect/previewPanel.test.ts @@ -20,7 +20,7 @@ describe('getPreviewPanelFor', () => { }); test('When there are 1 or more filters, append to the ${filters} variable', () => { - const expected = 'avg(${metric}{${filters},__ignore_usage__=""} ${otel_join_query})'; + const expected = 'avg(${metric}{__ignore_usage__="",${filters}} ${otel_join_query})'; for (let i = 1; i < 10; ++i) { const expr = callAndGetExpr(1); diff --git a/public/app/features/trails/MetricSelect/previewPanel.ts b/public/app/features/trails/MetricSelect/previewPanel.ts index 92007959ac0..7206d3bb8e5 100644 --- a/public/app/features/trails/MetricSelect/previewPanel.ts +++ b/public/app/features/trails/MetricSelect/previewPanel.ts @@ -54,7 +54,7 @@ export function getPreviewPanelFor( function convertPreviewQueriesToIgnoreUsage(query: PromQuery, currentFilterCount: number) { // If there are filters, we append to the list. Otherwise, we replace the empty list. - const replacement = currentFilterCount > 0 ? '${filters},__ignore_usage__=""' : '__ignore_usage__=""'; + const replacement = currentFilterCount > 0 ? '__ignore_usage__="",${filters}' : '__ignore_usage__=""'; const expr = query.expr?.replace('${filters}', replacement); diff --git a/public/app/features/trails/autoQuery/getAutoQueriesForMetric.ts b/public/app/features/trails/autoQuery/getAutoQueriesForMetric.ts index 732fd6ed4b3..01b74f22ace 100644 --- a/public/app/features/trails/autoQuery/getAutoQueriesForMetric.ts +++ b/public/app/features/trails/autoQuery/getAutoQueriesForMetric.ts @@ -1,3 +1,5 @@ +import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support'; + import { createDefaultMetricQueryDefs } from './queryGenerators/default'; import { createHistogramMetricQueryDefs } from './queryGenerators/histogram'; import { createSummaryMetricQueryDefs } from './queryGenerators/summary'; @@ -5,7 +7,7 @@ import { AutoQueryContext, AutoQueryInfo } from './types'; import { getUnit } from './units'; export function getAutoQueriesForMetric(metric: string, nativeHistogram?: boolean): AutoQueryInfo { - const isUtf8Metric = false; + const isUtf8Metric = !isValidLegacyName(metric); const metricParts = metric.split('_'); const suffix = metricParts.at(-1); diff --git a/public/app/features/trails/autoQuery/queryGenerators/common.ts b/public/app/features/trails/autoQuery/queryGenerators/common.ts index b0e82b4baa3..d77a8ce1de1 100644 --- a/public/app/features/trails/autoQuery/queryGenerators/common.ts +++ b/public/app/features/trails/autoQuery/queryGenerators/common.ts @@ -24,6 +24,7 @@ export function generateCommonAutoQueryInfo({ refId: 'A', expr: mainQueryExpr, legendFormat: description, + fromExploreMetrics: true, }; const main = { @@ -48,6 +49,7 @@ export function generateCommonAutoQueryInfo({ refId: 'A', expr: breakdownQueryExpr, legendFormat: `{{${VAR_GROUP_BY_EXP}}}`, + fromExploreMetrics: true, }, ], vizBuilder: () => simpleGraphBuilder(breakdown), diff --git a/public/app/features/trails/autoQuery/queryGenerators/histogram.ts b/public/app/features/trails/autoQuery/queryGenerators/histogram.ts index c4fc17ad331..015ecdc8441 100644 --- a/public/app/features/trails/autoQuery/queryGenerators/histogram.ts +++ b/public/app/features/trails/autoQuery/queryGenerators/histogram.ts @@ -44,6 +44,7 @@ export function createHistogramMetricQueryDefs(context: AutoQueryContext) { isUtf8Metric: context.isUtf8Metric, groupings: ['le'], }), + fromExploreMetrics: true, format: 'heatmap', }, ], @@ -73,5 +74,6 @@ function percentileQuery(context: AutoQueryContext, percentile: number, grouping refId: `Percentile${percentile}`, expr: `histogram_quantile(${percent}, ${query})`, legendFormat, + fromExploreMetrics: true, }; } diff --git a/public/app/features/trails/helpers/MetricDatasourceHelper.ts b/public/app/features/trails/helpers/MetricDatasourceHelper.ts index 4ecfb09db53..b77a28d2c3c 100644 --- a/public/app/features/trails/helpers/MetricDatasourceHelper.ts +++ b/public/app/features/trails/helpers/MetricDatasourceHelper.ts @@ -149,6 +149,7 @@ export class MetricDatasourceHelper { const ds = await this.getDatasource(); if (ds instanceof PrometheusDatasource) { + options.key = unwrapQuotes(options.key); const keys = await ds.getTagValues(options); return keys; } @@ -172,3 +173,15 @@ export function getMetricDescription(metadata?: PromMetricsMetadataItem) { return lines.join('\n\n'); } + +function unwrapQuotes(value: string): string { + if (value === '' || !isWrappedInQuotes(value)) { + return value; + } + return value.slice(1, -1); +} + +function isWrappedInQuotes(value: string): boolean { + const wrappedInQuotes = /^".*"$/; + return wrappedInQuotes.test(value); +} diff --git a/public/app/features/trails/otel/api.ts b/public/app/features/trails/otel/api.ts index 3399b0f7fe9..083d1bef15e 100644 --- a/public/app/features/trails/otel/api.ts +++ b/public/app/features/trails/otel/api.ts @@ -1,5 +1,6 @@ import { RawTimeRange, Scope } from '@grafana/data'; import { getPrometheusTime } from '@grafana/prometheus/src/language_utils'; +import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support'; import { config, getBackendSrv } from '@grafana/runtime'; import { callSuggestionsApi } from '../utils'; @@ -40,7 +41,10 @@ export async function totalOtelResources( ): Promise { const start = getPrometheusTime(timeRange.from, false); const end = getPrometheusTime(timeRange.to, true); - + // check that the metric is utf8 before doing a resource query + if (metric && !isValidLegacyName(metric)) { + metric = `{"${metric}"}`; + } const query = metric ? metricOtelJobInstanceQuery(metric) : otelTargetInfoQuery(filters); const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/query`; @@ -204,7 +208,13 @@ export async function getFilteredResourceAttributes( // The match param for the metric to get all possible labels for this metric const metricMatchTerms = limitOtelMatchTerms([], metricResources.jobs, metricResources.instances); - let metricMatchParam = `${metric}{${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`; + let metricMatchParam = ''; + // check metric is utf8 to give corrrect syntax + if (!isValidLegacyName(metric)) { + metricMatchParam = `{'${metric}',${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`; + } else { + metricMatchParam = `${metric}{${metricMatchTerms.jobsRegex},${metricMatchTerms.instancesRegex}}`; + } const start = getPrometheusTime(timeRange.from, false); const end = getPrometheusTime(timeRange.to, true); diff --git a/public/app/features/trails/otel/util.ts b/public/app/features/trails/otel/util.ts index d5c2c0b9886..b0f95dabd51 100644 --- a/public/app/features/trails/otel/util.ts +++ b/public/app/features/trails/otel/util.ts @@ -1,4 +1,5 @@ import { AdHocVariableFilter, MetricFindValue, RawTimeRange, VariableHide } from '@grafana/data'; +import { isValidLegacyName } from '@grafana/prometheus/src/utf8_support'; import { config } from '@grafana/runtime'; import { AdHocFiltersVariable, ConstantVariable, sceneGraph, SceneObject } from '@grafana/scenes'; @@ -110,7 +111,13 @@ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: strin // add the other OTEL resource filters for (let i = 0; i < otelFilters?.length; i++) { - const labelName = otelFilters[i].key; + let labelName = otelFilters[i].key; + + // when adding an otel resource filter with utfb + if (!isValidLegacyName(labelName)) { + labelName = `'${labelName}'`; + } + const op = otelFilters[i].operator; const labelValue = otelFilters[i].value; @@ -281,8 +288,15 @@ export async function updateOtelJoinWithGroupLeft(trail: DataTrail, metric: stri ); // here we start to add the attributes to the group left if (attributes.length > 0) { + // loop through attributes to check for utf8 + const utf8Attributes = attributes.map((a) => { + if (!isValidLegacyName(a)) { + return `'${a}'`; + } + return a; + }); // update the group left variable that contains all the filtered resource attributes - otelGroupLeft.setState({ value: attributes.join(',') }); + otelGroupLeft.setState({ value: utf8Attributes.join(',') }); // get the new otel join query that includes the group left attributes const resourceObject = getOtelResourcesObject(trail); const otelJoinQuery = getOtelJoinQuery(resourceObject, trail);