diff --git a/public/app/features/trails/Breakdown/AddToFiltersGraphAction.tsx b/public/app/features/trails/Breakdown/AddToFiltersGraphAction.tsx index b4b53e80ae8..faf7ef9542b 100644 --- a/public/app/features/trails/Breakdown/AddToFiltersGraphAction.tsx +++ b/public/app/features/trails/Breakdown/AddToFiltersGraphAction.tsx @@ -9,7 +9,7 @@ import { import { Button } from '@grafana/ui'; import { reportExploreMetrics } from '../interactions'; -import { VAR_OTEL_GROUP_LEFT, VAR_OTEL_RESOURCES } from '../shared'; +import { VAR_OTEL_AND_METRIC_FILTERS, VAR_OTEL_GROUP_LEFT, VAR_OTEL_RESOURCES } from '../shared'; import { getTrailFor } from '../utils'; export interface AddToFiltersGraphActionState extends SceneObjectState { @@ -38,7 +38,6 @@ export class AddToFiltersGraphAction extends SceneObjectBase ({ totalOtelResources: jest.fn(() => ({ job: 'oteldemo', instance: 'instance' })), - getDeploymentEnvironments: jest.fn(() => ['production', 'staging']), isOtelStandardization: jest.fn(() => true), })); @@ -481,15 +480,18 @@ describe('DataTrail', () => { describe('OTel resources attributes', () => { let trail: DataTrail; + + // selecting a non promoted resource from VAR_OTEL_AND_METRICS will automatically update the otel resources var + const nonPromotedOtelResources = ['deployment_environment']; const preTrailUrl = '/trail?from=now-1h&to=now&var-ds=edwxqcebl0cg0c&var-deployment_environment=oteldemo01&var-otel_resources=k8s_cluster_name%7C%3D%7Cappo11ydev01&var-filters=&refresh=&metricPrefix=all&metricSearch=http&actionView=breakdown&var-groupby=$__all&metric=http_client_duration_milliseconds_bucket'; - function getOtelDepEnvVar(trail: DataTrail) { - const variable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, trail); - if (variable instanceof CustomVariable) { + function getOtelAndMetricsVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, trail); + if (variable instanceof AdHocFiltersVariable) { return variable; } - throw new Error('getDepEnvVar failed'); + throw new Error('getOtelAndMetricsVar failed'); } function getOtelJoinQueryVar(trail: DataTrail) { @@ -497,7 +499,7 @@ describe('DataTrail', () => { if (variable instanceof ConstantVariable) { return variable; } - throw new Error('getDepEnvVar failed'); + throw new Error('getOtelJoinQueryVar failed'); } function getOtelResourcesVar(trail: DataTrail) { @@ -513,26 +515,34 @@ describe('DataTrail', () => { if (variable instanceof ConstantVariable) { return variable; } - throw new Error('getOtelResourcesVar failed'); + throw new Error('getOtelGroupLeftVar failed'); + } + + function getFilterVar() { + const variable = sceneGraph.lookupVariable(VAR_FILTERS, trail); + if (variable instanceof AdHocFiltersVariable) { + return variable; + } + throw new Error('getFilterVar failed'); } beforeEach(() => { - trail = new DataTrail({}); + trail = new DataTrail({ + nonPromotedOtelResources, + // before checking, things should be hidden + initialOtelCheckComplete: false, + }); locationService.push(preTrailUrl); activateFullSceneTree(trail); - getOtelResourcesVar(trail).setState({ filters: [{ key: 'service_name', operator: '=', value: 'adservice' }] }); - getOtelDepEnvVar(trail).changeValueTo('production'); getOtelGroupLeftVar(trail).setState({ value: 'attribute1,attribute2' }); }); - - it('should start with hidden dep env variable', () => { - const depEnvVarHide = getOtelDepEnvVar(trail).state.hide; - expect(depEnvVarHide).toBe(VariableHide.hideVariable); - }); - - it('should start with hidden otel resources variable', () => { - const resourcesVarHide = getOtelResourcesVar(trail).state.hide; - expect(resourcesVarHide).toBe(VariableHide.hideVariable); + // default otel experience to off + it('clicking start button should start with OTel off and showing var filters', () => { + trail.setState({ startButtonClicked: true }); + const otelResourcesHide = getOtelResourcesVar(trail).state.hide; + const varFiltersHide = getFilterVar().state.hide; + expect(otelResourcesHide).toBe(VariableHide.hideVariable); + expect(varFiltersHide).toBe(VariableHide.hideLabel); }); it('should start with hidden otel join query variable', () => { @@ -540,25 +550,54 @@ describe('DataTrail', () => { expect(joinQueryVarHide).toBe(VariableHide.hideVariable); }); - it('should add history step for when updating the otel resource variable', () => { - expect(trail.state.history.state.steps[2].type).toBe('resource'); + it('should have a group left variable for resource attributes', () => { + expect(getOtelGroupLeftVar(trail).state.value).toBe('attribute1,attribute2'); }); - it('Should have otel resource attribute selected as "service_name=adservice"', () => { - expect(getOtelResourcesVar(trail).state.filters[0].key).toBe('service_name'); - expect(getOtelResourcesVar(trail).state.filters[0].value).toBe('adservice'); - }); + describe('resetting the OTel experience', () => { + it('should display with hideLabel var filters and hide VAR_OTEL_AND_METRIC_FILTERS when resetting otel experience', () => { + trail.resetOtelExperience(); + expect(getFilterVar().state.hide).toBe(VariableHide.hideLabel); + expect(getOtelAndMetricsVar(trail).state.hide).toBe(VariableHide.hideVariable); + }); - it('Should have deployment environment selected as "production"', () => { - expect(getOtelDepEnvVar(trail).getValue()).toBe('production'); + // it should preserve var filters when it resets }); - it('should add history step for when updating the dep env variable', () => { - expect(trail.state.history.state.steps[3].type).toBe('dep_env'); - }); + describe('when otel is on the subscription to Otel and metrics var should update other variables', () => { + beforeEach(() => { + trail.setState({ initialOtelCheckComplete: true, useOtelExperience: true }); + }); - it('should have a group left variable for resource attributes', () => { - expect(getOtelGroupLeftVar(trail).state.value).toBe('attribute1,attribute2'); + it('should automatically update the otel resources var when a non promoted resource has been selected from VAR_OTEL_AND_METRICS', () => { + getOtelAndMetricsVar(trail).setState({ + filters: [{ key: 'deployment_environment', operator: '=', value: 'production' }], + }); + + const otelResourcesVar = getOtelResourcesVar(trail); + const otelResourcesFilter = otelResourcesVar.state.filters[0]; + expect(otelResourcesFilter.key).toBe('deployment_environment'); + expect(otelResourcesFilter.value).toBe('production'); + }); + + it('should add history step of type "resource" when adding a non promoted otel resource', () => { + getOtelAndMetricsVar(trail).setState({ + filters: [{ key: 'deployment_environment', operator: '=', value: 'production' }], + }); + expect(trail.state.history.state.steps[2].type).toBe('resource'); + }); + + it('should automatically update the var filters when a promoted resource has been selected from VAR_OTEL_AND_METRICS', () => { + getOtelAndMetricsVar(trail).setState({ filters: [{ key: 'promoted', operator: '=', value: 'resource' }] }); + const varFilters = getFilterVar().state.filters[0]; + expect(varFilters.key).toBe('promoted'); + expect(varFilters.value).toBe('resource'); + }); + + it('should add history step of type "filters" when adding a non promoted otel resource', () => { + getOtelAndMetricsVar(trail).setState({ filters: [{ key: 'promoted', operator: '=', value: 'resource' }] }); + expect(trail.state.history.state.steps[2].type).toBe('filters'); + }); }); }); }); diff --git a/public/app/features/trails/DataTrail.tsx b/public/app/features/trails/DataTrail.tsx index 6e98e2514e1..f9f6d0bee19 100644 --- a/public/app/features/trails/DataTrail.tsx +++ b/public/app/features/trails/DataTrail.tsx @@ -1,15 +1,7 @@ import { css } from '@emotion/css'; -import { useEffect } from 'react'; +import { useEffect, useRef } from 'react'; -import { - AdHocVariableFilter, - GetTagResponse, - GrafanaTheme2, - MetricFindValue, - RawTimeRange, - urlUtil, - VariableHide, -} from '@grafana/data'; +import { AdHocVariableFilter, GrafanaTheme2, RawTimeRange, urlUtil, VariableHide } from '@grafana/data'; import { PromQuery } from '@grafana/prometheus'; import { locationService, useChromeHeaderHeight } from '@grafana/runtime'; import { @@ -48,17 +40,11 @@ import { MetricSelectScene } from './MetricSelect/MetricSelectScene'; import { MetricsHeader } from './MetricsHeader'; import { getTrailStore } from './TrailStore/TrailStore'; import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper'; -import { reportChangeInLabelFilters } from './interactions'; -import { getDeploymentEnvironments, TARGET_INFO_FILTER, totalOtelResources } from './otel/api'; -import { OtelResourcesObject, OtelTargetType } from './otel/types'; -import { - getOtelJoinQuery, - getOtelResourcesObject, - getProdOrDefaultOption, - sortResources, - updateOtelJoinWithGroupLeft, -} from './otel/util'; -import { getOtelExperienceToggleState } from './services/store'; +import { reportChangeInLabelFilters, reportExploreMetrics } from './interactions'; +import { migrateOtelDeploymentEnvironment } from './migrations/otelDeploymentEnvironment'; +import { getDeploymentEnvironments, getNonPromotedOtelResources, totalOtelResources } from './otel/api'; +import { OtelTargetType } from './otel/types'; +import { manageOtelAndMetricFilters, updateOtelData, updateOtelJoinWithGroupLeft } from './otel/util'; import { getVariablesWithOtelJoinQueryConstant, MetricSelectedEvent, @@ -67,6 +53,7 @@ import { VAR_DATASOURCE_EXPR, VAR_FILTERS, VAR_MISSING_OTEL_TARGETS, + VAR_OTEL_AND_METRIC_FILTERS, VAR_OTEL_DEPLOYMENT_ENV, VAR_OTEL_GROUP_LEFT, VAR_OTEL_JOIN_QUERY, @@ -92,6 +79,13 @@ export interface DataTrailState extends SceneObjectState { otelTargets?: OtelTargetType; // all the targets with job and instance regex, job=~"|"", instance=~"|" otelJoinQuery?: string; isStandardOtel?: boolean; + nonPromotedOtelResources?: string[]; + initialOtelCheckComplete?: boolean; // updated after the first otel check + startButtonClicked?: boolean; // from original landing page + afterFirstOtelCheck?: boolean; // when starting there is always a DS var change from variable dependency + resettingOtel?: boolean; // when switching OTel off from the switch + isUpdatingOtel?: boolean; + addingLabelFromBreakdown?: boolean; // do not use the otel and metrics var subscription when adding label from the breakdown // moved into settings showPreviews?: boolean; @@ -133,6 +127,9 @@ export class DataTrail extends SceneObjectBase implements SceneO } public _onActivate() { + const urlParams = urlUtil.getUrlSearchParams(); + migrateOtelDeploymentEnvironment(this, urlParams); + if (!this.state.topScene) { this.setState({ topScene: getTopSceneFor(this.state.metric) }); } @@ -151,6 +148,41 @@ export class DataTrail extends SceneObjectBase implements SceneO ); } + // This is for OTel consolidation filters + // whenever the otel and metric filter is updated, + // we need to add that filter to the correct otel resource var or var filter + // so the filter can be interpolated in the query correctly + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, this); + const otelFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, this); + if ( + otelAndMetricsFiltersVariable instanceof AdHocFiltersVariable && + otelFiltersVariable instanceof AdHocFiltersVariable && + filtersVariable instanceof AdHocFiltersVariable + ) { + this._subs.add( + otelAndMetricsFiltersVariable?.subscribeToState((newState, prevState) => { + // identify the added, updated or removed variables and update the correct filter, + // either the otel resource or the var filter + // do not update on switching on otel experience or the initial check + // do not update when selecting a label from metric scene breakdown + if ( + this.state.useOtelExperience && + this.state.initialOtelCheckComplete && + !this.state.addingLabelFromBreakdown + ) { + const nonPromotedOtelResources = this.state.nonPromotedOtelResources ?? []; + manageOtelAndMetricFilters( + newState.filters, + prevState.filters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + } + }) + ); + } + // Save the current trail as a recent (if the browser closes or reloads) if user selects a metric OR applies filters to metric select view const saveRecentTrail = () => { const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, this); @@ -170,39 +202,30 @@ export class DataTrail extends SceneObjectBase implements SceneO } protected _variableDependency = new VariableDependencyConfig(this, { - variableNames: [VAR_DATASOURCE, VAR_OTEL_RESOURCES, VAR_OTEL_DEPLOYMENT_ENV, VAR_OTEL_JOIN_QUERY], + variableNames: [VAR_DATASOURCE, VAR_OTEL_RESOURCES, VAR_OTEL_JOIN_QUERY, VAR_OTEL_AND_METRIC_FILTERS], onReferencedVariableValueChanged: async (variable: SceneVariable) => { const { name } = variable.state; if (name === VAR_DATASOURCE) { this.datasourceHelper.reset(); + if (this.state.afterFirstOtelCheck) { + // we need a new check for OTel + this.setState({ initialOtelCheckComplete: false }); + // clear out the OTel filters, do not clear out var filters + this.resetOtelExperience(); + } // fresh check for otel experience this.checkDataSourceForOTelResources(); } // update otel variables when changed - if (this.state.useOtelExperience && (name === VAR_OTEL_DEPLOYMENT_ENV || name === VAR_OTEL_RESOURCES)) { + if (this.state.useOtelExperience && name === VAR_OTEL_RESOURCES && this.state.initialOtelCheckComplete) { // for state and variables const timeRange: RawTimeRange | undefined = this.state.$timeRange?.state; const datasourceUid = sceneGraph.interpolate(this, VAR_DATASOURCE_EXPR); - const otelDepEnvVariable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, this); - const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, this); - const otelJoinQueryVariable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, this); - - if ( - timeRange && - otelResourcesVariable instanceof AdHocFiltersVariable && - otelJoinQueryVariable instanceof ConstantVariable && - otelDepEnvVariable instanceof CustomVariable - ) { - this.updateOtelData( - datasourceUid, - timeRange, - otelDepEnvVariable, - otelResourcesVariable, - otelJoinQueryVariable - ); + if (timeRange) { + updateOtelData(this, datasourceUid, timeRange); } } }, @@ -215,14 +238,20 @@ export class DataTrail extends SceneObjectBase implements SceneO */ public addFilterWithoutReportingInteraction(filter: AdHocVariableFilter) { const variable = sceneGraph.lookupVariable('filters', this); - if (!(variable instanceof AdHocFiltersVariable)) { + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, this); + if ( + !(variable instanceof AdHocFiltersVariable) || + !(otelAndMetricsFiltersVariable instanceof AdHocFiltersVariable) + ) { return; } this._addingFilterWithoutReportingInteraction = true; - - variable.setState({ filters: [...variable.state.filters, filter] }); - + if (this.state.useOtelExperience) { + otelAndMetricsFiltersVariable.setState({ filters: [...otelAndMetricsFiltersVariable.state.filters, filter] }); + } else { + variable.setState({ filters: [...variable.state.filters, filter] }); + } this._addingFilterWithoutReportingInteraction = false; } @@ -320,7 +349,7 @@ export class DataTrail extends SceneObjectBase implements SceneO * Check that the data source is standard for OTEL * Show a warning if not * Update the following variables: - * deployment_environment (first filter), otelResources (filters), otelJoinQuery (used in the query) + * otelResources (filters), otelJoinQuery (used in the query) * Enable the otel experience * * @returns @@ -333,267 +362,102 @@ export class DataTrail extends SceneObjectBase implements SceneO const timeRange: RawTimeRange | undefined = trail.state.$timeRange?.state; if (timeRange) { - const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, this); - const otelDepEnvVariable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, this); - const otelJoinQueryVariable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, this); - const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, this); - const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR); - const otelTargets = await totalOtelResources(datasourceUid, timeRange); const deploymentEnvironments = await getDeploymentEnvironments(datasourceUid, timeRange, getSelectedScopes()); const hasOtelResources = otelTargets.jobs.length > 0 && otelTargets.instances.length > 0; + // loading from the url with otel resources selected will result in turning on OTel experience + const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, this); + let previouslyUsedOtelResources = false; + if (otelResourcesVariable instanceof AdHocFiltersVariable) { + previouslyUsedOtelResources = otelResourcesVariable.state.filters.length > 0; + } + + // Future refactor: non promoted resources could be the full check + // - remove hasOtelResources + // - remove deployment environments as a check + const nonPromotedOtelResources = await getNonPromotedOtelResources(datasourceUid, timeRange); + + // This is the function that will turn on OTel for the entire app. + // The conditions to use this function are + // 1. must be an otel data source + // 2. Do not turn it on if the start button was clicked + // 3. Url or bookmark has previous otel filters + // 4. We are restting OTel with the toggle switch if ( - otelResourcesVariable instanceof AdHocFiltersVariable && - otelDepEnvVariable instanceof CustomVariable && - otelJoinQueryVariable instanceof ConstantVariable && - filtersVariable instanceof AdHocFiltersVariable + hasOtelResources && + nonPromotedOtelResources && // it is an otel data source + !this.state.startButtonClicked && // we are not starting from the start button + (previouslyUsedOtelResources || this.state.resettingOtel) // there are otel filters or we are restting ) { // HERE WE START THE OTEL EXPERIENCE ENGINE // 1. Set deployment variable values // 2. update all other variables and state - if (hasOtelResources && deploymentEnvironments.length > 0) { - // apply VAR FILTERS manually - // otherwise they will appear anywhere the query contains {} characters - filtersVariable.setState({ - addFilterButtonText: 'Select metric attributes', - label: 'Select metric attribute', - }); - - // 1. set deployment variable values - let varQuery = ''; - const options = deploymentEnvironments.map((env) => { - varQuery += env + ','; - return { value: env, label: env }; - }); - // We have to have a default value because custom variable requires it - // we choose one default value to help filter metrics - // The work flow for OTel begins with users selecting a deployment environment - // default to production - let defaultDepEnv = getProdOrDefaultOption(options) ?? ''; - // On starting the explore metrics workflow, the custom variable has no value - // Even if there is state, the value is always '' - // The only reference to state values are in the text - const otelDepEnvValue = otelDepEnvVariable.state.text; - - // TypeScript issue: VariableValue is either a string or array but does not have any string or array methods on it to check that it is empty - const notInitialvalue = otelDepEnvValue !== '' && otelDepEnvValue.toLocaleString() !== ''; - - const depEnvInitialValue = notInitialvalue ? otelDepEnvValue : defaultDepEnv; - - otelDepEnvVariable?.setState({ - value: depEnvInitialValue, - options: options, - hide: VariableHide.dontHide, - }); - - this.updateOtelData( - datasourceUid, - timeRange, - otelDepEnvVariable, - otelResourcesVariable, - otelJoinQueryVariable, - deploymentEnvironments, - hasOtelResources - ); - } else { - // reset filters to apply auto, anywhere there are {} characters - this.resetOtelExperience( - otelResourcesVariable, - otelDepEnvVariable, - otelJoinQueryVariable, - filtersVariable, - hasOtelResources, - deploymentEnvironments - ); - } + updateOtelData( + this, + datasourceUid, + timeRange, + deploymentEnvironments, + hasOtelResources, + nonPromotedOtelResources + ); + } else { + this.resetOtelExperience(hasOtelResources, nonPromotedOtelResources); } } } - /** - * This function is used to update state and otel variables. - * - * 1. Set the otelResources adhoc tagKey and tagValues filter functions - * 2. Get the otel join query for state and variable - * 3. Update state with the following - * - otel join query - * - otelTargets used to filter metrics - * For initialization we also update the following - * - has otel resources flag - * - isStandardOtel flag (for enabliing the otel experience toggle) - * - and useOtelExperience - * - * This function is called on start and when variables change. - * On start will provide the deploymentEnvironments and hasOtelResources parameters. - * In the variable change case, we will not provide these parameters. It is assumed that the - * data source has been checked for otel resources and standardization and the otel variables are enabled at this point. - * @param datasourceUid - * @param timeRange - * @param otelDepEnvVariable - * @param otelResourcesVariable - * @param otelJoinQueryVariable - * @param deploymentEnvironments - * @param hasOtelResources - */ - async updateOtelData( - datasourceUid: string, - timeRange: RawTimeRange, - otelDepEnvVariable: CustomVariable, - otelResourcesVariable: AdHocFiltersVariable, - otelJoinQueryVariable: ConstantVariable, - deploymentEnvironments?: string[], - hasOtelResources?: boolean - ) { - // 1. Set the otelResources adhoc tagKey and tagValues filter functions - // get the labels for otel resources - // collection of filters for the otel resource variable - // filter label names and label values - // the first filter is {__name__="target_info"} - let filters: AdHocVariableFilter[] = [TARGET_INFO_FILTER]; - - // always start with the deployment environment - const depEnvValue = '' + otelDepEnvVariable?.getValue(); - - if (depEnvValue) { - // update the operator if more than one - const op = depEnvValue.includes(',') ? '=~' : '='; - // the second filter is deployment_environment - const filter = { - key: 'deployment_environment', - value: depEnvValue.split(',').join('|'), - operator: op, - }; - - filters.push(filter); - } - // next we check the otel resources adhoc variable for filters - const values = otelResourcesVariable.getValue(); - - if (values && otelResourcesVariable.state.filters.length > 0) { - filters = filters.concat(otelResourcesVariable.state.filters); - } - // the datasourceHelper will give us access to the - // Prometheus functions getTagKeys and getTagValues - // because we can access the ds - const datasourceHelper = this.datasourceHelper; - // now we reset the override tagKeys and tagValues functions of the adhoc variable - otelResourcesVariable.setState({ - getTagKeysProvider: async ( - variable: AdHocFiltersVariable, - currentKey: string | null - ): Promise<{ - replace?: boolean; - values: GetTagResponse | MetricFindValue[]; - }> => { - // apply filters here - // we're passing the queries so we get the labels that adhere to the queries - // we're also passing the scopes so we get the labels that adhere to the scopes filters - let values = await datasourceHelper.getTagKeys({ - filters, - scopes: getSelectedScopes(), - queries: this.getQueries(), - }); - values = sortResources(values, filters.map((f) => f.key).concat(currentKey ?? '')); - return { replace: true, values }; - }, - getTagValuesProvider: async ( - variable: AdHocFiltersVariable, - filter: AdHocVariableFilter - ): Promise<{ - replace?: boolean; - values: GetTagResponse | MetricFindValue[]; - }> => { - // apply filters here - // remove current selected filter if refiltering - filters = filters.filter((f) => f.key !== filter.key); - // we're passing the queries so we get the label values that adhere to the queries - // we're also passing the scopes so we get the label values that adhere to the scopes filters - const values = await datasourceHelper.getTagValues({ - key: filter.key, - filters, - scopes: getSelectedScopes(), - queries: this.getQueries(), - }); - return { replace: true, values }; - }, - hide: VariableHide.hideLabel, - }); + resetOtelExperience(hasOtelResources?: boolean, nonPromotedResources?: string[]) { + const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, this); + const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, this); + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, this); + const otelJoinQueryVariable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, this); - // 2. Get the otel join query for state and variable - // Because we need to define the deployment environment variable - // we also need to update the otel join query state and variable - const resourcesObject: OtelResourcesObject = getOtelResourcesObject(this); - const otelJoinQuery = getOtelJoinQuery(resourcesObject); - - // update the otel join query variable too - otelJoinQueryVariable.setState({ value: otelJoinQuery }); - - // 3. Update state with the following - // - otel join query - // - otelTargets used to filter metrics - // now we can filter target_info targets by deployment_environment="somevalue" - // and use these new targets to reduce the metrics - // for initialization we also update the following - // - has otel resources flag - // - and default to useOtelExperience - const otelTargets = await totalOtelResources(datasourceUid, timeRange, resourcesObject.filters); - - // we pass in deploymentEnvironments and hasOtelResources on start - if (hasOtelResources && deploymentEnvironments) { - const isEnabledInLocalStorage = getOtelExperienceToggleState(); - this.setState({ - otelTargets, - otelJoinQuery, - hasOtelResources, - isStandardOtel: deploymentEnvironments.length > 0, - useOtelExperience: isEnabledInLocalStorage, - }); - } else { - // we are updating on variable changes - this.setState({ - otelTargets, - otelJoinQuery, - }); + if ( + !( + otelResourcesVariable instanceof AdHocFiltersVariable && + filtersVariable instanceof AdHocFiltersVariable && + otelAndMetricsFiltersVariable instanceof AdHocFiltersVariable && + otelJoinQueryVariable instanceof ConstantVariable + ) + ) { + return; } - } - resetOtelExperience( - otelResourcesVariable: AdHocFiltersVariable, - otelDepEnvVariable: CustomVariable, - otelJoinQueryVariable: ConstantVariable, - filtersVariable: AdHocFiltersVariable, - hasOtelResources?: boolean, - deploymentEnvironments?: string[] - ) { - // reset filters to apply auto, anywhere there are {} characters + // show the var filters normally filtersVariable.setState({ addFilterButtonText: 'Add label', label: 'Select label', + hide: VariableHide.hideLabel, + }); + // Resetting the otel experience filters means clearing both the otel resources var and the otelMetricsVar + // hide the super otel and metric filter and reset it + otelAndMetricsFiltersVariable.setState({ + filters: [], + hide: VariableHide.hideVariable, }); // if there are no resources reset the otel variables and otel state // or if not standard otelResourcesVariable.setState({ + filters: [], defaultKeys: [], hide: VariableHide.hideVariable, }); - otelDepEnvVariable.setState({ - value: '', - hide: VariableHide.hideVariable, - }); - otelJoinQueryVariable.setState({ value: '' }); - // full reset when a data source fails the check - if (hasOtelResources && deploymentEnvironments) { + // potential full reset when a data source fails the check or is the initial check with turning off + if (hasOtelResources && nonPromotedResources) { this.setState({ hasOtelResources, - isStandardOtel: deploymentEnvironments.length > 0, + isStandardOtel: nonPromotedResources.length > 0, useOtelExperience: false, otelTargets: { jobs: [], instances: [] }, otelJoinQuery: '', + afterFirstOtelCheck: true, + initialOtelCheckComplete: true, + isUpdatingOtel: false, }); } else { // partial reset when a user turns off the otel experience @@ -601,6 +465,9 @@ export class DataTrail extends SceneObjectBase implements SceneO otelTargets: { jobs: [], instances: [] }, otelJoinQuery: '', useOtelExperience: false, + afterFirstOtelCheck: true, + initialOtelCheckComplete: true, + isUpdatingOtel: false, }); } } @@ -629,22 +496,13 @@ export class DataTrail extends SceneObjectBase implements SceneO const showHeaderForFirstTimeUsers = getTrailStore().recent.length < 2; useEffect(() => { - // check if the otel experience has been enabled - if (!useOtelExperience) { + if (model.state.addingLabelFromBreakdown) { + return; + } + + if (!useOtelExperience && model.state.afterFirstOtelCheck) { // if the experience has been turned off, reset the otel variables - const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, model); - const otelDepEnvVariable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, model); - const otelJoinQueryVariable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, model); - const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, model); - - if ( - otelResourcesVariable instanceof AdHocFiltersVariable && - otelDepEnvVariable instanceof CustomVariable && - otelJoinQueryVariable instanceof ConstantVariable && - filtersVariable instanceof AdHocFiltersVariable - ) { - model.resetOtelExperience(otelResourcesVariable, otelDepEnvVariable, otelJoinQueryVariable, filtersVariable); - } + model.resetOtelExperience(); } else { // if experience is enabled, check standardization and update the otel variables model.checkDataSourceForOTelResources(); @@ -653,9 +511,18 @@ export class DataTrail extends SceneObjectBase implements SceneO useEffect(() => { const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, model); + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, model); + const limitedFilterVariable = useOtelExperience ? otelAndMetricsFiltersVariable : filtersVariable; const datasourceHelper = model.datasourceHelper; - limitAdhocProviders(model, filtersVariable, datasourceHelper); - }, [model]); + limitAdhocProviders(model, limitedFilterVariable, datasourceHelper); + }, [model, useOtelExperience]); + + const reportOtelExperience = useRef(false); + // only report otel experience once + if (useOtelExperience && !reportOtelExperience.current) { + reportExploreMetrics('otel_experience_used', {}); + reportOtelExperience.current = true; + } return (
@@ -702,14 +569,6 @@ function getVariableSet( value: initialDS, pluginId: 'prometheus', }), - new CustomVariable({ - name: VAR_OTEL_DEPLOYMENT_ENV, - label: 'Deployment environment', - hide: VariableHide.hideVariable, - value: undefined, - placeholder: 'Select', - isMulti: true, - }), new AdHocFiltersVariable({ name: VAR_OTEL_RESOURCES, label: 'Select resource attributes', @@ -724,6 +583,7 @@ function getVariableSet( name: VAR_FILTERS, addFilterButtonText: 'Add label', datasource: trailDS, + // default to use var filters and have otel off hide: VariableHide.hideLabel, layout: 'vertical', filters: initialFilters ?? [], @@ -743,6 +603,30 @@ function getVariableSet( hide: VariableHide.hideVariable, value: false, }), + new AdHocFiltersVariable({ + name: VAR_OTEL_AND_METRIC_FILTERS, + addFilterButtonText: 'Filter', + datasource: trailDS, + hide: VariableHide.hideVariable, + layout: 'vertical', + filters: initialFilters ?? [], + baseFilters: getBaseFiltersForMetric(metric), + applyMode: 'manual', + // since we only support prometheus datasources, this is always true + supportsMultiValueOperators: true, + // skipUrlSync: true + }), + // Legacy variable needed for bookmarking which is necessary because + // url sync method does not handle multiple dep env values + // Remove this when the rudderstack event "deployment_environment_migrated" tapers off + new CustomVariable({ + name: VAR_OTEL_DEPLOYMENT_ENV, + label: 'Deployment environment', + hide: VariableHide.hideVariable, + value: undefined, + placeholder: 'Select', + isMulti: true, + }), ], }); } diff --git a/public/app/features/trails/DataTrailsHistory.tsx b/public/app/features/trails/DataTrailsHistory.tsx index aa66384eb4f..bfd7af07344 100644 --- a/public/app/features/trails/DataTrailsHistory.tsx +++ b/public/app/features/trails/DataTrailsHistory.tsx @@ -127,6 +127,7 @@ export class DataTrailHistory extends SceneObjectBase { this.setState({ filtersApplied }); } + // TEST THE MIGRATION OF REMOVING THE VAR_OTEL_DEPLOYMENT_ENV if (evt.payload.state.name === VAR_OTEL_DEPLOYMENT_ENV) { const otelDepEnvs = this.state.otelDepEnvs; const urlState = sceneUtils.getUrlState(trail); diff --git a/public/app/features/trails/DataTrailsHome.tsx b/public/app/features/trails/DataTrailsHome.tsx index c0c4d6d5995..4d21a05239c 100644 --- a/public/app/features/trails/DataTrailsHome.tsx +++ b/public/app/features/trails/DataTrailsHome.tsx @@ -25,7 +25,7 @@ export class DataTrailsHome extends SceneObjectBase { public onNewMetricsTrail = () => { const app = getAppFor(this); - const trail = newMetricsTrail(getDatasourceForNewTrail()); + const trail = newMetricsTrail(getDatasourceForNewTrail(), true); reportExploreMetrics('exploration_started', { cause: 'new_clicked' }); app.goToUrlForTrail(trail); }; diff --git a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx index edaf14c9533..1eb1deacf45 100644 --- a/public/app/features/trails/MetricSelect/MetricSelectScene.tsx +++ b/public/app/features/trails/MetricSelect/MetricSelectScene.tsx @@ -25,7 +25,7 @@ import { SceneVariableSet, VariableDependencyConfig, } from '@grafana/scenes'; -import { Alert, Field, Icon, IconButton, InlineSwitch, Input, Select, Tooltip, useStyles2 } from '@grafana/ui'; +import { Alert, Badge, Field, Icon, IconButton, InlineSwitch, Input, Select, Tooltip, useStyles2 } from '@grafana/ui'; import { Trans } from 'app/core/internationalization'; import { getSelectedScopes } from 'app/features/scopes'; @@ -486,9 +486,19 @@ export class MetricSelectScene extends SceneObjectBase i public onToggleOtelExperience = () => { const trail = getTrailFor(this); const useOtelExperience = trail.state.useOtelExperience; - + // set the startButtonClicked to null as we have gone past the owrkflow this is needed for + let startButtonClicked = false; + let resettingOtel = true; + if (useOtelExperience) { + reportExploreMetrics('otel_experience_toggled', { value: 'off' }); + // if turning off OTel + resettingOtel = false; + trail.resetOtelExperience(); + } else { + reportExploreMetrics('otel_experience_toggled', { value: 'on' }); + } setOtelExperienceToggleState(!useOtelExperience); - trail.setState({ useOtelExperience: !useOtelExperience }); + trail.setState({ useOtelExperience: !useOtelExperience, resettingOtel, startButtonClicked }); }; public static Component = ({ model }: SceneComponentProps) => { @@ -564,19 +574,28 @@ export class MetricSelectScene extends SceneObjectBase i {hasOtelResources && ( - Filter by - - This switch enables filtering by OTel resources for OTel native data sources. - - } - /> -
+ <> +
+ Filter by + + This switch enables filtering by OTel resources for OTel native data sources. + + } + /> +
+ New} + color={'blue'} + className={styles.badgeStyle} + > +
+
+ } className={styles.displayOption} > @@ -668,6 +687,17 @@ function getStyles(theme: GrafanaTheme2) { warningIcon: css({ color: theme.colors.warning.main, }), + badgeStyle: css({ + display: 'flex', + height: '1rem', + padding: '0rem 0.25rem 0 0.30rem', + alignItems: 'center', + borderRadius: theme.shape.radius.pill, + border: `1px solid ${theme.colors.info.text}`, + background: theme.colors.info.transparent, + marginTop: '4px', + marginLeft: '-3px', + }), }; } diff --git a/public/app/features/trails/TrailStore/TrailStore.test.ts b/public/app/features/trails/TrailStore/TrailStore.test.ts index c018a19ba52..ba41d6c6014 100644 --- a/public/app/features/trails/TrailStore/TrailStore.test.ts +++ b/public/app/features/trails/TrailStore/TrailStore.test.ts @@ -510,9 +510,10 @@ describe('TrailStore', () => { to: 'now', timezone, 'var-ds': 'prom-mock', - 'var-deployment_environment': ['undefined'], 'var-otel_resources': [''], 'var-filters': [], + 'var-otel_and_metric_filters': [''], + 'var-deployment_environment': [''], refresh: '', }, type: 'time', @@ -716,9 +717,10 @@ describe('TrailStore', () => { to: 'now', timezone, 'var-ds': 'prom-mock', - 'var-deployment_environment': ['undefined'], 'var-otel_resources': [''], 'var-filters': [], + 'var-otel_and_metric_filters': [''], + 'var-deployment_environment': [''], refresh: '', }, type: 'start', @@ -730,9 +732,10 @@ describe('TrailStore', () => { to: 'now', timezone, 'var-ds': 'prom-mock', - 'var-deployment_environment': ['undefined'], 'var-otel_resources': [''], 'var-filters': [], + 'var-otel_and_metric_filters': [''], + 'var-deployment_environment': [''], refresh: '', }, type: 'time', @@ -744,9 +747,10 @@ describe('TrailStore', () => { to: 'now', timezone, 'var-ds': 'prom-mock', - 'var-deployment_environment': ['undefined'], 'var-otel_resources': [''], 'var-filters': [], + 'var-otel_and_metric_filters': [''], + 'var-deployment_environment': [''], refresh: '', }, type: 'metric', @@ -766,9 +770,10 @@ describe('TrailStore', () => { to: 'now', timezone, 'var-ds': 'prom-mock', - 'var-deployment_environment': ['undefined'], 'var-otel_resources': [''], 'var-filters': [], + 'var-otel_and_metric_filters': [''], + 'var-deployment_environment': [''], refresh: '', }, type: 'time', diff --git a/public/app/features/trails/helpers/MetricDatasourceHelper.ts b/public/app/features/trails/helpers/MetricDatasourceHelper.ts index c1e25cfbaed..d9b84d76b82 100644 --- a/public/app/features/trails/helpers/MetricDatasourceHelper.ts +++ b/public/app/features/trails/helpers/MetricDatasourceHelper.ts @@ -34,7 +34,7 @@ export class MetricDatasourceHelper { return ds; } - private _metricsMetadata?: Promise; + _metricsMetadata?: Promise; private async _getMetricsMetadata() { const ds = await this.getDatasource(); @@ -62,10 +62,7 @@ export class MetricDatasourceHelper { } /** - * Used for filtering label names for OTel resources to add custom match filters - * - target_info metric - * - deployment_environment label - * - all other OTel filters + * Used for additional filtering for adhoc vars labels in Explore metrics. * @param options * @returns */ @@ -81,10 +78,7 @@ export class MetricDatasourceHelper { } /** - * Used for filtering label values for OTel resources to add custom match filters - * - target_info metric - * - deployment_environment label - * - all other OTel filters + * Used for additional filtering for adhoc vars label values in Explore metrics. * @param options * @returns */ diff --git a/public/app/features/trails/interactions.ts b/public/app/features/trails/interactions.ts index 181b321dfc7..968742feb35 100644 --- a/public/app/features/trails/interactions.ts +++ b/public/app/features/trails/interactions.ts @@ -25,6 +25,7 @@ type Interactions = { label: string; action: 'added' | 'removed' | 'changed'; cause: 'breakdown' | 'adhoc_filter'; + otel_resource_attribute?: boolean; }; // User changed the breakdown layout breakdown_layout_changed: { layout: BreakdownLayoutType }; @@ -126,6 +127,11 @@ type Interactions = { missing_otel_labels_by_truncating_job_and_instance: { metric?: string; }, + deployment_environment_migrated: {}, + otel_experience_used: {}, + otel_experience_toggled: { + value: ('on'| 'off') + } }; const PREFIX = 'grafana_explore_metrics_'; @@ -135,7 +141,11 @@ export function reportExploreMetrics { + describe('migrateOtelDeploymentEnvironment', () => { + let trail = {} as DataTrail; + beforeEach(() => { + trail = new DataTrail({ + useOtelExperience: true, + }); + }); + it('should not be called if var-otel_and_metric_filters is present with label', () => { + // this variable being present indicates it has already been migrated + const urlParams: UrlQueryMap = { + 'var-otel_and_metric_filters': ['key|=|value'], + }; + + migrateOtelDeploymentEnvironment(trail, urlParams); + + const otelMetricsVar = getOtelAndMetricsVar(trail); + + expect(otelMetricsVar.state.filters).toEqual([]); + }); + + it('should not be called if starting a new trail', () => { + // new trails do not need to be migrated + trail.setState({ startButtonClicked: true }); + + const urlParams: UrlQueryMap = { + 'var-otel_and_metric_filters': [''], + }; + + migrateOtelDeploymentEnvironment(trail, urlParams); + + const otelMetricsVar = getOtelAndMetricsVar(trail); + + expect(otelMetricsVar.state.filters).toEqual([]); + }); + + it('should not migrate if var-deployment_environment is not present', () => { + const urlParams: UrlQueryMap = {}; + + migrateOtelDeploymentEnvironment(trail, urlParams); + + const otelMetricsVar = getOtelAndMetricsVar(trail); + + expect(otelMetricsVar.state.filters).toEqual([]); + }); + + it('should migrate deployment environment and set filters correctly', () => { + const urlParams: UrlQueryMap = { + 'var-deployment_environment': ['env1', 'env2'], + 'var-otel_resources': ['otelResource|=|value'], + 'var-filters': ['metricFilter|=|value'], + }; + + migrateOtelDeploymentEnvironment(trail, urlParams); + + const expectedFilters = [ + { + key: 'deployment_environment', + operator: '=~', + value: 'env1|env2', + }, + { + key: 'otelResource', + operator: '=', + value: 'value', + }, + { + key: 'metricFilter', + operator: '=', + value: 'value', + }, + // Add expected otelFilters and metricFilters here + ]; + + const otelMetricsVar = getOtelAndMetricsVar(trail); + expect(otelMetricsVar.state.filters).toEqual(expectedFilters); + // should clear out the dep env var + const depEnvVar = getDepEnvVar(trail); + expect(depEnvVar.state.value).toBe(''); + }); + }); + + describe('migrateAdHocFilters', () => { + it('should return empty array when urlFilter is not present', () => { + const urlFilter: UrlQueryValue = null; + const filters: AdHocVariableFilter[] = []; + migrateAdHocFilters(urlFilter, filters); + expect(filters).toEqual([]); + }); + + it('should return filters when urlFilter is present', () => { + const urlFilter: UrlQueryValue = ['someKey|=|someValue']; + const filters: AdHocVariableFilter[] = []; + migrateAdHocFilters(urlFilter, filters); + const expected = [ + { + key: 'someKey', + operator: '=', + value: 'someValue', + }, + ]; + expect(filters).toEqual(expected); + }); + }); + + function getOtelAndMetricsVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, trail); + if (variable instanceof AdHocFiltersVariable) { + return variable; + } + throw new Error('getOtelAndMetricsVar failed'); + } + + function getDepEnvVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, trail); + if (variable instanceof CustomVariable) { + return variable; + } + throw new Error('getDepVar failed'); + } +}); diff --git a/public/app/features/trails/migrations/otelDeploymentEnvironment.ts b/public/app/features/trails/migrations/otelDeploymentEnvironment.ts new file mode 100644 index 00000000000..969c95bf662 --- /dev/null +++ b/public/app/features/trails/migrations/otelDeploymentEnvironment.ts @@ -0,0 +1,111 @@ +import { AdHocVariableFilter, UrlQueryValue, UrlQueryMap } from '@grafana/data'; +import { sceneGraph, AdHocFiltersVariable, CustomVariable } from '@grafana/scenes'; + +import { DataTrail } from '../DataTrail'; +import { reportExploreMetrics } from '../interactions'; +import { VAR_OTEL_AND_METRIC_FILTERS, VAR_OTEL_DEPLOYMENT_ENV } from '../shared'; + +/** + * Migration for the otel deployment environment variable. + * When the deployment environment is present in the url, "var-deployment_environment", + * it is migrated to the new variable "var-otel_and_metric_filters." + * + * We check if the otel resources vars are also present in the url, "var-otel_resources" + * and if the metric filters are present in the url, "var-filters". + * + * Once all the variables are migrated to "var-otel_and_metric_filters", the rest is handled in trail.updateOtelData. + * + * @param trail + * @returns + */ +export function migrateOtelDeploymentEnvironment(trail: DataTrail, urlParams: UrlQueryMap) { + const deploymentEnv = urlParams['var-deployment_environment']; + // does not need to be migrated + const otelMetricsVar = urlParams['var-otel_and_metric_filters']; + + // this check is if it has already been migrated + if ( + otelMetricsVar && + Array.isArray(otelMetricsVar) && + otelMetricsVar.length > 0 && + (trail.state.startButtonClicked || otelMetricsVar[0] !== '') + ) { + return; + } + // if there is no dep env, does not need to be migrated + if (!deploymentEnv) { + return; + } + + let filters: AdHocVariableFilter[] = []; + // if there is a deployment environment, we must also migrate the otel resources to the new variable + const otelResources = urlParams['var-otel_resources']; + const metricVarfilters = urlParams['var-filters']; + if ( + Array.isArray(deploymentEnv) && + deploymentEnv.length > 0 && + deploymentEnv[0] !== '' && + deploymentEnv.every((r) => r && typeof r === 'string') + ) { + // all the values are strings because they are prometheus labels + // so we can safely cast them to strings + const stringDepEnv = deploymentEnv.map((r) => r.toString()); + const value = stringDepEnv.join('|'); + + filters.push({ + key: 'deployment_environment', + operator: deploymentEnv.length > 1 ? '=~' : '=', + value, + }); + } + + // mutate the filters and add to them if we need to + migrateAdHocFilters(otelResources, filters); + migrateAdHocFilters(metricVarfilters, filters); + + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, trail); + const deploymentEnvironmentVariable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, trail); + + if ( + !( + otelAndMetricsFiltersVariable instanceof AdHocFiltersVariable && + deploymentEnvironmentVariable instanceof CustomVariable + ) + ) { + return; + } + + otelAndMetricsFiltersVariable.setState({ + filters, + }); + // clear the deployment environment to not migrate it again + reportExploreMetrics('deployment_environment_migrated', {}); + deploymentEnvironmentVariable.setState({ + value: '', + }); +} + +export function migrateAdHocFilters(urlFilter: UrlQueryValue, filters: AdHocVariableFilter[]) { + if ( + !( + urlFilter && // is present + Array.isArray(urlFilter) && // is an array + urlFilter.length > 0 && // has values + urlFilter[0] !== '' && // empty vars can contain '' + urlFilter.every((r) => r && typeof r === 'string') // vars are of any type but ours are all strings + ) + ) { + return filters; + } + + urlFilter.forEach((filter) => { + const parts = filter.toString().split('|'); + filters.push({ + key: parts[0].toString(), + operator: parts[1].toString(), + value: parts[2].toString(), + }); + }); + + return filters; +} diff --git a/public/app/features/trails/otel/api.test.ts b/public/app/features/trails/otel/api.test.ts index b5469478c5a..97e5f4a01ae 100644 --- a/public/app/features/trails/otel/api.test.ts +++ b/public/app/features/trails/otel/api.test.ts @@ -1,13 +1,7 @@ import { RawTimeRange } from '@grafana/data'; import { BackendSrvRequest } from '@grafana/runtime'; -import { - getOtelResources, - totalOtelResources, - isOtelStandardization, - getDeploymentEnvironments, - getFilteredResourceAttributes, -} from './api'; +import { totalOtelResources, getDeploymentEnvironments, getFilteredResourceAttributes } from './api'; jest.mock('./util', () => ({ ...jest.requireActual('./util'), @@ -37,9 +31,7 @@ jest.mock('@grafana/runtime', () => ({ options?: Partial ) => { // explore-metrics-otel-resources - if (requestId === 'explore-metrics-otel-resources') { - return Promise.resolve({ data: ['job', 'instance', 'deployment_environment'] }); - } else if ( + if ( requestId === 'explore-metrics-otel-check-total-count(target_info{}) by (job, instance)' || requestId === 'explore-metrics-otel-check-total-count(metric) by (job, instance)' ) { @@ -51,12 +43,6 @@ jest.mock('@grafana/runtime', () => ({ ], }, }); - } else if (requestId === 'explore-metrics-otel-check-standard') { - return Promise.resolve({ - data: { - result: [{ metric: { job: 'job1', instance: 'instance1' } }], - }, - }); } else if (requestId === 'explore-metrics-otel-resources-deployment-env') { return Promise.resolve({ data: ['env1', 'env2'] }); } else if ( @@ -89,14 +75,6 @@ describe('OTEL API', () => { jest.clearAllMocks(); }); - describe('getOtelResources', () => { - it('should fetch and filter OTEL resources', async () => { - const resources = await getOtelResources(dataSourceUid, timeRange); - - expect(resources).toEqual(['job', 'instance']); - }); - }); - describe('totalOtelResources', () => { it('should fetch total OTEL resources', async () => { const result = await totalOtelResources(dataSourceUid, timeRange); @@ -108,18 +86,6 @@ describe('OTEL API', () => { }); }); - describe('isOtelStandardization', () => { - // keeping for reference because standardization for OTel by series on target_info for job&instance is not consistent - // There is a bug currently where there is stale data in Prometheus resulting in duplicate series for job&instance at random times - // When this is resolved, we can check for standardization again - xit('should check if OTEL standardization is met when there are no duplicate series on target_info for job&instance', async () => { - // will return duplicates, see mock above - const isStandard = await isOtelStandardization(dataSourceUid, timeRange); - - expect(isStandard).toBe(false); - }); - }); - describe('getDeploymentEnvironments', () => { it('should fetch deployment environments', async () => { const environments = await getDeploymentEnvironments(dataSourceUid, timeRange, []); diff --git a/public/app/features/trails/otel/api.ts b/public/app/features/trails/otel/api.ts index b1c1b8a09f8..3399b0f7fe9 100644 --- a/public/app/features/trails/otel/api.ts +++ b/public/app/features/trails/otel/api.ts @@ -7,7 +7,7 @@ import { callSuggestionsApi } from '../utils'; import { OtelResponse, LabelResponse, OtelTargetType } from './types'; import { limitOtelMatchTerms, sortResources } from './util'; -const OTEL_RESOURCE_EXCLUDED_FILTERS = ['__name__', 'deployment_environment']; // name is handled by metric search metrics bar +const OTEL_RESOURCE_EXCLUDED_FILTERS = ['__name__']; // name is handled by metric search metrics bar /** * Function used to test for OTEL * When filters are added, we can also get a list of otel targets used to reduce the metric list @@ -18,41 +18,14 @@ const metricOtelJobInstanceQuery = (metric: string) => `count(${metric}) by (job export const TARGET_INFO_FILTER = { key: '__name__', value: 'target_info', operator: '=' }; /** - * Query the DS for target_info matching job and instance. - * Parse the results to get label filters. - * @param dataSourceUid - * @param timeRange - * @param excludedFilters - * @param matchFilters - * @returns OtelResourcesType[], labels for the query result requesting matching job and instance on target_info metric - */ -export async function getOtelResources( - dataSourceUid: string, - timeRange: RawTimeRange, - excludedFilters?: string[], - matchFilters?: string -): Promise { - const allExcludedFilters = (excludedFilters ?? []).concat(OTEL_RESOURCE_EXCLUDED_FILTERS); - - const start = getPrometheusTime(timeRange.from, false); - const end = getPrometheusTime(timeRange.to, true); - - const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/labels`; - const params: Record = { - start, - end, - 'match[]': `{__name__="target_info"${matchFilters ? `,${matchFilters}` : ''}}`, - }; - - const response = await getBackendSrv().get(url, params, 'explore-metrics-otel-resources'); - - // exclude __name__ or deployment_environment or previously chosen filters - return response.data?.filter((resource) => !allExcludedFilters.includes(resource)).map((el: string) => el); -} - -/** - * Get the total amount of job/instance pairs on a metric. - * Can be used for target_info. + * Get the total amount of job/instance for target_info or for a metric. + * + * If used for target_info, this is the metric preview scene with many panels and + * the job/instance pairs will be used to filter the metric list. + * + * If used for a metric, this is the metric preview scene with a single panel and + * the job/instance pairs will be used to identify otel resource attributes for the metric + * and distinguish between resource attributes and promoted attributes. * * @param dataSourceUid * @param timeRange @@ -105,37 +78,9 @@ export async function totalOtelResources( }; } -/** - * Look for duplicated series in target_info metric by job and instance labels - * If each job&instance combo is unique, the data source is otel standardized. - * If there is a count by job&instance on target_info greater than one, - * the data source is not standardized - * - * @param dataSourceUid - * @param timeRange - * @returns - */ -export async function isOtelStandardization(dataSourceUid: string, timeRange: RawTimeRange): Promise { - const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/query`; - - const start = getPrometheusTime(timeRange.from, false); - const end = getPrometheusTime(timeRange.to, true); - - const paramsTargets: Record = { - start, - end, - // any data source with duplicated series will have a count > 1 - query: `${otelTargetInfoQuery()} > 1`, - }; - - const response = await getBackendSrv().get(url, paramsTargets, 'explore-metrics-otel-check-standard'); - - // the response should be not greater than zero if it is standard - return !(response.data.result.length > 0); -} - /** * Query the DS for deployment environment label values. + * The deployment environment can be either on target_info or promoted to metrics. * * @param dataSourceUid * @param timeRange @@ -172,7 +117,8 @@ export async function getDeploymentEnvironmentsWithoutScopes( const params: Record = { start, end, - 'match[]': '{__name__="target_info"}', + // we are ok if deployment_environment has been promoted to metrics so we don't need the match + // 'match[]': '{__name__="target_info"}', }; const response = await getBackendSrv().get( @@ -181,7 +127,7 @@ export async function getDeploymentEnvironmentsWithoutScopes( 'explore-metrics-otel-resources-deployment-env' ); - // exclude __name__ or deployment_environment or previously chosen filters + // exclude __name__ or previously chosen filters return response.data; } @@ -203,17 +149,19 @@ export async function getDeploymentEnvironmentsWithScopes( timeRange, scopes, [ - { - key: '__name__', - operator: '=', - value: 'target_info', - }, + // we are ok if deployment_environment has been promoted to metrics so we don't need the match + // 'match[]': '{__name__="target_info"}', + // { + // key: '__name__', + // operator: '=', + // value: 'target_info', + // }, ], 'deployment_environment', undefined, 'explore-metrics-otel-resources-deployment-env' ); - // exclude __name__ or deployment_environment or previously chosen filters + // exclude __name__ or previously chosen filters return response.data.data; } @@ -221,11 +169,16 @@ export async function getDeploymentEnvironmentsWithScopes( * For OTel, get the resource attributes for a metric. * Handle filtering on both OTel resources as well as metric labels. * + * 1. Does not include resources promoted to metrics + * 2. Does not include __name__ or previously chosen filters + * 3. Sorts the resources, surfacing the blessedlist on top + * 4. Identifies if missing targets if the job/instance list is too long for the label values endpoint request + * * @param datasourceUid * @param timeRange * @param metric * @param excludedFilters - * @returns + * @returns attributes: string[], missingOtelTargets: boolean */ export async function getFilteredResourceAttributes( datasourceUid: string, @@ -295,7 +248,7 @@ export async function getFilteredResourceAttributes( // first filters out metric labels from the resource attributes const firstFilter = targetInfoAttributes.filter((resource) => !metricLabels.includes(resource)); - // exclude __name__ or deployment_environment or previously chosen filters + // exclude __name__ or previously chosen filters const secondFilter = firstFilter .filter((resource) => !allExcludedFilters.includes(resource)) .map((el) => ({ text: el })); @@ -307,3 +260,58 @@ export async function getFilteredResourceAttributes( return { attributes: resourceAttributes, missingOtelTargets: metricMatchTerms.missingOtelTargets }; } + +/** + * This function gets otel resources that only exist in target_info and + * do not exist on metrics as promoted labels. + * + * This is used when selecting a label from the list that includes both otel resources and metric labels. + * This list helps identify that a selected lbel/resource must be stored in VAR_OTEL_RESOURCES or VAR_FILTERS to be interpolated correctly in the queries. + */ +export async function getNonPromotedOtelResources(datasourceUid: string, timeRange: RawTimeRange) { + const start = getPrometheusTime(timeRange.from, false); + const end = getPrometheusTime(timeRange.to, true); + // The URL for the labels endpoint + const url = `/api/datasources/uid/${datasourceUid}/resources/api/v1/labels`; + // GET TARGET_INFO LABELS + const targetInfoParams: Record = { + start, + end, + 'match[]': `{__name__="target_info"}`, + }; + + // these are the resource attributes that come from target_info, + // filtered by the metric job and instance + const targetInfoResponse = getBackendSrv().get( + url, + targetInfoParams, + `explore-metrics-all-otel-resources-on-target_info` + ); + + // all labels in all metrics + const metricParams: Record = { + start, + end, + 'match[]': `{name!="",__name__!~"target_info"}`, + }; + + // Get the metric labels but exclude any labels found on target_info. + // We prioritize metric attributes over resource attributes. + // If a label is present in both metric and target_info, we exclude it from the resource attributes. + // This prevents errors in the join query. + const metricResponse = await getBackendSrv().get( + url, + metricParams, + `explore-metrics-all-metric-labels-not-otel-resource-attributes` + ); + const promResponses = await Promise.all([targetInfoResponse, metricResponse]); + // otel resource attributes + const targetInfoLabels = promResponses[0].data ?? []; + // the metric labels here + const metricLabels = new Set(promResponses[1].data ?? []); + + // get all the resource attributes that are not present on metrics (have been promoted to metrics) + const nonPromotedResources = targetInfoLabels.filter((item) => !metricLabels.has(item)); + + return nonPromotedResources; +} diff --git a/public/app/features/trails/otel/util.ts b/public/app/features/trails/otel/util.ts index c8633e2df95..d5c2c0b9886 100644 --- a/public/app/features/trails/otel/util.ts +++ b/public/app/features/trails/otel/util.ts @@ -1,19 +1,21 @@ -import { MetricFindValue } from '@grafana/data'; +import { AdHocVariableFilter, MetricFindValue, RawTimeRange, VariableHide } from '@grafana/data'; import { config } from '@grafana/runtime'; -import { AdHocFiltersVariable, ConstantVariable, CustomVariable, sceneGraph, SceneObject } from '@grafana/scenes'; +import { AdHocFiltersVariable, ConstantVariable, sceneGraph, SceneObject } from '@grafana/scenes'; import { DataTrail } from '../DataTrail'; +import { reportChangeInLabelFilters } from '../interactions'; +import { getOtelExperienceToggleState } from '../services/store'; import { VAR_DATASOURCE_EXPR, VAR_FILTERS, VAR_MISSING_OTEL_TARGETS, - VAR_OTEL_DEPLOYMENT_ENV, + VAR_OTEL_AND_METRIC_FILTERS, VAR_OTEL_GROUP_LEFT, VAR_OTEL_JOIN_QUERY, VAR_OTEL_RESOURCES, } from '../shared'; -import { getFilteredResourceAttributes } from './api'; +import { getFilteredResourceAttributes, totalOtelResources } from './api'; import { OtelResourcesObject } from './types'; export const blessedList = (): Record => { @@ -81,10 +83,9 @@ export function getOtelJoinQuery(otelResourcesObject: OtelResourcesObject, scene } let otelResourcesJoinQuery = ''; - if (otelResourcesObject.filters && otelResourcesObject.labels) { - // add support for otel data sources that are not standardized, i.e., have non unique target_info series by job, instance - otelResourcesJoinQuery = `* on (job, instance) group_left(${groupLeft}) topk by (job, instance) (1, target_info{${otelResourcesObject.filters}})`; - } + // add support for otel data sources that are not standardized, i.e., have non unique target_info series by job, instance + // target_info does not have to be filtered by deployment environment + otelResourcesJoinQuery = `* on (job, instance) group_left(${groupLeft}) topk by (job, instance) (1, target_info{${otelResourcesObject.filters}})`; return otelResourcesJoinQuery; } @@ -98,34 +99,14 @@ export function getOtelJoinQuery(otelResourcesObject: OtelResourcesObject, scene */ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: string): OtelResourcesObject { const otelResources = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, scene); - // add deployment env to otel resource filters - const otelDepEnv = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, scene); - let otelResourcesObject = { labels: '', filters: '' }; - if (otelResources instanceof AdHocFiltersVariable && otelDepEnv instanceof CustomVariable) { + if (otelResources instanceof AdHocFiltersVariable) { // get the collection of adhoc filters const otelFilters = otelResources.state.filters; - // get the value for deployment_environment variable - let otelDepEnvValue = String(otelDepEnv.getValue()); - // check if there are multiple environments - const isMulti = otelDepEnvValue.includes(','); - // start with the default label filters for deployment_environment - let op = '='; - let val = firstQueryVal ? firstQueryVal : otelDepEnvValue; - // update the filters if multiple deployment environments selected - if (isMulti) { - op = '=~'; - val = val.split(',').join('|'); - } - - // start with the deployment environment - let allFilters = `deployment_environment${op}"${val}"`; - if (config.featureToggles.prometheusSpecialCharsInLabelValues) { - allFilters = `deployment_environment${op}'${val}'`; - } - let allLabels = 'deployment_environment'; + let allFilters = ''; + let allLabels = ''; // add the other OTEL resource filters for (let i = 0; i < otelFilters?.length; i++) { @@ -133,16 +114,20 @@ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: strin const op = otelFilters[i].operator; const labelValue = otelFilters[i].value; + if (i > 0) { + allFilters += ','; + } + if (config.featureToggles.prometheusSpecialCharsInLabelValues) { - allFilters += `,${labelName}${op}'${labelValue}'`; + allFilters += `${labelName}${op}'${labelValue}'`; } else { - allFilters += `,${labelName}${op}"${labelValue}"`; + allFilters += `${labelName}${op}"${labelValue}"`; } const addLabelToGroupLeft = labelName !== 'job' && labelName !== 'instance'; if (addLabelToGroupLeft) { - allLabels += `,${labelName}`; + allLabels += `${labelName}`; } } @@ -262,7 +247,8 @@ export async function updateOtelJoinWithGroupLeft(trail: DataTrail, metric: stri return; } // Remove the group left - if (!metric) { + // if the metric is target_info, it already has all resource attributes + if (!metric || metric === 'target_info') { // if the metric is not present, that means we are in the metric select scene // and that should have no group left because it may interfere with queries. otelGroupLeft.setState({ value: '' }); @@ -271,10 +257,6 @@ export async function updateOtelJoinWithGroupLeft(trail: DataTrail, metric: stri otelJoinQueryVariable.setState({ value: otelJoinQuery }); return; } - // if the metric is target_info, it already has all resource attributes - if (metric === 'target_info') { - return; - } // Add the group left const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, trail); @@ -312,16 +294,334 @@ export async function updateOtelJoinWithGroupLeft(trail: DataTrail, metric: stri } /** - * Returns the option value that is like 'prod'. + * Returns the environment that is like 'prod'. * If there are no options, returns null. * * @param options * @returns */ -export function getProdOrDefaultOption(options: Array<{ value: string; label: string }>): string | null { - if (options.length === 0) { +export function getProdOrDefaultEnv(envs: string[]): string | null { + if (envs.length === 0) { return null; } - return options.find((option) => option.value.toLowerCase().indexOf('prod') > -1)?.value ?? options[0].value; + return envs.find((env) => env.toLowerCase().indexOf('prod') > -1) ?? envs[0]; +} + +/** + * This function is used to update state and otel variables. + * + * 1. Set the otelResources adhoc tagKey and tagValues filter functions + * 2. Get the otel join query for state and variable + * 3. Update state with the following + * - otel join query + * - otelTargets used to filter metrics + * For initialization we also update the following + * - has otel resources flag + * - isStandardOtel flag (for enabliing the otel experience toggle) + * - and useOtelExperience + * + * This function is called on start and when variables change. + * On start will provide the deploymentEnvironments and hasOtelResources parameters. + * In the variable change case, we will not provide these parameters. It is assumed that the + * data source has been checked for otel resources and standardization and the otel variables are enabled at this point. + * @param datasourceUid + * @param timeRange + * @param deploymentEnvironments + * @param hasOtelResources + * @param nonPromotedOtelResources + * @param fromDataSourceChanged + */ +export async function updateOtelData( + trail: DataTrail, + datasourceUid: string, + timeRange: RawTimeRange, + deploymentEnvironments?: string[], + hasOtelResources?: boolean, + nonPromotedOtelResources?: string[] +) { + // currently need isUpdatingOtel check for variable race conditions and state changes + // future refactor project + // - checkDataSourceForOTelResources for state changes + // - otel resources var for variable dependency listeners + if (trail.state.isUpdatingOtel) { + return; + } + trail.setState({ isUpdatingOtel: true }); + + const otelResourcesVariable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, trail); + const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, trail); + const otelAndMetricsFiltersVariable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, trail); + const otelJoinQueryVariable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, trail); + const initialOtelCheckComplete = trail.state.initialOtelCheckComplete; + const resettingOtel = trail.state.resettingOtel; + + if ( + !( + otelResourcesVariable instanceof AdHocFiltersVariable && + filtersVariable instanceof AdHocFiltersVariable && + otelAndMetricsFiltersVariable instanceof AdHocFiltersVariable && + otelJoinQueryVariable instanceof ConstantVariable + ) + ) { + return; + } + // Set deployment environment variable as a new otel & metric filter. + // We choose one default value at the beginning of the OTel experience. + // This is because the work flow for OTel begins with users selecting a deployment environment + // default to production. + let defaultDepEnv = getProdOrDefaultEnv(deploymentEnvironments ?? []) ?? ''; + + const isEnabledInLocalStorage = getOtelExperienceToggleState(); + + // We respect that if users have it turned off in local storage we keep it off unless the toggle is switched + if (!isEnabledInLocalStorage) { + trail.resetOtelExperience(hasOtelResources, nonPromotedOtelResources); + } else { + // 1. Cases of how to add filters to the otelmetricsvar + // -- when we set these on instantiation, we need to check that we are not double setting them + // 1.0. legacy, check url values for dep env and otel resources and migrate to otelmetricvar + // -- do not duplicate + // 1.1. NONE If the otel metrics var has no filters, set the default value + // 1.2. VAR_FILTERS If the var filters has filters, add to otemetricsvar + // -- do not duplicate when adding to otelmtricsvar + // 1.3. OTEL_FILTERS If the otel resources var has filters, add to otelmetricsvar + // -- do not duplicate when adding to otelmtricsvar + + // 1. switching data source + // the previous var filters are not reset so even if they don't apply to the new data source we want to keep them + // 2. on load with url values, check isInitial CheckComplete + // Set otelmetrics var, distinguish if these are var filters or otel resources, then place in correct filter + let prevVarFilters = resettingOtel ? filtersVariable.state.filters : []; + // only look at url values for otelmetricsvar if the initial check is NOT YET complete + const urlOtelAndMetricsFilters = + initialOtelCheckComplete && !resettingOtel ? [] : otelAndMetricsFiltersVariable.state.filters; + // url vars should override the deployment environment variable + const urlVarsObject = checkLabelPromotion(urlOtelAndMetricsFilters, nonPromotedOtelResources); + const urlOtelResources = initialOtelCheckComplete ? [] : urlVarsObject.nonPromoted; + const urlVarFilters = initialOtelCheckComplete ? [] : urlVarsObject.promoted; + + // set the vars if the following conditions + if (!initialOtelCheckComplete || resettingOtel) { + // if the default dep env value like 'prod' is missing OR + // if we are loading from the url and the default dep env is missing + // there are no prev deployment environments from url + const hasPreviousDepEnv = urlOtelAndMetricsFilters.filter((f) => f.key === 'deployment_environment').length > 0; + const doNotSetDepEvValue = defaultDepEnv === '' || hasPreviousDepEnv; + // we do not have to set the dep env value if the default is missing + const defaultDepEnvFilter = doNotSetDepEvValue + ? [] + : [ + { + key: 'deployment_environment', + value: defaultDepEnv, + operator: defaultDepEnv.includes(',') ? '=~' : '=', + }, + ]; + + const notPromoted = nonPromotedOtelResources?.includes('deployment_environment'); + // Next, the previous data source filters may include the default dep env but in the wrong filter + // i.e., dep env is not promoted to metrics but in the previous DS, it was, so it will exist in the VAR FILTERS + // and we will see a duplication in the OTELMETRICSVAR + // remove the duplication + prevVarFilters = notPromoted ? prevVarFilters.filter((f) => f.key !== 'deployment_environment') : prevVarFilters; + + // previous var filters are handled but what about previous otel resources filters? + // need to add the prev otel resources to the otelmetricsvar filters + otelAndMetricsFiltersVariable?.setState({ + filters: [...defaultDepEnvFilter, ...prevVarFilters, ...urlOtelAndMetricsFilters], + hide: VariableHide.hideLabel, + }); + + // update the otel resources if the dep env has not been promoted + const otelDepEnvFilters = notPromoted ? defaultDepEnvFilter : []; + const otelFilters = [...otelDepEnvFilters, ...urlOtelResources]; + otelResourcesVariable.setState({ + filters: otelFilters, + hide: VariableHide.hideVariable, + }); + + const isPromoted = !notPromoted; + // if the dep env IS PROMOTED + // we need to ask, does var filters already contain it? + // keep previous filters if they are there + // add the dep env to var filters if not present and isPromoted + const depEnvFromVarFilters = prevVarFilters.filter((f) => f.key === 'deployment_environment'); + + // if promoted and no dep env has been chosen yet, set the default + if (isPromoted && depEnvFromVarFilters.length === 0) { + prevVarFilters = [...prevVarFilters, ...defaultDepEnvFilter]; + } + + prevVarFilters = [...prevVarFilters, ...urlVarFilters]; + + filtersVariable.setState({ + filters: prevVarFilters, + hide: VariableHide.hideVariable, + }); + } + } + // 1. Get the otel join query for state and variable + // Because we need to define the deployment environment variable + // we also need to update the otel join query state and variable + const resourcesObject: OtelResourcesObject = getOtelResourcesObject(trail); + // THIS ASSUMES THAT WE ALWAYS HAVE DEPLOYMENT ENVIRONMENT! + // FIX THIS SO THAT WE HAVE SOME QUERY EVEN IF THERE ARE NO OTEL FILTERS + const otelJoinQuery = getOtelJoinQuery(resourcesObject); + + // update the otel join query variable too + otelJoinQueryVariable.setState({ value: otelJoinQuery }); + + // 2. Update state with the following + // - otel join query + // - otelTargets used to filter metrics + // now we can filter target_info targets by deployment_environment="somevalue" + // and use these new targets to reduce the metrics + // for initialization we also update the following + // - has otel resources flag + // - and default to useOtelExperience + const otelTargets = await totalOtelResources(datasourceUid, timeRange, resourcesObject.filters); + + // we pass in deploymentEnvironments and hasOtelResources on start + // RETHINK We may be able to get rid of this check + // a non standard data source is more missing job and instance matchers + if (hasOtelResources && deploymentEnvironments && !initialOtelCheckComplete) { + trail.setState({ + otelTargets, + otelJoinQuery, + hasOtelResources, + // Previously checking standardization for having deployment environments + // Now we check that there are target_info labels that are not promoted + isStandardOtel: (nonPromotedOtelResources ?? []).length > 0, + useOtelExperience: isEnabledInLocalStorage, + nonPromotedOtelResources, + initialOtelCheckComplete: true, + resettingOtel: false, + afterFirstOtelCheck: true, + isUpdatingOtel: false, + }); + } else { + // we are updating on variable changes + trail.setState({ + otelTargets, + otelJoinQuery, + resettingOtel: false, + afterFirstOtelCheck: true, + isUpdatingOtel: false, + }); + } +} + +function checkLabelPromotion(filters: AdHocVariableFilter[], nonPromotedOtelResources: string[] = []) { + const nonPromotedResources = new Set(nonPromotedOtelResources); + const nonPromoted = filters.filter((f) => nonPromotedResources.has(f.key)); + const promoted = filters.filter((f) => !nonPromotedResources.has(f.key)); + + return { + nonPromoted, + promoted, + }; +} + +/** + * When a new filter is chosen from the consolidated filters, VAR_OTEL_AND_METRIC_FILTERS, + * we need to identify the following: + * + * 1. Is the filter a non-promoted otel resource or a metric filter? + * 2. Is the filter being added or removed? + * + * Once we know this, we can add the selected filter to either the + * VAR_OTEL_RESOURCES or VAR_FILTERS variable. + * + * When the correct variable is updated, the rest of the explore metrics behavior will remain the same. + * + * @param newStateFilters + * @param prevStateFilters + * @param nonPromotedOtelResources + * @param otelFiltersVariable + * @param filtersVariable + */ +export function manageOtelAndMetricFilters( + newStateFilters: AdHocVariableFilter[], + prevStateFilters: AdHocVariableFilter[], + nonPromotedOtelResources: string[], + otelFiltersVariable: AdHocFiltersVariable, + filtersVariable: AdHocFiltersVariable +) { + // add filter + if (newStateFilters.length > prevStateFilters.length) { + const newFilter = newStateFilters[newStateFilters.length - 1]; + // check that the filter is a non-promoted otel resource + if (nonPromotedOtelResources?.includes(newFilter.key)) { + // add to otel filters + otelFiltersVariable.setState({ + filters: [...otelFiltersVariable.state.filters, newFilter], + }); + reportChangeInLabelFilters(newStateFilters, prevStateFilters, true); + } else { + // add to metric filters + filtersVariable.setState({ + filters: [...filtersVariable.state.filters, newFilter], + }); + } + return; + } + // remove filter + if (newStateFilters.length < prevStateFilters.length) { + // get the removed filter + const removedFilter = prevStateFilters.filter((f) => !newStateFilters.includes(f))[0]; + if (nonPromotedOtelResources?.includes(removedFilter.key)) { + // remove from otel filters + otelFiltersVariable.setState({ + filters: otelFiltersVariable.state.filters.filter((f) => f.key !== removedFilter.key), + }); + reportChangeInLabelFilters(newStateFilters, prevStateFilters, true); + } else { + // remove from metric filters + filtersVariable.setState({ + filters: filtersVariable.state.filters.filter((f) => f.key !== removedFilter.key), + }); + } + return; + } + // a filter has been changed + let updatedFilter: AdHocVariableFilter[] = []; + if ( + newStateFilters.length === prevStateFilters.length && + newStateFilters.some((filter, i) => { + const newKey = filter.key; + const newValue = filter.value; + const isUpdatedFilter = prevStateFilters[i].key === newKey && prevStateFilters[i].value !== newValue; + if (isUpdatedFilter) { + updatedFilter.push(filter); + } + return isUpdatedFilter; + }) + ) { + // check if the filter is a non-promoted otel resource + if (nonPromotedOtelResources?.includes(updatedFilter[0].key)) { + // add to otel filters + otelFiltersVariable.setState({ + // replace the updated filter + filters: otelFiltersVariable.state.filters.map((f) => { + if (f.key === updatedFilter[0].key) { + return updatedFilter[0]; + } + return f; + }), + }); + reportChangeInLabelFilters(newStateFilters, prevStateFilters, true); + } else { + // add to metric filters + filtersVariable.setState({ + // replace the updated filter + filters: filtersVariable.state.filters.map((f) => { + if (f.key === updatedFilter[0].key) { + return updatedFilter[0]; + } + return f; + }), + }); + } + } } diff --git a/public/app/features/trails/otel/utils.test.ts b/public/app/features/trails/otel/utils.test.ts index 11d8657aa6b..572643831d7 100644 --- a/public/app/features/trails/otel/utils.test.ts +++ b/public/app/features/trails/otel/utils.test.ts @@ -1,12 +1,18 @@ -import { MetricFindValue } from '@grafana/data'; +import { AdHocVariableFilter, MetricFindValue } from '@grafana/data'; import { locationService, setDataSourceSrv } from '@grafana/runtime'; -import { AdHocFiltersVariable, ConstantVariable, CustomVariable, sceneGraph } from '@grafana/scenes'; +import { AdHocFiltersVariable, ConstantVariable, sceneGraph } from '@grafana/scenes'; import { mockDataSource, MockDataSourceSrv } from 'app/features/alerting/unified/mocks'; import { DataSourceType } from 'app/features/alerting/unified/utils/datasource'; import { activateFullSceneTree } from 'app/features/dashboard-scene/utils/test-utils'; import { DataTrail } from '../DataTrail'; -import { VAR_OTEL_DEPLOYMENT_ENV, VAR_OTEL_GROUP_LEFT, VAR_OTEL_JOIN_QUERY, VAR_OTEL_RESOURCES } from '../shared'; +import { + VAR_FILTERS, + VAR_OTEL_AND_METRIC_FILTERS, + VAR_OTEL_GROUP_LEFT, + VAR_OTEL_JOIN_QUERY, + VAR_OTEL_RESOURCES, +} from '../shared'; import { sortResources, @@ -14,7 +20,9 @@ import { blessedList, limitOtelMatchTerms, updateOtelJoinWithGroupLeft, - getProdOrDefaultOption, + getProdOrDefaultEnv, + updateOtelData, + manageOtelAndMetricFilters, } from './util'; jest.mock('./api', () => ({ @@ -54,7 +62,7 @@ describe('getOtelJoinQuery', () => { ); }); - it('should return an empty string if filters or labels are missing', () => { + it('should return a join query if filters or labels are missing', () => { const otelResourcesObject = { filters: '', labels: '', @@ -62,7 +70,7 @@ describe('getOtelJoinQuery', () => { const result = getOtelJoinQuery(otelResourcesObject); - expect(result).toBe(''); + expect(result).toBe('* on (job, instance) group_left() topk by (job, instance) (1, target_info{})'); }); }); @@ -116,32 +124,6 @@ describe('sortResources', () => { }); }); -describe('getOtelJoinQuery', () => { - it('should return the correct join query', () => { - const otelResourcesObject = { - filters: 'job="test-job",instance="test-instance"', - labels: 'deployment_environment,custom_label', - }; - - const result = getOtelJoinQuery(otelResourcesObject); - - expect(result).toBe( - '* on (job, instance) group_left() topk by (job, instance) (1, target_info{job="test-job",instance="test-instance"})' - ); - }); - - it('should return an empty string if filters or labels are missing', () => { - const otelResourcesObject = { - filters: '', - labels: '', - }; - - const result = getOtelJoinQuery(otelResourcesObject); - - expect(result).toBe(''); - }); -}); - describe('limitOtelMatchTerms', () => { it('should limit the OTel match terms if the total match term character count exceeds 2000', () => { // the initial match is 1980 characters @@ -213,14 +195,6 @@ describe('updateOtelJoinWithGroupLeft', () => { const preTrailUrl = '/trail?from=now-1h&to=now&var-ds=edwxqcebl0cg0c&var-deployment_environment=oteldemo01&var-otel_resources=k8s_cluster_name%7C%3D%7Cappo11ydev01&var-filters=&refresh=&metricPrefix=all&metricSearch=http&actionView=breakdown&var-groupby=$__all&metric=http_client_duration_milliseconds_bucket'; - function getOtelDepEnvVar(trail: DataTrail) { - const variable = sceneGraph.lookupVariable(VAR_OTEL_DEPLOYMENT_ENV, trail); - if (variable instanceof CustomVariable) { - return variable; - } - throw new Error('getDepEnvVar failed'); - } - function getOtelJoinQueryVar(trail: DataTrail) { const variable = sceneGraph.lookupVariable(VAR_OTEL_JOIN_QUERY, trail); if (variable instanceof ConstantVariable) { @@ -229,22 +203,6 @@ describe('updateOtelJoinWithGroupLeft', () => { throw new Error('getDepEnvVar failed'); } - function getOtelResourcesVar(trail: DataTrail) { - const variable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, trail); - if (variable instanceof AdHocFiltersVariable) { - return variable; - } - throw new Error('getOtelResourcesVar failed'); - } - - function getOtelGroupLeftVar(trail: DataTrail) { - const variable = sceneGraph.lookupVariable(VAR_OTEL_GROUP_LEFT, trail); - if (variable instanceof ConstantVariable) { - return variable; - } - throw new Error('getOtelResourcesVar failed'); - } - beforeEach(() => { jest.spyOn(DataTrail.prototype, 'checkDataSourceForOTelResources').mockImplementation(() => Promise.resolve()); setDataSourceSrv( @@ -255,12 +213,12 @@ describe('updateOtelJoinWithGroupLeft', () => { }), }) ); - trail = new DataTrail({}); + trail = new DataTrail({ + useOtelExperience: true, + nonPromotedOtelResources: ['service_name'], + }); locationService.push(preTrailUrl); activateFullSceneTree(trail); - getOtelResourcesVar(trail).setState({ filters: [{ key: 'service_name', operator: '=', value: 'adservice' }] }); - getOtelDepEnvVar(trail).changeValueTo('production'); - getOtelGroupLeftVar(trail).setState({ value: 'attribute1,attribute2' }); }); it('should update OTel join query with the group left resource attributes', async () => { @@ -268,53 +226,414 @@ describe('updateOtelJoinWithGroupLeft', () => { const otelJoinQueryVar = getOtelJoinQueryVar(trail); // this will include the group left resource attributes expect(otelJoinQueryVar.getValue()).toBe( - '* on (job, instance) group_left(resourceAttribute) topk by (job, instance) (1, target_info{deployment_environment="production",service_name="adservice"})' + '* on (job, instance) group_left(resourceAttribute) topk by (job, instance) (1, target_info{})' ); }); it('should not update OTel join query with the group left resource attributes when the metric is target_info', async () => { await updateOtelJoinWithGroupLeft(trail, 'target_info'); const otelJoinQueryVar = getOtelJoinQueryVar(trail); - - expect(otelJoinQueryVar.getValue()).toBe(''); + const emptyGroupLeftClause = 'group_left()'; + const otelJoinQuery = otelJoinQueryVar.state.value; + if (typeof otelJoinQuery === 'string') { + expect(otelJoinQuery.includes(emptyGroupLeftClause)).toBe(true); + } }); }); -describe('getProdOrDefaultOption', () => { +describe('getProdOrDefaultEnv', () => { it('should return the value of the option containing "prod"', () => { - const options = [ - { value: 'test1', label: 'Test 1' }, - { value: 'prod2', label: 'Prod 2' }, - { value: 'test3', label: 'Test 3' }, - ]; - expect(getProdOrDefaultOption(options)).toBe('prod2'); + const options = ['test1', 'prod2', 'test3']; + + expect(getProdOrDefaultEnv(options)).toBe('prod2'); }); it('should return the first option value if no option contains "prod"', () => { - const options = [ - { value: 'test1', label: 'Test 1' }, - { value: 'test2', label: 'Test 2' }, - { value: 'test3', label: 'Test 3' }, - ]; - expect(getProdOrDefaultOption(options)).toBe('test1'); + const options = ['test1', 'test2', 'test3']; + + expect(getProdOrDefaultEnv(options)).toBe('test1'); }); it('should handle case insensitivity', () => { - const options = [ - { value: 'test1', label: 'Test 1' }, - { value: 'PROD2', label: 'Prod 2' }, - { value: 'test3', label: 'Test 3' }, - ]; - expect(getProdOrDefaultOption(options)).toBe('PROD2'); + const options = ['test1', 'PROD2', 'test3']; + + expect(getProdOrDefaultEnv(options)).toBe('PROD2'); }); it('should return null if the options array is empty', () => { - const options: Array<{ value: string; label: string }> = []; - expect(getProdOrDefaultOption(options)).toBeNull(); + const options: string[] = []; + expect(getProdOrDefaultEnv(options)).toBeNull(); }); it('should return the first option value if the options array has one element', () => { - const options = [{ value: 'test1', label: 'Test 1' }]; - expect(getProdOrDefaultOption(options)).toBe('test1'); + const options = ['test1']; + expect(getProdOrDefaultEnv(options)).toBe('test1'); + }); +}); + +describe('util functions that rely on trail and variable setup', () => { + beforeAll(() => { + jest.spyOn(DataTrail.prototype, 'checkDataSourceForOTelResources').mockImplementation(() => Promise.resolve()); + setDataSourceSrv( + new MockDataSourceSrv({ + prom: mockDataSource({ + name: 'Prometheus', + type: DataSourceType.Prometheus, + }), + }) + ); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + let trail: DataTrail; + const defaultTimeRange = { from: 'now-1h', to: 'now' }; + // selecting a non promoted resource from VAR_OTEL_AND_METRICS will automatically update the otel resources var + const nonPromotedOtelResources = ['deployment_environment']; + const preTrailUrl = + '/trail?from=now-1h&to=now&var-ds=edwxqcebl0cg0c&var-deployment_environment=oteldemo01&var-otel_resources=k8s_cluster_name%7C%3D%7Cappo11ydev01&var-filters=&refresh=&metricPrefix=all&metricSearch=http&actionView=breakdown&var-groupby=$__all&metric=http_client_duration_milliseconds_bucket'; + + function getOtelAndMetricsVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_AND_METRIC_FILTERS, trail); + if (variable instanceof AdHocFiltersVariable) { + return variable; + } + throw new Error('getOtelAndMetricsVar failed'); + } + + function getOtelResourcesVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_RESOURCES, trail); + if (variable instanceof AdHocFiltersVariable) { + return variable; + } + throw new Error('getOtelResourcesVar failed'); + } + + function getOtelGroupLeftVar(trail: DataTrail) { + const variable = sceneGraph.lookupVariable(VAR_OTEL_GROUP_LEFT, trail); + if (variable instanceof ConstantVariable) { + return variable; + } + throw new Error('getOtelGroupLeftVar failed'); + } + + function getFilterVar() { + const variable = sceneGraph.lookupVariable(VAR_FILTERS, trail); + if (variable instanceof AdHocFiltersVariable) { + return variable; + } + throw new Error('getFilterVar failed'); + } + + beforeEach(() => { + trail = new DataTrail({ + useOtelExperience: true, + nonPromotedOtelResources, + }); + locationService.push(preTrailUrl); + activateFullSceneTree(trail); + getOtelGroupLeftVar(trail).setState({ value: 'attribute1,attribute2' }); + }); + + afterEach(() => { + trail.setState({ initialOtelCheckComplete: false }); + }); + describe('updateOtelData', () => { + it('should automatically add the deployment environment on loading a data trail from start', () => { + trail.setState({ startButtonClicked: true }); + const autoSelectedDepEnvValue = 'production'; + const deploymentEnvironments = [autoSelectedDepEnvValue]; + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + nonPromotedOtelResources + ); + const otelMetricsVar = getOtelAndMetricsVar(trail); + const otelMetricsKey = otelMetricsVar.state.filters[0].key; + const otelMetricsValue = otelMetricsVar.state.filters[0].value; + + const otelResourcesVar = getOtelResourcesVar(trail); + const otelResourcesKey = otelResourcesVar.state.filters[0].key; + const otelResourcesValue = otelResourcesVar.state.filters[0].value; + + expect(otelMetricsKey).toBe('deployment_environment'); + expect(otelMetricsValue).toBe(autoSelectedDepEnvValue); + + expect(otelResourcesKey).toBe('deployment_environment'); + expect(otelResourcesValue).toBe(autoSelectedDepEnvValue); + }); + + it('should use the deployment environment from url when loading a trail and not automatically load it', () => { + const autoSelectedDeploymentEnvironmentValue = 'production'; + const deploymentEnvironments = [autoSelectedDeploymentEnvironmentValue]; + // the url loads the deployment environment into otelmetricsvar + const prevUrlDepEnvValue = 'from_url'; + getOtelAndMetricsVar(trail).setState({ + filters: [{ key: 'deployment_environment', operator: '=', value: prevUrlDepEnvValue }], + }); + + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + nonPromotedOtelResources + ); + const otelMetricsVar = getOtelAndMetricsVar(trail); + const otelMetricsKey = otelMetricsVar.state.filters[0].key; + const otelMetricsValue = otelMetricsVar.state.filters[0].value; + + const otelResourcesVar = getOtelResourcesVar(trail); + const otelResourcesKey = otelResourcesVar.state.filters[0].key; + const otelResourcesValue = otelResourcesVar.state.filters[0].value; + + expect(otelMetricsKey).toBe('deployment_environment'); + expect(otelMetricsValue).toBe(prevUrlDepEnvValue); + + expect(otelResourcesKey).toBe('deployment_environment'); + expect(otelResourcesValue).toBe(prevUrlDepEnvValue); + }); + + it('should load all filters based on the url for VAR_OTEL_AND_METRICS_FILTERS on initial load', () => { + const nonPromotedOtelResources = ['deployment_environment', 'resource']; + const depEnvFilter = { key: 'deployment_environment', operator: '=', value: 'from_url' }; + const otelResourceFilter = { key: 'resource', operator: '=', value: 'resource' }; + const promotedFilter = { key: 'promoted', operator: '=', value: 'promoted' }; + const metricFilter = { key: 'metric', operator: '=', value: 'metric' }; + + getOtelAndMetricsVar(trail).setState({ + filters: [depEnvFilter, otelResourceFilter, promotedFilter, metricFilter], + }); + + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + ['production'], + true, // hasOtelResources + nonPromotedOtelResources + ); + + const otelMetricsVar = getOtelAndMetricsVar(trail); + const otelResourcesVar = getOtelResourcesVar(trail); + const varFilters = getFilterVar(); + + // otelmetrics var will contain all three + expect(otelMetricsVar.state.filters).toEqual([depEnvFilter, otelResourceFilter, promotedFilter, metricFilter]); + // otel resources will contain only non promoted + expect(otelResourcesVar.state.filters).toEqual([depEnvFilter, otelResourceFilter]); + // var filters will contain promoted and metric labels + expect(varFilters.state.filters).toEqual([promotedFilter, metricFilter]); + }); + + it('should not automatically add the deployment environment on loading a data trail when there are no deployment environments in the data source', () => { + // no dep env values found in the data source + const deploymentEnvironments: string[] = []; + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + nonPromotedOtelResources + ); + const otelMetricsVar = getOtelAndMetricsVar(trail); + + const otelResourcesVar = getOtelResourcesVar(trail); + + expect(otelMetricsVar.state.filters.length).toBe(0); + expect(otelResourcesVar.state.filters.length).toBe(0); + }); + + it('should not automatically add the deployment environment on loading a data trail when loading from url and no dep env are present in the filters', () => { + // not from start + // no dep env values found in the data source + const deploymentEnvironments: string[] = []; + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + nonPromotedOtelResources + ); + const otelMetricsVar = getOtelAndMetricsVar(trail); + + const otelResourcesVar = getOtelResourcesVar(trail); + + expect(otelMetricsVar.state.filters.length).toBe(0); + expect(otelResourcesVar.state.filters.length).toBe(0); + }); + + it('should add the deployment environment to var filters if it has been promoted from start', () => { + trail.setState({ startButtonClicked: true }); + // the deployment environment has been promoted to a metric label + const deploymentEnvironments = ['production']; + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + [] //nonPromotedOtelResources + ); + const varFilters = getFilterVar().state.filters[0]; + expect(varFilters.key).toBe('deployment_environment'); + expect(varFilters.value).toBe('production'); + }); + + it('should preserve var filters when switching a data source but not initial load', () => { + trail.setState({ initialOtelCheckComplete: true }); + const deploymentEnvironments = ['production']; + getFilterVar().setState({ filters: [{ key: 'zone', operator: '=', value: 'a' }] }); + updateOtelData( + trail, + 'datasourceUid', + defaultTimeRange, + deploymentEnvironments, + true, // hasOtelResources + nonPromotedOtelResources + ); + const varFilters = getFilterVar().state.filters[0]; + expect(varFilters.key).toBe('zone'); + expect(varFilters.value).toBe('a'); + }); + }); + + describe('manageOtelAndMetricFilters', () => { + it('should add a new filter to otel filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = [{ key: 'otel_key', value: 'value', operator: '=' }]; + const prevStateFilters: AdHocVariableFilter[] = []; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + + const filtersVariable = getFilterVar(); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(otelFiltersVariable.state.filters).toEqual(newStateFilters); + }); + + it('should add a new filter to metric filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = [{ key: 'metric_key', value: 'value', operator: '=' }]; + const prevStateFilters: AdHocVariableFilter[] = []; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + + const filtersVariable = getFilterVar(); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(filtersVariable.state.filters).toEqual(newStateFilters); + }); + + it('should remove a filter from otel filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = []; + const prevStateFilters: AdHocVariableFilter[] = [{ key: 'otel_key', value: 'value', operator: '=' }]; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + + const filtersVariable = getFilterVar(); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(otelFiltersVariable.state.filters).toEqual(newStateFilters); + }); + + it('should remove a filter from metric filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = []; + const prevStateFilters: AdHocVariableFilter[] = [{ key: 'metric_key', value: 'value', operator: '=' }]; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + + const filtersVariable = getFilterVar(); + filtersVariable.setState({ filters: [{ key: 'metric_key', value: 'value', operator: '=' }] }); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(filtersVariable.state.filters).toEqual(newStateFilters); + }); + + it('should update a filter in otel filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = [{ key: 'otel_key', value: 'new_value', operator: '=' }]; + const prevStateFilters: AdHocVariableFilter[] = [{ key: 'otel_key', value: 'old_value', operator: '=' }]; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + otelFiltersVariable.setState({ filters: [{ key: 'otel_key', value: 'old_value', operator: '=' }] }); + + const filtersVariable = getFilterVar(); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(otelFiltersVariable.state.filters).toEqual(newStateFilters); + }); + + it('should update a filter in metric filters when VAR_OTEL_AND_METRIC_FILTERS is updated', () => { + const newStateFilters: AdHocVariableFilter[] = [{ key: 'metric_key', value: 'new_value', operator: '=' }]; + const prevStateFilters: AdHocVariableFilter[] = [{ key: 'metric_key', value: 'old_value', operator: '=' }]; + + const nonPromotedOtelResources = ['otel_key']; + + const otelFiltersVariable = getOtelResourcesVar(trail); + + const filtersVariable = getFilterVar(); + filtersVariable.setState({ filters: [{ key: 'metric_key', value: 'old_value', operator: '=' }] }); + + manageOtelAndMetricFilters( + newStateFilters, + prevStateFilters, + nonPromotedOtelResources, + otelFiltersVariable, + filtersVariable + ); + + expect(filtersVariable.state.filters).toEqual(newStateFilters); + }); }); }); diff --git a/public/app/features/trails/shared.ts b/public/app/features/trails/shared.ts index aab8bc4c736..41820ce0239 100644 --- a/public/app/features/trails/shared.ts +++ b/public/app/features/trails/shared.ts @@ -36,6 +36,9 @@ export const VAR_OTEL_GROUP_LEFT = 'otel_group_left'; export const VAR_OTEL_GROUP_LEFT_EXPR = '${otel_group_left}'; export const VAR_MISSING_OTEL_TARGETS = 'missing_otel_targets'; export const VAR_MISSING_OTEL_TARGETS_EXPR = '${missing_otel_targets}'; +// for consolidating otel and metric filters into one adhoc filter set +export const VAR_OTEL_AND_METRIC_FILTERS = 'otel_and_metric_filters'; +export const VAR_OTEL_AND_METRIC_FILTERS_EXPR = '${otel_and_metric_filters}'; export const LOGS_METRIC = '$__logs__'; export const KEY_SQR_METRIC_VIZ_QUERY = 'sqr-metric-viz-query'; diff --git a/public/app/features/trails/utils.test.ts b/public/app/features/trails/utils.test.ts index f4cab00bc54..04b15995982 100644 --- a/public/app/features/trails/utils.test.ts +++ b/public/app/features/trails/utils.test.ts @@ -5,6 +5,8 @@ import { getDatasourceSrv } from '../plugins/datasource_srv'; import { DataTrail } from './DataTrail'; import { getTrailStore } from './TrailStore/TrailStore'; import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper'; +import { sortResources } from './otel/util'; +import { VAR_OTEL_AND_METRIC_FILTERS } from './shared'; import { getDatasourceForNewTrail, limitAdhocProviders } from './utils'; jest.mock('./TrailStore/TrailStore', () => ({ @@ -15,8 +17,13 @@ jest.mock('../plugins/datasource_srv', () => ({ getDatasourceSrv: jest.fn(), })); +jest.mock('./otel/util', () => ({ + sortResources: jest.fn(), +})); + describe('limitAdhocProviders', () => { let filtersVariable: AdHocFiltersVariable; + let otelAndMetricsVariable: AdHocFiltersVariable; let datasourceHelper: MetricDatasourceHelper; let dataTrail: DataTrail; @@ -31,6 +38,12 @@ describe('limitAdhocProviders', () => { type: 'adhoc', }); + otelAndMetricsVariable = new AdHocFiltersVariable({ + name: VAR_OTEL_AND_METRIC_FILTERS, + label: 'Test Variable', + type: 'adhoc', + }); + datasourceHelper = { getTagKeys: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'key' })), getTagValues: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'value' })), @@ -70,6 +83,14 @@ describe('limitAdhocProviders', () => { expect(result.replace).toBe(true); } }); + + it('should call sort resources and sort the promoted otel resources list if using the otel and metrics filter', async () => { + limitAdhocProviders(dataTrail, otelAndMetricsVariable, datasourceHelper); + if (otelAndMetricsVariable instanceof AdHocFiltersVariable && otelAndMetricsVariable.state.getTagKeysProvider) { + await otelAndMetricsVariable.state.getTagKeysProvider(otelAndMetricsVariable, null); + } + expect(sortResources).toHaveBeenCalled(); + }); }); describe('getDatasourceForNewTrail', () => { diff --git a/public/app/features/trails/utils.ts b/public/app/features/trails/utils.ts index c96e5099f41..abacfb7a5a6 100644 --- a/public/app/features/trails/utils.ts +++ b/public/app/features/trails/utils.ts @@ -32,7 +32,8 @@ import { DataTrailSettings } from './DataTrailSettings'; import { MetricScene } from './MetricScene'; import { getTrailStore } from './TrailStore/TrailStore'; import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper'; -import { LOGS_METRIC, TRAILS_ROUTE, VAR_DATASOURCE_EXPR } from './shared'; +import { sortResources } from './otel/util'; +import { LOGS_METRIC, TRAILS_ROUTE, VAR_DATASOURCE_EXPR, VAR_OTEL_AND_METRIC_FILTERS } from './shared'; export function getTrailFor(model: SceneObject): DataTrail { return sceneGraph.getAncestor(model, DataTrail); @@ -42,11 +43,12 @@ export function getTrailSettings(model: SceneObject): DataTrailSettings { return sceneGraph.getAncestor(model, DataTrail).state.settings; } -export function newMetricsTrail(initialDS?: string): DataTrail { +export function newMetricsTrail(initialDS?: string, startButtonClicked?: boolean): DataTrail { return new DataTrail({ initialDS, $timeRange: new SceneTimeRange({ from: 'now-1h', to: 'now' }), embedded: false, + startButtonClicked, }); } @@ -145,19 +147,19 @@ const MAX_ADHOC_VARIABLE_OPTIONS = 10000; * This function still uses these functions from inside the data source helper. * * @param dataTrail - * @param filtersVariable + * @param limitedFilterVariable Depending on otel experience flag, either filtersVar or otelAndMetricsVar * @param datasourceHelper */ export function limitAdhocProviders( dataTrail: DataTrail, - filtersVariable: SceneVariable | null, + limitedFilterVariable: SceneVariable | null, datasourceHelper: MetricDatasourceHelper ) { - if (!(filtersVariable instanceof AdHocFiltersVariable)) { + if (!(limitedFilterVariable instanceof AdHocFiltersVariable)) { return; } - filtersVariable.setState({ + limitedFilterVariable.setState({ getTagKeysProvider: async ( variable: AdHocFiltersVariable, currentKey: string | null @@ -170,7 +172,7 @@ export function limitAdhocProviders( // to use in the query to filter the response // using filters, e.g. {previously_selected_label:"value"}, // as the series match[] parameter in Prometheus labels endpoint - const filters = filtersVariable.state.filters; + const filters = limitedFilterVariable.state.filters; // call getTagKeys and truncate the response // we're passing the queries so we get the labels that adhere to the queries // we're also passing the scopes so we get the labels that adhere to the scopes filters @@ -187,7 +189,15 @@ export function limitAdhocProviders( opts.queries = []; } - const values = (await datasourceHelper.getTagKeys(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS); + let values = (await datasourceHelper.getTagKeys(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS); + + // sort the values for otel resources at the top + if (limitedFilterVariable.state.name === VAR_OTEL_AND_METRIC_FILTERS) { + values = sortResources( + values, + filters.map((f) => f.key) + ); + } // use replace: true to override the default lookup in adhoc filter variable return { replace: true, values }; }, @@ -203,7 +213,7 @@ export function limitAdhocProviders( // to use in the query to filter the response // using filters, e.g. {previously_selected_label:"value"}, // as the series match[] parameter in Prometheus label values endpoint - const filtersValues = filtersVariable.state.filters; + const filtersValues = limitedFilterVariable.state.filters; // remove current selected filter if updating a chosen filter const filters = filtersValues.filter((f) => f.key !== filter.key); // call getTagValues and truncate the response diff --git a/public/locales/en-US/grafana.json b/public/locales/en-US/grafana.json index d4ecfb90224..7645b538e9b 100644 --- a/public/locales/en-US/grafana.json +++ b/public/locales/en-US/grafana.json @@ -3320,6 +3320,7 @@ }, "metric-select": { "filter-by": "Filter by", + "new-badge": "New", "otel-switch": "This switch enables filtering by OTel resources for OTel native data sources." }, "recent-metrics": { diff --git a/public/locales/pseudo-LOCALE/grafana.json b/public/locales/pseudo-LOCALE/grafana.json index 340bd89b16d..7df520bca8f 100644 --- a/public/locales/pseudo-LOCALE/grafana.json +++ b/public/locales/pseudo-LOCALE/grafana.json @@ -3320,6 +3320,7 @@ }, "metric-select": { "filter-by": "Fįľŧęř þy", + "new-badge": "Ńęŵ", "otel-switch": "Ŧĥįş şŵįŧčĥ ęʼnäþľęş ƒįľŧęřįʼnģ þy ØŦęľ řęşőūřčęş ƒőř ØŦęľ ʼnäŧįvę đäŧä şőūřčęş." }, "recent-metrics": {