Scopes: Add scopes to metrics explore (#94802)

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -199,6 +199,7 @@ export interface FeatureToggles {
   failWrongDSUID?: boolean;
   zanzana?: boolean;
   reloadDashboardsOnParamsChange?: boolean;
+  enableScopesInMetricsExplore?: boolean;
   alertingApiServer?: boolean;
   cloudWatchRoundUpEndTime?: boolean;
   cloudwatchMetricInsightsCrossAccount?: boolean;

pkg/promlib/resource/resource.go

@@ -177,12 +177,17 @@ func (r *Resource) GetSuggestions(ctx context.Context, req *backend.CallResource
 	}
 
 	values := url.Values{}
-
 	for _, s := range selectorList {
 		vs := parser.VectorSelector{Name: s, LabelMatchers: matchers}
 		values.Add("match[]", vs.String())
 	}
 
+	// if no timeserie name is provided, but scopes are, the scope is still rendered and passed as match param.
+	if len(selectorList) == 0 && len(sugReq.Scopes) > 0 {
+		vs := parser.VectorSelector{LabelMatchers: matchers}
+		values.Add("match[]", vs.String())
+	}
+
 	if sugReq.Start != "" {
 		values.Add("start", sugReq.Start)
 	}

pkg/services/featuremgmt/registry.go

@@ -1373,6 +1373,17 @@ var (
 			HideFromDocs:      true,
 			HideFromAdminPage: true,
 		},
+		{
+			Name:              "enableScopesInMetricsExplore",
+			Description:       "Enables the scopes usage in Metrics Explore",
+			FrontendOnly:      false,
+			Stage:             FeatureStageExperimental,
+			Owner:             grafanaDashboardsSquad,
+			RequiresRestart:   false,
+			AllowSelfServe:    false,
+			HideFromDocs:      true,
+			HideFromAdminPage: true,
+		},
 		{
 			Name:            "alertingApiServer",
 			Description:     "Register Alerting APIs with the K8s API server",

pkg/services/featuremgmt/toggles_gen.csv

@@ -180,6 +180,7 @@ ssoSettingsLDAP,preview,@grafana/identity-access-team,false,true,false
 failWrongDSUID,experimental,@grafana/plugins-platform-backend,false,false,false
 zanzana,experimental,@grafana/identity-access-team,false,false,false
 reloadDashboardsOnParamsChange,experimental,@grafana/dashboards-squad,false,false,false
+enableScopesInMetricsExplore,experimental,@grafana/dashboards-squad,false,false,false
 alertingApiServer,experimental,@grafana/alerting-squad,false,true,false
 cloudWatchRoundUpEndTime,GA,@grafana/aws-datasources,false,false,false
 cloudwatchMetricInsightsCrossAccount,GA,@grafana/aws-datasources,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -731,6 +731,10 @@ const (
 	// Enables reload of dashboards on scopes, time range and variables changes
 	FlagReloadDashboardsOnParamsChange = "reloadDashboardsOnParamsChange"
 
+	// FlagEnableScopesInMetricsExplore
+	// Enables the scopes usage in Metrics Explore
+	FlagEnableScopesInMetricsExplore = "enableScopesInMetricsExplore"
+
 	// FlagAlertingApiServer
 	// Register Alerting APIs with the K8s API server
 	FlagAlertingApiServer = "alertingApiServer"

pkg/services/featuremgmt/toggles_gen.json

@@ -1231,6 +1231,20 @@
         "hideFromAdminPage": true
       }
     },
+    {
+      "metadata": {
+        "name": "enableScopesInMetricsExplore",
+        "resourceVersion": "1729765731452",
+        "creationTimestamp": "2024-10-24T10:28:51Z"
+      },
+      "spec": {
+        "description": "Enables the scopes usage in Metrics Explore",
+        "stage": "experimental",
+        "codeowner": "@grafana/dashboards-squad",
+        "hideFromAdminPage": true,
+        "hideFromDocs": true
+      }
+    },
     {
       "metadata": {
         "name": "exploreContentOutline",

public/app/features/trails/Breakdown/LabelBreakdownScene.tsx

@@ -42,6 +42,7 @@ import { getSortByPreference } from '../services/store';
 import { ALL_VARIABLE_VALUE } from '../services/variables';
 import {
   MDP_METRIC_PREVIEW,
+  RefreshMetricsEvent,
   trailDS,
   VAR_FILTERS,
   VAR_GROUP_BY,
@@ -98,6 +99,14 @@ export class LabelBreakdownScene extends SceneObjectBase<LabelBreakdownSceneStat
 
     const variable = this.getVariable();
 
+    if (config.featureToggles.enableScopesInMetricsExplore) {
+      this._subs.add(
+        this.subscribeToEvent(RefreshMetricsEvent, () => {
+          this.updateBody(this.getVariable());
+        })
+      );
+    }
+
     variable.subscribeToState((newState, oldState) => {
       if (
         newState.options !== oldState.options ||

public/app/features/trails/DataTrail.tsx

@@ -10,6 +10,7 @@ import {
   VariableHide,
   urlUtil,
 } from '@grafana/data';
+import { PromQuery } from '@grafana/prometheus';
 import { locationService, useChromeHeaderHeight } from '@grafana/runtime';
 import {
   AdHocFiltersVariable,
@@ -24,6 +25,7 @@ import {
   SceneObjectState,
   SceneObjectUrlSyncConfig,
   SceneObjectUrlValues,
+  SceneQueryRunner,
   SceneRefreshPicker,
   SceneTimePicker,
   SceneTimeRange,
@@ -35,6 +37,7 @@ import {
   VariableValueSelectors,
 } from '@grafana/scenes';
 import { useStyles2 } from '@grafana/ui';
+import { getSelectedScopes } from 'app/features/scopes';
 
 import { DataTrailSettings } from './DataTrailSettings';
 import { DataTrailHistory } from './DataTrailsHistory';
@@ -325,7 +328,7 @@ export class DataTrail extends SceneObjectBase<DataTrailState> {
       const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR);
 
       const otelTargets = await totalOtelResources(datasourceUid, timeRange);
-      const deploymentEnvironments = await getDeploymentEnvironments(datasourceUid, timeRange);
+      const deploymentEnvironments = await getDeploymentEnvironments(datasourceUid, timeRange, getSelectedScopes());
       const hasOtelResources = otelTargets.jobs.length > 0 && otelTargets.instances.length > 0;
       if (
         otelResourcesVariable instanceof AdHocFiltersVariable &&
@@ -469,7 +472,13 @@ export class DataTrail extends SceneObjectBase<DataTrailState> {
         values: GetTagResponse | MetricFindValue[];
       }> => {
         // apply filters here
-        let values = await datasourceHelper.getTagKeys({ filters });
+        // we're passing the queries so we get the labels that adhere to the queries
+        // we're also passing the scopes so we get the labels that adhere to the scopes filters
+        let values = await datasourceHelper.getTagKeys({
+          filters,
+          scopes: getSelectedScopes(),
+          queries: this.getQueries(),
+        });
         values = sortResources(values, filters.map((f) => f.key).concat(currentKey ?? ''));
         return { replace: true, values };
       },
@@ -483,7 +492,14 @@ export class DataTrail extends SceneObjectBase<DataTrailState> {
         // apply filters here
         // remove current selected filter if refiltering
         filters = filters.filter((f) => f.key !== filter.key);
-        const values = await datasourceHelper.getTagValues({ key: filter.key, filters });
+        // we're passing the queries so we get the label values that adhere to the queries
+        // we're also passing the scopes so we get the label values that adhere to the scopes filters
+        const values = await datasourceHelper.getTagValues({
+          key: filter.key,
+          filters,
+          scopes: getSelectedScopes(),
+          queries: this.getQueries(),
+        });
         return { replace: true, values };
       },
       hide: VariableHide.hideLabel,
@@ -573,6 +589,22 @@ export class DataTrail extends SceneObjectBase<DataTrailState> {
     }
   }
 
+  public getQueries(): PromQuery[] {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const sqrs = sceneGraph.findAllObjects(this, (b) => b instanceof SceneQueryRunner) as SceneQueryRunner[];
+
+    return sqrs.reduce<PromQuery[]>((acc, sqr) => {
+      acc.push(
+        ...sqr.state.queries.map((q) => ({
+          ...q,
+          expr: sceneGraph.interpolate(sqr, q.expr),
+        }))
+      );
+
+      return acc;
+    }, []);
+  }
+
   static Component = ({ model }: SceneComponentProps<DataTrail>) => {
     const { controls, topScene, history, settings, useOtelExperience, hasOtelResources } = model.useState();
 
@@ -606,7 +638,7 @@ export class DataTrail extends SceneObjectBase<DataTrailState> {
     useEffect(() => {
       const filtersVariable = sceneGraph.lookupVariable(VAR_FILTERS, model);
       const datasourceHelper = model.datasourceHelper;
-      limitAdhocProviders(filtersVariable, datasourceHelper);
+      limitAdhocProviders(model, filtersVariable, datasourceHelper);
     }, [model]);
 
     return (

public/app/features/trails/DataTrailsApp.tsx

@@ -1,15 +1,25 @@
+import { css } from '@emotion/css';
 import { useEffect, useState } from 'react';
 import { Routes, Route } from 'react-router-dom-v5-compat';
 
-import { PageLayoutType } from '@grafana/data';
-import { locationService } from '@grafana/runtime';
+import {
+  DataQueryRequest,
+  DataSourceGetTagKeysOptions,
+  DataSourceGetTagValuesOptions,
+  PageLayoutType,
+} from '@grafana/data';
+import { config, locationService } from '@grafana/runtime';
 import { SceneComponentProps, SceneObjectBase, SceneObjectState, UrlSyncContextProvider } from '@grafana/scenes';
+import { useStyles2 } from '@grafana/ui/';
 import { Page } from 'app/core/components/Page/Page';
+import { getClosestScopesFacade, ScopesFacade, ScopesSelector } from 'app/features/scopes';
+
+import { AppChromeUpdate } from '../../core/components/AppChrome/AppChromeUpdate';
 
 import { DataTrail } from './DataTrail';
 import { DataTrailsHome } from './DataTrailsHome';
 import { getTrailStore } from './TrailStore/TrailStore';
-import { HOME_ROUTE, TRAILS_ROUTE } from './shared';
+import { HOME_ROUTE, RefreshMetricsEvent, TRAILS_ROUTE } from './shared';
 import { getMetricName, getUrlForTrail, newMetricsTrail } from './utils';
 
 export interface DataTrailsAppState extends SceneObjectState {
@@ -18,8 +28,32 @@ export interface DataTrailsAppState extends SceneObjectState {
 }
 
 export class DataTrailsApp extends SceneObjectBase<DataTrailsAppState> {
+  private _scopesFacade: ScopesFacade | null;
+
   public constructor(state: DataTrailsAppState) {
     super(state);
+
+    this._scopesFacade = getClosestScopesFacade(this);
+  }
+
+  public enrichDataRequest(): Partial<DataQueryRequest> {
+    if (!config.featureToggles.promQLScope) {
+      return {};
+    }
+
+    return {
+      scopes: this._scopesFacade?.value,
+    };
+  }
+
+  public enrichFiltersRequest(): Partial<DataSourceGetTagKeysOptions | DataSourceGetTagValuesOptions> {
+    if (!config.featureToggles.promQLScope) {
+      return {};
+    }
+
+    return {
+      scopes: this._scopesFacade?.value,
+    };
   }
 
   goToUrlForTrail(trail: DataTrail) {
@@ -54,6 +88,7 @@ export class DataTrailsApp extends SceneObjectBase<DataTrailsAppState> {
 }
 
 function DataTrailView({ trail }: { trail: DataTrail }) {
+  const styles = useStyles2(getStyles);
   const [isInitialized, setIsInitialized] = useState(false);
   const { metric } = trail.useState();
 
@@ -73,6 +108,15 @@ function DataTrailView({ trail }: { trail: DataTrail }) {
   return (
     <UrlSyncContextProvider scene={trail}>
       <Page navId="explore/metrics" pageNav={{ text: getMetricName(metric) }} layout={PageLayoutType.Custom}>
+        {config.featureToggles.singleTopNav && config.featureToggles.enableScopesInMetricsExplore && (
+          <AppChromeUpdate
+            actions={
+              <div className={styles.topNavContainer}>
+                <ScopesSelector />
+              </div>
+            }
+          />
+        )}
         <trail.Component model={trail} />
       </Page>
     </UrlSyncContextProvider>
@@ -83,11 +127,36 @@ let dataTrailsApp: DataTrailsApp;
 
 export function getDataTrailsApp() {
   if (!dataTrailsApp) {
+    const $behaviors = config.featureToggles.enableScopesInMetricsExplore
+      ? [
+          new ScopesFacade({
+            handler: (facade) => {
+              const trail = facade.parent && 'trail' in facade.parent.state ? facade.parent.state.trail : undefined;
+
+              if (trail instanceof DataTrail) {
+                trail.publishEvent(new RefreshMetricsEvent());
+                trail.checkDataSourceForOTelResources();
+              }
+            },
+          }),
+        ]
+      : undefined;
+
     dataTrailsApp = new DataTrailsApp({
       trail: newMetricsTrail(),
       home: new DataTrailsHome({}),
+      $behaviors,
     });
   }
 
   return dataTrailsApp;
 }
+
+const getStyles = () => ({
+  topNavContainer: css({
+    width: '100%',
+    display: 'flex',
+    flexDirection: 'row',
+    justifyItems: 'flex-start',
+  }),
+});

public/app/features/trails/MetricScene.tsx

@@ -32,6 +32,7 @@ import {
   getVariablesWithMetricConstant,
   MakeOptional,
   MetricSelectedEvent,
+  RefreshMetricsEvent,
   trailDS,
   VAR_GROUP_BY,
   VAR_METRIC_EXPR,
@@ -69,6 +70,16 @@ export class MetricScene extends SceneObjectBase<MetricSceneState> {
     if (this.state.actionView === undefined) {
       this.setActionView('overview');
     }
+
+    if (config.featureToggles.enableScopesInMetricsExplore) {
+      // Push the scopes change event to the tabs
+      // The event is not propagated because the tabs are not part of the scene graph
+      this._subs.add(
+        this.subscribeToEvent(RefreshMetricsEvent, (event) => {
+          this.state.body.state.selectedTab?.publishEvent(event);
+        })
+      );
+    }
   }
 
   getUrlState() {

public/app/features/trails/MetricSelect/MetricSelectScene.tsx

@@ -2,8 +2,8 @@ import { css } from '@emotion/css';
 import { debounce, isEqual } from 'lodash';
 import { SyntheticEvent, useReducer } from 'react';
 
-import { GrafanaTheme2, RawTimeRange, SelectableValue } from '@grafana/data';
-import { isFetchError } from '@grafana/runtime';
+import { AdHocVariableFilter, GrafanaTheme2, RawTimeRange, SelectableValue } from '@grafana/data';
+import { config, isFetchError } from '@grafana/runtime';
 import {
   AdHocFiltersVariable,
   PanelBuilders,
@@ -22,22 +22,22 @@ import {
   SceneObjectUrlValues,
   SceneObjectWithUrlSync,
   SceneTimeRange,
-  SceneVariable,
   SceneVariableSet,
   VariableDependencyConfig,
 } from '@grafana/scenes';
 import { Alert, Field, Icon, IconButton, InlineSwitch, Input, Select, Tooltip, useStyles2 } from '@grafana/ui';
 import { Trans } from 'app/core/internationalization';
+import { getSelectedScopes } from 'app/features/scopes';
 
 import { MetricScene } from '../MetricScene';
 import { StatusWrapper } from '../StatusWrapper';
 import { Node, Parser } from '../groop/parser';
 import { getMetricDescription } from '../helpers/MetricDatasourceHelper';
 import { reportExploreMetrics } from '../interactions';
-import { limitOtelMatchTerms } from '../otel/util';
 import {
   getVariablesWithMetricConstant,
   MetricSelectedEvent,
+  RefreshMetricsEvent,
   VAR_DATASOURCE,
   VAR_DATASOURCE_EXPR,
   VAR_FILTERS,
@@ -104,7 +104,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
   protected _urlSync = new SceneObjectUrlSyncConfig(this, { keys: ['metricPrefix'] });
   protected _variableDependency = new VariableDependencyConfig(this, {
     variableNames: [VAR_DATASOURCE, VAR_FILTERS],
-    onReferencedVariableValueChanged: (variable: SceneVariable) => {
+    onReferencedVariableValueChanged: () => {
       // In all cases, we want to reload the metric names
       this._debounceRefreshMetricNames();
     },
@@ -194,7 +194,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
     );
 
     this._subs.add(
-      trail.subscribeToState(({ useOtelExperience }, oldState) => {
+      trail.subscribeToState(() => {
         // users will most likely not switch this off but for now,
         // update metric names when changing useOtelExperience
         this._debounceRefreshMetricNames();
@@ -202,13 +202,21 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
     );
 
     this._subs.add(
-      trail.subscribeToState(({ showPreviews }, oldState) => {
+      trail.subscribeToState(() => {
         // move showPreviews into the settings
         // build layout when toggled
         this.buildLayout();
       })
     );
 
+    if (config.featureToggles.enableScopesInMetricsExplore) {
+      this._subs.add(
+        trail.subscribeToEvent(RefreshMetricsEvent, () => {
+          this._debounceRefreshMetricNames();
+        })
+      );
+    }
+
     this._debounceRefreshMetricNames();
   }
 
@@ -220,45 +228,39 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
       return;
     }
 
-    const matchTerms: string[] = [];
+    const filters: AdHocVariableFilter[] = [];
 
     const filtersVar = sceneGraph.lookupVariable(VAR_FILTERS, this);
-    const hasFilters = filtersVar instanceof AdHocFiltersVariable && filtersVar.getValue()?.valueOf();
-    if (hasFilters) {
-      matchTerms.push(sceneGraph.interpolate(trail, '${filters}'));
+    const adhocFilters = filtersVar instanceof AdHocFiltersVariable ? (filtersVar?.state.filters ?? []) : [];
+    if (adhocFilters.length > 0) {
+      filters.push(...adhocFilters);
     }
 
     const metricSearchRegex = createPromRegExp(trail.state.metricSearch);
     if (metricSearchRegex) {
-      matchTerms.push(`__name__=~"${metricSearchRegex}"`);
-    }
-
-    let noOtelMetrics = false;
-    let missingOtelTargets = false;
-
-    if (trail.state.useOtelExperience) {
-      const jobsList = trail.state.otelTargets?.jobs;
-      const instancesList = trail.state.otelTargets?.instances;
-      // no targets have this combination of filters so there are no metrics that can be joined
-      // show no metrics
-      if (jobsList && jobsList.length > 0 && instancesList && instancesList.length > 0) {
-        const otelMatches = limitOtelMatchTerms(matchTerms, jobsList, instancesList, missingOtelTargets);
-
-        missingOtelTargets = otelMatches.missingOtelTargets;
-
-        matchTerms.push(otelMatches.jobsRegex);
-        matchTerms.push(otelMatches.instancesRegex);
-      } else {
-        noOtelMetrics = true;
-      }
+      filters.push({
+        key: '__name__',
+        operator: '=~',
+        value: metricSearchRegex,
+      });
     }
 
-    const match = `{${matchTerms.join(',')}}`;
     const datasourceUid = sceneGraph.interpolate(trail, VAR_DATASOURCE_EXPR);
     this.setState({ metricNamesLoading: true, metricNamesError: undefined, metricNamesWarning: undefined });
 
     try {
-      const response = await getMetricNames(datasourceUid, timeRange, match, MAX_METRIC_NAMES);
+      const jobsList = trail.state.useOtelExperience ? (trail.state.otelTargets?.jobs ?? []) : [];
+      const instancesList = trail.state.useOtelExperience ? (trail.state.otelTargets?.instances ?? []) : [];
+
+      const response = await getMetricNames(
+        datasourceUid,
+        timeRange,
+        getSelectedScopes(),
+        filters,
+        jobsList,
+        instancesList,
+        MAX_METRIC_NAMES
+      );
       const searchRegex = createJSRegExpFromSearchTerms(getMetricSearch(this));
       let metricNames = searchRegex
         ? response.data.filter((metric) => !searchRegex || searchRegex.test(metric))
@@ -281,21 +283,19 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
         : undefined;
 
       // if there are no otel targets for otel resources, there will be no labels
-      if (noOtelMetrics) {
+      if (trail.state.useOtelExperience && (jobsList.length === 0 || instancesList.length === 0)) {
         metricNames = [];
         metricNamesWarning = undefined;
       }
 
-      if (missingOtelTargets) {
+      if (response.missingOtelTargets) {
         metricNamesWarning = `${metricNamesWarning ?? ''} The list of metrics is not complete. Select more OTel resource attributes to see a full list of metrics.`;
       }
 
       let bodyLayout = this.state.body;
 
-      let rootGroupNode = this.state.rootGroup;
-
       // generate groups based on the search metrics input
-      rootGroupNode = await this.generateGroups(filteredMetricNames);
+      let rootGroupNode = await this.generateGroups(filteredMetricNames);
 
       this.setState({
         metricNames,
@@ -364,9 +364,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> i
 
       const oldPanel = this.previewCache[metricName];
 
-      const panel = oldPanel || { name: metricName, index, loaded: false };
-
-      metricsMap[metricName] = panel;
+      metricsMap[metricName] = oldPanel || { name: metricName, index, loaded: false };
     }
 
     try {

public/app/features/trails/MetricSelect/api.ts

@@ -1,17 +1,48 @@
-import { RawTimeRange } from '@grafana/data';
+import { AdHocVariableFilter, RawTimeRange, Scope } from '@grafana/data';
 import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
-import { getBackendSrv } from '@grafana/runtime';
+import { config, getBackendSrv } from '@grafana/runtime';
 
-type MetricValuesResponse = {
-  data: string[];
-  status: 'success' | 'error';
-  error?: 'string';
-  warnings?: string[];
-};
+import { limitOtelMatchTerms } from '../otel/util';
+import { callSuggestionsApi, SuggestionsResponse } from '../utils';
 
 const LIMIT_REACHED = 'results truncated due to limit';
 
-export async function getMetricNames(dataSourceUid: string, timeRange: RawTimeRange, filters: string, limit?: number) {
+export async function getMetricNames(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  scopes: Scope[],
+  filters: AdHocVariableFilter[],
+  jobs: string[],
+  instances: string[],
+  limit?: number
+): Promise<SuggestionsResponse & { limitReached: boolean; missingOtelTargets: boolean }> {
+  if (!config.featureToggles.enableScopesInMetricsExplore) {
+    return await getMetricNamesWithoutScopes(dataSourceUid, timeRange, filters, jobs, instances, limit);
+  }
+
+  return getMetricNamesWithScopes(dataSourceUid, timeRange, scopes, filters, jobs, instances, limit);
+}
+
+export async function getMetricNamesWithoutScopes(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  adhocFilters: AdHocVariableFilter[],
+  jobs: string[],
+  instances: string[],
+  limit?: number
+) {
+  const matchTerms = adhocFilters.map((filter) => `${filter.key}${filter.operator}"${filter.value}"`);
+  let missingOtelTargets = false;
+
+  if (jobs.length > 0 && instances.length > 0) {
+    const otelMatches = limitOtelMatchTerms(matchTerms, jobs, instances);
+    missingOtelTargets = otelMatches.missingOtelTargets;
+    matchTerms.push(otelMatches.jobsRegex);
+    matchTerms.push(otelMatches.instancesRegex);
+  }
+
+  const filters = `{${matchTerms.join(',')}}`;
+
   const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/label/__name__/values`;
   const params: Record<string, string | number> = {
     start: getPrometheusTime(timeRange.from, false),
@@ -20,11 +51,51 @@ export async function getMetricNames(dataSourceUid: string, timeRange: RawTimeRa
     ...(limit ? { limit } : {}),
   };
 
-  const response = await getBackendSrv().get<MetricValuesResponse>(url, params, 'explore-metrics-names');
+  const response = await getBackendSrv().get<SuggestionsResponse>(url, params, 'explore-metrics-names');
 
   if (limit && response.warnings?.includes(LIMIT_REACHED)) {
-    return { ...response, limitReached: true };
+    return { ...response, limitReached: true, missingOtelTargets };
   }
 
-  return { ...response, limitReached: false };
+  return { ...response, limitReached: false, missingOtelTargets };
+}
+
+export async function getMetricNamesWithScopes(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  scopes: Scope[],
+  filters: AdHocVariableFilter[],
+  jobs: string[],
+  instances: string[],
+  limit?: number
+) {
+  const response = await callSuggestionsApi(
+    dataSourceUid,
+    timeRange,
+    scopes,
+    filters,
+    '__name__',
+    limit,
+    'explore-metrics-names'
+  );
+
+  if (jobs.length > 0 && instances.length > 0) {
+    filters.push({
+      key: 'job',
+      operator: '=~',
+      value: jobs?.join('|') || '',
+    });
+
+    filters.push({
+      key: 'instance',
+      operator: '=~',
+      value: instances?.join('|') || '',
+    });
+  }
+
+  return {
+    ...response.data,
+    limitReached: !!limit && !!response.data.warnings?.includes(LIMIT_REACHED),
+    missingOtelTargets: false,
+  };
 }

public/app/features/trails/otel/api.test.ts

@@ -10,7 +10,9 @@ import {
 } from './api';
 
 jest.mock('@grafana/runtime', () => ({
+  ...jest.requireActual('@grafana/runtime'),
   config: {
+    ...jest.requireActual('@grafana/runtime').config,
     publicDashboardAccessToken: '123',
   },
   getBackendSrv: () => {
@@ -107,7 +109,7 @@ describe('OTEL API', () => {
 
   describe('getDeploymentEnvironments', () => {
     it('should fetch deployment environments', async () => {
-      const environments = await getDeploymentEnvironments(dataSourceUid, timeRange);
+      const environments = await getDeploymentEnvironments(dataSourceUid, timeRange, []);
 
       expect(environments).toEqual(['env1', 'env2']);
     });

public/app/features/trails/otel/api.ts

@@ -1,6 +1,8 @@
-import { RawTimeRange } from '@grafana/data';
+import { RawTimeRange, Scope } from '@grafana/data';
 import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
-import { getBackendSrv } from '@grafana/runtime';
+import { config, getBackendSrv } from '@grafana/runtime';
+
+import { callSuggestionsApi } from '../utils';
 
 import { OtelResponse, LabelResponse, OtelTargetType } from './types';
 import { sortResources } from './util';
@@ -45,9 +47,7 @@ export async function getOtelResources(
   const response = await getBackendSrv().get<LabelResponse>(url, params, 'explore-metrics-otel-resources');
 
   // exclude __name__ or deployment_environment or previously chosen filters
-  const resources = response.data?.filter((resource) => !allExcludedFilters.includes(resource)).map((el: string) => el);
-
-  return resources;
+  return response.data?.filter((resource) => !allExcludedFilters.includes(resource)).map((el: string) => el);
 }
 
 /**
@@ -56,7 +56,7 @@ export async function getOtelResources(
  *
  * @param dataSourceUid
  * @param timeRange
- * @param expr
+ * @param filters
  * @returns
  */
 export async function totalOtelResources(
@@ -99,12 +99,10 @@ export async function totalOtelResources(
     }
   });
 
-  const otelTargets: OtelTargetType = {
+  return {
     jobs,
     instances,
   };
-
-  return otelTargets;
 }
 
 /**
@@ -115,14 +113,9 @@ export async function totalOtelResources(
  *
  * @param dataSourceUid
  * @param timeRange
- * @param expr
  * @returns
  */
-export async function isOtelStandardization(
-  dataSourceUid: string,
-  timeRange: RawTimeRange,
-  expr?: string
-): Promise<boolean> {
+export async function isOtelStandardization(dataSourceUid: string, timeRange: RawTimeRange): Promise<boolean> {
   const url = `/api/datasources/uid/${dataSourceUid}/resources/api/v1/query`;
 
   const start = getPrometheusTime(timeRange.from, false);
@@ -138,9 +131,27 @@ export async function isOtelStandardization(
   const response = await getBackendSrv().get<OtelResponse>(url, paramsTargets, 'explore-metrics-otel-check-standard');
 
   // the response should be not greater than zero if it is standard
-  const checkStandard = !(response.data.result.length > 0);
+  return !(response.data.result.length > 0);
+}
+
+/**
+ * Query the DS for deployment environment label values.
+ *
+ * @param dataSourceUid
+ * @param timeRange
+ * @param scopes
+ * @returns string[], values for the deployment_environment label
+ */
+export async function getDeploymentEnvironments(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  scopes: Scope[]
+): Promise<string[]> {
+  if (!config.featureToggles.enableScopesInMetricsExplore) {
+    return getDeploymentEnvironmentsWithoutScopes(dataSourceUid, timeRange);
+  }
 
-  return checkStandard;
+  return getDeploymentEnvironmentsWithScopes(dataSourceUid, timeRange, scopes);
 }
 
 /**
@@ -150,7 +161,10 @@ export async function isOtelStandardization(
  * @param timeRange
  * @returns string[], values for the deployment_environment label
  */
-export async function getDeploymentEnvironments(dataSourceUid: string, timeRange: RawTimeRange): Promise<string[]> {
+export async function getDeploymentEnvironmentsWithoutScopes(
+  dataSourceUid: string,
+  timeRange: RawTimeRange
+): Promise<string[]> {
   const start = getPrometheusTime(timeRange.from, false);
   const end = getPrometheusTime(timeRange.to, true);
 
@@ -168,9 +182,39 @@ export async function getDeploymentEnvironments(dataSourceUid: string, timeRange
   );
 
   // exclude __name__ or deployment_environment or previously chosen filters
-  const resources = response.data;
+  return response.data;
+}
 
-  return resources;
+/**
+ * Query the DS for deployment environment label values.
+ *
+ * @param dataSourceUid
+ * @param timeRange
+ * @param scopes
+ * @returns string[], values for the deployment_environment label
+ */
+export async function getDeploymentEnvironmentsWithScopes(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  scopes: Scope[]
+): Promise<string[]> {
+  const response = await callSuggestionsApi(
+    dataSourceUid,
+    timeRange,
+    scopes,
+    [
+      {
+        key: '__name__',
+        operator: '=',
+        value: 'target_info',
+      },
+    ],
+    'deployment_environment',
+    undefined,
+    'explore-metrics-otel-resources-deployment-env'
+  );
+  // exclude __name__ or deployment_environment or previously chosen filters
+  return response.data.data;
 }
 
 /**

public/app/features/trails/otel/util.ts

@@ -153,15 +153,14 @@ export function getOtelResourcesObject(scene: SceneObject, firstQueryVal?: strin
  * @param matchTerms __name__ and other Prom filters
  * @param jobsList list of jobs in target_info
  * @param instancesList list of instances in target_info
- * @param missingOtelTargets flag to indicate truncated job and instance filters
  * @returns
  */
 export function limitOtelMatchTerms(
   matchTerms: string[],
   jobsList: string[],
-  instancesList: string[],
-  missingOtelTargets: boolean
+  instancesList: string[]
 ): { missingOtelTargets: boolean; jobsRegex: string; instancesRegex: string } {
+  let missingOtelTargets = false;
   const charLimit = 2000;
 
   let initialCharAmount = matchTerms.join(',').length;

public/app/features/trails/otel/utils.test.ts

@@ -152,9 +152,7 @@ describe('limitOtelMatchTerms', () => {
     const jobs = ['a', 'b', 'c'];
     const instances = ['d', 'e', 'f'];
 
-    const missingOtelTargets = false;
-
-    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets);
+    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances);
 
     expect(result.missingOtelTargets).toEqual(true);
     expect(result.jobsRegex).toEqual('job=~"a"');
@@ -179,9 +177,7 @@ describe('limitOtelMatchTerms', () => {
     const jobs = ['a', 'b', 'c'];
     const instances = ['d', 'e', 'f'];
 
-    const missingOtelTargets = false;
-
-    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets);
+    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances);
 
     expect(result.missingOtelTargets).toEqual(true);
     expect(result.jobsRegex).toEqual('job=~"a|b"');
@@ -195,9 +191,7 @@ describe('limitOtelMatchTerms', () => {
 
     const instances = ['instance1', 'instance2', 'instance3', 'instance4', 'instance5'];
 
-    const missingOtelTargets = false;
-
-    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances, missingOtelTargets);
+    const result = limitOtelMatchTerms(promMatchTerms, jobs, instances);
 
     expect(result.missingOtelTargets).toEqual(false);
     expect(result.jobsRegex).toEqual('job=~"job1|job2|job3|job4|job5"');

public/app/features/trails/shared.ts

@@ -1,4 +1,4 @@
-import { BusEventWithPayload } from '@grafana/data';
+import { BusEventBase, BusEventWithPayload } from '@grafana/data';
 import { ConstantVariable, SceneObject } from '@grafana/scenes';
 import { VariableHide } from '@grafana/schema';
 
@@ -73,3 +73,7 @@ export function getVariablesWithOtelJoinQueryConstant(otelJoinQuery: string) {
 export class MetricSelectedEvent extends BusEventWithPayload<string | undefined> {
   public static type = 'metric-selected-event';
 }
+
+export class RefreshMetricsEvent extends BusEventBase {
+  public static type = 'refresh-metrics-event';
+}

public/app/features/trails/utils.test.ts

@@ -1,11 +1,13 @@
 import { AdHocFiltersVariable } from '@grafana/scenes';
 
+import { DataTrail } from './DataTrail';
 import { MetricDatasourceHelper } from './helpers/MetricDatasourceHelper';
 import { limitAdhocProviders } from './utils';
 
 describe('limitAdhocProviders', () => {
   let filtersVariable: AdHocFiltersVariable;
   let datasourceHelper: MetricDatasourceHelper;
+  let dataTrail: DataTrail;
 
   beforeEach(() => {
     // disable console.log called in Scenes for this test
@@ -22,6 +24,10 @@ describe('limitAdhocProviders', () => {
       getTagKeys: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'key' })),
       getTagValues: jest.fn().mockResolvedValue(Array(20000).fill({ text: 'value' })),
     } as unknown as MetricDatasourceHelper;
+
+    dataTrail = {
+      getQueries: jest.fn().mockReturnValue([]),
+    } as unknown as DataTrail;
   });
 
   afterAll(() => {
@@ -29,7 +35,7 @@ describe('limitAdhocProviders', () => {
   });
 
   it('should limit the number of tag keys returned in the variable to 10000', async () => {
-    limitAdhocProviders(filtersVariable, datasourceHelper);
+    limitAdhocProviders(dataTrail, filtersVariable, datasourceHelper);
 
     if (filtersVariable instanceof AdHocFiltersVariable && filtersVariable.state.getTagKeysProvider) {
       console.log = jest.fn();
@@ -41,7 +47,7 @@ describe('limitAdhocProviders', () => {
   });
 
   it('should limit the number of tag values returned in the variable to 10000', async () => {
-    limitAdhocProviders(filtersVariable, datasourceHelper);
+    limitAdhocProviders(dataTrail, filtersVariable, datasourceHelper);
 
     if (filtersVariable instanceof AdHocFiltersVariable && filtersVariable.state.getTagValuesProvider) {
       const result = await filtersVariable.state.getTagValuesProvider(filtersVariable, {

public/app/features/trails/utils.ts

@@ -1,5 +1,17 @@
-import { AdHocVariableFilter, GetTagResponse, MetricFindValue, urlUtil } from '@grafana/data';
-import { config, getDataSourceSrv } from '@grafana/runtime';
+import { lastValueFrom } from 'rxjs';
+
+import {
+  AdHocVariableFilter,
+  GetTagResponse,
+  MetricFindValue,
+  RawTimeRange,
+  Scope,
+  scopeFilterOperatorMap,
+  ScopeSpecFilter,
+  urlUtil,
+} from '@grafana/data';
+import { getPrometheusTime } from '@grafana/prometheus/src/language_utils';
+import { config, FetchResponse, getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
 import {
   AdHocFiltersVariable,
   sceneGraph,
@@ -11,6 +23,7 @@ import {
   SceneVariable,
   SceneVariableState,
 } from '@grafana/scenes';
+import { getClosestScopesFacade } from 'app/features/scopes';
 
 import { getDatasourceSrv } from '../plugins/datasource_srv';
 
@@ -129,10 +142,12 @@ const MAX_ADHOC_VARIABLE_OPTIONS = 10000;
  * The current provider functions for adhoc filter variables are the functions getTagKeys and getTagValues in the data source.
  * This function still uses these functions from inside the data source helper.
  *
+ * @param dataTrail
  * @param filtersVariable
  * @param datasourceHelper
  */
 export function limitAdhocProviders(
+  dataTrail: DataTrail,
   filtersVariable: SceneVariable<SceneVariableState> | null,
   datasourceHelper: MetricDatasourceHelper
 ) {
@@ -155,7 +170,22 @@ export function limitAdhocProviders(
       // as the series match[] parameter in Prometheus labels endpoint
       const filters = filtersVariable.state.filters;
       // call getTagKeys and truncate the response
-      const values = (await datasourceHelper.getTagKeys({ filters })).slice(0, MAX_ADHOC_VARIABLE_OPTIONS);
+      // we're passing the queries so we get the labels that adhere to the queries
+      // we're also passing the scopes so we get the labels that adhere to the scopes filters
+
+      const opts = {
+        filters,
+        scopes: getClosestScopesFacade(variable)?.value,
+        queries: dataTrail.getQueries(),
+      };
+
+      // if there are too many queries it takes to much time to process the requests.
+      // In this case we favour responsiveness over reducing the number of options.
+      if (opts.queries.length > 20) {
+        opts.queries = [];
+      }
+
+      const values = (await datasourceHelper.getTagKeys(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS);
       // use replace: true to override the default lookup in adhoc filter variable
       return { replace: true, values };
     },
@@ -175,12 +205,73 @@ export function limitAdhocProviders(
       // remove current selected filter if updating a chosen filter
       const filters = filtersValues.filter((f) => f.key !== filter.key);
       // call getTagValues and truncate the response
-      const values = (await datasourceHelper.getTagValues({ key: filter.key, filters })).slice(
-        0,
-        MAX_ADHOC_VARIABLE_OPTIONS
-      );
+      // we're passing the queries so we get the label values that adhere to the queries
+      // we're also passing the scopes so we get the label values that adhere to the scopes filters
+
+      const opts = {
+        key: filter.key,
+        filters,
+        scopes: getClosestScopesFacade(variable)?.value,
+        queries: dataTrail.getQueries(),
+      };
+
+      // if there are too many queries it takes to much time to process the requests.
+      // In this case we favour responsiveness over reducing the number of options.
+      if (opts.queries.length > 20) {
+        opts.queries = [];
+      }
+
+      const values = (await datasourceHelper.getTagValues(opts)).slice(0, MAX_ADHOC_VARIABLE_OPTIONS);
       // use replace: true to override the default lookup in adhoc filter variable
       return { replace: true, values };
     },
   });
 }
+
+export type SuggestionsResponse = {
+  data: string[];
+  status: 'success' | 'error';
+  error?: 'string';
+  warnings?: string[];
+};
+
+// Suggestions API is an API that receives adhoc filters, scopes and queries and returns the labels or label values that match the provided parameters
+// Under the hood it does exactly what the label and label values API where doing but the processing is done in the BE rather than in the FE
+export async function callSuggestionsApi(
+  dataSourceUid: string,
+  timeRange: RawTimeRange,
+  scopes: Scope[],
+  adHocVariableFilters: AdHocVariableFilter[],
+  labelName: string | undefined,
+  limit: number | undefined,
+  requestId: string
+): Promise<FetchResponse<SuggestionsResponse>> {
+  return await lastValueFrom(
+    getBackendSrv().fetch<SuggestionsResponse>({
+      url: `/api/datasources/uid/${dataSourceUid}/resources/suggestions`,
+      data: {
+        labelName,
+        queries: [],
+        scopes: scopes.reduce<ScopeSpecFilter[]>((acc, scope) => {
+          acc.push(...scope.spec.filters);
+
+          return acc;
+        }, []),
+        adhocFilters: adHocVariableFilters.map((filter) => ({
+          key: filter.key,
+          operator: scopeFilterOperatorMap[filter.operator],
+          value: filter.value,
+          values: filter.values,
+        })),
+        start: getPrometheusTime(timeRange.from, false).toString(),
+        end: getPrometheusTime(timeRange.to, true).toString(),
+        limit,
+      },
+      requestId,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    })
+  );
+}