[release-11.5.4] Azure Monitor: Filter namespaces by resource group (#103654)

Azure Monitor: Filter namespaces by resource group (#100325)

(cherry picked from commit 4c5a906c83)

Co-authored-by: Alyssa (Bull) Joyner <58453566+alyssabull@users.noreply.github.com>

.betterer.results

@@ -6562,9 +6562,6 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"],
       [0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"]
     ],
-    "public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts:5381": [
-      [0, 0, 0, "Do not use any type assertions.", "0"]
-    ],
     "public/app/plugins/datasource/azuremonitor/components/ArgQueryEditor/index.tsx:5381": [
       [0, 0, 0, "Do not re-export imported variable (\`./ArgQueryEditor\`)", "0"]
     ],
@@ -6611,6 +6608,9 @@ exports[`better eslint`] = {
     "public/app/plugins/datasource/azuremonitor/types/templateVariables.ts:5381": [
       [0, 0, 0, "Do not re-export imported variable (\`../dataquery.gen\`)", "0"]
     ],
+    "public/app/plugins/datasource/azuremonitor/utils/common.ts:5381": [
+      [0, 0, 0, "Do not use any type assertions.", "0"]
+    ],
     "public/app/plugins/datasource/azuremonitor/utils/messageFromError.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"]
     ],

docs/sources/datasources/azure-monitor/template-variables/index.md

@@ -49,10 +49,10 @@ For an introduction to templating and template variables, refer to the [Templati
 You can specify these Azure Monitor data source queries in the Variable edit view's **Query Type** field.
 
 | Name                    | Description                                                                                                                                    |
-| ----------------------- | ------------------------------------------------------------------------------------------------------------------ |
+| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
 | **Subscriptions**       | Returns subscriptions.                                                                                                                         |
-| **Resource Groups**     | Returns resource groups for a specified. Supports multi-value. subscription.                                       |
-| **Namespaces**          | Returns metric namespaces for the specified subscription and resource group.                                       |
+| **Resource Groups**     | Returns resource groups for a specified subscription. Supports multi-value.                                                                    |
+| **Namespaces**          | Returns metric namespaces for the specified subscription. If a resource group is provided, only the namespaces within that group are returned. |
 | **Regions**             | Returns regions for the specified subscription                                                                                                 |
 | **Resource Names**      | Returns a list of resource names for a specified subscription, resource group and namespace. Supports multi-value.                             |
 | **Metric Names**        | Returns a list of metric names for a resource.                                                                                                 |

public/app/plugins/datasource/azuremonitor/__mocks__/datasource.ts

@@ -74,6 +74,8 @@ export default function createMockDatasource(overrides?: DeepPartial<Datasource>
       getResourceURIFromWorkspace: jest.fn().mockReturnValue(''),
       getResourceURIDisplayProperties: jest.fn().mockResolvedValue({}),
     },
+
+    azureResourceGraphDatasource: {},
     getVariablesRaw: jest.fn().mockReturnValue([]),
     currentUserAuth: false,
     ...overrides,

public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts

@@ -2,7 +2,7 @@ import { find, startsWith } from 'lodash';
 
 import { AzureCredentials } from '@grafana/azure-sdk';
 import { ScopedVars } from '@grafana/data';
-import { DataSourceWithBackend, getTemplateSrv, TemplateSrv, VariableInterpolation } from '@grafana/runtime';
+import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime';
 
 import { getCredentials } from '../credentials';
 import TimegrainConverter from '../time_grain_converter';
@@ -27,7 +27,7 @@ import {
   Metric,
   MetricNamespace,
 } from '../types';
-import { routeNames } from '../utils/common';
+import { replaceTemplateVariables, routeNames } from '../utils/common';
 import migrateQuery from '../utils/migrateQuery';
 
 import ResponseParser from './response_parser';
@@ -123,7 +123,9 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend<
       migratedTarget.subscription || this.defaultSubscriptionId,
       scopedVars
     );
-    const resources = migratedQuery.resources?.map((r) => this.replaceTemplateVariables(r, scopedVars)).flat();
+    const resources = migratedQuery.resources
+      ?.map((r) => replaceTemplateVariables(this.templateSrv, r, scopedVars))
+      .flat();
     const metricNamespace = this.templateSrv.replace(migratedQuery.metricNamespace, scopedVars);
     const customNamespace = this.templateSrv.replace(migratedQuery.customNamespace, scopedVars);
     const timeGrain = this.templateSrv.replace((migratedQuery.timeGrain || '').toString(), scopedVars);
@@ -185,7 +187,7 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend<
   }
 
   async getResourceNames(query: AzureGetResourceNamesQuery, skipToken?: string) {
-    const promises = this.replaceTemplateVariables(query).map(
+    const promises = replaceTemplateVariables(this.templateSrv, query).map(
       ({ metricNamespace, subscriptionId, resourceGroup, region }) => {
         const validMetricNamespace = startsWith(metricNamespace?.toLowerCase(), 'microsoft.storage/storageaccounts/')
           ? 'microsoft.storage/storageaccounts'
@@ -348,47 +350,7 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend<
     // { resourceGroup: 'rg1', resourceName: 'res1' } which is valid but
     // { resourceGroup: ['rg1', 'rg2'], resourceName: ['res2'] } would result in
     // { resourceGroup: 'rg1', resourceName: 'res2' } which is not.
-    return this.replaceTemplateVariables(query, scopedVars)[0];
-  }
-
-  private replaceTemplateVariables<T extends { [K in keyof T]: string }>(query: T, scopedVars?: ScopedVars) {
-    const workingQueries: Array<{ [K in keyof T]: string }> = [{ ...query }];
-    const keys = Object.keys(query) as Array<keyof T>;
-    keys.forEach((key) => {
-      const rawValue = workingQueries[0][key];
-      let interpolated: VariableInterpolation[] = [];
-      const replaced = this.templateSrv.replace(rawValue, scopedVars, 'raw', interpolated);
-      if (interpolated.length > 0) {
-        for (const variable of interpolated) {
-          if (variable.found === false) {
-            continue;
-          }
-          if (variable.value.includes(',')) {
-            const multiple = variable.value.split(',');
-            const currentQueries = [...workingQueries];
-            multiple.forEach((value, i) => {
-              currentQueries.forEach((q) => {
-                if (i === 0) {
-                  q[key] = rawValue.replace(variable.match, value);
-                } else {
-                  workingQueries.push({ ...q, [key]: rawValue.replace(variable.match, value) });
-                }
-              });
-            });
-          } else {
-            workingQueries.forEach((q) => {
-              q[key] = replaced;
-            });
-          }
-        }
-      } else {
-        workingQueries.forEach((q) => {
-          q[key] = replaced;
-        });
-      }
-    });
-
-    return workingQueries;
+    return replaceTemplateVariables(this.templateSrv, query, scopedVars)[0];
   }
 
   async getProvider(providerName: string) {

public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.test.ts

@@ -3,11 +3,14 @@ import { set, get } from 'lodash';
 import { CustomVariableModel } from '@grafana/data';
 
 import { Context, createContext } from '../__mocks__/datasource';
+import { createMockInstanceSetttings } from '../__mocks__/instanceSettings';
 import createMockQuery from '../__mocks__/query';
 import { createTemplateVariables } from '../__mocks__/utils';
 import { multiVariable, singleVariable, subscriptionsVariable } from '../__mocks__/variables';
 import { AzureQueryType } from '../types';
 
+import AzureResourceGraphDatasource from './azure_resource_graph_datasource';
+
 let getTempVars = () => [] as CustomVariableModel[];
 let replace = () => '';
 
@@ -150,4 +153,27 @@ describe('AzureResourceGraphDatasource', () => {
       })
     );
   });
+
+  describe('pagedResourceGraphRequest', () => {
+    it('makes multiple requests when it is returned a skip token', async () => {
+      const instanceSettings = createMockInstanceSetttings();
+      const datasource = new AzureResourceGraphDatasource(instanceSettings);
+      const postResource = jest.fn();
+      datasource.postResource = postResource;
+      const mockResponses = [
+        { data: ['some resource data'], $skipToken: 'skipToken' },
+        { data: ['some more resource data'] },
+      ];
+      for (const response of mockResponses) {
+        postResource.mockResolvedValueOnce(response);
+      }
+
+      await datasource.pagedResourceGraphRequest('some query');
+
+      expect(postResource).toHaveBeenCalledTimes(2);
+      const secondCall = postResource.mock.calls[1];
+      const [_, postBody] = secondCall;
+      expect(postBody.options.$skipToken).toEqual('skipToken');
+    });
+  });
 });

public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.ts

@@ -2,15 +2,34 @@
 import _ from 'lodash';
 
 import { ScopedVars } from '@grafana/data';
-import { getTemplateSrv, DataSourceWithBackend } from '@grafana/runtime';
+import { getTemplateSrv, DataSourceWithBackend, TemplateSrv } from '@grafana/runtime';
 
-import { AzureMonitorQuery, AzureMonitorDataSourceJsonData, AzureQueryType } from '../types';
-import { interpolateVariable } from '../utils/common';
+import { resourceTypes } from '../azureMetadata';
+import {
+  AzureMonitorQuery,
+  AzureMonitorDataSourceJsonData,
+  AzureQueryType,
+  AzureMonitorDataSourceInstanceSettings,
+  RawAzureResourceItem,
+  AzureGraphResponse,
+  AzureResourceGraphOptions,
+} from '../types';
+import { interpolateVariable, replaceTemplateVariables, routeNames } from '../utils/common';
 
 export default class AzureResourceGraphDatasource extends DataSourceWithBackend<
   AzureMonitorQuery,
   AzureMonitorDataSourceJsonData
 > {
+  resourcePath: string;
+  resourceGraphURL = '/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01';
+  constructor(
+    instanceSettings: AzureMonitorDataSourceInstanceSettings,
+    private readonly templateSrv: TemplateSrv = getTemplateSrv()
+  ) {
+    super(instanceSettings);
+    this.resourcePath = routeNames.resourceGraph;
+  }
+
   filterQuery(item: AzureMonitorQuery): boolean {
     return !!item.azureResourceGraph?.query && !!item.subscriptions && item.subscriptions.length > 0;
   }
@@ -43,4 +62,66 @@ export default class AzureResourceGraphDatasource extends DataSourceWithBackend<
       },
     };
   }
+
+  async pagedResourceGraphRequest<T = unknown>(query: string, maxRetries = 1): Promise<T[]> {
+    try {
+      let allFetched = false;
+      let $skipToken = undefined;
+      let response: T[] = [];
+      while (!allFetched) {
+        // The response may include several pages
+        let options: Partial<AzureResourceGraphOptions> = {};
+        if ($skipToken) {
+          options = {
+            $skipToken,
+          };
+        }
+        const queryResponse = await this.postResource<AzureGraphResponse<T[]>>(
+          this.resourcePath + this.resourceGraphURL,
+          {
+            query: query,
+            options: {
+              resultFormat: 'objectArray',
+              ...options,
+            },
+          }
+        );
+        response = response.concat(queryResponse.data);
+        $skipToken = queryResponse.$skipToken;
+        allFetched = !$skipToken;
+      }
+
+      return response;
+    } catch (error) {
+      if (maxRetries > 0) {
+        return this.pagedResourceGraphRequest(query, maxRetries - 1);
+      }
+
+      throw error;
+    }
+  }
+
+  // Retrieve metric namespaces relevant to a subscription/resource group/resource
+  async getMetricNamespaces(resourceUri: string) {
+    const promises = replaceTemplateVariables(this.templateSrv, { resourceUri }).map(async ({ resourceUri }) => {
+      const namespacesFilter = resourceTypes.map((type) => `"${type}"`).join(',');
+      const query = `
+        resources
+        | where id hasprefix "${resourceUri}"
+        | where type in (${namespacesFilter})
+        | project type
+        | distinct type
+        | order by tolower(type) asc`;
+
+      const namespaces = await this.pagedResourceGraphRequest<RawAzureResourceItem>(query);
+
+      return namespaces.map((r) => {
+        return {
+          text: r.type,
+          value: r.type,
+        };
+      });
+    });
+    return (await Promise.all(promises)).flat();
+  }
 }

public/app/plugins/datasource/azuremonitor/components/VariableEditor/VariableEditor.tsx

@@ -158,7 +158,7 @@ const VariableEditor = (props: Props) => {
   // When resource group is set, retrieve metric namespaces (aka resource types for a custom metric and custom metric namespace query)
   useEffect(() => {
     if (subscription && resourceGroup) {
-      datasource.getMetricNamespaces(subscription, resourceGroup).then((rgs) => {
+      datasource.getMetricNamespaces(subscription, resourceGroup, undefined, false, true).then((rgs) => {
         setNamespaces(rgs.map((s) => ({ label: s.text, value: s.value })));
       });
     }

public/app/plugins/datasource/azuremonitor/datasource.ts

@@ -47,8 +47,8 @@ export default class Datasource extends DataSourceWithBackend<AzureMonitorQuery,
   ) {
     super(instanceSettings);
     this.azureMonitorDatasource = new AzureMonitorDatasource(instanceSettings);
-    this.azureLogAnalyticsDatasource = new AzureLogAnalyticsDatasource(instanceSettings);
     this.azureResourceGraphDatasource = new AzureResourceGraphDatasource(instanceSettings);
+    this.azureLogAnalyticsDatasource = new AzureLogAnalyticsDatasource(instanceSettings);
     this.resourcePickerData = new ResourcePickerData(instanceSettings, this.azureMonitorDatasource);
 
     this.pseudoDatasource = {
@@ -168,7 +168,13 @@ export default class Datasource extends DataSourceWithBackend<AzureMonitorQuery,
     return this.azureMonitorDatasource.getResourceGroups(this.templateSrv.replace(subscriptionId));
   }
 
-  getMetricNamespaces(subscriptionId: string, resourceGroup?: string, resourceUri?: string, custom?: boolean) {
+  getMetricNamespaces(
+    subscriptionId: string,
+    resourceGroup?: string,
+    resourceUri?: string,
+    custom?: boolean,
+    variableQuery?: boolean
+  ) {
     let url = `/subscriptions/${subscriptionId}`;
     if (resourceGroup) {
       url += `/resourceGroups/${resourceGroup}`;
@@ -176,6 +182,14 @@ export default class Datasource extends DataSourceWithBackend<AzureMonitorQuery,
     if (resourceUri) {
       url = resourceUri;
     }
+
+    // For variable queries it's more efficient to use resource graph
+    // Using resource graph allows us to return namespaces irrespective of a users permissions
+    // This also ensure the returned namespaces are filtered to the selected resource group when specified
+    if (variableQuery) {
+      return this.azureResourceGraphDatasource.getMetricNamespaces(url);
+    }
+
     return this.azureMonitorDatasource.getMetricNamespaces(
       { resourceUri: url },
       // If custom namespaces are being queried we do not issue the query against the global region

public/app/plugins/datasource/azuremonitor/utils/common.ts

@@ -1,6 +1,7 @@
 import { map } from 'lodash';
 
-import { SelectableValue, VariableWithMultiSupport } from '@grafana/data';
+import { ScopedVars, SelectableValue, VariableWithMultiSupport } from '@grafana/data';
+import { TemplateSrv, VariableInterpolation } from '@grafana/runtime';
 
 import { AzureMonitorOption, VariableOptionGroup } from '../types';
 
@@ -72,3 +73,47 @@ export function interpolateVariable(
   });
   return quotedValues.join(',');
 }
+
+export function replaceTemplateVariables<T extends { [K in keyof T]: string }>(
+  templateSrv: TemplateSrv,
+  query: T,
+  scopedVars?: ScopedVars
+) {
+  const workingQueries: Array<{ [K in keyof T]: string }> = [{ ...query }];
+  const keys = Object.keys(query) as Array<keyof T>;
+  keys.forEach((key) => {
+    const rawValue = workingQueries[0][key];
+    let interpolated: VariableInterpolation[] = [];
+    const replaced = templateSrv.replace(rawValue, scopedVars, 'raw', interpolated);
+    if (interpolated.length > 0) {
+      for (const variable of interpolated) {
+        if (variable.found === false) {
+          continue;
+        }
+        if (variable.value.includes(',')) {
+          const multiple = variable.value.split(',');
+          const currentQueries = [...workingQueries];
+          multiple.forEach((value, i) => {
+            currentQueries.forEach((q) => {
+              if (i === 0) {
+                q[key] = rawValue.replace(variable.match, value);
+              } else {
+                workingQueries.push({ ...q, [key]: rawValue.replace(variable.match, value) });
+              }
+            });
+          });
+        } else {
+          workingQueries.forEach((q) => {
+            q[key] = replaced;
+          });
+        }
+      }
+    } else {
+      workingQueries.forEach((q) => {
+        q[key] = replaced;
+      });
+    }
+  });
+
+  return workingQueries;
+}

public/app/plugins/datasource/azuremonitor/variables.ts

@@ -53,9 +53,15 @@ export class VariableSupport extends CustomVariableSupport<DataSource, AzureMoni
             return { data: [] };
           case AzureQueryType.NamespacesQuery:
             if (queryObj.subscription && this.hasValue(queryObj.subscription)) {
-              const rgs = await this.datasource.getMetricNamespaces(queryObj.subscription, queryObj.resourceGroup);
+              const namespaces = await this.datasource.getMetricNamespaces(
+                queryObj.subscription,
+                queryObj.resourceGroup,
+                undefined,
+                false,
+                true
+              );
               return {
-                data: rgs?.length ? [toDataFrame(rgs)] : [],
+                data: namespaces?.length ? [toDataFrame(namespaces)] : [],
               };
             }
             return { data: [] };