diff --git a/.betterer.results b/.betterer.results index 7e77e31e128..59dba053ba6 100644 --- a/.betterer.results +++ b/.betterer.results @@ -6562,9 +6562,6 @@ exports[`better eslint`] = { [0, 0, 0, "Do not use export all (\`export * from ...\`)", "0"], [0, 0, 0, "Do not use export all (\`export * from ...\`)", "1"] ], - "public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"] - ], "public/app/plugins/datasource/azuremonitor/components/ArgQueryEditor/index.tsx:5381": [ [0, 0, 0, "Do not re-export imported variable (\`./ArgQueryEditor\`)", "0"] ], @@ -6611,6 +6608,9 @@ exports[`better eslint`] = { "public/app/plugins/datasource/azuremonitor/types/templateVariables.ts:5381": [ [0, 0, 0, "Do not re-export imported variable (\`../dataquery.gen\`)", "0"] ], + "public/app/plugins/datasource/azuremonitor/utils/common.ts:5381": [ + [0, 0, 0, "Do not use any type assertions.", "0"] + ], "public/app/plugins/datasource/azuremonitor/utils/messageFromError.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], diff --git a/docs/sources/datasources/azure-monitor/template-variables/index.md b/docs/sources/datasources/azure-monitor/template-variables/index.md index 73711ee701a..1db472a4251 100644 --- a/docs/sources/datasources/azure-monitor/template-variables/index.md +++ b/docs/sources/datasources/azure-monitor/template-variables/index.md @@ -48,18 +48,18 @@ For an introduction to templating and template variables, refer to the [Templati You can specify these Azure Monitor data source queries in the Variable edit view's **Query Type** field. -| Name | Description | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------ | -| **Subscriptions** | Returns subscriptions. | -| **Resource Groups** | Returns resource groups for a specified. Supports multi-value. subscription. | -| **Namespaces** | Returns metric namespaces for the specified subscription and resource group. | -| **Regions** | Returns regions for the specified subscription | -| **Resource Names** | Returns a list of resource names for a specified subscription, resource group and namespace. Supports multi-value. | -| **Metric Names** | Returns a list of metric names for a resource. | -| **Workspaces** | Returns a list of workspaces for the specified subscription. | -| **Logs** | Use a KQL query to return values. | -| **Custom Namespaces** | Returns metric namespaces for the specified resource. | -| **Custom Metric Names** | Returns a list of custom metric names for the specified resource. | +| Name | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| **Subscriptions** | Returns subscriptions. | +| **Resource Groups** | Returns resource groups for a specified subscription. Supports multi-value. | +| **Namespaces** | Returns metric namespaces for the specified subscription. If a resource group is provided, only the namespaces within that group are returned. | +| **Regions** | Returns regions for the specified subscription | +| **Resource Names** | Returns a list of resource names for a specified subscription, resource group and namespace. Supports multi-value. | +| **Metric Names** | Returns a list of metric names for a resource. | +| **Workspaces** | Returns a list of workspaces for the specified subscription. | +| **Logs** | Use a KQL query to return values. | +| **Custom Namespaces** | Returns metric namespaces for the specified resource. | +| **Custom Metric Names** | Returns a list of custom metric names for the specified resource. | {{< admonition type="note" >}} Custom metrics cannot be emitted against a subscription or resource group. Select resources only when you need to retrieve custom metric namespaces or custom metric names associated with a specific resource. diff --git a/public/app/plugins/datasource/azuremonitor/__mocks__/datasource.ts b/public/app/plugins/datasource/azuremonitor/__mocks__/datasource.ts index f4f4dcaf0d5..f3ced557fde 100644 --- a/public/app/plugins/datasource/azuremonitor/__mocks__/datasource.ts +++ b/public/app/plugins/datasource/azuremonitor/__mocks__/datasource.ts @@ -74,6 +74,8 @@ export default function createMockDatasource(overrides?: DeepPartial getResourceURIFromWorkspace: jest.fn().mockReturnValue(''), getResourceURIDisplayProperties: jest.fn().mockResolvedValue({}), }, + + azureResourceGraphDatasource: {}, getVariablesRaw: jest.fn().mockReturnValue([]), currentUserAuth: false, ...overrides, diff --git a/public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts b/public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts index 42da2efb178..38a10e79a3a 100644 --- a/public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts +++ b/public/app/plugins/datasource/azuremonitor/azure_monitor/azure_monitor_datasource.ts @@ -2,7 +2,7 @@ import { find, startsWith } from 'lodash'; import { AzureCredentials } from '@grafana/azure-sdk'; import { ScopedVars } from '@grafana/data'; -import { DataSourceWithBackend, getTemplateSrv, TemplateSrv, VariableInterpolation } from '@grafana/runtime'; +import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; import { getCredentials } from '../credentials'; import TimegrainConverter from '../time_grain_converter'; @@ -27,7 +27,7 @@ import { Metric, MetricNamespace, } from '../types'; -import { routeNames } from '../utils/common'; +import { replaceTemplateVariables, routeNames } from '../utils/common'; import migrateQuery from '../utils/migrateQuery'; import ResponseParser from './response_parser'; @@ -123,7 +123,9 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend< migratedTarget.subscription || this.defaultSubscriptionId, scopedVars ); - const resources = migratedQuery.resources?.map((r) => this.replaceTemplateVariables(r, scopedVars)).flat(); + const resources = migratedQuery.resources + ?.map((r) => replaceTemplateVariables(this.templateSrv, r, scopedVars)) + .flat(); const metricNamespace = this.templateSrv.replace(migratedQuery.metricNamespace, scopedVars); const customNamespace = this.templateSrv.replace(migratedQuery.customNamespace, scopedVars); const timeGrain = this.templateSrv.replace((migratedQuery.timeGrain || '').toString(), scopedVars); @@ -185,7 +187,7 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend< } async getResourceNames(query: AzureGetResourceNamesQuery, skipToken?: string) { - const promises = this.replaceTemplateVariables(query).map( + const promises = replaceTemplateVariables(this.templateSrv, query).map( ({ metricNamespace, subscriptionId, resourceGroup, region }) => { const validMetricNamespace = startsWith(metricNamespace?.toLowerCase(), 'microsoft.storage/storageaccounts/') ? 'microsoft.storage/storageaccounts' @@ -348,47 +350,7 @@ export default class AzureMonitorDatasource extends DataSourceWithBackend< // { resourceGroup: 'rg1', resourceName: 'res1' } which is valid but // { resourceGroup: ['rg1', 'rg2'], resourceName: ['res2'] } would result in // { resourceGroup: 'rg1', resourceName: 'res2' } which is not. - return this.replaceTemplateVariables(query, scopedVars)[0]; - } - - private replaceTemplateVariables(query: T, scopedVars?: ScopedVars) { - const workingQueries: Array<{ [K in keyof T]: string }> = [{ ...query }]; - const keys = Object.keys(query) as Array; - keys.forEach((key) => { - const rawValue = workingQueries[0][key]; - let interpolated: VariableInterpolation[] = []; - const replaced = this.templateSrv.replace(rawValue, scopedVars, 'raw', interpolated); - if (interpolated.length > 0) { - for (const variable of interpolated) { - if (variable.found === false) { - continue; - } - if (variable.value.includes(',')) { - const multiple = variable.value.split(','); - const currentQueries = [...workingQueries]; - multiple.forEach((value, i) => { - currentQueries.forEach((q) => { - if (i === 0) { - q[key] = rawValue.replace(variable.match, value); - } else { - workingQueries.push({ ...q, [key]: rawValue.replace(variable.match, value) }); - } - }); - }); - } else { - workingQueries.forEach((q) => { - q[key] = replaced; - }); - } - } - } else { - workingQueries.forEach((q) => { - q[key] = replaced; - }); - } - }); - - return workingQueries; + return replaceTemplateVariables(this.templateSrv, query, scopedVars)[0]; } async getProvider(providerName: string) { diff --git a/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.test.ts b/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.test.ts index 4da9bab3078..d5e1818f193 100644 --- a/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.test.ts +++ b/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.test.ts @@ -3,11 +3,14 @@ import { set, get } from 'lodash'; import { CustomVariableModel } from '@grafana/data'; import { Context, createContext } from '../__mocks__/datasource'; +import { createMockInstanceSetttings } from '../__mocks__/instanceSettings'; import createMockQuery from '../__mocks__/query'; import { createTemplateVariables } from '../__mocks__/utils'; import { multiVariable, singleVariable, subscriptionsVariable } from '../__mocks__/variables'; import { AzureQueryType } from '../types'; +import AzureResourceGraphDatasource from './azure_resource_graph_datasource'; + let getTempVars = () => [] as CustomVariableModel[]; let replace = () => ''; @@ -150,4 +153,27 @@ describe('AzureResourceGraphDatasource', () => { }) ); }); + + describe('pagedResourceGraphRequest', () => { + it('makes multiple requests when it is returned a skip token', async () => { + const instanceSettings = createMockInstanceSetttings(); + const datasource = new AzureResourceGraphDatasource(instanceSettings); + const postResource = jest.fn(); + datasource.postResource = postResource; + const mockResponses = [ + { data: ['some resource data'], $skipToken: 'skipToken' }, + { data: ['some more resource data'] }, + ]; + for (const response of mockResponses) { + postResource.mockResolvedValueOnce(response); + } + + await datasource.pagedResourceGraphRequest('some query'); + + expect(postResource).toHaveBeenCalledTimes(2); + const secondCall = postResource.mock.calls[1]; + const [_, postBody] = secondCall; + expect(postBody.options.$skipToken).toEqual('skipToken'); + }); + }); }); diff --git a/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.ts b/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.ts index c815517314d..7c0edc53294 100644 --- a/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.ts +++ b/public/app/plugins/datasource/azuremonitor/azure_resource_graph/azure_resource_graph_datasource.ts @@ -2,15 +2,34 @@ import _ from 'lodash'; import { ScopedVars } from '@grafana/data'; -import { getTemplateSrv, DataSourceWithBackend } from '@grafana/runtime'; +import { getTemplateSrv, DataSourceWithBackend, TemplateSrv } from '@grafana/runtime'; -import { AzureMonitorQuery, AzureMonitorDataSourceJsonData, AzureQueryType } from '../types'; -import { interpolateVariable } from '../utils/common'; +import { resourceTypes } from '../azureMetadata'; +import { + AzureMonitorQuery, + AzureMonitorDataSourceJsonData, + AzureQueryType, + AzureMonitorDataSourceInstanceSettings, + RawAzureResourceItem, + AzureGraphResponse, + AzureResourceGraphOptions, +} from '../types'; +import { interpolateVariable, replaceTemplateVariables, routeNames } from '../utils/common'; export default class AzureResourceGraphDatasource extends DataSourceWithBackend< AzureMonitorQuery, AzureMonitorDataSourceJsonData > { + resourcePath: string; + resourceGraphURL = '/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01'; + constructor( + instanceSettings: AzureMonitorDataSourceInstanceSettings, + private readonly templateSrv: TemplateSrv = getTemplateSrv() + ) { + super(instanceSettings); + this.resourcePath = routeNames.resourceGraph; + } + filterQuery(item: AzureMonitorQuery): boolean { return !!item.azureResourceGraph?.query && !!item.subscriptions && item.subscriptions.length > 0; } @@ -43,4 +62,66 @@ export default class AzureResourceGraphDatasource extends DataSourceWithBackend< }, }; } + + async pagedResourceGraphRequest(query: string, maxRetries = 1): Promise { + try { + let allFetched = false; + let $skipToken = undefined; + let response: T[] = []; + while (!allFetched) { + // The response may include several pages + let options: Partial = {}; + if ($skipToken) { + options = { + $skipToken, + }; + } + const queryResponse = await this.postResource>( + this.resourcePath + this.resourceGraphURL, + { + query: query, + options: { + resultFormat: 'objectArray', + ...options, + }, + } + ); + response = response.concat(queryResponse.data); + $skipToken = queryResponse.$skipToken; + allFetched = !$skipToken; + } + + return response; + } catch (error) { + if (maxRetries > 0) { + return this.pagedResourceGraphRequest(query, maxRetries - 1); + } + + throw error; + } + } + + // Retrieve metric namespaces relevant to a subscription/resource group/resource + async getMetricNamespaces(resourceUri: string) { + const promises = replaceTemplateVariables(this.templateSrv, { resourceUri }).map(async ({ resourceUri }) => { + const namespacesFilter = resourceTypes.map((type) => `"${type}"`).join(','); + const query = ` + resources + | where id hasprefix "${resourceUri}" + | where type in (${namespacesFilter}) + | project type + | distinct type + | order by tolower(type) asc`; + + const namespaces = await this.pagedResourceGraphRequest(query); + + return namespaces.map((r) => { + return { + text: r.type, + value: r.type, + }; + }); + }); + return (await Promise.all(promises)).flat(); + } } diff --git a/public/app/plugins/datasource/azuremonitor/components/VariableEditor/VariableEditor.tsx b/public/app/plugins/datasource/azuremonitor/components/VariableEditor/VariableEditor.tsx index 24aad61c8b5..e238f4468f4 100644 --- a/public/app/plugins/datasource/azuremonitor/components/VariableEditor/VariableEditor.tsx +++ b/public/app/plugins/datasource/azuremonitor/components/VariableEditor/VariableEditor.tsx @@ -158,7 +158,7 @@ const VariableEditor = (props: Props) => { // When resource group is set, retrieve metric namespaces (aka resource types for a custom metric and custom metric namespace query) useEffect(() => { if (subscription && resourceGroup) { - datasource.getMetricNamespaces(subscription, resourceGroup).then((rgs) => { + datasource.getMetricNamespaces(subscription, resourceGroup, undefined, false, true).then((rgs) => { setNamespaces(rgs.map((s) => ({ label: s.text, value: s.value }))); }); } diff --git a/public/app/plugins/datasource/azuremonitor/datasource.ts b/public/app/plugins/datasource/azuremonitor/datasource.ts index ce4b2966519..0735d1a891f 100644 --- a/public/app/plugins/datasource/azuremonitor/datasource.ts +++ b/public/app/plugins/datasource/azuremonitor/datasource.ts @@ -47,8 +47,8 @@ export default class Datasource extends DataSourceWithBackend( + templateSrv: TemplateSrv, + query: T, + scopedVars?: ScopedVars +) { + const workingQueries: Array<{ [K in keyof T]: string }> = [{ ...query }]; + const keys = Object.keys(query) as Array; + keys.forEach((key) => { + const rawValue = workingQueries[0][key]; + let interpolated: VariableInterpolation[] = []; + const replaced = templateSrv.replace(rawValue, scopedVars, 'raw', interpolated); + if (interpolated.length > 0) { + for (const variable of interpolated) { + if (variable.found === false) { + continue; + } + if (variable.value.includes(',')) { + const multiple = variable.value.split(','); + const currentQueries = [...workingQueries]; + multiple.forEach((value, i) => { + currentQueries.forEach((q) => { + if (i === 0) { + q[key] = rawValue.replace(variable.match, value); + } else { + workingQueries.push({ ...q, [key]: rawValue.replace(variable.match, value) }); + } + }); + }); + } else { + workingQueries.forEach((q) => { + q[key] = replaced; + }); + } + } + } else { + workingQueries.forEach((q) => { + q[key] = replaced; + }); + } + }); + + return workingQueries; +} diff --git a/public/app/plugins/datasource/azuremonitor/variables.ts b/public/app/plugins/datasource/azuremonitor/variables.ts index 794f0283ab8..49486e077fa 100644 --- a/public/app/plugins/datasource/azuremonitor/variables.ts +++ b/public/app/plugins/datasource/azuremonitor/variables.ts @@ -53,9 +53,15 @@ export class VariableSupport extends CustomVariableSupport