AzureMonitor: Apply query migrations in QueryEditor (#37704)

* move query migrations out of the angular controller

* Migrate queries in QueryEditor

* finish up migrations

* update deprecated comment

* remove comment

package.json

@@ -250,6 +250,7 @@
     "dangerously-set-html-content": "1.0.6",
     "debounce-promise": "3.1.2",
     "eventemitter3": "4.0.0",
+    "fast-deep-equal": "^3.1.3",
     "fast-json-patch": "2.2.1",
     "fast-text-encoding": "^1.0.0",
     "file-saver": "2.0.2",

public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/QueryEditor.tsx

@@ -18,7 +18,7 @@ import ApplicationInsightsEditor from '../ApplicationInsightsEditor';
 import InsightsAnalyticsEditor from '../InsightsAnalyticsEditor';
 import { Space } from '../Space';
 import { debounce } from 'lodash';
-import useDefaultQuery from './useDefaultQuery';
+import usePreparedQuery from './usePreparedQuery';
 
 export type AzureMonitorQueryEditorProps = QueryEditorProps<
   AzureMonitorDatasource,
@@ -43,7 +43,7 @@ const QueryEditor: React.FC<AzureMonitorQueryEditorProps> = ({
     [onChange, onRunQuery]
   );
 
-  const query = useDefaultQuery(baseQuery, onQueryChange);
+  const query = usePreparedQuery(baseQuery, onQueryChange);
 
   const subscriptionId = query.subscription || datasource.azureMonitorDatasource.defaultSubscriptionId;
   const variableOptionGroup = {

public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/useDefaultQuery.ts

@@ -1,34 +0,0 @@
-import { useEffect, useMemo } from 'react';
-import { AzureMonitorQuery, AzureQueryType } from '../../types';
-
-const DEFAULT_QUERY_TYPE = AzureQueryType.AzureMonitor;
-
-const createQueryWithDefaults = (query: AzureMonitorQuery) => {
-  // A quick and easy way to set just the default query type. If we want to set any other defaults,
-  // we might want to look into something more robust
-  if (!query.queryType) {
-    return {
-      ...query,
-      queryType: query.queryType ?? DEFAULT_QUERY_TYPE,
-    };
-  }
-
-  return query;
-};
-
-/**
- * Returns queries with some defaults, and calls onChange function to notify if it changes
- */
-const useDefaultQuery = (query: AzureMonitorQuery, onChangeQuery: (newQuery: AzureMonitorQuery) => void) => {
-  const queryWithDefaults = useMemo(() => createQueryWithDefaults(query), [query]);
-
-  useEffect(() => {
-    if (queryWithDefaults !== query) {
-      onChangeQuery(queryWithDefaults);
-    }
-  }, [queryWithDefaults, query, onChangeQuery]);
-
-  return queryWithDefaults;
-};
-
-export default useDefaultQuery;

public/app/plugins/datasource/grafana-azure-monitor-datasource/components/QueryEditor/usePreparedQuery.ts

@@ -0,0 +1,36 @@
+import { useEffect, useMemo } from 'react';
+import { defaults } from 'lodash';
+import { AzureMonitorQuery, AzureQueryType } from '../../types';
+import deepEqual from 'fast-deep-equal';
+import migrateQuery from '../../utils/migrateQuery';
+
+const DEFAULT_QUERY = {
+  queryType: AzureQueryType.AzureMonitor,
+};
+
+const prepareQuery = (query: AzureMonitorQuery) => {
+  // Note: _.defaults does not apply default values deeply.
+  const withDefaults = defaults({}, query, DEFAULT_QUERY);
+  const migratedQuery = migrateQuery(withDefaults);
+
+  // If we didn't make any changes to the object, then return the original object to keep the
+  // identity the same, and not trigger any other useEffects or anything.
+  return deepEqual(migratedQuery, query) ? query : migratedQuery;
+};
+
+/**
+ * Returns queries with some defaults + migrations, and calls onChange function to notify if it changes
+ */
+const usePreparedQuery = (query: AzureMonitorQuery, onChangeQuery: (newQuery: AzureMonitorQuery) => void) => {
+  const preparedQuery = useMemo(() => prepareQuery(query), [query]);
+
+  useEffect(() => {
+    if (preparedQuery !== query) {
+      onChangeQuery(preparedQuery);
+    }
+  }, [preparedQuery, query, onChangeQuery]);
+
+  return preparedQuery;
+};
+
+export default usePreparedQuery;

public/app/plugins/datasource/grafana-azure-monitor-datasource/datasource.ts

@@ -3,13 +3,7 @@ import AzureMonitorDatasource from './azure_monitor/azure_monitor_datasource';
 import AppInsightsDatasource from './app_insights/app_insights_datasource';
 import AzureLogAnalyticsDatasource from './azure_log_analytics/azure_log_analytics_datasource';
 import ResourcePickerData from './resourcePicker/resourcePickerData';
-import {
+import { AzureDataSourceJsonData, AzureMonitorQuery, AzureQueryType, DatasourceValidationResult } from './types';
-  AzureDataSourceJsonData,
-  AzureMonitorQuery,
-  AzureQueryType,
-  DatasourceValidationResult,
-  InsightsAnalyticsQuery,
-} from './types';
 import {
   DataFrame,
   DataQueryRequest,
@@ -22,7 +16,7 @@ import {
 import { forkJoin, Observable, of } from 'rxjs';
 import { getTemplateSrv, TemplateSrv } from '@grafana/runtime';
 import InsightsAnalyticsDatasource from './insights_analytics/insights_analytics_datasource';
-import { migrateMetricsDimensionFilters } from './query_ctrl';
+import { datasourceMigrations } from './utils/migrateQuery';
 import { map } from 'rxjs/operators';
 import AzureResourceGraphDatasource from './azure_resource_graph/azure_resource_graph_datasource';
 import { getAzureCloud } from './credentials';
@@ -82,9 +76,9 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
   query(options: DataQueryRequest<AzureMonitorQuery>): Observable<DataQueryResponse> {
     const byType = new Map<AzureQueryType, DataQueryRequest<AzureMonitorQuery>>();
 
-    for (const target of options.targets) {
+    for (const baseTarget of options.targets) {
-      // Migrate old query structure
+      // Migrate old query structures
-      migrateQuery(target);
+      const target = datasourceMigrations(baseTarget);
 
       // Skip hidden or invalid queries or ones without properties
       if (!target.queryType || target.hide || !hasQueryForType(target)) {
@@ -298,23 +292,6 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
   }
 }
 
-function migrateQuery(target: AzureMonitorQuery) {
-  if (target.queryType === AzureQueryType.ApplicationInsights) {
-    if ((target.appInsights as any).rawQuery) {
-      target.queryType = AzureQueryType.InsightsAnalytics;
-      target.insightsAnalytics = (target.appInsights as unknown) as InsightsAnalyticsQuery;
-      delete target.appInsights;
-    }
-  }
-  if (!target.queryType) {
-    target.queryType = AzureQueryType.AzureMonitor;
-  }
-
-  if (target.queryType === AzureQueryType.AzureMonitor && target.azureMonitor) {
-    migrateMetricsDimensionFilters(target.azureMonitor);
-  }
-}
-
 function hasQueryForType(query: AzureMonitorQuery): boolean {
   switch (query.queryType) {
     case AzureQueryType.AzureMonitor:

public/app/plugins/datasource/grafana-azure-monitor-datasource/types/query.ts

@@ -36,8 +36,11 @@ export interface AzureMonitorQuery extends DataQuery {
  */
 export interface AzureMetricQuery {
   resourceGroup?: string;
-  resourceName?: string;
+
+  /** Resource type */
   metricDefinition?: string;
+
+  resourceName?: string;
   metricNamespace?: string;
   metricName?: string;
   timeGrain?: string;
@@ -51,6 +54,12 @@ export interface AzureMetricQuery {
 
   /** @deprecated Remove this once angular is removed */
   allowedTimeGrainsMs?: number[];
+
+  /** @deprecated This property was migrated to dimensionFilters and should only be accessed in the migration */
+  dimension?: string;
+
+  /** @deprecated This property was migrated to dimensionFilters and should only be accessed in the migration */
+  dimensionFilter?: string;
 }
 
 /**
@@ -86,6 +95,9 @@ export interface ApplicationInsightsQuery {
   dimension?: string[]; // Was string before 7.1
   dimensionFilter?: string;
   alias?: string;
+
+  /** @deprecated Migrated to Insights Analytics query  */
+  rawQuery?: string;
 }
 
 /**

public/app/plugins/datasource/grafana-azure-monitor-datasource/utils/migrateQuery.test.ts

@@ -0,0 +1,40 @@
+import { AzureMonitorQuery, AzureQueryType } from '../types';
+import migrateQuery from './migrateQuery';
+
+const modernMetricsQuery: AzureMonitorQuery = {
+  appInsights: { dimension: [], metricName: 'select', timeGrain: 'auto' },
+  azureLogAnalytics: {
+    query:
+      '//change this example to create your own time series query\n<table name>                                                              //the table to query (e.g. Usage, Heartbeat, Perf)\n| where $__timeFilter(TimeGenerated)                                      //this is a macro used to show the full chart’s time range, choose the datetime column here\n| summarize count() by <group by column>, bin(TimeGenerated, $__interval) //change “group by column” to a column in your table, such as “Computer”. The $__interval macro is used to auto-select the time grain. Can also use 1h, 5m etc.\n| order by TimeGenerated asc',
+    resultFormat: 'time_series',
+    workspace: 'e3fe4fde-ad5e-4d60-9974-e2f3562ffdf2',
+  },
+  azureMonitor: {
+    aggregation: 'Average',
+    alias: '{{ dimensionvalue }}',
+    allowedTimeGrainsMs: [60000, 300000, 900000, 1800000, 3600000, 21600000, 43200000, 86400000],
+    dimensionFilters: [{ dimension: 'dependency/success', filter: '', operator: 'eq' }],
+    metricDefinition: 'microsoft.insights/components',
+    metricName: 'dependencies/duration',
+    metricNamespace: 'microsoft.insights/components',
+    resourceGroup: 'cloud-datasources',
+    resourceName: 'AppInsightsTestData',
+    timeGrain: 'PT5M',
+    top: '10',
+  },
+  azureResourceGraph: { resultFormat: 'table' },
+  insightsAnalytics: { query: '', resultFormat: 'time_series' },
+  queryType: AzureQueryType.AzureMonitor,
+  refId: 'A',
+  subscription: '44693801-6ee6-49de-9b2d-9106972f9572',
+  subscriptions: ['44693801-6ee6-49de-9b2d-9106972f9572'],
+};
+
+describe('AzureMonitor: migrateQuery', () => {
+  it('modern queries should not change', () => {
+    const result = migrateQuery(modernMetricsQuery);
+
+    // MUST use .toBe because we want to assert that the identity of unmigrated queries remains the same
+    expect(modernMetricsQuery).toBe(result);
+  });
+});

public/app/plugins/datasource/grafana-azure-monitor-datasource/utils/migrateQuery.ts

@@ -0,0 +1,165 @@
+import { AzureMonitorQuery, AzureQueryType } from '../types';
+import TimegrainConverter from '../time_grain_converter';
+import {
+  appendDimensionFilter,
+  setTimeGrain as setMetricsTimeGrain,
+} from '../components/MetricsQueryEditor/setQueryValue';
+import { setKustoQuery } from '../components/LogsQueryEditor/setQueryValue';
+
+const OLD_DEFAULT_DROPDOWN_VALUE = 'select';
+
+export default function migrateQuery(query: AzureMonitorQuery): AzureMonitorQuery {
+  let workingQuery = query;
+
+  // The old angular controller also had a `migrateApplicationInsightsKeys` migraiton that
+  // migrated old properties to other properties that still do not appear to be used anymore, so
+  // we decided to not include that migration anymore
+  // See https://github.com/grafana/grafana/blob/a6a09add/public/app/plugins/datasource/grafana-azure-monitor-datasource/query_ctrl.ts#L269-L288
+
+  workingQuery = migrateTimeGrains(workingQuery);
+  workingQuery = migrateLogAnalyticsToFromTimes(workingQuery);
+  workingQuery = migrateToDefaultNamespace(workingQuery);
+  workingQuery = migrateApplicationInsightsDimensions(workingQuery);
+  workingQuery = migrateMetricsDimensionFilters(workingQuery);
+
+  return workingQuery;
+}
+
+function migrateTimeGrains(query: AzureMonitorQuery): AzureMonitorQuery {
+  let workingQuery = query;
+
+  if (workingQuery.azureMonitor?.timeGrainUnit && workingQuery.azureMonitor.timeGrain !== 'auto') {
+    const newTimeGrain = TimegrainConverter.createISO8601Duration(
+      workingQuery.azureMonitor.timeGrain ?? 'auto',
+      workingQuery.azureMonitor.timeGrainUnit
+    );
+    workingQuery = setMetricsTimeGrain(workingQuery, newTimeGrain);
+
+    delete workingQuery.azureMonitor?.timeGrainUnit;
+  }
+
+  if (workingQuery.appInsights?.timeGrainUnit && workingQuery.appInsights.timeGrain !== 'auto') {
+    const appInsights = {
+      ...workingQuery.appInsights,
+    };
+
+    if (workingQuery.appInsights.timeGrainCount) {
+      appInsights.timeGrain = TimegrainConverter.createISO8601Duration(
+        workingQuery.appInsights.timeGrainCount,
+        workingQuery.appInsights.timeGrainUnit
+      );
+    } else {
+      appInsights.timeGrainCount = workingQuery.appInsights.timeGrain;
+
+      if (workingQuery.appInsights.timeGrain) {
+        appInsights.timeGrain = TimegrainConverter.createISO8601Duration(
+          workingQuery.appInsights.timeGrain,
+          workingQuery.appInsights.timeGrainUnit
+        );
+      }
+    }
+
+    workingQuery = {
+      ...workingQuery,
+      appInsights: appInsights,
+    };
+  }
+
+  return workingQuery;
+}
+
+function migrateLogAnalyticsToFromTimes(query: AzureMonitorQuery): AzureMonitorQuery {
+  let workingQuery = query;
+
+  if (workingQuery.azureLogAnalytics?.query?.match(/\$__from\s/gi)) {
+    workingQuery = setKustoQuery(
+      workingQuery,
+      workingQuery.azureLogAnalytics.query.replace(/\$__from\s/gi, '$__timeFrom() ')
+    );
+  }
+
+  if (workingQuery.azureLogAnalytics?.query?.match(/\$__to\s/gi)) {
+    workingQuery = setKustoQuery(
+      workingQuery,
+      workingQuery.azureLogAnalytics.query.replace(/\$__to\s/gi, '$__timeTo() ')
+    );
+  }
+
+  return workingQuery;
+}
+
+function migrateToDefaultNamespace(query: AzureMonitorQuery): AzureMonitorQuery {
+  const haveMetricNamespace =
+    query.azureMonitor?.metricNamespace && query.azureMonitor.metricNamespace !== OLD_DEFAULT_DROPDOWN_VALUE;
+
+  if (!haveMetricNamespace && query.azureMonitor?.metricDefinition) {
+    return {
+      ...query,
+      azureMonitor: {
+        ...query.azureMonitor,
+        metricNamespace: query.azureMonitor.metricDefinition,
+      },
+    };
+  }
+
+  return query;
+}
+
+function migrateApplicationInsightsDimensions(query: AzureMonitorQuery): AzureMonitorQuery {
+  const dimension = query?.appInsights?.dimension as unknown;
+
+  if (dimension && typeof dimension === 'string') {
+    return {
+      ...query,
+      appInsights: {
+        ...query.appInsights,
+        dimension: [dimension],
+      },
+    };
+  }
+
+  return query;
+}
+
+// Exported because its also used directly in the datasource.ts for some reason
+function migrateMetricsDimensionFilters(query: AzureMonitorQuery): AzureMonitorQuery {
+  let workingQuery = query;
+
+  const oldDimension = workingQuery.azureMonitor?.dimension;
+  if (oldDimension && oldDimension !== 'None') {
+    workingQuery = appendDimensionFilter(workingQuery, oldDimension, 'eq', workingQuery.azureMonitor?.dimensionFilter);
+  }
+
+  return workingQuery;
+}
+
+// datasource.ts also contains some migrations, which have been moved to here. Unsure whether
+// they should also do all the other migrations...
+export function datasourceMigrations(query: AzureMonitorQuery): AzureMonitorQuery {
+  let workingQuery = query;
+
+  if (workingQuery.queryType === AzureQueryType.ApplicationInsights && workingQuery.appInsights?.rawQuery) {
+    workingQuery = {
+      ...workingQuery,
+      queryType: AzureQueryType.InsightsAnalytics,
+      appInsights: undefined,
+      insightsAnalytics: {
+        query: workingQuery.appInsights.rawQuery,
+        resultFormat: 'time_series',
+      },
+    };
+  }
+
+  if (!workingQuery.queryType) {
+    workingQuery = {
+      ...workingQuery,
+      queryType: AzureQueryType.AzureMonitor,
+    };
+  }
+
+  if (workingQuery.queryType === AzureQueryType.AzureMonitor && workingQuery.azureMonitor) {
+    workingQuery = migrateMetricsDimensionFilters(workingQuery);
+  }
+
+  return workingQuery;
+}