Usage insights query caching (#47893)

* Updates queryResponse tests to include new cachedResponse bool

* Adds cachedResponse bool to QueryResponse

* Adds tests to assert the correct query counts (totalQueries and cachedQueries) are dispatched for data-requests

* Adds totalQueries and cachedQueries counts to the data-request events

* Adds new metrics and their descriptions to docs

* uses more descriptive variable name

* changes naming

* removes hyphen in docs

* extracts calculations to own lines prior to assignment

docs/sources/enterprise/usage-insights/export-logs.md

@@ -45,6 +45,8 @@ Logs of usage insights contain the following fields, where the fields followed b
 | `tokenId`\* | number | ID of the user’s authentication token. |
 | `username`\* | string | Name of the Grafana user that made the request. |
 | `userId`\* | number | ID of the Grafana user that made the request. |
+| `totalQueries`\* | number | Number of queries executed for the data request. |
+| `cachedQueries`\* | number | Number of fetched queries that came from the cache. |
 
 ## Configuration
 

packages/grafana-data/src/types/data.ts

@@ -47,6 +47,9 @@ export interface QueryResultMeta {
   /** The path for live stream updates for this frame */
   channel?: string;
 
+  /** Did the query response come from the cache */
+  isCachedResponse?: boolean;
+
   /**
    * Optionally identify which topic the frame should be assigned to.
    * A value specified in the response will override what the request asked for.

packages/grafana-runtime/src/types/analytics.ts

@@ -55,6 +55,8 @@ export interface DashboardViewEventPayload extends DashboardInfo {
  */
 export interface DataRequestEventPayload extends DataRequestInfo {
   eventName: MetaAnalyticsEventName.DataRequest;
+  totalQueries?: number;
+  cachedQueries?: number;
 }
 
 /**

packages/grafana-runtime/src/utils/queryResponse.test.ts

@@ -298,10 +298,12 @@ describe('Query Response parser', () => {
       };
     });
 
-    test('adds notice for responses with X-Cache: HIT header', () => {
+    test('adds notice and cached boolean for responses with X-Cache: HIT header', () => {
       const queries: DataQuery[] = [{ refId: 'A' }];
       resp.headers.set('X-Cache', 'HIT');
-      expect(toDataQueryResponse(resp, queries).data[0].meta.notices).toStrictEqual([cachedResponseNotice]);
+      const meta = toDataQueryResponse(resp, queries).data[0].meta;
+      expect(meta.notices).toStrictEqual([cachedResponseNotice]);
+      expect(meta.isCachedResponse).toBeTruthy();
     });
 
     test('does not remove existing notices', () => {
@@ -314,10 +316,11 @@ describe('Query Response parser', () => {
       ]);
     });
 
-    test('does not add notice for responses with X-Cache: MISS header', () => {
+    test('does not add notice or cached response boolean for responses with X-Cache: MISS header', () => {
       const queries: DataQuery[] = [{ refId: 'A' }];
       resp.headers.set('X-Cache', 'MISS');
       expect(toDataQueryResponse(resp, queries).data[0].meta?.notices).toBeUndefined();
+      expect(toDataQueryResponse(resp, queries).data[0].meta?.isCachedResponse).toBeUndefined();
     });
 
     test('does not add notice for responses without X-Cache header', () => {

packages/grafana-runtime/src/utils/queryResponse.ts

@@ -152,6 +152,7 @@ function addCacheNotice(frame: DataFrameJSON): DataFrameJSON {
       meta: {
         ...frame.schema?.meta,
         notices: [...(frame.schema?.meta?.notices ?? []), cachedResponseNotice],
+        isCachedResponse: true,
       },
     },
   };

public/app/features/query/state/queryAnalytics.test.ts

@@ -1,7 +1,7 @@
 import { MetaAnalyticsEventName, reportMetaAnalytics } from '@grafana/runtime';
-import { CoreApp, DataQueryRequest, DataSourceApi, dateTime, LoadingState, PanelData } from '@grafana/data';
+import { CoreApp, DataFrame, DataQueryRequest, DataSourceApi, dateTime, LoadingState, PanelData } from '@grafana/data';
 import { emitDataRequestEvent } from './queryAnalytics';
-import { DashboardModel } from '../../dashboard/state/DashboardModel';
+import { DashboardModel } from '../../dashboard/state';
 
 beforeEach(() => {
   jest.clearAllMocks();
@@ -40,7 +40,39 @@ jest.mock('@grafana/data', () => ({
   },
 }));
 
-function getTestData(requestApp: string): PanelData {
+const partiallyCachedSeries = [
+  {
+    refId: 'A',
+    meta: {
+      isCachedResponse: true,
+    },
+    fields: [],
+    length: 0,
+  },
+  {
+    refId: 'B',
+    fields: [],
+    length: 0,
+  },
+];
+
+const multipleDataframesWithSameRefId = [
+  {
+    refId: 'A',
+    meta: {
+      isCachedResponse: true,
+    },
+    fields: [],
+    length: 0,
+  },
+  {
+    refId: 'A',
+    fields: [],
+    length: 0,
+  },
+];
+
+function getTestData(requestApp: string, series: DataFrame[] = []): PanelData {
   const now = dateTime();
   return {
     request: {
@@ -50,7 +82,7 @@ function getTestData(requestApp: string): PanelData {
       startTime: now.unix(),
       endTime: now.add(1, 's').unix(),
     } as DataQueryRequest,
-    series: [],
+    series,
     state: LoadingState.Done,
     timeRange: {
       from: dateTime(),
@@ -61,10 +93,9 @@ function getTestData(requestApp: string): PanelData {
 }
 
 describe('emitDataRequestEvent - from a dashboard panel', () => {
-  const data = getTestData(CoreApp.Dashboard);
-  const fn = emitDataRequestEvent(datasource);
   it('Should report meta analytics', () => {
-    fn(data);
+    const data = getTestData(CoreApp.Dashboard);
+    emitDataRequestEvent(datasource)(data);
 
     expect(reportMetaAnalytics).toBeCalledTimes(1);
     expect(reportMetaAnalytics).toBeCalledWith(
@@ -79,19 +110,71 @@ describe('emitDataRequestEvent - from a dashboard panel', () => {
         folderName: 'Test Folder',
         dataSize: 0,
         duration: 1,
+        totalQueries: 0,
+        cachedQueries: 0,
+      })
+    );
+  });
+
+  it('Should report meta analytics with counts for cached and total queries', () => {
+    const data = getTestData(CoreApp.Dashboard, partiallyCachedSeries);
+    emitDataRequestEvent(datasource)(data);
+
+    expect(reportMetaAnalytics).toBeCalledTimes(1);
+    expect(reportMetaAnalytics).toBeCalledWith(
+      expect.objectContaining({
+        eventName: MetaAnalyticsEventName.DataRequest,
+        datasourceName: datasource.name,
+        datasourceId: datasource.id,
+        panelId: 2,
+        dashboardId: 1,
+        dashboardName: 'Test Dashboard',
+        dashboardUid: 'test',
+        folderName: 'Test Folder',
+        dataSize: 2,
+        duration: 1,
+        totalQueries: 2,
+        cachedQueries: 1,
+      })
+    );
+  });
+
+  it('Should report meta analytics with counts for cached and total queries when same refId spread across multiple DataFrames', () => {
+    const data = getTestData(CoreApp.Dashboard, multipleDataframesWithSameRefId);
+    emitDataRequestEvent(datasource)(data);
+
+    expect(reportMetaAnalytics).toBeCalledTimes(1);
+    expect(reportMetaAnalytics).toBeCalledWith(
+      expect.objectContaining({
+        eventName: MetaAnalyticsEventName.DataRequest,
+        datasourceName: datasource.name,
+        datasourceId: datasource.id,
+        panelId: 2,
+        dashboardId: 1,
+        dashboardName: 'Test Dashboard',
+        dashboardUid: 'test',
+        folderName: 'Test Folder',
+        dataSize: 2,
+        duration: 1,
+        totalQueries: 1,
+        cachedQueries: 1,
       })
     );
   });
 
   it('Should not report meta analytics twice if the request receives multiple responses', () => {
+    const data = getTestData(CoreApp.Dashboard);
+    const fn = emitDataRequestEvent(datasource);
     fn(data);
-    expect(reportMetaAnalytics).not.toBeCalled();
+    fn(data);
+    expect(reportMetaAnalytics).toBeCalledTimes(1);
   });
 
   it('Should not report meta analytics in edit mode', () => {
     mockGetUrlSearchParams.mockImplementationOnce(() => {
       return { editPanel: 2 };
     });
+    const data = getTestData(CoreApp.Dashboard);
     emitDataRequestEvent(datasource)(data);
     expect(reportMetaAnalytics).not.toBeCalled();
   });

public/app/features/query/state/queryAnalytics.ts

@@ -19,6 +19,16 @@ export function emitDataRequestEvent(datasource: DataSourceApi) {
       return;
     }
 
+    const queryCacheStatus: { [key: string]: boolean } = {};
+    for (let i = 0; i < data.series.length; i++) {
+      const refId = data.series[i].refId;
+      if (refId && !queryCacheStatus[refId]) {
+        queryCacheStatus[refId] = data.series[i].meta?.isCachedResponse ?? false;
+      }
+    }
+    const totalQueries = Object.keys(queryCacheStatus).length;
+    const cachedQueries = Object.values(queryCacheStatus).filter((val) => val === true).length;
+
     const eventData: DataRequestEventPayload = {
       eventName: MetaAnalyticsEventName.DataRequest,
       datasourceName: datasource.name,
@@ -28,6 +38,8 @@ export function emitDataRequestEvent(datasource: DataSourceApi) {
       dashboardId: data.request.dashboardId,
       dataSize: 0,
       duration: data.request.endTime! - data.request.startTime,
+      totalQueries,
+      cachedQueries,
     };
 
     // enrich with dashboard info