Loki/ES: Add further items to query tracking (#60347)

* add more tracking to queries from ES and Loki

* improve tracking elasticsearch

* update type assertion

* remove unused import

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -638,7 +638,8 @@ export class ElasticDatasource
     const shouldRunTroughBackend =
       request.app === CoreApp.Explore && config.featureToggles.elasticsearchBackendMigration;
     if (shouldRunTroughBackend) {
-      return super.query(request).pipe(tap((response) => trackQuery(response, request.targets, request.app)));
+      const start = new Date();
+      return super.query(request).pipe(tap((response) => trackQuery(response, request, start)));
     }
     let payload = '';
     const targets = this.interpolateVariablesInQueries(cloneDeep(request.targets), request.scopedVars);
@@ -705,6 +706,7 @@ export class ElasticDatasource
 
     const url = this.getMultiSearchUrl();
 
+    const start = new Date();
     return this.post(url, payload).pipe(
       map((res) => {
         const er = new ElasticResponse(sentTargets, res);
@@ -721,7 +723,7 @@ export class ElasticDatasource
 
         return er.getTimeSeries();
       }),
-      tap((response) => trackQuery(response, request.targets, request.app))
+      tap((response) => trackQuery(response, request, start))
     );
   }
 

public/app/plugins/datasource/elasticsearch/tracking.ts

@@ -1,5 +1,5 @@
-import { CoreApp, DashboardLoadedEvent, DataQueryResponse } from '@grafana/data';
-import { reportInteraction } from '@grafana/runtime';
+import { CoreApp, DashboardLoadedEvent, DataQueryRequest, DataQueryResponse } from '@grafana/data';
+import { config, reportInteraction } from '@grafana/runtime';
 import { variableRegex } from 'app/features/variables/utils';
 
 import { REF_ID_STARTER_LOG_VOLUME } from './datasource';
@@ -110,7 +110,12 @@ const shouldNotReportBasedOnRefId = (refId: string): boolean => {
   return false;
 };
 
-export function trackQuery(response: DataQueryResponse, queries: ElasticsearchQuery[], app: string): void {
+export function trackQuery(
+  response: DataQueryResponse,
+  request: DataQueryRequest<ElasticsearchQuery> & { targets: ElasticsearchQuery[] },
+  startTime: Date
+): void {
+  const { targets: queries, app } = request;
   if (app === CoreApp.Dashboard || app === CoreApp.PanelViewer) {
     return;
   }
@@ -121,6 +126,7 @@ export function trackQuery(response: DataQueryResponse, queries: ElasticsearchQu
     }
     reportInteraction('grafana_elasticsearch_query_executed', {
       app,
+      grafana_version: config.buildInfo.version,
       with_lucene_query: query.query ? true : false,
       query_type: getQueryType(query),
       line_limit: getLineLimit(query),
@@ -128,6 +134,9 @@ export function trackQuery(response: DataQueryResponse, queries: ElasticsearchQu
       has_error: response.error !== undefined,
       has_data: response.data.some((frame) => frame.length > 0),
       simultaneously_sent_query_count: queries.length,
+      time_range_from: request?.range?.from?.toISOString(),
+      time_range_to: request?.range?.to?.toISOString(),
+      time_taken: Date.now() - startTime.getTime(),
     });
   }
 }

public/app/plugins/datasource/loki/datasource.ts

@@ -170,7 +170,7 @@ export class LokiDatasource
       .map(getNormalizedLokiQuery) // "fix" the `.queryType` prop
       .map((q) => ({ ...q, maxLines: q.maxLines || this.maxLines })); // set maxLines if not set
 
-    const fixedRequest = {
+    const fixedRequest: DataQueryRequest<LokiQuery> & { targets: LokiQuery[] } = {
       ...request,
       targets: queries,
     };
@@ -197,12 +197,13 @@ export class LokiDatasource
     if (fixedRequest.liveStreaming) {
       return this.runLiveQueryThroughBackend(fixedRequest);
     } else {
+      const startTime = new Date();
       return super.query(fixedRequest).pipe(
         // in case of an empty query, this is somehow run twice. `share()` is no workaround here as the observable is generated from `of()`.
         map((response) =>
           transformBackendResult(response, fixedRequest.targets, this.instanceSettings.jsonData.derivedFields ?? [])
         ),
-        tap((response) => trackQuery(response, fixedRequest.targets, fixedRequest.app))
+        tap((response) => trackQuery(response, fixedRequest, startTime))
       );
     }
   }

public/app/plugins/datasource/loki/tracking.ts

@@ -1,5 +1,5 @@
-import { CoreApp, DashboardLoadedEvent, DataQueryResponse } from '@grafana/data';
-import { reportInteraction } from '@grafana/runtime';
+import { CoreApp, DashboardLoadedEvent, DataQueryRequest, DataQueryResponse } from '@grafana/data';
+import { reportInteraction, config } from '@grafana/runtime';
 import { variableRegex } from 'app/features/variables/utils';
 
 import { QueryEditorMode } from '../prometheus/querybuilder/shared/types';
@@ -132,18 +132,35 @@ const shouldNotReportBasedOnRefId = (refId: string): boolean => {
   return false;
 };
 
-export function trackQuery(response: DataQueryResponse, queries: LokiQuery[], app: string): void {
+export function trackQuery(
+  response: DataQueryResponse,
+  request: DataQueryRequest<LokiQuery> & { targets: LokiQuery[] },
+  startTime: Date
+): void {
   // We only want to track usage for these specific apps
+  const { app, targets: queries } = request;
+
   if (app === CoreApp.Dashboard || app === CoreApp.PanelViewer) {
     return;
   }
 
+  let totalBytes = 0;
+  for (const frame of response.data) {
+    const byteKey = frame.meta?.custom?.lokiQueryStatKey;
+    if (byteKey) {
+      totalBytes +=
+        frame.meta?.stats?.find((stat: { displayName: string }) => stat.displayName === byteKey)?.value ?? 0;
+    }
+  }
+
   for (const query of queries) {
     if (shouldNotReportBasedOnRefId(query.refId)) {
       return;
     }
+
     reportInteraction('grafana_loki_query_executed', {
       app,
+      grafana_version: config.buildInfo.version,
       editor_mode: query.editorMode,
       has_data: response.data.some((frame) => frame.length > 0),
       has_error: response.error !== undefined,
@@ -154,6 +171,10 @@ export function trackQuery(response: DataQueryResponse, queries: LokiQuery[], ap
       query_vector_type: query.queryType,
       resolution: query.resolution,
       simultaneously_sent_query_count: queries.length,
+      time_range_from: request?.range?.from?.toISOString(),
+      time_range_to: request?.range?.to?.toISOString(),
+      time_taken: Date.now() - startTime.getTime(),
+      bytes_processed: totalBytes,
     });
   }
 }