SQL: Add timeFilter macro to query builder (#74575)

* Add timeFilter macro to query builder

* Only render SQLWhereRow when fields are there

* Change the default query to timeseries and remove 50 limit

* Add timeFilter macro for the first time when timeSeries

* Add test for timeFilter macro

* Lint fix

* Annotation query format should be table

* Set order by as default

* Revert changes that made time series default

* Fix e2e test

* Fix e2e test

* Make sure to reset the date value when operator is changed

* Add docs

docs/sources/datasources/mssql/query-editor/index.md

@@ -120,6 +120,8 @@ To filter on more columns, click the plus (`+`) button to the right of the condi
 
 To remove a filter, click the `x` button next to that filter's dropdown.
 
+After selecting a date type column, you can choose Macros from the operators list and select timeFilter which will add the $\_\_timeFilter macro to the query with the selected date column.
+
 ### Group results
 
 To group results by column, toggle the **Group** switch at the top of the editor.

docs/sources/datasources/mysql/_index.md

@@ -201,10 +201,17 @@ Add further value columns by clicking the plus button and another column dropdow
 
 ### Filter data (WHERE)
 
-To add a filter, flip the switch at the top of the editor.
-Using the first dropdown, select if all the filters need to match (AND) or if only one of the filters needs to match (OR).
+To add a filter, toggle the **Filter** switch at the top of the editor.
+This reveals a **Filter by column value** section with two dropdown selectors.
 
-To add more columns to filter on use the plus button.
+Use the first dropdown to choose whether all of the filters need to match (`AND`), or if only one of the filters needs to match (`OR`).
+Use the second dropdown to choose a filter.
+
+To filter on more columns, click the plus (`+`) button to the right of the condition dropdown.
+
+To remove a filter, click the `x` button next to that filter's dropdown.
+
+After selecting a date type column, you can choose Macros from the operators list and select timeFilter which will add the $\_\_timeFilter macro to the query with the selected date column.
 
 ### Group By
 

docs/sources/datasources/postgres/_index.md

@@ -114,10 +114,17 @@ Add further value columns by clicking the plus button and another column dropdow
 
 ### Filter data (WHERE)
 
-To add a filter, flip the switch at the top of the editor.
-Using the first dropdown, select if all the filters need to match (AND) or if only one of the filters needs to match (OR).
+To add a filter, toggle the **Filter** switch at the top of the editor.
+This reveals a **Filter by column value** section with two dropdown selectors.
 
-To add more columns to filter on use the plus button.
+Use the first dropdown to choose whether all of the filters need to match (`AND`), or if only one of the filters needs to match (`OR`).
+Use the second dropdown to choose a filter.
+
+To filter on more columns, click the plus (`+`) button to the right of the condition dropdown.
+
+To remove a filter, click the `x` button next to that filter's dropdown.
+
+After selecting a date type column, you can choose Macros from the operators list and select timeFilter which will add the $\_\_timeFilter macro to the query with the selected date column.
 
 ### Group By
 

e2e/various-suite/mysql.spec.ts

@@ -9,10 +9,6 @@ const normalTableName = tablesResponse.results.tables.frames[0].data.values[0][0
 
 describe('MySQL datasource', () => {
   beforeEach(() => {
-    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-  });
-
-  it('code editor autocomplete should handle table name escaping/quoting', () => {
     cy.intercept('POST', '/api/ds/query', (req) => {
       if (req.body.queries[0].refId === 'datasets') {
         req.alias = 'datasets';
@@ -31,11 +27,14 @@ describe('MySQL datasource', () => {
         });
       }
     });
-
+    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
     e2e.pages.Explore.visit();
 
     e2e.components.DataSourcePicker.container().should('be.visible').type('gdev-mysql{enter}');
+    cy.wait('@datasets');
+  });
 
+  it('code editor autocomplete should handle table name escaping/quoting', () => {
     cy.get("label[for^='option-code']").should('be.visible').click();
     cy.get('textarea').type('S{downArrow}{enter}');
     cy.wait('@tables');
@@ -60,4 +59,61 @@ describe('MySQL datasource', () => {
     cy.get('textarea').type('.');
     cy.get('.suggest-widget').contains('No suggestions.').should('be.visible');
   });
+
+  describe('visual query builder', () => {
+    it('should be able to add timeFilter macro', () => {
+      cy.get("[aria-label='Table selector']").should('be.visible').click();
+      selectOption(normalTableName);
+      // Open column selector
+      cy.get("[id^='select-column-0']").should('be.visible').click();
+      selectOption('createdAt');
+
+      // Toggle where row
+      cy.get("label[for^='sql-filter']").last().should('be.visible').click();
+
+      // Click add filter button
+      cy.get('button[title="Add filter"]').should('be.visible').click();
+      cy.get('button[title="Add filter"]').should('be.visible').click(); // For some reason we need to click twice
+
+      // Open field selector
+      cy.get("[aria-label='Field']").should('be.visible').click();
+      selectOption('createdAt');
+
+      // Open operator selector
+      cy.get("[aria-label='Operator']").should('be.visible').click();
+      selectOption('Macros');
+
+      // Open macros value selector
+      cy.get("[aria-label='Macros value selector']").should('be.visible').click();
+      selectOption('timeFilter');
+
+      // Validate that the timeFilter macro was added
+
+      e2e.components.CodeEditor.container()
+        .get('textarea')
+        .should(
+          'have.value',
+          `SELECT\n  createdAt\nFROM\n  DataMaker.normalTable\nWHERE\n  $__timeFilter(createdAt)\nLIMIT\n  50`
+        );
+
+      // Validate that the timeFilter macro was removed when changed to equals operator
+
+      // For some reason the input is not visible the second time so we need to force the click
+      cy.get("[aria-label='Operator']").click({ force: true });
+      selectOption('==');
+
+      e2e.components.DateTimePicker.input().should('be.visible').click().blur();
+
+      e2e.components.CodeEditor.container()
+        .get('textarea')
+        .should(
+          'not.have.value',
+          `SELECT\n  createdAt\nFROM\n  DataMaker.normalTable\nWHERE\n  $__timeFilter(createdAt)\nLIMIT\n  50`
+        );
+    });
+  });
 });
+
+function selectOption(option: string) {
+  cy.get("[aria-label='Select option']").contains(option).should('be.visible').click();
+}

packages/grafana-e2e-selectors/src/selectors/components.ts

@@ -32,6 +32,9 @@ export const Components = {
     rolePicker: 'Built-in role picker',
     permissionLevel: 'Permission Level',
   },
+  DateTimePicker: {
+    input: 'data-testid date-time-input',
+  },
   DataSource: {
     TestData: {
       QueryTab: {

packages/grafana-ui/src/components/DateTimePickers/DateTimePicker/DateTimePicker.test.tsx

@@ -3,6 +3,7 @@ import userEvent from '@testing-library/user-event';
 import React from 'react';
 
 import { dateTime } from '@grafana/data';
+import { Components } from '@grafana/e2e-selectors';
 
 import { DateTimePicker, Props } from './DateTimePicker';
 
@@ -33,7 +34,7 @@ describe('Date time picker', () => {
   it('should update date onblur', async () => {
     const onChangeInput = jest.fn();
     render(<DateTimePicker date={dateTime('2021-05-05 12:00:00')} onChange={onChangeInput} />);
-    const dateTimeInput = screen.getByTestId('date-time-input');
+    const dateTimeInput = screen.getByTestId(Components.DateTimePicker.input);
     await userEvent.clear(dateTimeInput);
     await userEvent.type(dateTimeInput, '2021-07-31 12:30:30');
     expect(dateTimeInput).toHaveDisplayValue('2021-07-31 12:30:30');
@@ -44,7 +45,7 @@ describe('Date time picker', () => {
   it('should not update onblur if invalid date', async () => {
     const onChangeInput = jest.fn();
     render(<DateTimePicker date={dateTime('2021-05-05 12:00:00')} onChange={onChangeInput} />);
-    const dateTimeInput = screen.getByTestId('date-time-input');
+    const dateTimeInput = screen.getByTestId(Components.DateTimePicker.input);
     await userEvent.clear(dateTimeInput);
     await userEvent.type(dateTimeInput, '2021:05:05 12-00-00');
     expect(dateTimeInput).toHaveDisplayValue('2021:05:05 12-00-00');

packages/grafana-ui/src/components/DateTimePickers/DateTimePicker/DateTimePicker.tsx

@@ -8,6 +8,7 @@ import { usePopper } from 'react-popper';
 import { useMedia } from 'react-use';
 
 import { dateTimeFormat, DateTime, dateTime, GrafanaTheme2, isDateTime } from '@grafana/data';
+import { Components } from '@grafana/e2e-selectors';
 
 import { useStyles2, useTheme2 } from '../../../themes';
 import { Button } from '../../Button/Button';
@@ -232,7 +233,7 @@ const DateTimeInput = React.forwardRef<HTMLInputElement, InputProps>(
           addonAfter={icon}
           value={internalDate.value}
           onBlur={onBlur}
-          data-testid="date-time-input"
+          data-testid={Components.DateTimePicker.input}
           placeholder="Select date/time"
           ref={ref}
         />

public/app/features/plugins/sql/components/visual-query-builder/AwesomeQueryBuilder.tsx

@@ -2,7 +2,6 @@ import {
   AnyObject,
   BasicConfig,
   Config,
-  JsonItem,
   JsonTree,
   Operator,
   Settings,
@@ -24,38 +23,14 @@ const buttonLabels = {
   remove: 'Remove',
 };
 
-export const emptyInitValue: JsonItem = {
-  id: Utils.uuid(),
-  type: 'group' as const,
-  children1: {
-    [Utils.uuid()]: {
-      type: 'rule',
-      properties: {
-        field: null,
-        operator: null,
-        value: [],
-        valueSrc: [],
-      },
-    },
-  },
-};
-
 export const emptyInitTree: JsonTree = {
   id: Utils.uuid(),
-  type: 'group' as const,
-  children1: {
-    [Utils.uuid()]: {
-      type: 'rule',
-      properties: {
-        field: null,
-        operator: null,
-        value: [],
-        valueSrc: [],
-      },
-    },
-  },
+  type: 'group',
 };
 
+const TIME_FILTER = 'timeFilter';
+const macros = [TIME_FILTER];
+
 export const widgets: Widgets = {
   ...BasicConfig.widgets,
   text: {
@@ -86,15 +61,47 @@ export const widgets: Widgets = {
   datetime: {
     ...BasicConfig.widgets.datetime,
     factory: function DateTimeInput(props) {
+      if (props?.operator === Op.MACROS) {
+        return (
+          <Select
+            id={props.id}
+            aria-label="Macros value selector"
+            menuShouldPortal
+            options={macros.map(toOption)}
+            value={props?.value}
+            onChange={(val) => props.setValue(val.value)}
+          />
+        );
+      }
+      const dateValue = dateTime(props?.value).isValid() ? dateTime(props?.value).utc() : undefined;
       return (
         <DateTimePicker
           onChange={(e) => {
             props?.setValue(e.format(BasicConfig.widgets.datetime.valueFormat));
           }}
-          date={dateTime(props?.value).utc()}
+          date={dateValue}
         />
       );
     },
+    sqlFormatValue: (val, field, widget, operator, operatorDefinition, rightFieldDef) => {
+      if (operator === Op.MACROS) {
+        if (macros.includes(val)) {
+          return val;
+        }
+        return undefined;
+      }
+
+      // This is just satisfying the type checker, this should never happen
+      if (
+        typeof BasicConfig.widgets.datetime.sqlFormatValue === 'string' ||
+        typeof BasicConfig.widgets.datetime.sqlFormatValue === 'object'
+      ) {
+        return undefined;
+      }
+      const func = BasicConfig.widgets.datetime.sqlFormatValue;
+      // We need to pass the ctx to this function this way so *this* is correct
+      return func?.call(BasicConfig.ctx, val, field, widget, operator, operatorDefinition, rightFieldDef) || '';
+    },
   },
 };
 
@@ -173,6 +180,7 @@ export const settings: Settings = {
 const enum Op {
   IN = 'select_any_in',
   NOT_IN = 'select_not_any_in',
+  MACROS = 'macros',
 }
 // eslint-ignore
 const customOperators = getCustomOperators(BasicConfig) as typeof BasicConfig.operators;
@@ -192,6 +200,16 @@ const customTypes = {
       text: customTextWidget,
     },
   },
+  datetime: {
+    ...BasicConfig.types.datetime,
+    widgets: {
+      ...BasicConfig.types.datetime.widgets,
+      datetime: {
+        ...BasicConfig.types.datetime.widgets.datetime,
+        operators: [Op.MACROS, ...(BasicConfig.types.datetime.widgets.datetime.operators || [])],
+      },
+    },
+  },
 };
 
 export const raqbConfig: Config = {
@@ -284,6 +302,15 @@ function getCustomOperators(config: BasicConfig) {
       ...supportedOperators[Op.NOT_IN],
       sqlFormatOp: customSqlNotInFormatter,
     },
+    [Op.MACROS]: {
+      label: 'Macros',
+      sqlFormatOp: (field: string, _operator: string, value: string | List<string>) => {
+        if (value === TIME_FILTER) {
+          return `$__timeFilter(${field})`;
+        }
+        return value;
+      },
+    },
   };
 
   return customOperators;

public/app/features/plugins/sql/datasource/SqlDatasource.ts

@@ -57,7 +57,7 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
     const settingsData = instanceSettings.jsonData || {};
     this.interval = settingsData.timeInterval || '1m';
     this.db = this.getDB();
-    /* 
+    /*
       The `settingsData.database` will be defined if a default database has been defined in either
       1) the ConfigurationEditor.tsx, OR 2) the provisioning config file, either under `jsondata.database`, or simply `database`.
     */
@@ -167,7 +167,7 @@ export abstract class SqlDatasource extends DataSourceWithBackend<SQLQuery, SQLO
     // No need to check for database change/update issues if the datasource is being used in Explore.
     if (request.app !== CoreApp.Explore) {
       /*
-        If a preconfigured datasource database has been added/updated - and the user has built ANY number of queries using a 
+        If a preconfigured datasource database has been added/updated - and the user has built ANY number of queries using a
         database OTHER than the preconfigured one, return a database issue - since those databases are no longer available.
         The user will need to update their queries to use the preconfigured database.
       */