mirror of https://github.com/grafana/grafana
E2E: Add plugin-e2e scenario verification tests (#79969)
* add playwright test and plugin-e2e * run tests in ci * add ds config tests * add panel edit tests * add annotation test * add variable edit page tests * add explore page tests * add panel plugin tests * add readme * remove comments * fix broken test * remove user.json * remove newline in starlark * fix lint issue * ignore failure of playwright tests * update code owners * add detailed error messages in every expect * update message frame * fix link * upload report to gcp * echo url * add playwright developer guide * bump plugin-e2e * add custom provisioning dir * update plugin-e2e * remove not used imports * fix typo * minor fixes * use latest version of plugin-e2e * fix broken link * use latest plugin-e2e * add feature toggle scenario verification tests * bump version * use auth file from package * fix type error * add panel data assertions * rename parent dir and bump version * fix codeowners * reset files * remove not used file * update plugin-e2e * separate tests per role * pass prov dir * skip using provisioning fixture * wip * fix permission test * move to e2e dir * fix path to readme * post comment with report url * format starlark * post comment with report url * post comment with report url * fix token * make test fail * fix exit code * bump version * bump to latest plugin-e2e * revert reporting message * remove comments * readding report comment * change exit code * format starlark * force test to fail * add new step that posts comment * fix link * use latest playwright image * fix failing test * format starlark * remove unused fixture Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com> --------- Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>pull/83299/head
Erik Sundell
1 year ago
committed by
GitHub
parent
b25667223c
commit
3e456127cb
@ -0,0 +1,8 @@ |
||||
# @grafana/plugin-e2e API tests |
||||
|
||||
The purpose of the E2E tests in this directory is not to test the plugins per se - it's to verify that the fixtures, models and expect matchers provided by the [`@grafana/plugin-e2e`](https://github.com/grafana/plugin-tools/tree/main/packages/plugin-e2e) package are compatible with the latest version of Grafana. If you find that any of these tests are failing, it's probably due to one of the following reasons: |
||||
|
||||
- you have changed a value of a selector defined in @grafana/e2e-selector |
||||
- you have made structural changes to the UI |
||||
|
||||
For information on how to address this, follow the instructions in the [contributing guidelines](https://github.com/grafana/plugin-tools/blob/main/packages/plugin-e2e/CONTRIBUTING.md#how-to-fix-broken-test-scenarios-after-changes-in-grafana) for the @grafana/plugin-e2e package in the plugin-tools repository. |
||||
@ -0,0 +1,15 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
import { successfulAnnotationQuery } from '../mocks/queries'; |
||||
|
||||
test('annotation query data with mocked response', async ({ annotationEditPage, page }) => { |
||||
annotationEditPage.mockQueryDataResponse(successfulAnnotationQuery); |
||||
await annotationEditPage.datasource.set('gdev-testdata'); |
||||
await page.getByLabel('Scenario').last().fill('CSV Content'); |
||||
await page.keyboard.press('Tab'); |
||||
await expect( |
||||
annotationEditPage.runQuery(), |
||||
formatExpectError('Expected annotation query to execute successfully') |
||||
).toBeOK(); |
||||
}); |
||||
@ -0,0 +1,35 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
|
||||
test.describe('test createDataSourceConfigPage fixture, saveAndTest and toBeOK matcher', () => { |
||||
test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => { |
||||
const configPage = await createDataSourceConfigPage({ type: 'prometheus' }); |
||||
await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090'); |
||||
await expect( |
||||
configPage.saveAndTest(), |
||||
formatExpectError('Expected save data source config to fail when Prometheus server is not running') |
||||
).not.toBeOK(); |
||||
}); |
||||
|
||||
test('valid credentials should return a 200 status code', async ({ createDataSourceConfigPage, page }) => { |
||||
const configPage = await createDataSourceConfigPage({ type: 'prometheus' }); |
||||
configPage.mockHealthCheckResponse({ status: 200 }); |
||||
await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090'); |
||||
await expect( |
||||
configPage.saveAndTest(), |
||||
formatExpectError('Expected data source config to be successfully saved') |
||||
).toBeOK(); |
||||
}); |
||||
}); |
||||
|
||||
test.describe('test data source with frontend only health check', () => { |
||||
test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage }) => { |
||||
const configPage = await createDataSourceConfigPage({ type: 'testdata' }); |
||||
await configPage.saveAndTest({ skipWaitForResponse: true }); |
||||
await expect( |
||||
configPage, |
||||
formatExpectError('Expected data source config to display success alert after save') |
||||
).toHaveAlert('success', { hasNotText: 'Datasource updated' }); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,15 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
|
||||
test('query data response should be OK when query is valid', async ({ explorePage }) => { |
||||
await explorePage.datasource.set('gdev-testdata'); |
||||
await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to execute successfully')).toBeOK(); |
||||
}); |
||||
|
||||
test('query data response should not be OK when query is invalid', async ({ explorePage }) => { |
||||
await explorePage.datasource.set('gdev-testdata'); |
||||
const queryEditorRow = await explorePage.getQueryEditorRow('A'); |
||||
await queryEditorRow.getByLabel('Labels').fill('invalid-label-format'); |
||||
await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to fail')).not.toBeOK(); |
||||
}); |
||||
@ -0,0 +1,17 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
const TRUTHY_CUSTOM_TOGGLE = 'custom_toggle1'; |
||||
const FALSY_CUSTOM_TOGGLE = 'custom_toggle2'; |
||||
|
||||
// override the feature toggles defined in playwright.config.ts only for tests in this file
|
||||
test.use({ |
||||
featureToggles: { |
||||
[TRUTHY_CUSTOM_TOGGLE]: true, |
||||
[FALSY_CUSTOM_TOGGLE]: false, |
||||
}, |
||||
}); |
||||
|
||||
test('should set and check feature toggles correctly', async ({ isFeatureToggleEnabled }) => { |
||||
expect(await isFeatureToggleEnabled(TRUTHY_CUSTOM_TOGGLE)).toBeTruthy(); |
||||
expect(await isFeatureToggleEnabled(FALSY_CUSTOM_TOGGLE)).toBeFalsy(); |
||||
}); |
||||
@ -0,0 +1,110 @@ |
||||
import { expect, test, PanelEditPage, DashboardPage } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
import { successfulDataQuery } from '../mocks/queries'; |
||||
|
||||
const REACT_TABLE_DASHBOARD = { uid: 'U_bZIMRMk' }; |
||||
|
||||
test.describe('panel edit page', () => { |
||||
test('table panel data assertions with provisioned dashboard', async ({ |
||||
page, |
||||
selectors, |
||||
grafanaVersion, |
||||
request, |
||||
}) => { |
||||
const panelEditPage = new PanelEditPage( |
||||
{ page, selectors, grafanaVersion, request }, |
||||
{ dashboard: REACT_TABLE_DASHBOARD, id: '4' } |
||||
); |
||||
await panelEditPage.goto(); |
||||
await expect( |
||||
panelEditPage.panel.locator, |
||||
formatExpectError('Could not locate panel in panel edit page') |
||||
).toBeVisible(); |
||||
await expect( |
||||
panelEditPage.panel.fieldNames, |
||||
formatExpectError('Could not locate header elements in table panel') |
||||
).toContainText(['Field', 'Max', 'Mean', 'Last']); |
||||
}); |
||||
|
||||
test('table panel data assertions', async ({ panelEditPage }) => { |
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200); |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
await panelEditPage.setVisualization('Table'); |
||||
await panelEditPage.refreshPanel(); |
||||
await expect( |
||||
panelEditPage.panel.locator, |
||||
formatExpectError('Could not locate panel in panel edit page') |
||||
).toBeVisible(); |
||||
await expect( |
||||
panelEditPage.panel.fieldNames, |
||||
formatExpectError('Could not locate header elements in table panel') |
||||
).toContainText(['col1', 'col2']); |
||||
await expect(panelEditPage.panel.data, formatExpectError('Could not locate headers in table panel')).toContainText([ |
||||
'val1', |
||||
'val2', |
||||
'val3', |
||||
'val4', |
||||
]); |
||||
}); |
||||
|
||||
test('timeseries panel - table view assertions', async ({ panelEditPage }) => { |
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200); |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
await panelEditPage.setVisualization('Time series'); |
||||
await panelEditPage.refreshPanel(); |
||||
await panelEditPage.toggleTableView(); |
||||
await expect( |
||||
panelEditPage.panel.locator, |
||||
formatExpectError('Could not locate panel in panel edit page') |
||||
).toBeVisible(); |
||||
await expect( |
||||
panelEditPage.panel.fieldNames, |
||||
formatExpectError('Could not locate header elements in table panel') |
||||
).toContainText(['col1', 'col2']); |
||||
await expect( |
||||
panelEditPage.panel.data, |
||||
formatExpectError('Could not locate data elements in table panel') |
||||
).toContainText(['val1', 'val2', 'val3', 'val4']); |
||||
}); |
||||
}); |
||||
|
||||
test.describe('dashboard page', () => { |
||||
test('getting panel by title', async ({ page, selectors, grafanaVersion, request }) => { |
||||
const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD); |
||||
await dashboardPage.goto(); |
||||
const panel = await dashboardPage.getPanelByTitle('Colored background'); |
||||
await expect(panel.fieldNames).toContainText(['Field', 'Max', 'Mean', 'Last']); |
||||
}); |
||||
|
||||
test('getting panel by id', async ({ page, selectors, grafanaVersion, request }) => { |
||||
const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD); |
||||
await dashboardPage.goto(); |
||||
const panel = await dashboardPage.getPanelById('4'); |
||||
await expect(panel.fieldNames, formatExpectError('Could not locate header elements in table panel')).toContainText([ |
||||
'Field', |
||||
'Max', |
||||
'Mean', |
||||
'Last', |
||||
]); |
||||
}); |
||||
}); |
||||
|
||||
test.describe('explore page', () => { |
||||
test('table panel', async ({ explorePage }) => { |
||||
const url = |
||||
'left=%7B"datasource":"grafana","queries":%5B%7B"queryType":"randomWalk","refId":"A","datasource":%7B"type":"datasource","uid":"grafana"%7D%7D%5D,"range":%7B"from":"1547161200000","to":"1576364400000"%7D%7D&orgId=1'; |
||||
await explorePage.goto({ |
||||
queryParams: new URLSearchParams(url), |
||||
}); |
||||
await expect( |
||||
explorePage.timeSeriesPanel.locator, |
||||
formatExpectError('Could not locate time series panel in explore page') |
||||
).toBeVisible(); |
||||
await expect( |
||||
explorePage.tablePanel.locator, |
||||
formatExpectError('Could not locate table panel in explore page') |
||||
).toBeVisible(); |
||||
await expect(explorePage.tablePanel.fieldNames).toContainText(['time', 'A-series']); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,85 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
import { successfulDataQuery } from '../mocks/queries'; |
||||
import { scenarios } from '../mocks/resources'; |
||||
|
||||
const PANEL_TITLE = 'Table panel E2E test'; |
||||
const TABLE_VIZ_NAME = 'Table'; |
||||
const STANDARD_OTIONS_CATEGORY = 'Standard options'; |
||||
const DISPLAY_NAME_LABEL = 'Display name'; |
||||
|
||||
test.describe('query editor query data', () => { |
||||
test('query data response should be OK when query is valid', async ({ panelEditPage }) => { |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
await expect( |
||||
panelEditPage.refreshPanel(), |
||||
formatExpectError('Expected panel query to execute successfully') |
||||
).toBeOK(); |
||||
}); |
||||
|
||||
test('query data response should not be OK and panel error should be displayed when query is invalid', async ({ |
||||
panelEditPage, |
||||
}) => { |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
const queryEditorRow = await panelEditPage.getQueryEditorRow('A'); |
||||
await queryEditorRow.getByLabel('Labels').fill('invalid-label-format'); |
||||
await expect(panelEditPage.refreshPanel(), formatExpectError('Expected panel query to fail')).not.toBeOK(); |
||||
await expect( |
||||
panelEditPage.panel.getErrorIcon(), |
||||
formatExpectError('Expected panel error to be displayed after query execution') |
||||
).toBeVisible(); |
||||
}); |
||||
}); |
||||
|
||||
test.describe('query editor with mocked responses', () => { |
||||
test('and resource `scenarios` is mocked', async ({ panelEditPage, selectors }) => { |
||||
await panelEditPage.mockResourceResponse('scenarios', scenarios); |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
const queryEditorRow = await panelEditPage.getQueryEditorRow('A'); |
||||
await queryEditorRow.getByLabel('Scenario').last().click(); |
||||
await expect( |
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Select.option), |
||||
formatExpectError('Expected certain select options to be displayed after clicking on the select input') |
||||
).toHaveText(scenarios.map((s) => s.name)); |
||||
}); |
||||
|
||||
test('mocked query data response', async ({ panelEditPage, selectors }) => { |
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200); |
||||
await panelEditPage.datasource.set('gdev-testdata'); |
||||
await panelEditPage.setVisualization(TABLE_VIZ_NAME); |
||||
await panelEditPage.refreshPanel(); |
||||
await expect( |
||||
panelEditPage.panel.getErrorIcon(), |
||||
formatExpectError('Did not expect panel error to be displayed after query execution') |
||||
).not.toBeVisible(); |
||||
await expect( |
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Visualization.Table.body), |
||||
formatExpectError('Expected certain select options to be displayed after clicking on the select input') |
||||
).toHaveText('val1val2val3val4'); |
||||
}); |
||||
}); |
||||
|
||||
test.describe('edit panel plugin settings', () => { |
||||
test('change viz to table panel, set panel title and collapse section', async ({ |
||||
panelEditPage, |
||||
selectors, |
||||
page, |
||||
}) => { |
||||
await panelEditPage.setVisualization(TABLE_VIZ_NAME); |
||||
await expect( |
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.PanelEditor.toggleVizPicker), |
||||
formatExpectError('Expected panel visualization to be set to table') |
||||
).toHaveText(TABLE_VIZ_NAME); |
||||
await panelEditPage.setPanelTitle(PANEL_TITLE); |
||||
await expect( |
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Panel.title(PANEL_TITLE)), |
||||
formatExpectError('Expected panel title to be updated') |
||||
).toBeVisible(); |
||||
await panelEditPage.collapseSection(STANDARD_OTIONS_CATEGORY); |
||||
await expect( |
||||
page.getByText(DISPLAY_NAME_LABEL), |
||||
formatExpectError('Expected section to be collapsed') |
||||
).toBeVisible(); |
||||
}); |
||||
}); |
||||
@ -0,0 +1,16 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
import { formatExpectError } from '../errors'; |
||||
import { prometheusLabels } from '../mocks/resources'; |
||||
|
||||
test('variable query with mocked response', async ({ variableEditPage, page }) => { |
||||
variableEditPage.mockResourceResponse('api/v1/labels?*', prometheusLabels); |
||||
await variableEditPage.datasource.set('gdev-prometheus'); |
||||
await variableEditPage.getByTestIdOrAriaLabel('Query type').fill('Label names'); |
||||
await page.keyboard.press('Tab'); |
||||
await variableEditPage.runQuery(); |
||||
await expect( |
||||
variableEditPage, |
||||
formatExpectError('Expected variable edit page to display certain label names after query execution') |
||||
).toDisplayPreviews(prometheusLabels.data); |
||||
}); |
||||
@ -0,0 +1,15 @@ |
||||
import { expect, test } from '@grafana/plugin-e2e'; |
||||
|
||||
test('should redirect to start page when permissions to navigate to page is missing', async ({ page }) => { |
||||
await page.goto('/'); |
||||
const homePageTitle = await page.title(); |
||||
await page.goto('/datasources', { waitUntil: 'networkidle' }); |
||||
expect(await page.title()).toEqual(homePageTitle); |
||||
}); |
||||
|
||||
test('current user should have viewer role', async ({ page, request }) => { |
||||
await page.goto('/'); |
||||
const response = await request.get('/api/user/orgs'); |
||||
await expect(response).toBeOK(); |
||||
await expect(await response.json()).toContainEqual(expect.objectContaining({ role: 'Viewer' })); |
||||
}); |
||||
@ -0,0 +1,4 @@ |
||||
export const formatExpectError = (message: string) => { |
||||
return `Error while verifying @grafana/plugin-e2e scenarios: ${message}.
|
||||
See https://github.com/grafana/grafana/blob/main/plugin-e2e/plugin-e2e-api-tests/README.md for more information.`;
|
||||
}; |
||||
@ -0,0 +1,77 @@ |
||||
export const successfulDataQuery = { |
||||
results: { |
||||
A: { |
||||
status: 200, |
||||
frames: [ |
||||
{ |
||||
schema: { |
||||
refId: 'A', |
||||
fields: [ |
||||
{ |
||||
name: 'col1', |
||||
type: 'string', |
||||
typeInfo: { |
||||
frame: 'string', |
||||
nullable: true, |
||||
}, |
||||
}, |
||||
{ |
||||
name: 'col2', |
||||
type: 'string', |
||||
typeInfo: { |
||||
frame: 'string', |
||||
nullable: true, |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
data: { |
||||
values: [ |
||||
['val1', 'val3'], |
||||
['val2', 'val4'], |
||||
], |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
}, |
||||
}; |
||||
|
||||
export const successfulAnnotationQuery = { |
||||
results: { |
||||
Anno: { |
||||
status: 200, |
||||
frames: [ |
||||
{ |
||||
schema: { |
||||
refId: 'Anno', |
||||
fields: [ |
||||
{ |
||||
name: 'time', |
||||
type: 'time', |
||||
typeInfo: { |
||||
frame: 'time.Time', |
||||
nullable: true, |
||||
}, |
||||
}, |
||||
{ |
||||
name: 'col2', |
||||
type: 'string', |
||||
typeInfo: { |
||||
frame: 'string', |
||||
nullable: true, |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
data: { |
||||
values: [ |
||||
[1702973084093, 1702973084099], |
||||
['val1', 'val2'], |
||||
], |
||||
}, |
||||
}, |
||||
], |
||||
}, |
||||
}, |
||||
}; |
||||
@ -0,0 +1,19 @@ |
||||
export const scenarios = [ |
||||
{ |
||||
description: '', |
||||
id: 'annotations', |
||||
name: 'Annotations', |
||||
stringInput: '', |
||||
}, |
||||
{ |
||||
description: '', |
||||
id: 'arrow', |
||||
name: 'Load Apache Arrow Data', |
||||
stringInput: '', |
||||
}, |
||||
]; |
||||
|
||||
export const prometheusLabels = { |
||||
status: 'success', |
||||
data: ['__name__', 'action', 'active', 'address'], |
||||
}; |
||||
@ -0,0 +1,65 @@ |
||||
import { defineConfig, devices } from '@playwright/test'; |
||||
import path, { dirname } from 'path'; |
||||
|
||||
import { PluginOptions } from '@grafana/plugin-e2e'; |
||||
|
||||
const testDirRoot = 'e2e/plugin-e2e/plugin-e2e-api-tests/'; |
||||
|
||||
export default defineConfig<PluginOptions>({ |
||||
fullyParallel: true, |
||||
/* Retry on CI only */ |
||||
retries: process.env.CI ? 2 : 0, |
||||
/* Opt out of parallel tests on CI. */ |
||||
workers: process.env.CI ? 1 : undefined, |
||||
reporter: 'html', |
||||
use: { |
||||
baseURL: `http://${process.env.HOST || 'localhost'}:${process.env.PORT || 3000}`, |
||||
trace: 'on-first-retry', |
||||
httpCredentials: { |
||||
username: 'admin', |
||||
password: 'admin', |
||||
}, |
||||
provisioningRootDir: path.join(process.cwd(), process.env.PROV_DIR ?? 'conf/provisioning'), |
||||
}, |
||||
projects: [ |
||||
// Login to Grafana with admin user and store the cookie on disk for use in other tests
|
||||
{ |
||||
name: 'authenticate', |
||||
testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`, |
||||
testMatch: [/.*\.js/], |
||||
}, |
||||
// Login to Grafana with new user with viewer role and store the cookie on disk for use in other tests
|
||||
{ |
||||
name: 'createUserAndAuthenticate', |
||||
testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`, |
||||
testMatch: [/.*\.js/], |
||||
use: { |
||||
user: { |
||||
user: 'viewer', |
||||
password: 'password', |
||||
role: 'Viewer', |
||||
}, |
||||
}, |
||||
}, |
||||
// Run all tests in parallel using user with admin role
|
||||
{ |
||||
name: 'admin', |
||||
testDir: path.join(testDirRoot, '/as-admin-user'), |
||||
use: { |
||||
...devices['Desktop Chrome'], |
||||
storageState: 'playwright/.auth/admin.json', |
||||
}, |
||||
dependencies: ['authenticate'], |
||||
}, |
||||
// Run all tests in parallel using user with viewer role
|
||||
{ |
||||
name: 'viewer', |
||||
testDir: path.join(testDirRoot, '/as-viewer-user'), |
||||
use: { |
||||
...devices['Desktop Chrome'], |
||||
storageState: 'playwright/.auth/viewer.json', |
||||
}, |
||||
dependencies: ['createUserAndAuthenticate'], |
||||
}, |
||||
], |
||||
}); |
||||
Loading…
Reference in new issue