From 74487726fca073a24e0ff735fd7ed849894b1933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Wed, 16 Apr 2025 06:21:06 +0200 Subject: [PATCH] e2e: improves flakiness and speed (#103533) * e2e: improves flakiness * Chore: refactor flaky test * e2e: more refactor * e2e: do not keep successfull test run videos * e2e: do not log selectors * chore: updates after pr feedback * chore: updates after pr feedback * chore: adds retries to flaky e2e test * e2e: skip flaky tests * e2e: skip flaky tests * e2e: revert back to timeout * chore: removes all the skips --- cypress.config.js | 18 ++++ .../panels-suite/geomap-map-controls.spec.ts | 56 +++++++----- e2e/old-arch/utils/support/types.ts | 4 +- e2e/panels-suite/geomap-map-controls.spec.ts | 56 +++++++----- e2e/utils/support/types.ts | 4 +- e2e/various-suite/bookmarks.spec.ts | 85 +++++++++++-------- 6 files changed, 143 insertions(+), 80 deletions(-) diff --git a/cypress.config.js b/cypress.config.js index a03692e2ac0..a76a3d562b5 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -13,6 +13,9 @@ module.exports = defineConfig({ viewportWidth: 1920, viewportHeight: 1080, + env: { + LOG_SELECTORS_INFO: false, + }, e2e: { supportFile: './e2e/cypress/support/e2e.js', setupNodeEvents(on, config) { @@ -82,6 +85,21 @@ module.exports = defineConfig({ // IMPORTANT: return the updated browser launch options return launchOptions; }); + + on('after:spec', (_, results) => { + if (!results || !results.video || !results.tests) { + return; + } + + // Do we have failures for any retry attempts? + const failures = results.tests.some((test) => test.attempts.some((attempt) => attempt.state === 'failed')); + if (failures) { + return; + } + + // delete the video if the spec passed and no tests retried + fs.unlinkSync(results.video); + }); }, }, }); diff --git a/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts b/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts index fbbc1c2c516..b344dae5abe 100644 --- a/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts +++ b/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts @@ -1,5 +1,6 @@ import { e2e } from '../utils'; const DASHBOARD_ID = 'P2jR04WVk'; +const TIMEOUT = 45000; describe('Geomap layer controls options', () => { beforeEach(() => { @@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => { it('Tests map controls options', () => { e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } }); - e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist'); + // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv' + e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible'); + e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible'); - // Show zoom + // Show zoom field e2e.components.PanelEditor.showZoomField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.contains('+'); - cy.get('.ol-zoom').should('exist'); - // Show attribution e2e.components.PanelEditor.showAttributionField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.get('.ol-attribution').should('exist'); - // Show scale e2e.components.PanelEditor.showScaleField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.get('.ol-scale-line').should('exist'); - // Show measure tool e2e.components.PanelEditor.showMeasureField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - e2e.components.PanelEditor.measureButton().should('exist'); - // Show debug e2e.components.PanelEditor.showDebugField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); + }); + + e2e.components.Panels.Panel.content({ timeout: TIMEOUT }) + .should('be.visible') + .within(() => { + // Verify zoom + cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible'); + + // Verify attribution + cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible'); + + // Verify scale + cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible'); + + // Verify measure tool + e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible'); + + // Verify debug tool + e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible'); }); - e2e.components.DebugOverlay.wrapper().should('exist'); }); }); diff --git a/e2e/old-arch/utils/support/types.ts b/e2e/old-arch/utils/support/types.ts index e7ccfe91db7..11030ee3e80 100644 --- a/e2e/old-arch/utils/support/types.ts +++ b/e2e/old-arch/utils/support/types.ts @@ -32,7 +32,9 @@ export type E2EFactoryArgs = { selectors: S }; export type CypressOptions = Partial; const processSelectors = (e2eObjects: E2EFunctions, selectors: S): E2EFunctions => { - const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data); + const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO')); + const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {}; + const keys = Object.keys(selectors); for (let index = 0; index < keys.length; index++) { const key = keys[index]; diff --git a/e2e/panels-suite/geomap-map-controls.spec.ts b/e2e/panels-suite/geomap-map-controls.spec.ts index fbbc1c2c516..b344dae5abe 100644 --- a/e2e/panels-suite/geomap-map-controls.spec.ts +++ b/e2e/panels-suite/geomap-map-controls.spec.ts @@ -1,5 +1,6 @@ import { e2e } from '../utils'; const DASHBOARD_ID = 'P2jR04WVk'; +const TIMEOUT = 45000; describe('Geomap layer controls options', () => { beforeEach(() => { @@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => { it('Tests map controls options', () => { e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } }); - e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist'); + // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv' + e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible'); + e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible'); - // Show zoom + // Show zoom field e2e.components.PanelEditor.showZoomField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.contains('+'); - cy.get('.ol-zoom').should('exist'); - // Show attribution e2e.components.PanelEditor.showAttributionField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.get('.ol-attribution').should('exist'); - // Show scale e2e.components.PanelEditor.showScaleField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - cy.get('.ol-scale-line').should('exist'); - // Show measure tool e2e.components.PanelEditor.showMeasureField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); }); - e2e.components.PanelEditor.measureButton().should('exist'); - // Show debug e2e.components.PanelEditor.showDebugField() - .should('exist') + .should('be.visible') .within(() => { - cy.get('input[type="checkbox"]').check({ force: true }); + cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked'); + }); + + e2e.components.Panels.Panel.content({ timeout: TIMEOUT }) + .should('be.visible') + .within(() => { + // Verify zoom + cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible'); + + // Verify attribution + cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible'); + + // Verify scale + cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible'); + + // Verify measure tool + e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible'); + + // Verify debug tool + e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible'); }); - e2e.components.DebugOverlay.wrapper().should('exist'); }); }); diff --git a/e2e/utils/support/types.ts b/e2e/utils/support/types.ts index e7ccfe91db7..11030ee3e80 100644 --- a/e2e/utils/support/types.ts +++ b/e2e/utils/support/types.ts @@ -32,7 +32,9 @@ export type E2EFactoryArgs = { selectors: S }; export type CypressOptions = Partial; const processSelectors = (e2eObjects: E2EFunctions, selectors: S): E2EFunctions => { - const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data); + const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO')); + const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {}; + const keys = Object.keys(selectors); for (let index = 0; index < keys.length; index++) { const key = keys[index]; diff --git a/e2e/various-suite/bookmarks.spec.ts b/e2e/various-suite/bookmarks.spec.ts index bda56fc763b..95781d1e6e2 100644 --- a/e2e/various-suite/bookmarks.spec.ts +++ b/e2e/various-suite/bookmarks.spec.ts @@ -3,56 +3,73 @@ import { fromBaseUrl } from '../utils/support/url'; describe('Pin nav items', () => { beforeEach(() => { - cy.viewport(1280, 800); e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD')); - cy.visit(fromBaseUrl('/')); }); + afterEach(() => { e2e.flows.setDefaultUserPreferences(); }); it('should pin the selected menu item and add it as a Bookmarks menu item child', () => { - // Check if the mega menu is visible - e2e.components.NavMenu.Menu().should('be.visible'); - - // Check if the Bookmark section is visible - const bookmarkSection = cy.get('[href="/bookmarks"]'); - bookmarkSection.should('be.visible'); - - // Click on the pin icon to add Administration to the Bookmarks section - const adminItem = cy.contains('a', 'Administration'); - const bookmarkPinIcon = adminItem.siblings('button').should('have.attr', 'aria-label', 'Add to Bookmarks'); - bookmarkPinIcon.click({ force: true }); - - // Check if the Administration menu item is visible in the Bookmarks section - cy.get('[aria-label="Expand section: Bookmarks"]').click(); - const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul'); - bookmarks.within(() => { - cy.get('a').should('contain.text', 'Administration'); + cy.visit(fromBaseUrl('/'), { + onBeforeLoad: (win) => { + win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked + }, }); + + e2e.components.NavMenu.Menu() + .should('be.visible') + .within(() => { + cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList'); + + // Check if the Bookmark section is visible + cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem'); + cy.get('@bookmarksItem').should('contain.text', 'Bookmarks'); + + // Check if the Adminstration section is visible + cy.get('@navList').children().last().should('be.visible').as('adminItem'); + cy.get('@adminItem').should('contain.text', 'Administration'); + cy.get('@adminItem').within(() => { + cy.get('button[aria-label="Add to Bookmarks"]').should('exist').click({ force: true }); + }); + + // Check if the Administration menu item is visible in the Bookmarks section + cy.get('@bookmarksItem').within(() => { + // Expand the Bookmarks section + cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true }); + cy.get('a').should('contain.text', 'Administration').should('be.visible'); + }); + }); }); it('should unpin the item and remove it from the Bookmarks section', () => { // Set Administration as a pinned item and reload the page e2e.flows.setUserPreferences({ navbar: { bookmarkUrls: ['/admin'] } }); - cy.reload(); - // Check if the mega menu is visible - e2e.components.NavMenu.Menu().should('be.visible'); + cy.visit(fromBaseUrl('/'), { + onBeforeLoad: (win) => { + win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked + }, + }); - // Check if the Bookmark section is visible and open it - cy.get('[href="/bookmarks"]').should('be.visible'); - cy.get('[aria-label="Expand section: Bookmarks"]').click(); + e2e.components.NavMenu.Menu() + .should('be.visible') + .within(() => { + cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList'); - // Check if the Administration menu item is visible in the Bookmarks section - const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul').children(); - const administrationIsPinned = bookmarks.filter('li').children().should('contain.text', 'Administration'); + // Check if the Bookmark section is visible + cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem'); + cy.get('@bookmarksItem').should('contain.text', 'Bookmarks'); + cy.get('@bookmarksItem').within(() => { + // Expand the Bookmarks section + cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true }); + cy.get('a').should('contain.text', 'Administration').should('be.visible'); + cy.get('button[aria-label="Remove from Bookmarks"]').should('exist').click({ force: true }); + }); - // Click on the pin icon to remove Administration from the Bookmarks section and check if it is removed - administrationIsPinned.within(() => { - cy.get('[aria-label="Remove from Bookmarks"]').click({ force: true }); - }); - cy.wait(500); - administrationIsPinned.should('not.exist'); + cy.get('@bookmarksItem', { timeout: 60000 }).within(() => { + cy.get('a').should('have.length', 1).should('not.contain.text', 'Administration'); + }); + }); }); });