e2e: improves flakiness and speed (#103533)

* e2e: improves flakiness

* Chore: refactor flaky test

* e2e: more refactor

* e2e: do not keep successfull test run videos

* e2e: do not log selectors

* chore: updates after pr feedback

* chore: updates after pr feedback

* chore: adds retries to flaky e2e test

* e2e: skip flaky tests

* e2e: skip flaky tests

* e2e: revert back to timeout

* chore: removes all the skips

cypress.config.js

@@ -13,6 +13,9 @@ module.exports = defineConfig({
   viewportWidth: 1920,
   viewportHeight: 1080,
 
+  env: {
+    LOG_SELECTORS_INFO: false,
+  },
   e2e: {
     supportFile: './e2e/cypress/support/e2e.js',
     setupNodeEvents(on, config) {
@@ -82,6 +85,21 @@ module.exports = defineConfig({
         // IMPORTANT: return the updated browser launch options
         return launchOptions;
       });
+
+      on('after:spec', (_, results) => {
+        if (!results || !results.video || !results.tests) {
+          return;
+        }
+
+        // Do we have failures for any retry attempts?
+        const failures = results.tests.some((test) => test.attempts.some((attempt) => attempt.state === 'failed'));
+        if (failures) {
+          return;
+        }
+
+        // delete the video if the spec passed and no tests retried
+        fs.unlinkSync(results.video);
+      });
     },
   },
 });

e2e/old-arch/panels-suite/geomap-map-controls.spec.ts

@@ -1,5 +1,6 @@
 import { e2e } from '../utils';
 const DASHBOARD_ID = 'P2jR04WVk';
+const TIMEOUT = 45000;
 
 describe('Geomap layer controls options', () => {
   beforeEach(() => {
@@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => {
 
   it('Tests map controls options', () => {
     e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } });
-    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist');
+    // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv'
+    e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible');
+    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible');
 
-    // Show zoom
+    // Show zoom field
     e2e.components.PanelEditor.showZoomField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.contains('+');
-    cy.get('.ol-zoom').should('exist');
-
     // Show attribution
     e2e.components.PanelEditor.showAttributionField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.get('.ol-attribution').should('exist');
-
     // Show scale
     e2e.components.PanelEditor.showScaleField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.get('.ol-scale-line').should('exist');
-
     // Show measure tool
     e2e.components.PanelEditor.showMeasureField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    e2e.components.PanelEditor.measureButton().should('exist');
-
     // Show debug
     e2e.components.PanelEditor.showDebugField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
+      });
+
+    e2e.components.Panels.Panel.content({ timeout: TIMEOUT })
+      .should('be.visible')
+      .within(() => {
+        // Verify zoom
+        cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify attribution
+        cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify scale
+        cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify measure tool
+        e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible');
+
+        // Verify debug tool
+        e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible');
       });
-    e2e.components.DebugOverlay.wrapper().should('exist');
   });
 });

e2e/old-arch/utils/support/types.ts

@@ -32,7 +32,9 @@ export type E2EFactoryArgs<S extends Selectors> = { selectors: S };
 export type CypressOptions = Partial<Cypress.Loggable & Cypress.Timeoutable & Cypress.Withinable & Cypress.Shadow>;
 
 const processSelectors = <S extends Selectors>(e2eObjects: E2EFunctions<S>, selectors: S): E2EFunctions<S> => {
-  const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data);
+  const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO'));
+  const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {};
+
   const keys = Object.keys(selectors);
   for (let index = 0; index < keys.length; index++) {
     const key = keys[index];

e2e/panels-suite/geomap-map-controls.spec.ts

@@ -1,5 +1,6 @@
 import { e2e } from '../utils';
 const DASHBOARD_ID = 'P2jR04WVk';
+const TIMEOUT = 45000;
 
 describe('Geomap layer controls options', () => {
   beforeEach(() => {
@@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => {
 
   it('Tests map controls options', () => {
     e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } });
-    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist');
+    // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv'
+    e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible');
+    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible');
 
-    // Show zoom
+    // Show zoom field
     e2e.components.PanelEditor.showZoomField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.contains('+');
-    cy.get('.ol-zoom').should('exist');
-
     // Show attribution
     e2e.components.PanelEditor.showAttributionField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.get('.ol-attribution').should('exist');
-
     // Show scale
     e2e.components.PanelEditor.showScaleField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    cy.get('.ol-scale-line').should('exist');
-
     // Show measure tool
     e2e.components.PanelEditor.showMeasureField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
       });
 
-    e2e.components.PanelEditor.measureButton().should('exist');
-
     // Show debug
     e2e.components.PanelEditor.showDebugField()
-      .should('exist')
+      .should('be.visible')
       .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
+      });
+
+    e2e.components.Panels.Panel.content({ timeout: TIMEOUT })
+      .should('be.visible')
+      .within(() => {
+        // Verify zoom
+        cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify attribution
+        cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify scale
+        cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify measure tool
+        e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible');
+
+        // Verify debug tool
+        e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible');
       });
-    e2e.components.DebugOverlay.wrapper().should('exist');
   });
 });

e2e/utils/support/types.ts

@@ -32,7 +32,9 @@ export type E2EFactoryArgs<S extends Selectors> = { selectors: S };
 export type CypressOptions = Partial<Cypress.Loggable & Cypress.Timeoutable & Cypress.Withinable & Cypress.Shadow>;
 
 const processSelectors = <S extends Selectors>(e2eObjects: E2EFunctions<S>, selectors: S): E2EFunctions<S> => {
-  const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data);
+  const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO'));
+  const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {};
+
   const keys = Object.keys(selectors);
   for (let index = 0; index < keys.length; index++) {
     const key = keys[index];

e2e/various-suite/bookmarks.spec.ts

@@ -3,56 +3,73 @@ import { fromBaseUrl } from '../utils/support/url';
 
 describe('Pin nav items', () => {
   beforeEach(() => {
-    cy.viewport(1280, 800);
     e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-    cy.visit(fromBaseUrl('/'));
   });
+
   afterEach(() => {
     e2e.flows.setDefaultUserPreferences();
   });
 
   it('should pin the selected menu item and add it as a Bookmarks menu item child', () => {
-    // Check if the mega menu is visible
+    cy.visit(fromBaseUrl('/'), {
-    e2e.components.NavMenu.Menu().should('be.visible');
+      onBeforeLoad: (win) => {
-
+        win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked
-    // Check if the Bookmark section is visible
+      },
-    const bookmarkSection = cy.get('[href="/bookmarks"]');
-    bookmarkSection.should('be.visible');
-
-    // Click on the pin icon to add Administration to the Bookmarks section
-    const adminItem = cy.contains('a', 'Administration');
-    const bookmarkPinIcon = adminItem.siblings('button').should('have.attr', 'aria-label', 'Add to Bookmarks');
-    bookmarkPinIcon.click({ force: true });
-
-    // Check if the Administration menu item is visible in the Bookmarks section
-    cy.get('[aria-label="Expand section: Bookmarks"]').click();
-    const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul');
-    bookmarks.within(() => {
-      cy.get('a').should('contain.text', 'Administration');
     });
+
+    e2e.components.NavMenu.Menu()
+      .should('be.visible')
+      .within(() => {
+        cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList');
+
+        // Check if the Bookmark section is visible
+        cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem');
+        cy.get('@bookmarksItem').should('contain.text', 'Bookmarks');
+
+        // Check if the Adminstration section is visible
+        cy.get('@navList').children().last().should('be.visible').as('adminItem');
+        cy.get('@adminItem').should('contain.text', 'Administration');
+        cy.get('@adminItem').within(() => {
+          cy.get('button[aria-label="Add to Bookmarks"]').should('exist').click({ force: true });
+        });
+
+        // Check if the Administration menu item is visible in the Bookmarks section
+        cy.get('@bookmarksItem').within(() => {
+          // Expand the Bookmarks section
+          cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true });
+          cy.get('a').should('contain.text', 'Administration').should('be.visible');
+        });
+      });
   });
 
   it('should unpin the item and remove it from the Bookmarks section', () => {
     // Set Administration as a pinned item and reload the page
     e2e.flows.setUserPreferences({ navbar: { bookmarkUrls: ['/admin'] } });
-    cy.reload();
 
-    // Check if the mega menu is visible
+    cy.visit(fromBaseUrl('/'), {
-    e2e.components.NavMenu.Menu().should('be.visible');
+      onBeforeLoad: (win) => {
+        win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked
+      },
+    });
 
-    // Check if the Bookmark section is visible and open it
+    e2e.components.NavMenu.Menu()
-    cy.get('[href="/bookmarks"]').should('be.visible');
+      .should('be.visible')
-    cy.get('[aria-label="Expand section: Bookmarks"]').click();
+      .within(() => {
+        cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList');
 
-    // Check if the Administration menu item is visible in the Bookmarks section
+        // Check if the Bookmark section is visible
-    const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul').children();
+        cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem');
-    const administrationIsPinned = bookmarks.filter('li').children().should('contain.text', 'Administration');
+        cy.get('@bookmarksItem').should('contain.text', 'Bookmarks');
+        cy.get('@bookmarksItem').within(() => {
+          // Expand the Bookmarks section
+          cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true });
+          cy.get('a').should('contain.text', 'Administration').should('be.visible');
+          cy.get('button[aria-label="Remove from Bookmarks"]').should('exist').click({ force: true });
+        });
 
-    // Click on the pin icon to remove Administration from the Bookmarks section and check if it is removed
+        cy.get('@bookmarksItem', { timeout: 60000 }).within(() => {
-    administrationIsPinned.within(() => {
+          cy.get('a').should('have.length', 1).should('not.contain.text', 'Administration');
-      cy.get('[aria-label="Remove from Bookmarks"]').click({ force: true });
+        });
-    });
+      });
-    cy.wait(500);
-    administrationIsPinned.should('not.exist');
   });
 });