From a4d287d2e15731becb289b5eae4ae202845c6ff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Wed, 8 May 2019 16:50:21 +0200 Subject: [PATCH] Tests: Adds end-to-end tests skeleton and basic smoke test scenario (#16901) * Chore: Adds neccessary packages * Wip: Initial dummy test in place * Feature: Downloads Chromium if needed * Fix: Adds global config object * Refactor: Adds basic e2eScenario * Build: Adds end to end tests to config * Build: Changes end to end job * Build: Adds browsers to image * Build: Adds failing test * Refactor: Adds first e2e-test scenario * Fix: Ignores test output in gitignore * Refactor: Adds compare screenshots ability * Refactor: Removes unnecessary code * Build: Removes jest-puppeteer * Fix: Replaces test snapshots * Refactor: Creates output dir if missing * Refactor: Changes aria-labels to be more consistent * Docs: Adds section about end to end tests * Fix: Fixes snapshots * Docs: Adds information about ENV variables --- .circleci/config.yml | 22 ++++ .gitignore | 2 + README.md | 35 +++++- jest.config.e2e.js | 15 +++ package.json | 13 +++ .../components/search/search_results.html | 2 +- .../AddPanelWidget/AddPanelWidget.tsx | 9 +- .../AddPanelWidget.test.tsx.snap | 2 + .../components/DashNav/DashNavButton.tsx | 6 +- .../SaveModals/SaveDashboardAsModalCtrl.ts | 11 +- .../components/ShareModal/template.html | 2 +- .../dashgrid/PanelHeader/PanelHeader.tsx | 7 +- .../PanelHeader/PanelHeaderMenuItem.tsx | 4 +- .../dashboard/panel_editor/PanelEditor.tsx | 2 +- .../datasources/NewDataSourcePage.tsx | 1 + .../datasources/settings/ButtonRow.tsx | 8 +- .../settings/DataSourceSettingsPage.tsx | 6 +- .../__snapshots__/ButtonRow.test.tsx.snap | 1 + public/app/features/panel/panel_directive.ts | 2 +- public/app/features/panel/panel_header.ts | 2 +- public/app/partials/login.html | 6 +- .../testdata/partials/query.editor.html | 2 +- .../app/plugins/panel/graph/axes_editor.html | 2 +- public/e2e-test/core/constants.ts | 6 + public/e2e-test/core/images.ts | 51 ++++++++ public/e2e-test/core/launcher.ts | 29 +++++ public/e2e-test/core/login.ts | 22 ++++ public/e2e-test/core/pageObjects.ts | 84 +++++++++++++ public/e2e-test/core/pages.ts | 110 ++++++++++++++++++ public/e2e-test/core/scenario.ts | 30 +++++ public/e2e-test/install/install.ts | 22 ++++ .../pages/dashboards/createDashboardPage.ts | 13 +++ .../pages/dashboards/dashboardsPage.ts | 14 +++ .../pages/dashboards/saveDashboardModal.ts | 20 ++++ .../pages/datasources/addDataSourcePage.ts | 13 +++ .../e2e-test/pages/datasources/dataSources.ts | 7 ++ .../pages/datasources/editDataSourcePage.ts | 22 ++++ public/e2e-test/pages/panels/editPanel.ts | 26 +++++ public/e2e-test/pages/panels/panel.ts | 14 +++ .../e2e-test/pages/panels/sharePanelModal.ts | 12 ++ public/e2e-test/pages/start/loginPage.ts | 23 ++++ public/e2e-test/scenarios/smoke.test.ts | 85 ++++++++++++++ .../theTruth/smoke-test-scenario.png | Bin 0 -> 29750 bytes tsconfig.json | 8 +- yarn.lock | 93 ++++++++++++++- 45 files changed, 838 insertions(+), 28 deletions(-) create mode 100644 jest.config.e2e.js create mode 100644 public/e2e-test/core/constants.ts create mode 100644 public/e2e-test/core/images.ts create mode 100644 public/e2e-test/core/launcher.ts create mode 100644 public/e2e-test/core/login.ts create mode 100644 public/e2e-test/core/pageObjects.ts create mode 100644 public/e2e-test/core/pages.ts create mode 100644 public/e2e-test/core/scenario.ts create mode 100644 public/e2e-test/install/install.ts create mode 100644 public/e2e-test/pages/dashboards/createDashboardPage.ts create mode 100644 public/e2e-test/pages/dashboards/dashboardsPage.ts create mode 100644 public/e2e-test/pages/dashboards/saveDashboardModal.ts create mode 100644 public/e2e-test/pages/datasources/addDataSourcePage.ts create mode 100644 public/e2e-test/pages/datasources/dataSources.ts create mode 100644 public/e2e-test/pages/datasources/editDataSourcePage.ts create mode 100644 public/e2e-test/pages/panels/editPanel.ts create mode 100644 public/e2e-test/pages/panels/panel.ts create mode 100644 public/e2e-test/pages/panels/sharePanelModal.ts create mode 100644 public/e2e-test/pages/start/loginPage.ts create mode 100644 public/e2e-test/scenarios/smoke.test.ts create mode 100644 public/e2e-test/screenShots/theTruth/smoke-test-scenario.png diff --git a/.circleci/config.yml b/.circleci/config.yml index 075105dd0d1..0cd47bfa6d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,28 @@ jobs: - run: name: cache server tests command: './scripts/circle-test-cache-servers.sh' + + end-to-end-test: + docker: + - image: circleci/node:8-browsers + - image: grafana/grafana:master + steps: + - run: dockerize -wait tcp://127.0.0.1:3000 -timeout 120s + - checkout + - restore_cache: + key: dependency-cache-{{ checksum "yarn.lock" }} + - run: + name: yarn install + command: 'yarn install --pure-lockfile --no-progress' + no_output_timeout: 5m + - save_cache: + key: dependency-cache-{{ checksum "yarn.lock" }} + paths: + - node_modules + - run: + name: run end-to-end tests + command: 'env BASE_URL=http://127.0.0.1:3000 yarn e2e-tests' + no_output_timeout: 5m codespell: docker: diff --git a/.gitignore b/.gitignore index a9547973fc8..50bfceb3686 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ debug.test /packages/**/dist /packages/**/compiled /packages/**/.rpt2_cache + +theOutput/ \ No newline at end of file diff --git a/README.md b/README.md index 84af184bda8..075b1dfd7e2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[Grafana](https://grafana.com) [![Circle CI](https://circleci.com/gh/grafana/grafana.svg?style=svg)](https://circleci.com/gh/grafana/grafana) [![Go Report Card](https://goreportcard.com/badge/github.com/grafana/grafana)](https://goreportcard.com/report/github.com/grafana/grafana) [![codecov](https://codecov.io/gh/grafana/grafana/branch/master/graph/badge.svg)](https://codecov.io/gh/grafana/grafana) -================ +# [Grafana](https://grafana.com) [![Circle CI](https://circleci.com/gh/grafana/grafana.svg?style=svg)](https://circleci.com/gh/grafana/grafana) [![Go Report Card](https://goreportcard.com/badge/github.com/grafana/grafana)](https://goreportcard.com/report/github.com/grafana/grafana) [![codecov](https://codecov.io/gh/grafana/grafana/branch/master/graph/badge.svg)](https://codecov.io/gh/grafana/grafana) + [Website](https://grafana.com) | [Twitter](https://twitter.com/grafana) | [Community & Forum](https://community.grafana.com) @@ -12,12 +12,15 @@ Graphite, Elasticsearch, OpenTSDB, Prometheus and InfluxDB. --> ## Installation + Head to [docs.grafana.org](http://docs.grafana.org/installation/) for documentation or [download](https://grafana.com/get) to get the latest release. ## Documentation & Support + Be sure to read the [getting started guide](http://docs.grafana.org/guides/gettingstarted/) and the other feature guides. ## Run from master + If you want to build a package yourself, or contribute - here is a guide for how to do that. You can always find the latest master builds [here](https://grafana.com/grafana/download) @@ -48,7 +51,7 @@ go run build.go build #### Frontend assets -*For this you need Node.js (LTS version).* +_For this you need Node.js (LTS version)._ ```bash yarn install --pure-lockfile @@ -80,7 +83,7 @@ yarn start:hot env GRAFANA_THEME=light yarn start:hot ``` -*Note: HMR for Angular is not supported. If you edit files in the Angular part of the app, the whole page will reload.* +_Note: HMR for Angular is not supported. If you edit files in the Angular part of the app, the whole page will reload._ Run tests and rebuild on source change: @@ -128,7 +131,9 @@ In your custom.ini uncomment (remove the leading `;`) sign. And set `app_mode = ### Running tests #### Frontend + Execute all frontend tests + ```bash yarn test ``` @@ -139,6 +144,7 @@ Writing & watching frontend tests - Jest will run all test files that end with the name ".test.ts" #### Backend + ```bash # Run Golang tests using sqlite3 as database (default) go test ./pkg/... @@ -150,6 +156,26 @@ GRAFANA_TEST_DB=mysql go test ./pkg/... GRAFANA_TEST_DB=postgres go test ./pkg/... ``` +#### End-to-end + +Execute all end-to-end tests + +```bash +yarn e2e-tests +``` + +Execute all end-to-end tests using using a specific url + +```bash +ENV BASE_URL=http://localhost:3333 yarn e2e-tests +``` + +Debugging all end-to-end tests (BROWSER=1 will start the browser and SLOWMO=1 will delay each puppeteer operation by 100ms) + +```bash +ENV BROWSER=1 SLOWMO=1 yarn e2e-tests +``` + ### Datasource and dashboard provisioning [Here](https://github.com/grafana/grafana/tree/master/devenv) you can find helpful scripts and docker-compose setup @@ -171,4 +197,3 @@ plugin development. ## License Grafana is distributed under [Apache 2.0 License](https://github.com/grafana/grafana/blob/master/LICENSE). - diff --git a/jest.config.e2e.js b/jest.config.e2e.js new file mode 100644 index 00000000000..feb6f7d4f3a --- /dev/null +++ b/jest.config.e2e.js @@ -0,0 +1,15 @@ +require('module-alias/register'); + +module.exports = { + verbose: false, + transform: { + '^.+\\.(ts|tsx)$': 'ts-jest', + }, + moduleDirectories: ['node_modules', 'public'], + roots: ['/public/e2e-test'], + testRegex: '(\\.|/)(test)\\.(jsx?|tsx?)$', + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'], + setupFiles: [], + globals: { 'ts-jest': { isolatedModules: true } }, + setupFilesAfterEnv: ['expect-puppeteer', '/public/e2e-test/install/install.ts'], +}; diff --git a/package.json b/package.json index 66f24e49f14..cd564bd3393 100644 --- a/package.json +++ b/package.json @@ -25,12 +25,16 @@ "@types/commander": "2.12.2", "@types/d3": "4.13.1", "@types/enzyme": "3.9.0", + "@types/expect-puppeteer": "3.3.1", "@types/inquirer": "0.0.43", "@types/jest": "24.0.11", "@types/jquery": "1.10.35", "@types/lodash": "4.14.123", "@types/node": "11.13.4", "@types/papaparse": "4.5.9", + "@types/pixelmatch": "4.0.0", + "@types/pngjs": "3.3.2", + "@types/puppeteer-core": "1.9.0", "@types/react": "16.8.16", "@types/react-dom": "16.8.4", "@types/react-grid-layout": "0.16.7", @@ -55,6 +59,7 @@ "es6-promise": "3.3.1", "es6-shim": "0.35.5", "execa": "1.0.0", + "expect-puppeteer": "4.1.1", "expect.js": "0.2.0", "expose-loader": "0.7.5", "file-loader": "3.0.1", @@ -85,6 +90,7 @@ "load-grunt-tasks": "3.5.2", "mini-css-extract-plugin": "0.5.0", "mocha": "4.1.0", + "module-alias": "2.2.0", "monaco-editor": "0.15.6", "ng-annotate-loader": "0.6.1", "ng-annotate-webpack-plugin": "0.3.0", @@ -94,10 +100,13 @@ "optimize-css-assets-webpack-plugin": "5.0.1", "ora": "3.2.0", "phantomjs-prebuilt": "2.1.16", + "pixelmatch": "4.0.2", + "pngjs": "3.4.0", "postcss-browser-reporter": "0.5.0", "postcss-loader": "3.0.0", "postcss-reporter": "6.0.1", "prettier": "1.16.4", + "puppeteer-core": "1.15.0", "react-hooks-testing-library": "0.3.7", "react-hot-loader": "4.8.0", "react-test-renderer": "16.8.4", @@ -140,6 +149,7 @@ "tslint": "tslint -c tslint.json --project tsconfig.json", "typecheck": "tsc --noEmit", "jest": "jest --notify --watch", + "e2e-tests": "jest --runInBand --config=jest.config.e2e.js", "api-tests": "jest --notify --watch --config=tests/api/jest.js", "storybook": "cd packages/grafana-ui && yarn storybook", "storybook:build": "cd packages/grafana-ui && yarn storybook:build", @@ -242,5 +252,8 @@ "**/@types/*", "**/@types/*/**" ] + }, + "_moduleAliases": { + "puppeteer": "node_modules/puppeteer-core" } } diff --git a/public/app/core/components/search/search_results.html b/public/app/core/components/search/search_results.html index 09d21ba80c0..a2337cfbc19 100644 --- a/public/app/core/components/search/search_results.html +++ b/public/app/core/components/search/search_results.html @@ -20,7 +20,7 @@
- +
{ dashboard.removePanel(this.props.panel); }; - renderOptionLink = (icon, text, onClick) => { + renderOptionLink = (icon: string, text: string, onClick) => { return (
- +
diff --git a/public/app/features/dashboard/components/AddPanelWidget/__snapshots__/AddPanelWidget.test.tsx.snap b/public/app/features/dashboard/components/AddPanelWidget/__snapshots__/AddPanelWidget.test.tsx.snap index 00faf48d8df..1908aa5f7f1 100644 --- a/public/app/features/dashboard/components/AddPanelWidget/__snapshots__/AddPanelWidget.test.tsx.snap +++ b/public/app/features/dashboard/components/AddPanelWidget/__snapshots__/AddPanelWidget.test.tsx.snap @@ -35,6 +35,7 @@ exports[`Render should render component 1`] = ` >
= ({ icon, tooltip, classSu if (onClick) { return ( - diff --git a/public/app/features/dashboard/components/SaveModals/SaveDashboardAsModalCtrl.ts b/public/app/features/dashboard/components/SaveModals/SaveDashboardAsModalCtrl.ts index 23ad97708b4..748acb94831 100644 --- a/public/app/features/dashboard/components/SaveModals/SaveDashboardAsModalCtrl.ts +++ b/public/app/features/dashboard/components/SaveModals/SaveDashboardAsModalCtrl.ts @@ -17,7 +17,7 @@ const template = `
- +
- + Cancel
diff --git a/public/app/features/dashboard/components/ShareModal/template.html b/public/app/features/dashboard/components/ShareModal/template.html index 86643043f4f..63f317fb442 100644 --- a/public/app/features/dashboard/components/ShareModal/template.html +++ b/public/app/features/dashboard/components/ShareModal/template.html @@ -92,7 +92,7 @@
- Direct link rendered image + Direct link rendered image
diff --git a/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeader.tsx b/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeader.tsx index b918acb8c81..e2344c35777 100644 --- a/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeader.tsx +++ b/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeader.tsx @@ -90,7 +90,12 @@ export class PanelHeader extends Component { error={error} />
-
+
diff --git a/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeaderMenuItem.tsx b/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeaderMenuItem.tsx index 66a942f0afc..a50af1eb56c 100644 --- a/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeaderMenuItem.tsx +++ b/public/app/features/dashboard/dashgrid/PanelHeader/PanelHeaderMenuItem.tsx @@ -14,7 +14,9 @@ export const PanelHeaderMenuItem: FC = props => {
  • {props.iconClassName && } - {props.text} + + {props.text} + {props.shortcut && {props.shortcut}} {props.children} diff --git a/public/app/features/dashboard/panel_editor/PanelEditor.tsx b/public/app/features/dashboard/panel_editor/PanelEditor.tsx index 5cf7ad78d1f..fd4a7858e2b 100644 --- a/public/app/features/dashboard/panel_editor/PanelEditor.tsx +++ b/public/app/features/dashboard/panel_editor/PanelEditor.tsx @@ -145,7 +145,7 @@ function TabItem({ tab, activeTab, onClick }: TabItemParams) { return (
    onClick(tab)}> - + diff --git a/public/app/features/datasources/NewDataSourcePage.tsx b/public/app/features/datasources/NewDataSourcePage.tsx index ae0eae9e02f..716c96280ad 100644 --- a/public/app/features/datasources/NewDataSourcePage.tsx +++ b/public/app/features/datasources/NewDataSourcePage.tsx @@ -54,6 +54,7 @@ class NewDataSourcePage extends PureComponent { onClick={() => this.onDataSourceTypeClicked(plugin)} className="add-data-source-grid-item" key={`${plugin.id}-${index}`} + aria-label={`${plugin.name} datasource plugin`} > {plugin.name} diff --git a/public/app/features/datasources/settings/ButtonRow.tsx b/public/app/features/datasources/settings/ButtonRow.tsx index 2c048a33e7a..16d25be293c 100644 --- a/public/app/features/datasources/settings/ButtonRow.tsx +++ b/public/app/features/datasources/settings/ButtonRow.tsx @@ -12,7 +12,13 @@ const ButtonRow: FC = ({ isReadOnly, onDelete, onSubmit, onTest }) => { return (
    {!isReadOnly && ( - )} diff --git a/public/app/features/datasources/settings/DataSourceSettingsPage.tsx b/public/app/features/datasources/settings/DataSourceSettingsPage.tsx index f06e66b98de..a4c8e60a99b 100644 --- a/public/app/features/datasources/settings/DataSourceSettingsPage.tsx +++ b/public/app/features/datasources/settings/DataSourceSettingsPage.tsx @@ -212,7 +212,7 @@ export class DataSourceSettingsPage extends PureComponent {
    {testingMessage && ( -
    +
    {testingStatus === 'error' ? ( @@ -221,7 +221,9 @@ export class DataSourceSettingsPage extends PureComponent { )}
    -
    {testingMessage}
    +
    + {testingMessage} +
    )} diff --git a/public/app/features/datasources/settings/__snapshots__/ButtonRow.test.tsx.snap b/public/app/features/datasources/settings/__snapshots__/ButtonRow.test.tsx.snap index 00251237f83..dc506449278 100644 --- a/public/app/features/datasources/settings/__snapshots__/ButtonRow.test.tsx.snap +++ b/public/app/features/datasources/settings/__snapshots__/ButtonRow.test.tsx.snap @@ -33,6 +33,7 @@ exports[`Render should render with buttons enabled 1`] = ` className="gf-form-button-row" >
    diff --git a/public/app/features/panel/panel_header.ts b/public/app/features/panel/panel_header.ts index e844a73cfa9..26139e23ac8 100644 --- a/public/app/features/panel/panel_header.ts +++ b/public/app/features/panel/panel_header.ts @@ -34,7 +34,7 @@ function renderMenuItem(item, ctrl) { } html += `>`; - html += `${item.text}`; + html += `${item.text}`; if (item.shortcut) { html += `${item.shortcut}`; diff --git a/public/app/partials/login.html b/public/app/partials/login.html index 33872c7d6d5..55bd8060a4f 100644 --- a/public/app/partials/login.html +++ b/public/app/partials/login.html @@ -8,15 +8,15 @@