Accessibility checks: Phase 2 - Adding Pa11y CI PR stage (#38556)

Co-authored-by: Hugo Häggmark <hugo.haggmark@grafana.com>

.drone.yml

@@ -146,6 +146,19 @@ steps:
   depends_on:
   - package
 
+- name: test-a11y-frontend-pr
+  image: buildkite/puppeteer
+  commands:
+  - yarn wait-on http://$HOST:$PORT
+  - yarn -s test:accessibility-pr
+  environment:
+    GRAFANA_MISC_STATS_API_KEY:
+      from_secret: grafana_misc_stats_api_key
+    HOST: end-to-end-tests-server
+    PORT: 3001
+  depends_on:
+  - end-to-end-tests-server
+
 - name: build-frontend-docs
   image: grafana/build-container:1.4.2
   commands:
@@ -3478,6 +3491,6 @@ get:
 
 ---
 kind: signature
-hmac: 1ed78e5f23e4e069c2a2cdb34d5220b556a76c812b23fe4bdfad09289363370b
+hmac: 502c06a59a21e0d44e32814700f0eb6cd37b8fe2b959b1b87d5ae0dc1658c3c4
 
 ...

.gitignore

@@ -123,6 +123,10 @@ compilation-stats.json
 !/e2e/**/screenshots/expected/*
 /e2e/**/videos/*
 
+# a11y tests
+/pa11y-ci-results.json
+/pa11y-ci-report
+
 # report dumping the whole system env
 /report.*.json
 

.pa11yci-pr.conf.js

@@ -0,0 +1,96 @@
+var config = {
+  defaults: {
+    concurrency: 1,
+    runners: ['axe'],
+    chromeLaunchConfig: {
+      args: ['--no-sandbox'],
+    },
+  },
+
+  urls: [
+    {
+      url: '${HOST}/login',
+      actions: [
+        "set field input[name='user'] to admin",
+        "set field input[name='password'] to admin",
+        "click element button[aria-label='Login button']",
+        "wait for element [aria-label='Skip change password button'] to be visible",
+      ],
+      threshold: 2,
+    },
+    {
+      url: '${HOST}/?orgId=1',
+      threshold: 7,
+    },
+    {
+      url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge',
+      hideElements: '.sidemenu',
+      threshold: 2,
+    },
+    {
+      url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
+      rootElement: '.dashboard-settings',
+      threshold: 10,
+    },
+    {
+      url: '${HOST}/?orgId=1&search=open',
+      rootElement: '.main-view',
+      threshold: 15,
+    },
+    {
+      url: '${HOST}/alerting/list',
+      rootElement: '.main-view',
+      threshold: 7,
+    },
+    {
+      url: '${HOST}/datasources',
+      rootElement: '.main-view',
+      threshold: 36,
+    },
+    {
+      url: '${HOST}/org/users',
+      rootElement: '.main-view',
+      threshold: 4,
+    },
+    {
+      url: '${HOST}/org/teams',
+      rootElement: '.main-view',
+      threshold: 1,
+    },
+    {
+      url: '${HOST}/plugins',
+      rootElement: '.main-view',
+      threshold: 41,
+    },
+    {
+      url: '${HOST}/org',
+      rootElement: '.main-view',
+      threshold: 2,
+    },
+    {
+      url: '${HOST}/org/apikeys',
+      rootElement: '.main-view',
+      threshold: 5,
+    },
+    {
+      url: '${HOST}/dashboards',
+      rootElement: '.main-view',
+      threshold: 8,
+    },
+  ],
+};
+
+function myPa11yCiConfiguration(urls, defaults) {
+  const HOST_SERVER = process.env.HOST || 'localhost';
+  const PORT_SERVER = process.env.PORT || '3000';
+  for (var idx = 0; idx < urls.length; idx++) {
+    urls[idx] = { ...urls[idx], url: urls[idx].url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`) };
+  }
+
+  return {
+    defaults: defaults,
+    urls: urls,
+  };
+}
+
+module.exports = myPa11yCiConfiguration(config.urls, config.defaults);

contribute/create-pull-request.md

@@ -55,6 +55,17 @@ Pull requests that create new UI components or modify existing ones must adhere
 - Use the [Grafana theme palette](/contribute/style-guides/themes.md) for styling. It contains colors with good contrast which aids accessibility.
 - Use [RTL](https://testing-library.com/docs/dom-testing-library/api-accessibility/) for writing unit tests. It helps to create accessible components.
 
+Pull requests that introduce accessibility(a11y) errors:
+
+We use [pa11y-ci](https://github.com/pa11y/pa11y-ci) to collect accessibility errors on [some URLs on the project](https://github.com/grafana/grafana/issues/36555), threshold errors are specified per URL.
+
+If the contribution introduces new a11y errors, our continuous integration will fail, preventing you to merge on the main branch. In those cases there are two alternatives for moving forward:
+
+- Check the error log on the pipeline step `test-a11y-frontend-pr`, identify what was the error, and fix it.
+- Locally run the command `yarn test:accessibility-report` that generates an HTML accessibility report, then go to the URL that contains your change, identify the error, and fix it. Keep in mind, a local Grafana instance needs to be running on `http://localhost:3000`.
+
+You can also prevent introducing a11y errors by installing an a11y plugin in your browser, for example, axe DevTools, Accessibility Insights for Web among others.
+
 ### Backend-specific guidelines
 
 Please refer to the [backend style guidelines](/contribute/style-guides/backend.md).

package.json

@@ -14,6 +14,8 @@
     "e2e:dev": "./e2e/start-and-run-suite dev",
     "test": "jest --notify --watch",
     "test:accessibility": "pa11y-ci --config .pa11yci.conf.js",
+    "test:accessibility-pr": "pa11y-ci --config .pa11yci-pr.conf.js",
+    "test:accessibility-report": "./scripts/generate-a11y-report.sh",
     "lint": "yarn run lint:ts && yarn run lint:sass",
     "lint:ts": "eslint . --ext .js,.tsx,.ts --cache",
     "lint:sass": "yarn run sass-lint -c public/sass/.sass-lint.yml 'public/sass/**/*.scss, packages/**/*.scss' -v -i '**/node_modules/**/*.scss'",
@@ -171,6 +173,7 @@
     "fs-extra": "9.1.0",
     "gaze": "1.1.3",
     "glob": "7.1.6",
+    "http-server": "13.0.1",
     "html-loader": "2.1.2",
     "html-webpack-harddisk-plugin": "2.0.0",
     "html-webpack-plugin": "5.3.2",

scripts/drone/pipelines/pr.star

@@ -23,6 +23,7 @@ load(
     'memcached_integration_tests_step',
     'benchmark_ldap_step',
     'validate_scuemata_step',
+    'test_a11y_frontend_step_pr',
 )
 
 load(
@@ -72,6 +73,7 @@ def pr_pipelines(edition):
         e2e_tests_server_step(edition=edition),
         e2e_tests_step(edition=edition),
         build_storybook_step(edition=edition, ver_mode=ver_mode),
+        test_a11y_frontend_step_pr(edition=edition),
         build_frontend_docs_step(edition=edition),
         build_docs_website_step(),
         copy_packages_for_docker_step(),

scripts/drone/steps/lib.star

@@ -423,6 +423,24 @@ def test_a11y_frontend_step(edition, port=3001):
         ],
     }
 
+def test_a11y_frontend_step_pr(edition, port=3001):
+    return {
+        'name': 'test-a11y-frontend-pr' + enterprise2_suffix(edition),
+        'image': 'buildkite/puppeteer',
+        'depends_on': [
+          'end-to-end-tests-server' + enterprise2_suffix(edition),
+        ],
+         'environment': {
+            'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'),
+            'HOST': 'end-to-end-tests-server' + enterprise2_suffix(edition),
+            'PORT': port,
+        },
+        'commands': [
+            'yarn wait-on http://$HOST:$PORT',
+            'yarn -s test:accessibility-pr',
+        ],
+    }
+
 def frontend_metrics_step(edition):
     if edition in ('enterprise', 'enterprise2'):
         return None

scripts/generate-a11y-report.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -x
+
+# Clean up old report
+jsonReport="pa11y-ci-results.json"
+if [ -f "$jsonReport" ] ; then
+    rm "$jsonReport"
+fi
+
+# Clean up old folder
+report="pa11y-ci-report/"
+
+if [ -d "$report" ] ; then
+    rm -R "$report"
+fi
+
+# Run accessibility command
+yarn wait-on http://localhost:3000
+
+yarn run -s test:accessibility --json > pa11y-ci-results.json
+
+# Generate HTML report
+pa11y-ci-reporter-html
+
+# Start local server
+yarn http-server pa11y-ci-report -p 1234

yarn.lock

@@ -7306,6 +7306,11 @@ base@^0.11.1:
     mixin-deep "^1.2.0"
     pascalcase "^0.1.1"
 
+basic-auth@^1.0.3:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-1.1.0.tgz#45221ee429f7ee1e5035be3f51533f1cdfd29884"
+  integrity sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ=
+
 batch-processor@1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/batch-processor/-/batch-processor-1.0.0.tgz#75c95c32b748e0850d10c2b168f6bdbe9891ace8"
@@ -8407,7 +8412,7 @@ colorette@^1.1.0, colorette@^1.2.1, colorette@^1.2.2, colorette@^1.3.0:
   resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.3.0.tgz#ff45d2f0edb244069d3b772adeb04fed38d0a0af"
   integrity sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w==
 
-colors@^1.1.2:
+colors@^1.1.2, colors@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
   integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==
@@ -8859,6 +8864,11 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
   resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
   integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
 
+corser@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87"
+  integrity sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c=
+
 cosmiconfig@^5.0.0:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a"
@@ -12919,7 +12929,7 @@ hastscript@^6.0.0:
     property-information "^5.0.0"
     space-separated-tokens "^1.0.0"
 
-he@1.2.0, he@1.2.x, he@^1.2.0:
+he@1.2.0, he@1.2.x, he@^1.1.0, he@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
@@ -13226,7 +13236,7 @@ http-proxy-middleware@^2.0.0:
     is-plain-obj "^3.0.0"
     micromatch "^4.0.2"
 
-http-proxy@^1.18.1:
+http-proxy@^1.18.0, http-proxy@^1.18.1:
   version "1.18.1"
   resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549"
   integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==
@@ -13235,6 +13245,24 @@ http-proxy@^1.18.1:
     follow-redirects "^1.0.0"
     requires-port "^1.0.0"
 
+http-server@13.0.1:
+  version "13.0.1"
+  resolved "https://registry.yarnpkg.com/http-server/-/http-server-13.0.1.tgz#e7340d082925c4b1d6484c905d0df29d7d8ec16f"
+  integrity sha512-ke9rphoNuqsOCHy4tA3b3W4Yuxy7VUIXcTHSLz6bkMDAJPQD4twjEatquelJBIPwNhZuC3+FYj/+dSaGHdKTCw==
+  dependencies:
+    basic-auth "^1.0.3"
+    colors "^1.4.0"
+    corser "^2.0.1"
+    he "^1.1.0"
+    http-proxy "^1.18.0"
+    mime "^1.6.0"
+    minimist "^1.2.5"
+    opener "^1.5.1"
+    portfinder "^1.0.25"
+    secure-compare "3.0.1"
+    union "~0.5.0"
+    url-join "^2.0.5"
+
 http-signature@~1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
@@ -16233,7 +16261,7 @@ mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.30, mime-types@^2.1.31,
   dependencies:
     mime-db "1.49.0"
 
-mime@1.6.0, mime@^1.4.1:
+mime@1.6.0, mime@^1.4.1, mime@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
@@ -17383,7 +17411,7 @@ opencollective-postinstall@^2.0.2:
   resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz#5657f1bede69b6e33a45939b061eb53d3c6c3a89"
   integrity sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==
 
-opener@^1.5.2:
+opener@^1.5.1, opener@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598"
   integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==
@@ -18146,7 +18174,7 @@ polished@^4.0.5:
   dependencies:
     "@babel/runtime" "^7.14.0"
 
-portfinder@^1.0.28:
+portfinder@^1.0.25, portfinder@^1.0.28:
   version "1.0.28"
   resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778"
   integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA==
@@ -19578,7 +19606,7 @@ qs@6.7.0:
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
   integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ==
 
-qs@^6.10.0, qs@^6.9.4:
+qs@^6.10.0, qs@^6.4.0, qs@^6.9.4:
   version "6.10.1"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.1.tgz#4931482fa8d647a5aab799c5271d2133b981fb6a"
   integrity sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg==
@@ -21511,6 +21539,11 @@ search-query-parser@1.5.4:
   resolved "https://registry.yarnpkg.com/search-query-parser/-/search-query-parser-1.5.4.tgz#b474185e02717bee95f408e1003d0c1c932fcf55"
   integrity sha512-Mw3BpGU9SCid5uaEUTAz5y2hu7LT0VZbwmLKOH7bgoH+vxnP52kvX0xRXVwl14iCeL668mDeL92RxVX2dVX6Pg==
 
+secure-compare@3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3"
+  integrity sha1-8aAymzCLIh+uN7mXTz1XjQypmeM=
+
 select-hose@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
@@ -23616,6 +23649,13 @@ union-value@^1.0.0:
     is-extendable "^0.1.1"
     set-value "^2.0.1"
 
+union@~0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075"
+  integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==
+  dependencies:
+    qs "^6.4.0"
+
 uniq@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff"
@@ -23817,6 +23857,11 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
+url-join@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/url-join/-/url-join-2.0.5.tgz#5af22f18c052a000a48d7b82c5e9c2e2feeda728"
+  integrity sha1-WvIvGMBSoACkjXuCxenC4v7tpyg=
+
 url-loader@^2.0.1:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-2.2.0.tgz#af321aece1fd0d683adc8aaeb27829f29c75b46e"