mirror of https://github.com/grafana/grafana
joshhunt
3 weeks ago
parent
9b8924fe74
commit
cea1f84437
@ -0,0 +1,146 @@ |
||||
name: Playwright E2E Tests |
||||
|
||||
on: |
||||
pull_request: |
||||
push: |
||||
branches: |
||||
- main |
||||
- release-*.*.* |
||||
|
||||
concurrency: |
||||
group: ${{ github.workflow }}-${{ github.ref }} |
||||
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} |
||||
|
||||
permissions: {} |
||||
|
||||
jobs: |
||||
build-grafana: |
||||
name: Build & Package Grafana |
||||
runs-on: ubuntu-latest-16-cores |
||||
permissions: |
||||
contents: read |
||||
outputs: |
||||
artifact: ${{ steps.artifact.outputs.artifact }} |
||||
steps: |
||||
- uses: actions/checkout@v4 |
||||
with: |
||||
path: ./grafana |
||||
persist-credentials: false |
||||
- uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e |
||||
with: |
||||
verb: run |
||||
args: go -C grafana run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}/grafana" > out.txt |
||||
- run: mv "$(cat out.txt)" grafana.tar.gz |
||||
- run: echo "artifact=grafana-playwright-${{github.run_number}}" >> "$GITHUB_OUTPUT" |
||||
id: artifact |
||||
- uses: actions/upload-artifact@v4 |
||||
id: upload |
||||
with: |
||||
retention-days: 1 |
||||
name: ${{ steps.artifact.outputs.artifact }} |
||||
path: grafana.tar.gz |
||||
|
||||
run-playwright-tests: |
||||
needs: |
||||
- build-grafana |
||||
strategy: |
||||
fail-fast: false |
||||
matrix: |
||||
include: |
||||
- suite: plugin-e2e-api-tests |
||||
project: admin |
||||
- suite: plugin-e2e-api-tests |
||||
project: viewer |
||||
- suite: plugin-e2e |
||||
project: elasticsearch |
||||
- suite: plugin-e2e |
||||
project: mysql |
||||
- suite: plugin-e2e |
||||
project: mssql |
||||
- suite: plugin-e2e |
||||
project: cloudwatch |
||||
- suite: plugin-e2e |
||||
project: azuremonitor |
||||
- suite: plugin-e2e |
||||
project: cloudmonitoring |
||||
- suite: plugin-e2e |
||||
project: graphite |
||||
- suite: plugin-e2e |
||||
project: influxdb |
||||
- suite: plugin-e2e |
||||
project: jaeger |
||||
- suite: plugin-e2e |
||||
project: zipkin |
||||
- suite: core-tests |
||||
project: panels |
||||
- suite: core-tests |
||||
project: smoke |
||||
- suite: core-tests |
||||
project: dashboards |
||||
- suite: core-tests |
||||
project: loki |
||||
- suite: test-plugins |
||||
project: extensions-test-app |
||||
- suite: test-plugins |
||||
project: grafana-e2etest-datasource |
||||
name: ${{ matrix.suite }}-${{ matrix.project }} |
||||
runs-on: ubuntu-latest-8-cores |
||||
permissions: |
||||
contents: read |
||||
|
||||
steps: |
||||
- uses: actions/checkout@v4 |
||||
with: |
||||
persist-credentials: false |
||||
- uses: actions/download-artifact@v4 |
||||
with: |
||||
name: ${{ needs.build-grafana.outputs.artifact }} |
||||
- name: Run Playwright E2E tests |
||||
uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e |
||||
with: |
||||
verb: run |
||||
args: go run ./pkg/build/daggerbuild/playwright run --package=grafana.tar.gz --project=${{ matrix.project }} |
||||
- name: Set suite name |
||||
id: set-suite-name |
||||
if: success() || failure() |
||||
env: |
||||
SUITE: ${{ matrix.suite }}-${{ matrix.project }} |
||||
run: | |
||||
set -euo pipefail |
||||
echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" |
||||
- uses: actions/upload-artifact@v4 |
||||
if: success() || failure() |
||||
with: |
||||
name: playwright-report-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} |
||||
path: playwright-report |
||||
retention-days: 3 |
||||
- uses: actions/upload-artifact@v4 |
||||
if: success() || failure() |
||||
with: |
||||
name: test-results-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} |
||||
path: test-results |
||||
retention-days: 3 |
||||
|
||||
# This is the job that is actually required by rulesets. |
||||
# We want to only require one job instead of all the individual tests. |
||||
required-playwright-tests: |
||||
needs: |
||||
- run-playwright-tests |
||||
# always() is the best function here. |
||||
# success() || failure() will skip this function if any need is also skipped. |
||||
# That means conditional test suites will fail the entire requirement check. |
||||
if: always() |
||||
|
||||
name: All Playwright E2E tests complete |
||||
runs-on: ubuntu-latest |
||||
steps: |
||||
- name: Check test suites |
||||
env: |
||||
NEEDS: ${{ toJson(needs) }} |
||||
run: | |
||||
FAILURES="$(echo "$NEEDS" | jq 'with_entries(select(.value.result == "failure")) | map_values(.result)')" |
||||
echo "$FAILURES" |
||||
if [ "$(echo "$FAILURES" | jq '. | length')" != "0" ]; then |
||||
exit 1 |
||||
fi |
||||
echo "All OK!" |
||||
@ -0,0 +1,22 @@ |
||||
# Pa11y accessability tests |
||||
|
||||
We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner. |
||||
|
||||
To run the tests locally: |
||||
|
||||
1. Install dagger locally https://docs.dagger.io/install/ |
||||
2. Grab the grafana.tar.gz artifact by either |
||||
1. Downloading it from the Github Action artifact from your PR |
||||
1. Build it locally with: |
||||
```sh |
||||
dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt |
||||
cat dist/files.txt # Will output the path to the grafana.tar.gz |
||||
``` |
||||
3. Run the dagger pipeline with: |
||||
```sh |
||||
dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json |
||||
``` |
||||
The JSON results file will be saved to the file from the `--results` arg |
||||
4. If they fail and you want to see the full output |
||||
1. Run the dagger command with `dagger -vE [...]` |
||||
2. At the end, arrow up to the exec pa11y-ci segment and hit Enter |
||||
@ -0,0 +1,177 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"log" |
||||
"os" |
||||
"os/signal" |
||||
|
||||
"dagger.io/dagger" |
||||
"github.com/urfave/cli/v3" |
||||
) |
||||
|
||||
var ( |
||||
grafanaHost = "grafana" |
||||
grafanaPort = 3001 |
||||
) |
||||
|
||||
func main() { |
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) |
||||
defer cancel() |
||||
|
||||
if err := NewApp().Run(ctx, os.Args); err != nil { |
||||
cancel() |
||||
fmt.Println(err) |
||||
os.Exit(1) |
||||
} |
||||
} |
||||
|
||||
func NewApp() *cli.Command { |
||||
return &cli.Command{ |
||||
Name: "a11y", |
||||
Usage: "Run Grafana playwright e2e tests", |
||||
Flags: []cli.Flag{ |
||||
&cli.StringFlag{ |
||||
Name: "grafana-dir", |
||||
Usage: "Path to the grafana/grafana clone directory", |
||||
Value: ".", |
||||
Validator: mustBeDir("grafana-dir"), |
||||
TakesFile: true, |
||||
}, |
||||
&cli.StringFlag{ |
||||
Name: "package", |
||||
Usage: "Path to the grafana tar.gz package", |
||||
Value: "grafana.tar.gz", |
||||
Validator: mustBeFile("package", false), |
||||
TakesFile: true, |
||||
}, |
||||
&cli.StringFlag{ |
||||
Name: "license", |
||||
Usage: "Path to the Grafana Enterprise license file (optional)", |
||||
Validator: mustBeFile("license", true), |
||||
TakesFile: true, |
||||
}, |
||||
// &cli.StringFlag{
|
||||
// Name: "config",
|
||||
// Usage: "Path to the pa11y config file to use",
|
||||
// Value: "e2e/pa11yci.conf.js",
|
||||
// Validator: mustBeFile("config", true),
|
||||
// TakesFile: true,
|
||||
// },
|
||||
// &cli.StringFlag{
|
||||
// Name: "results",
|
||||
// Usage: "Path to the pa11y results file to export",
|
||||
// TakesFile: true,
|
||||
// },
|
||||
// &cli.BoolFlag{
|
||||
// Name: "no-threshold-fail",
|
||||
// Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds",
|
||||
// Value: false,
|
||||
// },
|
||||
}, |
||||
Action: run, |
||||
} |
||||
} |
||||
|
||||
func run(ctx context.Context, cmd *cli.Command) error { |
||||
grafanaDir := cmd.String("grafana-dir") |
||||
targzPath := cmd.String("package") |
||||
licensePath := cmd.String("license") |
||||
// pa11yConfigPath := cmd.String("config")
|
||||
// pa11yResultsPath := cmd.String("results")
|
||||
// noThresholdFail := cmd.Bool("no-threshold-fail")
|
||||
|
||||
d, err := dagger.Connect(ctx) |
||||
if err != nil { |
||||
return fmt.Errorf("failed to connect to Dagger: %w", err) |
||||
} |
||||
|
||||
// Explicitly only the files used by the grafana-server service
|
||||
hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ |
||||
Include: []string{ |
||||
"./devenv", |
||||
"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
|
||||
"./scripts/grafana-server/custom.ini", |
||||
"./scripts/grafana-server/start-server", |
||||
"./scripts/grafana-server/kill-server", |
||||
"./scripts/grafana-server/variables", |
||||
}, |
||||
}) |
||||
|
||||
targz := d.Host().File(targzPath) |
||||
// pa11yConfig := d.Host().File(pa11yConfigPath)
|
||||
|
||||
var license *dagger.File |
||||
if licensePath != "" { |
||||
license = d.Host().File(licensePath) |
||||
} |
||||
|
||||
svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{ |
||||
HostSrc: hostSrc, |
||||
GrafanaTarGz: targz, |
||||
License: license, |
||||
}) |
||||
if err != nil { |
||||
return fmt.Errorf("failed to create Grafana service: %w", err) |
||||
} |
||||
|
||||
c, runErr := RunTest(ctx, d, svc) |
||||
if runErr != nil { |
||||
return fmt.Errorf("failed to run a11y test suite: %w", runErr) |
||||
} |
||||
|
||||
c, syncErr := c.Sync(ctx) |
||||
if syncErr != nil { |
||||
return fmt.Errorf("failed to sync a11y test suite: %w", syncErr) |
||||
} |
||||
|
||||
code, codeErr := c.ExitCode(ctx) |
||||
if codeErr != nil { |
||||
return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr) |
||||
} |
||||
|
||||
if code == 0 { |
||||
log.Printf("a11y tests passed with exit code %d", code) |
||||
} else { |
||||
return fmt.Errorf("a11y tests failed with exit code %d", code) |
||||
} |
||||
|
||||
log.Println("a11y tests completed successfully") |
||||
return nil |
||||
} |
||||
|
||||
func mustBeFile(arg string, emptyOk bool) func(string) error { |
||||
return func(s string) error { |
||||
if s == "" { |
||||
if emptyOk { |
||||
return nil |
||||
} |
||||
return cli.Exit(arg+" cannot be empty", 1) |
||||
} |
||||
stat, err := os.Stat(s) |
||||
if err != nil { |
||||
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) |
||||
} |
||||
if stat.IsDir() { |
||||
return cli.Exit(arg+" must be a file, not a directory: "+s, 1) |
||||
} |
||||
return nil |
||||
} |
||||
} |
||||
|
||||
func mustBeDir(arg string) func(string) error { |
||||
return func(s string) error { |
||||
if s == "" { |
||||
return cli.Exit(arg+" cannot be empty", 1) |
||||
} |
||||
stat, err := os.Stat(s) |
||||
if err != nil { |
||||
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) |
||||
} |
||||
if !stat.IsDir() { |
||||
return cli.Exit(arg+" must be a directory: "+s, 1) |
||||
} |
||||
return nil |
||||
} |
||||
} |
||||
@ -0,0 +1,79 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"strings" |
||||
|
||||
"dagger.io/dagger" |
||||
) |
||||
|
||||
// NodeVersionContainer returns a container whose `stdout` will return the node version from the '.nvmrc' file in the directory 'src'.
|
||||
func NodeVersion(d *dagger.Client, src *dagger.File) *dagger.Container { |
||||
return d.Container().From("alpine:3"). |
||||
WithMountedFile("/src/.nvmrc", src). |
||||
WithWorkdir("/src"). |
||||
WithExec([]string{"cat", ".nvmrc"}) |
||||
} |
||||
|
||||
func NodeImage(version string) string { |
||||
return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v")) |
||||
} |
||||
|
||||
func RunTest( |
||||
ctx context.Context, |
||||
d *dagger.Client, |
||||
grafanaService *dagger.Service, |
||||
) (*dagger.Container, error) { |
||||
|
||||
// Explicitly only the files u'sed by e2e tests
|
||||
hostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{ |
||||
Include: []string{ |
||||
// Include all files for a valid yarn workspace install
|
||||
"package.json", |
||||
"yarn.lock", |
||||
".yarnrc.yml", |
||||
".yarn", |
||||
"packages/*/package.json", |
||||
"public/app/plugins/*/*/package.json", |
||||
"e2e/test-plugins/*/package.json", |
||||
".nvmrc", |
||||
"public/app/types/*.d.ts", |
||||
|
||||
// packages we use in playwright tests
|
||||
"packages", |
||||
|
||||
// e2e files
|
||||
"e2e-playwright", |
||||
}, |
||||
Exclude: []string{ |
||||
"packages/*/dist", |
||||
}, |
||||
}) |
||||
|
||||
nodeVersion, err := NodeVersion(d, hostSrc.File(".nvmrc")).Stdout(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
yarnCache := d.CacheVolume("yarn-cache") |
||||
yarnCacheDir := "/yarn-cache" |
||||
|
||||
pa11yContainer := d.Container().From(NodeImage(nodeVersion)). |
||||
WithExec([]string{"npx", "-y", "playwright@1.52.0", "install", "--with-deps"}). // TODO: sync version from package.json
|
||||
WithWorkdir("/src"). |
||||
WithDirectory("/src", hostSrc). |
||||
WithMountedCache(yarnCacheDir, yarnCache). |
||||
WithExec([]string{"corepack", "enable"}). |
||||
WithExec([]string{"corepack", "install"}). |
||||
WithEnvVariable("YARN_CACHE_FOLDER", yarnCacheDir). |
||||
WithExec([]string{"yarn", "config", "get", "cacheFolder"}). |
||||
WithExec([]string{"yarn", "install", "--immutable"}). |
||||
// WithExec([]string{"yarn", "e2e:plugin:build"}).
|
||||
WithEnvVariable("HOST", grafanaHost). |
||||
WithEnvVariable("PORT", fmt.Sprint(grafanaPort)). |
||||
WithExec([]string{"yarn", "packages:build"}). |
||||
WithExec([]string{"yarn", "e2e:playwright"}) |
||||
|
||||
return pa11yContainer, nil |
||||
} |
||||
@ -0,0 +1,51 @@ |
||||
package main |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"os" |
||||
"strings" |
||||
|
||||
"dagger.io/dagger" |
||||
) |
||||
|
||||
type GrafanaServiceOpts struct { |
||||
HostSrc *dagger.Directory |
||||
GrafanaTarGz *dagger.File |
||||
License *dagger.File |
||||
} |
||||
|
||||
func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) { |
||||
container := d.Container().From("alpine:3"). |
||||
WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}). |
||||
WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz). |
||||
WithExec([]string{"mkdir", "-p", "/src/grafana"}). |
||||
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). |
||||
WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")). |
||||
WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")). |
||||
WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")). |
||||
WithWorkdir("/src/grafana"). |
||||
WithEnvVariable("GF_APP_MODE", "development"). |
||||
WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)). |
||||
WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1"). |
||||
WithExposedPort(grafanaPort) |
||||
|
||||
var licenseArg string |
||||
if opts.License != nil { |
||||
licenseArg = "/src/license.jwt" |
||||
container = container.WithMountedFile(licenseArg, opts.License) |
||||
} |
||||
|
||||
// We add all GF_ environment variables to allow for overriding Grafana configuration.
|
||||
// It is unlikely the runner has any such otherwise.
|
||||
for _, env := range os.Environ() { |
||||
if strings.HasPrefix(env, "GF_") { |
||||
parts := strings.SplitN(env, "=", 2) |
||||
container = container.WithEnvVariable(parts[0], parts[1]) |
||||
} |
||||
} |
||||
|
||||
svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}}) |
||||
|
||||
return svc, nil |
||||
} |
||||
Loading…
Reference in new issue