mirror of https://github.com/grafana/grafana
joshhunt
3 weeks ago
parent
9b8924fe74
commit
cea1f84437
@ -0,0 +1,146 @@ |
|||||||
|
name: Playwright E2E Tests |
||||||
|
|
||||||
|
on: |
||||||
|
pull_request: |
||||||
|
push: |
||||||
|
branches: |
||||||
|
- main |
||||||
|
- release-*.*.* |
||||||
|
|
||||||
|
concurrency: |
||||||
|
group: ${{ github.workflow }}-${{ github.ref }} |
||||||
|
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} |
||||||
|
|
||||||
|
permissions: {} |
||||||
|
|
||||||
|
jobs: |
||||||
|
build-grafana: |
||||||
|
name: Build & Package Grafana |
||||||
|
runs-on: ubuntu-latest-16-cores |
||||||
|
permissions: |
||||||
|
contents: read |
||||||
|
outputs: |
||||||
|
artifact: ${{ steps.artifact.outputs.artifact }} |
||||||
|
steps: |
||||||
|
- uses: actions/checkout@v4 |
||||||
|
with: |
||||||
|
path: ./grafana |
||||||
|
persist-credentials: false |
||||||
|
- uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e |
||||||
|
with: |
||||||
|
verb: run |
||||||
|
args: go -C grafana run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="${PWD}/grafana" > out.txt |
||||||
|
- run: mv "$(cat out.txt)" grafana.tar.gz |
||||||
|
- run: echo "artifact=grafana-playwright-${{github.run_number}}" >> "$GITHUB_OUTPUT" |
||||||
|
id: artifact |
||||||
|
- uses: actions/upload-artifact@v4 |
||||||
|
id: upload |
||||||
|
with: |
||||||
|
retention-days: 1 |
||||||
|
name: ${{ steps.artifact.outputs.artifact }} |
||||||
|
path: grafana.tar.gz |
||||||
|
|
||||||
|
run-playwright-tests: |
||||||
|
needs: |
||||||
|
- build-grafana |
||||||
|
strategy: |
||||||
|
fail-fast: false |
||||||
|
matrix: |
||||||
|
include: |
||||||
|
- suite: plugin-e2e-api-tests |
||||||
|
project: admin |
||||||
|
- suite: plugin-e2e-api-tests |
||||||
|
project: viewer |
||||||
|
- suite: plugin-e2e |
||||||
|
project: elasticsearch |
||||||
|
- suite: plugin-e2e |
||||||
|
project: mysql |
||||||
|
- suite: plugin-e2e |
||||||
|
project: mssql |
||||||
|
- suite: plugin-e2e |
||||||
|
project: cloudwatch |
||||||
|
- suite: plugin-e2e |
||||||
|
project: azuremonitor |
||||||
|
- suite: plugin-e2e |
||||||
|
project: cloudmonitoring |
||||||
|
- suite: plugin-e2e |
||||||
|
project: graphite |
||||||
|
- suite: plugin-e2e |
||||||
|
project: influxdb |
||||||
|
- suite: plugin-e2e |
||||||
|
project: jaeger |
||||||
|
- suite: plugin-e2e |
||||||
|
project: zipkin |
||||||
|
- suite: core-tests |
||||||
|
project: panels |
||||||
|
- suite: core-tests |
||||||
|
project: smoke |
||||||
|
- suite: core-tests |
||||||
|
project: dashboards |
||||||
|
- suite: core-tests |
||||||
|
project: loki |
||||||
|
- suite: test-plugins |
||||||
|
project: extensions-test-app |
||||||
|
- suite: test-plugins |
||||||
|
project: grafana-e2etest-datasource |
||||||
|
name: ${{ matrix.suite }}-${{ matrix.project }} |
||||||
|
runs-on: ubuntu-latest-8-cores |
||||||
|
permissions: |
||||||
|
contents: read |
||||||
|
|
||||||
|
steps: |
||||||
|
- uses: actions/checkout@v4 |
||||||
|
with: |
||||||
|
persist-credentials: false |
||||||
|
- uses: actions/download-artifact@v4 |
||||||
|
with: |
||||||
|
name: ${{ needs.build-grafana.outputs.artifact }} |
||||||
|
- name: Run Playwright E2E tests |
||||||
|
uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e |
||||||
|
with: |
||||||
|
verb: run |
||||||
|
args: go run ./pkg/build/daggerbuild/playwright run --package=grafana.tar.gz --project=${{ matrix.project }} |
||||||
|
- name: Set suite name |
||||||
|
id: set-suite-name |
||||||
|
if: success() || failure() |
||||||
|
env: |
||||||
|
SUITE: ${{ matrix.suite }}-${{ matrix.project }} |
||||||
|
run: | |
||||||
|
set -euo pipefail |
||||||
|
echo "suite=$(echo "$SUITE" | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" |
||||||
|
- uses: actions/upload-artifact@v4 |
||||||
|
if: success() || failure() |
||||||
|
with: |
||||||
|
name: playwright-report-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} |
||||||
|
path: playwright-report |
||||||
|
retention-days: 3 |
||||||
|
- uses: actions/upload-artifact@v4 |
||||||
|
if: success() || failure() |
||||||
|
with: |
||||||
|
name: test-results-${{ steps.set-suite-name.outputs.suite }}-${{ github.run_number }} |
||||||
|
path: test-results |
||||||
|
retention-days: 3 |
||||||
|
|
||||||
|
# This is the job that is actually required by rulesets. |
||||||
|
# We want to only require one job instead of all the individual tests. |
||||||
|
required-playwright-tests: |
||||||
|
needs: |
||||||
|
- run-playwright-tests |
||||||
|
# always() is the best function here. |
||||||
|
# success() || failure() will skip this function if any need is also skipped. |
||||||
|
# That means conditional test suites will fail the entire requirement check. |
||||||
|
if: always() |
||||||
|
|
||||||
|
name: All Playwright E2E tests complete |
||||||
|
runs-on: ubuntu-latest |
||||||
|
steps: |
||||||
|
- name: Check test suites |
||||||
|
env: |
||||||
|
NEEDS: ${{ toJson(needs) }} |
||||||
|
run: | |
||||||
|
FAILURES="$(echo "$NEEDS" | jq 'with_entries(select(.value.result == "failure")) | map_values(.result)')" |
||||||
|
echo "$FAILURES" |
||||||
|
if [ "$(echo "$FAILURES" | jq '. | length')" != "0" ]; then |
||||||
|
exit 1 |
||||||
|
fi |
||||||
|
echo "All OK!" |
||||||
@ -0,0 +1,22 @@ |
|||||||
|
# Pa11y accessability tests |
||||||
|
|
||||||
|
We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner. |
||||||
|
|
||||||
|
To run the tests locally: |
||||||
|
|
||||||
|
1. Install dagger locally https://docs.dagger.io/install/ |
||||||
|
2. Grab the grafana.tar.gz artifact by either |
||||||
|
1. Downloading it from the Github Action artifact from your PR |
||||||
|
1. Build it locally with: |
||||||
|
```sh |
||||||
|
dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt |
||||||
|
cat dist/files.txt # Will output the path to the grafana.tar.gz |
||||||
|
``` |
||||||
|
3. Run the dagger pipeline with: |
||||||
|
```sh |
||||||
|
dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json |
||||||
|
``` |
||||||
|
The JSON results file will be saved to the file from the `--results` arg |
||||||
|
4. If they fail and you want to see the full output |
||||||
|
1. Run the dagger command with `dagger -vE [...]` |
||||||
|
2. At the end, arrow up to the exec pa11y-ci segment and hit Enter |
||||||
@ -0,0 +1,177 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"log" |
||||||
|
"os" |
||||||
|
"os/signal" |
||||||
|
|
||||||
|
"dagger.io/dagger" |
||||||
|
"github.com/urfave/cli/v3" |
||||||
|
) |
||||||
|
|
||||||
|
var ( |
||||||
|
grafanaHost = "grafana" |
||||||
|
grafanaPort = 3001 |
||||||
|
) |
||||||
|
|
||||||
|
func main() { |
||||||
|
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) |
||||||
|
defer cancel() |
||||||
|
|
||||||
|
if err := NewApp().Run(ctx, os.Args); err != nil { |
||||||
|
cancel() |
||||||
|
fmt.Println(err) |
||||||
|
os.Exit(1) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func NewApp() *cli.Command { |
||||||
|
return &cli.Command{ |
||||||
|
Name: "a11y", |
||||||
|
Usage: "Run Grafana playwright e2e tests", |
||||||
|
Flags: []cli.Flag{ |
||||||
|
&cli.StringFlag{ |
||||||
|
Name: "grafana-dir", |
||||||
|
Usage: "Path to the grafana/grafana clone directory", |
||||||
|
Value: ".", |
||||||
|
Validator: mustBeDir("grafana-dir"), |
||||||
|
TakesFile: true, |
||||||
|
}, |
||||||
|
&cli.StringFlag{ |
||||||
|
Name: "package", |
||||||
|
Usage: "Path to the grafana tar.gz package", |
||||||
|
Value: "grafana.tar.gz", |
||||||
|
Validator: mustBeFile("package", false), |
||||||
|
TakesFile: true, |
||||||
|
}, |
||||||
|
&cli.StringFlag{ |
||||||
|
Name: "license", |
||||||
|
Usage: "Path to the Grafana Enterprise license file (optional)", |
||||||
|
Validator: mustBeFile("license", true), |
||||||
|
TakesFile: true, |
||||||
|
}, |
||||||
|
// &cli.StringFlag{
|
||||||
|
// Name: "config",
|
||||||
|
// Usage: "Path to the pa11y config file to use",
|
||||||
|
// Value: "e2e/pa11yci.conf.js",
|
||||||
|
// Validator: mustBeFile("config", true),
|
||||||
|
// TakesFile: true,
|
||||||
|
// },
|
||||||
|
// &cli.StringFlag{
|
||||||
|
// Name: "results",
|
||||||
|
// Usage: "Path to the pa11y results file to export",
|
||||||
|
// TakesFile: true,
|
||||||
|
// },
|
||||||
|
// &cli.BoolFlag{
|
||||||
|
// Name: "no-threshold-fail",
|
||||||
|
// Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds",
|
||||||
|
// Value: false,
|
||||||
|
// },
|
||||||
|
}, |
||||||
|
Action: run, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func run(ctx context.Context, cmd *cli.Command) error { |
||||||
|
grafanaDir := cmd.String("grafana-dir") |
||||||
|
targzPath := cmd.String("package") |
||||||
|
licensePath := cmd.String("license") |
||||||
|
// pa11yConfigPath := cmd.String("config")
|
||||||
|
// pa11yResultsPath := cmd.String("results")
|
||||||
|
// noThresholdFail := cmd.Bool("no-threshold-fail")
|
||||||
|
|
||||||
|
d, err := dagger.Connect(ctx) |
||||||
|
if err != nil { |
||||||
|
return fmt.Errorf("failed to connect to Dagger: %w", err) |
||||||
|
} |
||||||
|
|
||||||
|
// Explicitly only the files used by the grafana-server service
|
||||||
|
hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{ |
||||||
|
Include: []string{ |
||||||
|
"./devenv", |
||||||
|
"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
|
||||||
|
"./scripts/grafana-server/custom.ini", |
||||||
|
"./scripts/grafana-server/start-server", |
||||||
|
"./scripts/grafana-server/kill-server", |
||||||
|
"./scripts/grafana-server/variables", |
||||||
|
}, |
||||||
|
}) |
||||||
|
|
||||||
|
targz := d.Host().File(targzPath) |
||||||
|
// pa11yConfig := d.Host().File(pa11yConfigPath)
|
||||||
|
|
||||||
|
var license *dagger.File |
||||||
|
if licensePath != "" { |
||||||
|
license = d.Host().File(licensePath) |
||||||
|
} |
||||||
|
|
||||||
|
svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{ |
||||||
|
HostSrc: hostSrc, |
||||||
|
GrafanaTarGz: targz, |
||||||
|
License: license, |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return fmt.Errorf("failed to create Grafana service: %w", err) |
||||||
|
} |
||||||
|
|
||||||
|
c, runErr := RunTest(ctx, d, svc) |
||||||
|
if runErr != nil { |
||||||
|
return fmt.Errorf("failed to run a11y test suite: %w", runErr) |
||||||
|
} |
||||||
|
|
||||||
|
c, syncErr := c.Sync(ctx) |
||||||
|
if syncErr != nil { |
||||||
|
return fmt.Errorf("failed to sync a11y test suite: %w", syncErr) |
||||||
|
} |
||||||
|
|
||||||
|
code, codeErr := c.ExitCode(ctx) |
||||||
|
if codeErr != nil { |
||||||
|
return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr) |
||||||
|
} |
||||||
|
|
||||||
|
if code == 0 { |
||||||
|
log.Printf("a11y tests passed with exit code %d", code) |
||||||
|
} else { |
||||||
|
return fmt.Errorf("a11y tests failed with exit code %d", code) |
||||||
|
} |
||||||
|
|
||||||
|
log.Println("a11y tests completed successfully") |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func mustBeFile(arg string, emptyOk bool) func(string) error { |
||||||
|
return func(s string) error { |
||||||
|
if s == "" { |
||||||
|
if emptyOk { |
||||||
|
return nil |
||||||
|
} |
||||||
|
return cli.Exit(arg+" cannot be empty", 1) |
||||||
|
} |
||||||
|
stat, err := os.Stat(s) |
||||||
|
if err != nil { |
||||||
|
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) |
||||||
|
} |
||||||
|
if stat.IsDir() { |
||||||
|
return cli.Exit(arg+" must be a file, not a directory: "+s, 1) |
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func mustBeDir(arg string) func(string) error { |
||||||
|
return func(s string) error { |
||||||
|
if s == "" { |
||||||
|
return cli.Exit(arg+" cannot be empty", 1) |
||||||
|
} |
||||||
|
stat, err := os.Stat(s) |
||||||
|
if err != nil { |
||||||
|
return cli.Exit(arg+" does not exist or cannot be read: "+s, 1) |
||||||
|
} |
||||||
|
if !stat.IsDir() { |
||||||
|
return cli.Exit(arg+" must be a directory: "+s, 1) |
||||||
|
} |
||||||
|
return nil |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,79 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"strings" |
||||||
|
|
||||||
|
"dagger.io/dagger" |
||||||
|
) |
||||||
|
|
||||||
|
// NodeVersionContainer returns a container whose `stdout` will return the node version from the '.nvmrc' file in the directory 'src'.
|
||||||
|
func NodeVersion(d *dagger.Client, src *dagger.File) *dagger.Container { |
||||||
|
return d.Container().From("alpine:3"). |
||||||
|
WithMountedFile("/src/.nvmrc", src). |
||||||
|
WithWorkdir("/src"). |
||||||
|
WithExec([]string{"cat", ".nvmrc"}) |
||||||
|
} |
||||||
|
|
||||||
|
func NodeImage(version string) string { |
||||||
|
return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v")) |
||||||
|
} |
||||||
|
|
||||||
|
func RunTest( |
||||||
|
ctx context.Context, |
||||||
|
d *dagger.Client, |
||||||
|
grafanaService *dagger.Service, |
||||||
|
) (*dagger.Container, error) { |
||||||
|
|
||||||
|
// Explicitly only the files u'sed by e2e tests
|
||||||
|
hostSrc := d.Host().Directory(".", dagger.HostDirectoryOpts{ |
||||||
|
Include: []string{ |
||||||
|
// Include all files for a valid yarn workspace install
|
||||||
|
"package.json", |
||||||
|
"yarn.lock", |
||||||
|
".yarnrc.yml", |
||||||
|
".yarn", |
||||||
|
"packages/*/package.json", |
||||||
|
"public/app/plugins/*/*/package.json", |
||||||
|
"e2e/test-plugins/*/package.json", |
||||||
|
".nvmrc", |
||||||
|
"public/app/types/*.d.ts", |
||||||
|
|
||||||
|
// packages we use in playwright tests
|
||||||
|
"packages", |
||||||
|
|
||||||
|
// e2e files
|
||||||
|
"e2e-playwright", |
||||||
|
}, |
||||||
|
Exclude: []string{ |
||||||
|
"packages/*/dist", |
||||||
|
}, |
||||||
|
}) |
||||||
|
|
||||||
|
nodeVersion, err := NodeVersion(d, hostSrc.File(".nvmrc")).Stdout(ctx) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
yarnCache := d.CacheVolume("yarn-cache") |
||||||
|
yarnCacheDir := "/yarn-cache" |
||||||
|
|
||||||
|
pa11yContainer := d.Container().From(NodeImage(nodeVersion)). |
||||||
|
WithExec([]string{"npx", "-y", "playwright@1.52.0", "install", "--with-deps"}). // TODO: sync version from package.json
|
||||||
|
WithWorkdir("/src"). |
||||||
|
WithDirectory("/src", hostSrc). |
||||||
|
WithMountedCache(yarnCacheDir, yarnCache). |
||||||
|
WithExec([]string{"corepack", "enable"}). |
||||||
|
WithExec([]string{"corepack", "install"}). |
||||||
|
WithEnvVariable("YARN_CACHE_FOLDER", yarnCacheDir). |
||||||
|
WithExec([]string{"yarn", "config", "get", "cacheFolder"}). |
||||||
|
WithExec([]string{"yarn", "install", "--immutable"}). |
||||||
|
// WithExec([]string{"yarn", "e2e:plugin:build"}).
|
||||||
|
WithEnvVariable("HOST", grafanaHost). |
||||||
|
WithEnvVariable("PORT", fmt.Sprint(grafanaPort)). |
||||||
|
WithExec([]string{"yarn", "packages:build"}). |
||||||
|
WithExec([]string{"yarn", "e2e:playwright"}) |
||||||
|
|
||||||
|
return pa11yContainer, nil |
||||||
|
} |
||||||
@ -0,0 +1,51 @@ |
|||||||
|
package main |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"os" |
||||||
|
"strings" |
||||||
|
|
||||||
|
"dagger.io/dagger" |
||||||
|
) |
||||||
|
|
||||||
|
type GrafanaServiceOpts struct { |
||||||
|
HostSrc *dagger.Directory |
||||||
|
GrafanaTarGz *dagger.File |
||||||
|
License *dagger.File |
||||||
|
} |
||||||
|
|
||||||
|
func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) { |
||||||
|
container := d.Container().From("alpine:3"). |
||||||
|
WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}). |
||||||
|
WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz). |
||||||
|
WithExec([]string{"mkdir", "-p", "/src/grafana"}). |
||||||
|
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}). |
||||||
|
WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")). |
||||||
|
WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")). |
||||||
|
WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")). |
||||||
|
WithWorkdir("/src/grafana"). |
||||||
|
WithEnvVariable("GF_APP_MODE", "development"). |
||||||
|
WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)). |
||||||
|
WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1"). |
||||||
|
WithExposedPort(grafanaPort) |
||||||
|
|
||||||
|
var licenseArg string |
||||||
|
if opts.License != nil { |
||||||
|
licenseArg = "/src/license.jwt" |
||||||
|
container = container.WithMountedFile(licenseArg, opts.License) |
||||||
|
} |
||||||
|
|
||||||
|
// We add all GF_ environment variables to allow for overriding Grafana configuration.
|
||||||
|
// It is unlikely the runner has any such otherwise.
|
||||||
|
for _, env := range os.Environ() { |
||||||
|
if strings.HasPrefix(env, "GF_") { |
||||||
|
parts := strings.SplitN(env, "=", 2) |
||||||
|
container = container.WithEnvVariable(parts[0], parts[1]) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
svc := container.AsService(dagger.ContainerAsServiceOpts{Args: []string{"bash", "-x", "scripts/grafana-server/start-server", licenseArg}}) |
||||||
|
|
||||||
|
return svc, nil |
||||||
|
} |
||||||
Loading…
Reference in new issue