From 0fe200ce41456ba1d50ac9e5ceb6abd13502c870 Mon Sep 17 00:00:00 2001 From: sam boyer Date: Fri, 8 Oct 2021 08:47:55 -0400 Subject: [PATCH] Do codegen and check no-diff of all (non-blacklisted) CUE->TS codegen during CI (#39922) * Add file blacklist to `grafana-cli cue gen-ts` cmd * Add CI step checking all cuetsification is done * Add dummy command to make the next one fail * Generate drone bits * Check diff output failure * Echo list of untracked files, for failure locality * Move git cleanness checking into script * Blacklist of cue files is complete and correct * Remove news panel plugin from cuetsify blacklist * Dummy commit, check that untracked gen still fail * Tie off remaining errors * Re-add barchart to blacklist * Remove file left around by earlier pipeline * Commit generated news models.gen.ts * Include eslint as part of cuetsified output gen * Update pkg/cmd/grafana-cli/commands/cuetsify_command.go Co-authored-by: Ashley Harrison * Update scripts/drone/steps/lib.star Co-authored-by: Maria Alexandra <239999+axelavargas@users.noreply.github.com> * Update drone.yml * Last fix on .drone.yml Co-authored-by: Ashley Harrison Co-authored-by: Maria Alexandra <239999+axelavargas@users.noreply.github.com> --- .drone.yml | 114 +++++++++++++++++- .../grafana-cli/commands/cuetsify_command.go | 42 ++++++- public/app/plugins/panel/news/models.gen.ts | 7 +- scripts/clean-git-or-error.sh | 27 +++++ scripts/drone/pipelines/main.star | 2 + scripts/drone/pipelines/pr.star | 2 + scripts/drone/pipelines/release.star | 4 +- scripts/drone/steps/lib.star | 19 +++ 8 files changed, 210 insertions(+), 7 deletions(-) create mode 100755 scripts/clean-git-or-error.sh diff --git a/.drone.yml b/.drone.yml index 47d196a40ce..da6621713d4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,6 +28,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -105,6 +106,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: gen-version image: grafana/build-container:1.4.3 commands: @@ -298,6 +312,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -378,6 +393,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: gen-version image: grafana/build-container:1.4.3 commands: @@ -771,6 +799,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -854,6 +883,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: gen-version image: grafana/build-container:1.4.3 commands: @@ -1149,6 +1191,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -1232,6 +1275,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: lint-backend-enterprise2 image: grafana/build-container:1.4.3 commands: @@ -1729,6 +1785,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -1812,6 +1869,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: gen-version image: grafana/build-container:1.4.3 commands: @@ -2096,6 +2166,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -2179,6 +2250,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: lint-backend-enterprise2 image: grafana/build-container:1.4.3 commands: @@ -2669,6 +2753,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -2749,6 +2834,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: gen-version image: grafana/build-container:1.4.3 commands: @@ -3007,6 +3105,7 @@ steps: commands: - "echo -e \"unknwon\nreferer\nerrorstring\neror\niam\nwan\" > words_to_ignore.txt" - codespell -I words_to_ignore.txt docs/ + - rm words_to_ignore.txt depends_on: - initialize @@ -3087,6 +3186,19 @@ steps: depends_on: - build-backend +- name: ensure-cuetsified + image: grafana/build-container:1.4.3 + commands: + - ./bin/linux-amd64/grafana-cli cue gen-ts --grafana-root . + - "# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files." + - "# It is required that the generated Typescript be in sync with the input CUE files." + - "# ...Modulo eslint auto-fixes...:" + - ./node_modules/.bin/eslint . --ext .gen.ts --fix + - "# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result." + - ./scripts/clean-git-or-error.sh + depends_on: + - validate-scuemata + - name: lint-backend-enterprise2 image: grafana/build-container:1.4.3 commands: @@ -3542,6 +3654,6 @@ get: --- kind: signature -hmac: 5237fccbd3eb1fb753ffe2d52c75559311dc57df6b8875b24246cfc266d5895e +hmac: c4e7197d22d8534c2a43c32289ceb3db883b6d9e6f6dbbd1584d054e68f3887a ... diff --git a/pkg/cmd/grafana-cli/commands/cuetsify_command.go b/pkg/cmd/grafana-cli/commands/cuetsify_command.go index af8d4a8d437..cabd6120767 100644 --- a/pkg/cmd/grafana-cli/commands/cuetsify_command.go +++ b/pkg/cmd/grafana-cli/commands/cuetsify_command.go @@ -27,12 +27,38 @@ import ( var ctx = cuecontext.New() +// The only import statement we currently allow in any models.cue file const allowedImport = "github.com/grafana/grafana/packages/grafana-schema/src/schema" var importMap = map[string]string{ allowedImport: "@grafana/schema", } +// Hard-coded list of paths to skip. Remove a particular file as we're ready +// to rely on the TypeScript auto-generated by cuetsy for that particular file. +var skipPaths = []string{ + "public/app/plugins/panel/barchart/models.cue", + "public/app/plugins/panel/bargauge/models.cue", + "public/app/plugins/panel/canvas/models.cue", + "public/app/plugins/panel/dashlist/models.cue", + "public/app/plugins/panel/gauge/models.cue", + "public/app/plugins/panel/histogram/models.cue", + "public/app/plugins/panel/stat/models.cue", + "public/app/plugins/panel/state-timeline/models.cue", + "public/app/plugins/panel/status-history/models.cue", + "public/app/plugins/panel/table/models.cue", + "public/app/plugins/panel/text/models.cue", + "public/app/plugins/panel/timeseries/models.cue", + // All the cue files in this dir have to be individually excluded, even + // though the generator currently smooshes them all together + "packages/grafana-schema/src/schema/graph.cue", + "packages/grafana-schema/src/schema/legend.cue", + "packages/grafana-schema/src/schema/mudball.cue", + "packages/grafana-schema/src/schema/table.cue", + "packages/grafana-schema/src/schema/text.cue", + "packages/grafana-schema/src/schema/tooltip.cue", +} + const prefix = "/" func (cmd Command) generateTypescript(c utils.CommandLine) error { @@ -82,6 +108,20 @@ func (cmd Command) generateTypescript(c utils.CommandLine) error { "packages/grafana-schema/src/scuemata/dashboard/dist": true, } + exclude := func(path string) bool { + dir := filepath.Dir(path) + if excl[dir] { + return true + } + for _, p := range skipPaths { + if path == p { + return true + } + } + + return false + } + outfiles := make(map[string][]byte) cuetsify := func(in fs.FS) error { @@ -92,7 +132,7 @@ func (cmd Command) generateTypescript(c utils.CommandLine) error { } dir := filepath.Dir(path) - if d.IsDir() || filepath.Ext(d.Name()) != ".cue" || seen[dir] || excl[dir] { + if d.IsDir() || filepath.Ext(d.Name()) != ".cue" || seen[dir] || exclude(path) { return nil } seen[dir] = true diff --git a/public/app/plugins/panel/news/models.gen.ts b/public/app/plugins/panel/news/models.gen.ts index 4ef9f2dfd0e..82db65040ac 100644 --- a/public/app/plugins/panel/news/models.gen.ts +++ b/public/app/plugins/panel/news/models.gen.ts @@ -1,14 +1,13 @@ //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -// NOTE: This file will be auto generated from models.cue -// It is currenty hand written but will serve as the target for cuetsy +// This file was autogenerated by cuetsy. DO NOT EDIT! //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -export const modelVersion = Object.freeze([1, 0]); +export const modelVersion = Object.freeze([0, 0]); export interface PanelOptions { feedUrl?: string; - useProxy?: boolean; showImage?: boolean; + useProxy?: boolean; } export const defaultPanelOptions: PanelOptions = { diff --git a/scripts/clean-git-or-error.sh b/scripts/clean-git-or-error.sh new file mode 100755 index 00000000000..3e1b2bcfed6 --- /dev/null +++ b/scripts/clean-git-or-error.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +# Checks that a git working tree is clean. Exit 0 if it is, or exit 1 with a list of +# dirty files - including modifications, removals, and untracked+unignored files. + +# Primarily intended for ensuring in CI that codegen operations are a no-op/have +# already been performed. + +STAT="$(git status --porcelain 2>/dev/null)" +if [ -z "$STAT" ] +then + exit 0 +else + echo "$STAT" + exit 1 +fi + +# For safekeeping, alternative commands that meet a similar goal (in case the +# above approach ends up being problematic) +# +# List modified/removed files and exit nonzero if any exist: +# git diff --stat --exit-code +# +# List untracked files, and exit nonzero if any exist: +# git ls-files --others --exclude-standard', +# u="$(git ls-files --others --exclude-standard)" && test -z "$u" \ No newline at end of file diff --git a/scripts/drone/pipelines/main.star b/scripts/drone/pipelines/main.star index 24890af3956..dcece78f8d9 100644 --- a/scripts/drone/pipelines/main.star +++ b/scripts/drone/pipelines/main.star @@ -32,6 +32,7 @@ load( 'publish_packages_step', 'upload_cdn_step', 'validate_scuemata_step', + 'ensure_cuetsified_step', 'test_a11y_frontend_step' ) @@ -64,6 +65,7 @@ def get_steps(edition, is_downstream=False): build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream), build_plugins_step(edition=edition, sign=True), validate_scuemata_step(), + ensure_cuetsified_step(), ] # Have to insert Enterprise2 steps before they're depended on (in the gen-version step) diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index c8f11826248..3b0b04c89ca 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -24,6 +24,7 @@ load( 'memcached_integration_tests_step', 'benchmark_ldap_step', 'validate_scuemata_step', + 'ensure_cuetsified_step', 'test_a11y_frontend_step_pr', ) @@ -55,6 +56,7 @@ def pr_pipelines(edition): build_frontend_step(edition=edition, ver_mode=ver_mode), build_plugins_step(edition=edition), validate_scuemata_step(), + ensure_cuetsified_step(), ] # Have to insert Enterprise2 steps before they're depended on (in the gen-version step) diff --git a/scripts/drone/pipelines/release.star b/scripts/drone/pipelines/release.star index b975eb97975..e2cd1d7a60f 100644 --- a/scripts/drone/pipelines/release.star +++ b/scripts/drone/pipelines/release.star @@ -30,7 +30,8 @@ load( 'upload_packages_step', 'publish_packages_step', 'upload_cdn_step', - 'validate_scuemata_step' + 'validate_scuemata_step', + 'ensure_cuetsified_step' ) load( @@ -91,6 +92,7 @@ def get_steps(edition, ver_mode): build_frontend_step(edition=edition, ver_mode=ver_mode), build_plugins_step(edition=edition, sign=True), validate_scuemata_step(), + ensure_cuetsified_step(), ] # Have to insert Enterprise2 steps before they're depended on (in the gen-version step) diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 8f312c57d41..15fb70ba53e 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -480,6 +480,7 @@ def codespell_step(): # Important: all words have to be in lowercase, and separated by "\n". 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', 'codespell -I words_to_ignore.txt docs/', + 'rm words_to_ignore.txt', ], } @@ -1016,3 +1017,21 @@ def validate_scuemata_step(): './bin/linux-amd64/grafana-cli cue validate-schema --grafana-root .', ], } + +def ensure_cuetsified_step(): + return { + 'name': 'ensure-cuetsified', + 'image': build_image, + 'depends_on': [ + 'validate-scuemata', + ], + 'commands': [ + './bin/linux-amd64/grafana-cli cue gen-ts --grafana-root .', + '# The above command generates Typescript files (*.gen.ts) from all appropriate .cue files.', + '# It is required that the generated Typescript be in sync with the input CUE files.', + '# ...Modulo eslint auto-fixes...:', + './node_modules/.bin/eslint . --ext .gen.ts --fix', + '# If any filenames are emitted by the below script, run the generator command `grafana-cli cue gen-ts` locally and commit the result.', + './scripts/clean-git-or-error.sh', + ], + } \ No newline at end of file