Scuemata: Add test to validate devenv resources (#35810)

* Add test for devenv resources

* Refactor validation tests for grokkability

* Devenv dashboards error-tracking script

* Refactor to use cueerrors.Details()

* Further test refinement

* Close major elements of dashboard schema

* Centralize dashboard validation tests

General dashboard validation testing belongs in the load package.

* Better names for error context on glue CUE code

* Fixup validate-resource

Do only one of base or dist, and fix copied docs.

* Skip the devenv test

* Remove test for validateResources

* Fix shellcheck

* Backend linter

Co-authored-by: sam boyer <sdboyer@grafana.com>

cue/data/gen.cue

@@ -80,7 +80,6 @@ Family: scuemata.#Family & {
                 // synthetic Family to represent them in Go, for ease of generating
                 // e.g. JSON Schema.
                 #Panel: {
-                    ...
                     // The panel plugin type id. 
                     type: !=""
 
@@ -135,7 +134,6 @@ Family: scuemata.#Family & {
                     options: {...}
                     fieldConfig: {
                         defaults: {
-                            ...
                             // The display value for this field.  This supports template variables blank is auto
                             displayName?: string
 
@@ -189,7 +187,7 @@ Family: scuemata.#Family & {
                             // Can always exist. Valid fields within this are
                             // defined by the panel plugin - that's the
                             // PanelFieldConfig that comes from the plugin.
-                            custom?: {...}
+                            custom?: {}
                         }
                         overrides: [...{
                             matcher: {

pkg/cmd/grafana-cli/commands/commands.go

@@ -142,13 +142,18 @@ var cueCommands = []*cli.Command{
 	},
 	{
 		Name:   "validate-resource",
-		Usage:  "validate *.cue files in the project",
+		Usage:  "validate resource files (e.g. dashboard JSON) against schema",
 		Action: runPluginCommand(cmd.validateResources),
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "dashboard",
 				Usage: "dashboard JSON file to validate",
 			},
+			&cli.BoolFlag{
+				Name:  "base-only",
+				Usage: "validate using only base schema, not dist (includes plugin schema)",
+				Value: false,
+			},
 		},
 	},
 }

pkg/cmd/grafana-cli/commands/scuemata_validation_command.go

@@ -1,10 +1,12 @@
 package commands
 
 import (
+	gerrors "errors"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"cuelang.org/go/cue/errors"
 	"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
 	"github.com/grafana/grafana/pkg/schema"
 	"github.com/grafana/grafana/pkg/schema/load"
@@ -25,37 +27,31 @@ func (cmd Command) validateScuemataBasics(c utils.CommandLine) error {
 }
 
 func (cmd Command) validateResources(c utils.CommandLine) error {
-	resource := c.String("dashboard")
-	b, err := os.Open(filepath.Clean(resource))
-	if err != nil {
-		return err
-	}
-
-	if err := validateResources(b, paths, load.BaseDashboardFamily); err != nil {
-		return err
+	filename := c.String("dashboard")
+	baseonly := c.Bool("base-only")
+	if filename == "" {
+		return gerrors.New("must specify dashboard to validate with --dashboard")
 	}
-
-	if err := validateResources(b, paths, load.DistDashboardFamily); err != nil {
+	b, err := os.Open(filepath.Clean(filename))
+	res := schema.Resource{Value: b, Name: filename}
+	if err != nil {
 		return err
 	}
 
-	return nil
+	var sch schema.VersionedCueSchema
+	if baseonly {
+		sch, err = load.BaseDashboardFamily(paths)
+	} else {
+		sch, err = load.DistDashboardFamily(paths)
 	}
-
-func validateResources(resource interface{}, p load.BaseLoadPaths, loader func(p load.BaseLoadPaths) (schema.VersionedCueSchema, error)) error {
-	dash, err := loader(p)
 	if err != nil {
 		return fmt.Errorf("error while loading dashboard scuemata, err: %w", err)
 	}
 
-	// Validate checks that the resource is correct with respect to the schema.
-	if resource != nil {
-		err = dash.Validate(schema.Resource{Value: resource})
+	err = sch.Validate(res)
 	if err != nil {
-			return fmt.Errorf("failed validation: %w", err)
-		}
+		return gerrors.New(errors.Details(err, nil))
 	}
-
 	return nil
 }
 

pkg/cmd/grafana-cli/commands/scuemata_validation_command_test.go

@@ -1,9 +1,7 @@
 package commands
 
 import (
-	"io/fs"
 	"os"
-	"path/filepath"
 	"testing"
 	"testing/fstest"
 
@@ -67,55 +65,4 @@ func TestValidateScuemataBasics(t *testing.T) {
 		err = validateScuemata(baseLoadPaths, load.DistDashboardFamily)
 		assert.EqualError(t, err, "all schema should be valid with respect to basic CUE rules, Family.lineages.0.0: field #Panel not allowed")
 	})
-
-	t.Run("Testing validateResources against scuemata and resource inputs", func(t *testing.T) {
-		validPanel, err := os.ReadFile("testdata/panels/valid_resource_panel.json")
-		require.NoError(t, err)
-
-		invalidPanel, err := os.ReadFile("testdata/panels/invalid_resource_panel.json")
-		require.NoError(t, err)
-
-		filesystem := fstest.MapFS{
-			"valid.json":   &fstest.MapFile{Data: validPanel},
-			"invalid.json": &fstest.MapFile{Data: invalidPanel},
-		}
-		mergedFS := mergefs.Merge(filesystem, defaultBaseLoadPaths.BaseCueFS)
-
-		var baseLoadPaths = load.BaseLoadPaths{
-			BaseCueFS:       mergedFS,
-			DistPluginCueFS: defaultBaseLoadPaths.DistPluginCueFS,
-		}
-
-		require.NoError(t, fs.WalkDir(mergedFS, ".", func(path string, d fs.DirEntry, err error) error {
-			require.NoError(t, err)
-
-			if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
-				return nil
-			}
-
-			if d.Name() == "valid.json" {
-				t.Run(path, func(t *testing.T) {
-					b, err := mergedFS.Open(path)
-					require.NoError(t, err, "failed to open dashboard file")
-
-					err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
-					require.NoError(t, err, "error while loading base dashboard scuemata")
-
-					err = validateResources(b, baseLoadPaths, load.DistDashboardFamily)
-					require.NoError(t, err, "error while loading base dashboard scuemata")
-				})
-			}
-			if d.Name() == "invalid.json" {
-				t.Run(path, func(t *testing.T) {
-					b, err := mergedFS.Open(path)
-					require.NoError(t, err, "failed to open dashboard file")
-
-					err = validateResources(b, baseLoadPaths, load.BaseDashboardFamily)
-					assert.EqualError(t, err, "failed validation: Family.lineages.0.0.panels.0.type: incomplete value !=\"\"")
-				})
-			}
-
-			return nil
-		}))
-	})
 }

pkg/schema/load/dashboard.go

@@ -81,7 +81,7 @@ func DistDashboardFamily(p BaseLoadPaths) (schema.VersionedCueSchema, error) {
 	// Value.Fill() can't target definitions. Need new method based on cue.Path;
 	// a CL has been merged that creates FillPath and will be in the next
 	// release of CUE.
-	dummy, _ := rt.Compile("mergeStruct", `
+	dummy, _ := rt.Compile("glue-unifyPanelDashboard", `
 	obj: {}
 	dummy: {
 		#Panel: obj
@@ -125,7 +125,11 @@ type compositeDashboardSchema struct {
 
 // Validate checks that the resource is correct with respect to the schema.
 func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
-	rv, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rv, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return err
 	}

pkg/schema/load/generic.go

@@ -102,7 +102,11 @@ type genericVersionedSchema struct {
 
 // Validate checks that the resource is correct with respect to the schema.
 func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
-	rv, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rv, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return err
 	}

pkg/schema/load/load_test.go

@@ -1,13 +1,16 @@
 package load
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
 	"testing/fstest"
 
+	"cuelang.org/go/cue/errors"
 	"github.com/grafana/grafana/pkg/schema"
 	"github.com/laher/mergefs"
 	"github.com/stretchr/testify/require"
@@ -45,8 +48,11 @@ func TestScuemataBasics(t *testing.T) {
 	}
 }
 
-func TestDashboardValidity(t *testing.T) {
-	validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
+func TestDevenvDashboardValidity(t *testing.T) {
+	// TODO un-skip when tests pass on all devenv dashboards
+	t.Skip()
+	// validdir := os.DirFS(filepath.Join("..", "..", "..", "devenv", "dev-dashboards"))
+	validdir := filepath.Join("..", "..", "..", "devenv", "dev-dashboards")
 
 	dash, err := BaseDashboardFamily(p)
 	require.NoError(t, err, "error while loading base dashboard scuemata")
@@ -54,30 +60,56 @@ func TestDashboardValidity(t *testing.T) {
 	ddash, err := DistDashboardFamily(p)
 	require.NoError(t, err, "error while loading dist dashboard scuemata")
 
-	require.NoError(t, fs.WalkDir(validdir, ".", func(path string, d fs.DirEntry, err error) error {
+	doTest := func(sch schema.VersionedCueSchema) func(t *testing.T) {
+		return func(t *testing.T) {
+			t.Parallel()
+			require.NoError(t, filepath.Walk(validdir, func(path string, d fs.FileInfo, err error) error {
 				require.NoError(t, err)
 
 				if d.IsDir() || filepath.Ext(d.Name()) != ".json" {
 					return nil
 				}
 
-		t.Run(path, func(t *testing.T) {
-			b, err := validdir.Open(path)
+				// Ignore gosec warning G304 since it's a test
+				// nolint:gosec
+				b, err := os.Open(path)
 				require.NoError(t, err, "failed to open dashboard file")
 
-			t.Run("base", func(t *testing.T) {
-				_, err := schema.SearchAndValidate(dash, b)
-				require.NoError(t, err, "dashboard failed validation")
-			})
-			t.Run("dist", func(t *testing.T) {
-				_, err := schema.SearchAndValidate(ddash, b)
-				require.NoError(t, err, "dashboard failed validation")
-			})
+				// Only try to validate dashboards with schemaVersion >= 30
+				jtree := make(map[string]interface{})
+				byt, err := io.ReadAll(b)
+				if err != nil {
+					t.Fatal(err)
+				}
+				require.NoError(t, json.Unmarshal(byt, &jtree))
+				if oldschemav, has := jtree["schemaVersion"]; !has {
+					t.Logf("no schemaVersion in %s", path)
+					return nil
+				} else {
+					if !(oldschemav.(float64) > 29) {
+						t.Logf("schemaVersion is %v, older than 30, skipping %s", oldschemav, path)
+						return nil
+					}
+				}
+
+				t.Run(filepath.Base(path), func(t *testing.T) {
+					err := sch.Validate(schema.Resource{Value: byt, Name: path})
+					if err != nil {
+						// Testify trims errors to short length. We want the full text
+						t.Fatal(errors.Details(err, nil))
+					}
 				})
 
 				return nil
 			}))
 		}
+	}
+
+	// TODO will need to expand this appropriately when the scuemata contain
+	// more than one schema
+	t.Run("base", doTest(dash))
+	t.Run("dist", doTest(ddash))
+}
 
 func TestPanelValidity(t *testing.T) {
 	t.Skip()

pkg/schema/load/panel.go

@@ -16,7 +16,7 @@ import (
 // Returns a disjunction of structs representing each panel schema version
 // (post-mapping from on-disk #PanelModel form) from each scuemata in the map.
 func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Value, error) {
-	partsi, err := rt.Compile("panelDisjunction", `
+	partsi, err := rt.Compile("glue-panelDisjunction", `
 	allPanels: [Name=_]: {}
 	parts: or([for v in allPanels { v }])
 	`)
@@ -44,7 +44,7 @@ func disjunctPanelScuemata(scuemap map[string]schema.VersionedCueSchema) (cue.Va
 func mapPanelModel(id string, vcs schema.VersionedCueSchema) cue.Value {
 	maj, min := vcs.Version()
 	// Ignore err return, this can't fail to compile
-	inter, _ := rt.Compile("typedPanel", fmt.Sprintf(`
+	inter, _ := rt.Compile(fmt.Sprintf("%s-glue-panelComposition", id), fmt.Sprintf(`
 	in: {
 		type: %q
 		v: {

pkg/schema/schema.go

@@ -277,7 +277,11 @@ func Exact(maj, min int) SearchOption {
 // that are 1) missing in the Resource AND 2) specified by the schema,
 // filled with default values specified by the schema.
 func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) {
-	rv, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rv, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return r, err
 	}
@@ -306,7 +310,11 @@ func convertCUEValueToString(inputCUE cue.Value) (string, error) {
 // in the  where the values at those paths are the same as the default value
 // given in the schema.
 func TrimDefaults(r Resource, scue cue.Value) (Resource, error) {
-	rvInstance, err := rt.Compile("resource", r.Value)
+	name := r.Name
+	if name == "" {
+		name = "resource"
+	}
+	rvInstance, err := rt.Compile(name, r.Value)
 	if err != nil {
 		return r, err
 	}
@@ -329,7 +337,7 @@ func isCueValueEqual(inputdef cue.Value, input cue.Value) bool {
 func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
 	// To include all optional fields, we need to use inputdef for iteration,
 	// since the lookuppath with optional field doesn't work very well
-	rvInstance, err := rt.Compile("resource", []byte{})
+	rvInstance, err := rt.Compile("helper", []byte{})
 	if err != nil {
 		return input, false, err
 	}
@@ -421,6 +429,7 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
 // TODO this is a terrible way to do this, refactor
 type Resource struct {
 	Value interface{}
+	Name  string
 }
 
 // WrapCUEError is a wrapper for cueErrors that occur and are not self explanatory.
@@ -430,7 +439,7 @@ func WrapCUEError(err error) error {
 	var cErr errs.Error
 	m := make(map[int]string)
 	if ok := errors.As(err, &cErr); ok {
-		for _, e := range errs.Errors(err) {
+		for _, e := range errs.Errors(cErr) {
 			if e.Position().File() != nil {
 				line := e.Position().Line()
 				m[line] = fmt.Sprintf("%q: in file %s", err, e.Position().File().Name())

scripts/validate-devenv-dashboards.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Temporary - remove this script once the dashboard schema are mature
+
+# Remove the appropriate ellipses from the schema to check for unspecified
+# fields in the artifacts (validating "open")
+
+# Run from root of grafana repo
+CMD=${CLI:-bin/darwin-amd64/grafana-cli}
+FILES=$(grep -rl '"schemaVersion": 30' devenv)
+for DASH in ${FILES}; do echo "${DASH}"; ${CMD} cue validate-resource --dashboard "${DASH}"; done