Make "dist" version of dashboard scuemata directly importable in CUE (#39251)

* Update embeds with new paths, remove deprefixing

* Add dist dashboard scuemata, in CUE

Also add tests that ensure all panel plugins with models.cue are
imported by this file, so that we can't go out of sync.

* Remove comment detritus

* Check error return

* Remove hardcoded path prefix from cli validator

embed.go

@@ -2,16 +2,15 @@ package grafana
 
 import (
 	"embed"
-	"io/fs"
 )
 
 // CoreSchema embeds all core CUE files, which live in packages/grafana-schema/src
 //
-//go:embed cue.mod cue packages/grafana-schema/src/schema/*.cue packages/grafana-schema/src/scuemata/*/*.cue
+//go:embed cue.mod cue packages/grafana-schema/src/schema/*.cue packages/grafana-schema/src/scuemata/*/*.cue packages/grafana-schema/src/scuemata/*/*/*.cue
 var CoreSchema embed.FS
 
+// PluginSchema embeds all expected plugin CUE files and plugin metadata from
+// within the public/app/plugins subdirectory.
+//
 //go:embed public/app/plugins/*/*/*.cue public/app/plugins/*/*/plugin.json
-var base embed.FS
+var PluginSchema embed.FS
-
-// PluginSchema embeds all CUE files within the public/ subdirectory.
-var PluginSchema, _ = fs.Sub(base, "public/app/plugins")

packages/grafana-schema/src/scuemata/dashboard/dist/family.cue

@@ -0,0 +1,42 @@
+package dist
+
+import (
+    "github.com/grafana/grafana/packages/grafana-schema/src/scuemata/dashboard"
+    pbarchart "github.com/grafana/grafana/public/app/plugins/panel/barchart:grafanaschema"
+    pbargauge "github.com/grafana/grafana/public/app/plugins/panel/bargauge:grafanaschema"
+    pcanvas "github.com/grafana/grafana/public/app/plugins/panel/canvas:grafanaschema"
+    pdashlist "github.com/grafana/grafana/public/app/plugins/panel/dashlist:grafanaschema"
+    pgauge "github.com/grafana/grafana/public/app/plugins/panel/gauge:grafanaschema"
+    phistogram "github.com/grafana/grafana/public/app/plugins/panel/histogram:grafanaschema"
+    pnews "github.com/grafana/grafana/public/app/plugins/panel/news:grafanaschema"
+    pstat "github.com/grafana/grafana/public/app/plugins/panel/stat:grafanaschema"
+    st "github.com/grafana/grafana/public/app/plugins/panel/state-timeline:grafanaschema"
+    sh "github.com/grafana/grafana/public/app/plugins/panel/status-history:grafanaschema"
+    ptable "github.com/grafana/grafana/public/app/plugins/panel/table:grafanaschema"
+    ptext "github.com/grafana/grafana/public/app/plugins/panel/text:grafanaschema"
+    ptimeseries "github.com/grafana/grafana/public/app/plugins/panel/timeseries:grafanaschema"
+)
+
+// Family composes the base dashboard scuemata family with all Grafana core plugins -
+// the plugins that are dist[ributed] with Grafana. The resulting composed scuemata is
+// exactly equivalent to what's produced by the DistDashboardFamily() Go function.
+//
+// CUE programs should default to importing this dist variant over the base variant.
+Family: dashboard.Family & {
+    compose: Panel: {
+        // TODO do this with a loop once we include the panel type/plugin id in the model
+        barchart: pbarchart.Panel
+        bargauge: pbargauge.Panel
+        canvas: pcanvas.Panel
+        dashlist: pdashlist.Panel
+        gauge: pgauge.Panel
+        histogram: phistogram.Panel
+        news: pnews.Panel
+        stat: pstat.Panel
+        "state-timeline": st.Panel
+        "status-history": sh.Panel
+        text: ptext.Panel
+        table: ptable.Panel
+        timeseries: ptimeseries.Panel
+    }
+}

pkg/cmd/grafana-cli/commands/scuemata_validation_command.go

@@ -63,7 +63,7 @@ func (cmd Command) validateScuemata(c utils.CommandLine) error {
 	if err != nil {
 		return err
 	}
-	fspaths.DistPluginCueFS, err = populate(paths.DistPluginCueFS, "public/app/plugins")
+	fspaths.DistPluginCueFS, err = populate(paths.DistPluginCueFS, "")
 	if err != nil {
 		return err
 	}

pkg/schema/load/load_test.go

@@ -7,11 +7,15 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 	"testing/fstest"
 
+	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/load"
+	cuejson "cuelang.org/go/pkg/encoding/json"
 	"github.com/grafana/grafana/pkg/schema"
 	"github.com/laher/mergefs"
 	"github.com/stretchr/testify/require"
@@ -175,3 +179,45 @@ func TestCueErrorWrapper(t *testing.T) {
 	require.Contains(t, err.Error(), "in file")
 	require.Contains(t, err.Error(), "line: ")
 }
+
+func TestAllPluginsInDist(t *testing.T) {
+	overlay, err := defaultOverlay(p)
+	require.NoError(t, err)
+
+	cfg := &load.Config{
+		Overlay:    overlay,
+		ModuleRoot: prefix,
+		Module:     "github.com/grafana/grafana",
+		Dir:        filepath.Join(prefix, dashboardDir, "dist"),
+		Package:    "dist",
+	}
+	inst, err := rt.Build(load.Instances(nil, cfg)[0])
+	require.NoError(t, err)
+
+	dinst, err := rt.Compile("str", `
+	Family: compose: Panel: {}
+	typs: [for typ, _ in Family.compose.Panel {typ}]
+	`)
+	require.NoError(t, err)
+
+	typs := dinst.Value().Unify(inst.Value()).LookupPath(cue.MakePath(cue.Str("typs")))
+	j, err := cuejson.Marshal(typs)
+	require.NoError(t, err)
+
+	var importedPanelTypes, loadedPanelTypes []string
+	require.NoError(t, json.Unmarshal([]byte(j), &importedPanelTypes))
+
+	// TODO a more canonical way of getting all the dist plugin types with
+	// models.cue would be nice.
+	m, err := loadPanelScuemata(p)
+	require.NoError(t, err)
+
+	for typ := range m {
+		loadedPanelTypes = append(loadedPanelTypes, typ)
+	}
+
+	sort.Strings(importedPanelTypes)
+	sort.Strings(loadedPanelTypes)
+
+	require.Equal(t, loadedPanelTypes, importedPanelTypes, "%s/family.cue needs updating, it must compose the same set of panel plugin models that are found by the plugin loader", cfg.Dir)
+}