plugins: Introduce generated, static core plugin registry (#54118)

* Refactor towards template/codegen framework

* Add templates for plugin gen

* Add Go codegen for plugins; overhaul framework, too

* Add new codegen output; assorted framework fixes

* Regenerate after merge

* Remove accidental commit file, update templates

* Export the pfs.Tree loader from plugin types

* Print details from cuetsy errors

* Generate loaders for all plugins and list in registry

* Use pfs_gen.go over lineage_gen.go

* Un-un-ignore main file

* Introduce simple List static registry for plugins

* Last tweaks to codegen

* remove unused tvars

* Ensure loop-local instances for both vars

* Generate pfs parsing in-place in registry

* Stop generating pfs_gen.go

* Move Tree into pfs, rename subdir

* Change package name to match dir

* Ignore gocyclo on HTTPServer.getNavTree

.gitignore

@@ -169,6 +169,8 @@ compilation-stats.json
 !pkg/services/featuremgmt/toggles_gen.go
 !pkg/coremodel/**/*_gen.go
 !pkg/framework/**/*_gen.go
+!pkg/plugins/pfs/**/*_gen.go
+!public/app/plugins/**/*_gen.go
 
 # Auto-generated internationalization files
 public/locales/_build/

packages/grafana-schema/src/schema/dashboard/dashboard_experimental.gen.ts

@@ -1,8 +1,10 @@
 // This file is autogenerated. DO NOT EDIT.
 //
-// Run "make gen-cue" from repository root to regenerate.
+// Generated by pkg/framework/coremodel/gen.go
 //
-// Derived from the Thema lineage at pkg/coremodel/dashboard
+// Derived from the Thema lineage declared in pkg/coremodel/dashboard/coremodel.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
 
 
 // This model is a WIP and not yet canonical. Consequently, its members are

pkg/api/index.go

@@ -173,6 +173,7 @@ func (hs *HTTPServer) ReqCanAdminTeams(c *models.ReqContext) bool {
 	return c.OrgRole == org.RoleAdmin || (hs.Cfg.EditorsCanAdmin && c.OrgRole == org.RoleEditor)
 }
 
+//nolint:gocyclo
 func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool, prefs *pref.Preference) ([]*dtos.NavLink, error) {
 	hasAccess := ac.HasAccess(hs.AccessControl, c)
 	var navTree []*dtos.NavLink

pkg/codegen/coremodel.go

@@ -5,17 +5,14 @@ import (
 	"errors"
 	"fmt"
 	"go/ast"
-	"go/format"
-	"go/parser"
-	"go/token"
 	"io"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing/fstest"
-	"text/template"
 
+	cerrors "cuelang.org/go/cue/errors"
 	"cuelang.org/go/pkg/encoding/yaml"
 	"github.com/deepmap/oapi-codegen/pkg/codegen"
 	"github.com/getkin/kin-openapi/openapi3"
@@ -23,7 +20,6 @@ import (
 	"github.com/grafana/grafana/pkg/cuectx"
 	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/openapi"
-	"golang.org/x/tools/imports"
 )
 
 // ExtractedLineage contains the results of statically analyzing a Grafana
@@ -94,12 +90,14 @@ func ExtractLineage(path string, lib thema.Library) (*ExtractedLineage, error) {
 		},
 	}
 
-	ec.RelativePath, err = filepath.Rel(groot, filepath.Dir(path))
+	// ec.RelativePath, err = filepath.Rel(groot, filepath.Dir(path))
+	ec.RelativePath, err = filepath.Rel(groot, path)
 	if err != nil {
 		// should be unreachable, since we rootclimbed to find groot above
 		panic(err)
 	}
-	ec.Lineage, err = cuectx.LoadGrafanaInstancesWithThema(ec.RelativePath, fs, lib)
+	ec.RelativePath = filepath.ToSlash(ec.RelativePath)
+	ec.Lineage, err = cuectx.LoadGrafanaInstancesWithThema(filepath.Dir(ec.RelativePath), fs, lib)
 	if err != nil {
 		return ec, err
 	}
@@ -116,7 +114,7 @@ func (ls *ExtractedLineage) toTemplateObj() tplVars {
 	return tplVars{
 		Name:        lin.Name(),
 		LineagePath: ls.RelativePath,
-		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", ls.RelativePath)),
+		PkgPath:     filepath.ToSlash(filepath.Join("github.com/grafana/grafana", filepath.Dir(ls.RelativePath))),
 		TitleName:   strings.Title(lin.Name()), // nolint
 		LatestSeqv:  sch.Version()[0],
 		LatestSchv:  sch.Version()[1],
@@ -170,12 +168,19 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 		return nil, fmt.Errorf("loading generated openapi failed; %w", err)
 	}
 
+	var importbuf bytes.Buffer
+	if err = tmpls.Lookup("coremodel_imports.tmpl").Execute(&importbuf, tvars_coremodel_imports{
+		PackageName: lin.Name(),
+	}); err != nil {
+		return nil, fmt.Errorf("error executing imports template: %w", err)
+	}
+
 	gostr, err := codegen.Generate(oT, lin.Name(), codegen.Options{
 		GenerateTypes: true,
 		SkipPrune:     true,
 		SkipFmt:       true,
 		UserTemplates: map[string]string{
-			"imports.tmpl": fmt.Sprintf(tmplImports, ls.RelativePath),
+			"imports.tmpl": importbuf.String(),
 			"typedef.tmpl": tmplTypedef,
 		},
 	})
@@ -183,34 +188,34 @@ func (ls *ExtractedLineage) GenerateGoCoremodel(path string) (WriteDiffer, error
 		return nil, fmt.Errorf("openapi generation failed: %w", err)
 	}
 
-	vars := ls.toTemplateObj()
+	buf := new(bytes.Buffer)
-	var buuf bytes.Buffer
+	if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
-	err = tmplAddenda.Execute(&buuf, vars)
+		LineagePath:   ls.RelativePath,
-	if err != nil {
+		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
-		panic(err)
+	}); err != nil {
+		return nil, fmt.Errorf("error executing header template: %w", err)
 	}
 
-	fset := token.NewFileSet()
+	fmt.Fprint(buf, "\n", gostr)
-	fname := fmt.Sprintf("%s_gen.go", lin.Name())
-	gf, err := parser.ParseFile(fset, fname, gostr+buuf.String(), parser.ParseComments)
-	if err != nil {
-		return nil, fmt.Errorf("generated go file parsing failed: %w", err)
-	}
-	ast.Walk(prefixDropper(strings.Title(lin.Name())), gf)
 
-	var buf bytes.Buffer
+	vars := ls.toTemplateObj()
-	err = format.Node(&buf, fset, gf)
+	err = tmpls.Lookup("addenda.tmpl").Execute(buf, vars)
 	if err != nil {
-		return nil, fmt.Errorf("ast printing failed: %w", err)
+		panic(err)
 	}
 
-	byt, err := imports.Process(fname, buf.Bytes(), nil)
+	fullp := filepath.Join(path, fmt.Sprintf("%s_gen.go", lin.Name()))
+	byt, err := postprocessGoFile(genGoFile{
+		path:   fullp,
+		walker: makePrefixDropper(strings.Title(lin.Name()), "Model"),
+		in:     buf.Bytes(),
+	})
 	if err != nil {
-		return nil, fmt.Errorf("goimports processing failed: %w", err)
+		return nil, err
 	}
 
 	wd := NewWriteDiffer()
-	wd[filepath.Join(path, fname)] = byt
+	wd[fullp] = byt
 
 	return wd, nil
 }
@@ -238,7 +243,7 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 
 	top, err := cuetsy.GenerateSingleAST(strings.Title(ls.Lineage.Name()), schv, cuetsy.TypeInterface)
 	if err != nil {
-		return nil, fmt.Errorf("cuetsy top gen failed: %w", err)
+		return nil, fmt.Errorf("cuetsy top gen failed: %s", cerrors.Details(err, nil))
 	}
 
 	// TODO until cuetsy can toposort its outputs, put the top/parent type at the bottom of the file.
@@ -250,7 +255,12 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 	var strb strings.Builder
 	var str string
 	fpath := ls.Lineage.Name() + ".gen.ts"
-	strb.WriteString(fmt.Sprintf(genHeader, ls.RelativePath))
+	if err := tmpls.Lookup("autogen_header.tmpl").Execute(&strb, tvars_autogen_header{
+		LineagePath:   ls.RelativePath,
+		GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
+	}); err != nil {
+		return nil, fmt.Errorf("error executing header template: %w", err)
+	}
 
 	if !ls.IsCanonical {
 		fpath = fmt.Sprintf("%s_experimental.gen.ts", ls.Lineage.Name())
@@ -273,16 +283,34 @@ func (ls *ExtractedLineage) GenerateTypescriptCoremodel(path string) (WriteDiffe
 	return wd, nil
 }
 
-type prefixDropper string
+type prefixDropper struct {
+	str     string
+	base    string
+	rxp     *regexp.Regexp
+	rxpsuff *regexp.Regexp
+}
+
+func makePrefixDropper(str, base string) prefixDropper {
+	return prefixDropper{
+		str:     str,
+		base:    base,
+		rxpsuff: regexp.MustCompile(fmt.Sprintf(`%s([a-zA-Z_]*)`, str)),
+		rxp:     regexp.MustCompile(fmt.Sprintf(`%s([\s.,;-])`, str)),
+	}
+}
 
 func (d prefixDropper) Visit(n ast.Node) ast.Visitor {
-	asstr := string(d)
 	switch x := n.(type) {
 	case *ast.Ident:
-		if x.Name != asstr {
+		if x.Name != d.str {
-			x.Name = strings.TrimPrefix(x.Name, asstr)
+			x.Name = strings.TrimPrefix(x.Name, d.str)
 		} else {
-			x.Name = "Model"
+			x.Name = d.base
+		}
+	case *ast.CommentGroup:
+		for _, c := range x.List {
+			c.Text = d.rxp.ReplaceAllString(c.Text, d.base+"$1")
+			c.Text = d.rxpsuff.ReplaceAllString(c.Text, "$1")
 		}
 	}
 	return d
@@ -297,177 +325,33 @@ func GenerateCoremodelRegistry(path string, ecl []*ExtractedLineage) (WriteDiffe
 		cml = append(cml, ec.toTemplateObj())
 	}
 
-	var buf bytes.Buffer
+	buf := new(bytes.Buffer)
-	err := tmplRegistry.Execute(&buf, struct {
+	if err := tmpls.Lookup("coremodel_registry.tmpl").Execute(buf, tvars_coremodel_registry{
-		Coremodels []tplVars
+		Header: tvars_autogen_header{
-	}{
+			GeneratorPath: "pkg/framework/coremodel/gen.go", // FIXME hardcoding is not OK
+		},
 		Coremodels: cml,
-	})
+	}); err != nil {
-	if err != nil {
+		return nil, fmt.Errorf("failed executing coremodel registry template: %w", err)
-		return nil, fmt.Errorf("failed generating template: %w", err)
 	}
 
-	byt, err := imports.Process(path, buf.Bytes(), nil)
+	byt, err := postprocessGoFile(genGoFile{
+		path: path,
+		in:   buf.Bytes(),
+	})
 	if err != nil {
-		return nil, fmt.Errorf("goimports processing failed: %w", err)
+		return nil, err
 	}
-
 	wd := NewWriteDiffer()
 	wd[path] = byt
 	return wd, nil
 }
 
-var genHeader = `// This file is autogenerated. DO NOT EDIT.
-//
-// Run "make gen-cue" from repository root to regenerate.
-//
-// Derived from the Thema lineage at %s
-
-`
-
-var tmplImports = genHeader + `package {{ .PackageName }}
-
-import (
-	"embed"
-	"path/filepath"
-
-	"github.com/grafana/grafana/pkg/cuectx"
-	"github.com/grafana/grafana/pkg/framework/coremodel"
-	"github.com/grafana/thema"
-)
-`
-
-var tmplAddenda = template.Must(template.New("addenda").Parse(`
-//go:embed coremodel.cue
-var cueFS embed.FS
-
-// codegen ensures that this is always the latest Thema schema version
-var currentVersion = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
-
-// Lineage returns the Thema lineage representing a Grafana {{ .Name }}.
-//
-// The lineage is the canonical specification of the current {{ .Name }} schema,
-// all prior schema versions, and the mappings that allow migration between
-// schema versions.
-{{- if .IsComposed }}//
-// This is the base variant of the schema. It does not include any composed
-// plugin schemas.{{ end }}
-func Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
-	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("pkg", "coremodel", "{{ .Name }}"), cueFS, lib, opts...)
-}
-
-var _ thema.LineageFactory = Lineage
-var _ coremodel.Interface = &Coremodel{}
-
-// Coremodel contains the foundational schema declaration for {{ .Name }}s.
-// It implements coremodel.Interface.
-type Coremodel struct {
-	lin thema.Lineage
-}
-
-// Lineage returns the canonical {{ .Name }} Lineage.
-func (c *Coremodel) Lineage() thema.Lineage {
-	return c.lin
-}
-
-// CurrentSchema returns the current (latest) {{ .Name }} Thema schema.
-func (c *Coremodel) CurrentSchema() thema.Schema {
-	return thema.SchemaP(c.lin, currentVersion)
-}
-
-// GoType returns a pointer to an empty Go struct that corresponds to
-// the current Thema schema.
-func (c *Coremodel) GoType() interface{} {
-	return &Model{}
-}
-
-// New returns a new instance of the {{ .Name }} coremodel.
-//
-// Note that this function does not cache, and initially loading a Thema lineage
-// can be expensive. As such, the Grafana backend should prefer to access this
-// coremodel through a registry (pkg/framework/coremodel/registry), which does cache.
-func New(lib thema.Library) (*Coremodel, error) {
-	lin, err := Lineage(lib)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Coremodel{
-		lin: lin,
-	}, nil
-}
-`))
-
 var tmplTypedef = `{{range .Types}}
-{{ with .Schema.Description }}{{ . }}{{ else }}// {{.TypeName}} defines model for {{.JsonName}}.{{ end }}
+{{ with .Schema.Description }}{{ . }}{{ else }}// {{.TypeName}} is the Go representation of a {{.JsonName}}.{{ end }}
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type {{.TypeName}} {{if and (opts.AliasTypes) (.CanAlias)}}={{end}} {{.Schema.TypeDecl}}
 {{end}}
 `
-
-var tmplRegistry = template.Must(template.New("registry").Parse(`
-// This file is autogenerated. DO NOT EDIT.
-//
-// Generated by pkg/framework/coremodel/gen.go
-// Run "make gen-cue" from repository root to regenerate.
-
-package registry
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/google/wire"
-	{{range .Coremodels }}
-	"{{ .PkgPath }}"{{end}}
-	"github.com/grafana/grafana/pkg/cuectx"
-	"github.com/grafana/grafana/pkg/framework/coremodel"
-	"github.com/grafana/thema"
-)
-
-// Base is a registry of coremodel.Interface. It provides two modes for accessing
-// coremodels: individually via literal named methods, or as a slice returned from All().
-//
-// Prefer the individual named methods for use cases where the particular coremodel(s) that
-// are needed are known to the caller. For example, a dashboard linter can know that it
-// specifically wants the dashboard coremodel.
-//
-// Prefer All() when performing operations generically across all coremodels. For example,
-// a validation HTTP middleware for any coremodel-schematized object type.
-type Base struct {
-	all []coremodel.Interface
-	{{- range .Coremodels }}
-	{{ .Name }} *{{ .Name }}.Coremodel{{end}}
-}
-
-// type guards
-var (
-{{- range .Coremodels }}
-	_ coremodel.Interface = &{{ .Name }}.Coremodel{}{{end}}
-)
-
-{{range .Coremodels }}
-// {{ .TitleName }} returns the {{ .Name }} coremodel. The return value is guaranteed to
-// implement coremodel.Interface.
-func (s *Base) {{ .TitleName }}() *{{ .Name }}.Coremodel {
-	return s.{{ .Name }}
-}
-{{end}}
-
-func doProvideBase(lib thema.Library) *Base {
-	var err error
-	reg := &Base{}
-
-{{range .Coremodels }}
-	reg.{{ .Name }}, err = {{ .Name }}.New(lib)
-	if err != nil {
-		panic(fmt.Sprintf("error while initializing {{ .Name }} coremodel: %s", err))
-	}
-    reg.all = append(reg.all, reg.{{ .Name }})
-{{end}}
-
-	return reg
-}
-`))

pkg/codegen/pluggen.go

@@ -4,15 +4,20 @@ import (
 	"bytes"
 	"fmt"
 	"io/fs"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
-	"text/template"
 
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/pkg/encoding/yaml"
+	"github.com/deepmap/oapi-codegen/pkg/codegen"
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/grafana/cuetsy"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
 	"github.com/grafana/grafana/pkg/plugins/pfs"
 	"github.com/grafana/thema"
+	"github.com/grafana/thema/encoding/openapi"
 )
 
 // CUE import paths, mapped to corresponding TS import paths. An empty value
@@ -92,13 +97,20 @@ type PluginTreeOrErr struct {
 }
 
 // PluginTree is a pfs.Tree. It exists so we can add methods for code generation to it.
+//
+// It is, for now, tailored specifically to Grafana core's codegen needs.
 type PluginTree pfs.Tree
 
 func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 	t := (*pfs.Tree)(pt)
 
 	// TODO replace with cuetsy's TS AST
-	f := &tsFile{}
+	f := &tvars_cuetsy_multi{
+		Header: tvars_autogen_header{
+			GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			LineagePath:   "models.cue",
+		},
+	}
 
 	pi := t.RootPlugin()
 	slotimps := pi.SlotImplementations()
@@ -120,24 +132,18 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 			ModelName: slotname,
 		}
 
-		// TODO this is hardcoded for now, but should ultimately be a property of
+		if isGroupLineage(slotname) {
-		// whether the slot is a grouped lineage:
-		// https://github.com/grafana/thema/issues/62
-		switch slotname {
-		case "Panel", "DSConfig":
 			b, err := cuetsy.Generate(sch.UnwrapCUE(), cuetsy.Config{})
 			if err != nil {
 				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
 			}
 			sec.Body = string(b)
-		case "Query":
+		} else {
 			a, err := cuetsy.GenerateSingleAST(strings.Title(lin.Name()), sch.UnwrapCUE(), cuetsy.TypeInterface)
 			if err != nil {
 				return nil, fmt.Errorf("%s: error translating %s lineage to TypeScript: %w", path, slotname, err)
 			}
 			sec.Body = fmt.Sprint(a)
-		default:
-			panic("unrecognized slot name: " + slotname)
 		}
 
 		f.Sections = append(f.Sections, sec)
@@ -145,7 +151,7 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 
 	wd := NewWriteDiffer()
 	var buf bytes.Buffer
-	err := tsSectionTemplate.Execute(&buf, f)
+	err := tmpls.Lookup("cuetsy_multi.tmpl").Execute(&buf, f)
 	if err != nil {
 		return nil, fmt.Errorf("%s: error executing plugin TS generator template: %w", path, err)
 	}
@@ -153,6 +159,260 @@ func (pt *PluginTree) GenerateTS(path string) (WriteDiffer, error) {
 	return wd, nil
 }
 
+func isGroupLineage(slotname string) bool {
+	sl, has := coremodel.AllSlots()[slotname]
+	if !has {
+		panic("unknown slotname name: " + slotname)
+	}
+	return sl.IsGroup()
+}
+
+type GoGenConfig struct {
+	// Types indicates whether corresponding Go types should be generated from the
+	// latest version in the lineage(s).
+	Types bool
+
+	// ThemaBindings indicates whether Thema bindings (an implementation of
+	// ["github.com/grafana/thema".LineageFactory]) should be generated for
+	// lineage(s).
+	ThemaBindings bool
+
+	// DocPathPrefix allows the caller to optionally specify a path to be prefixed
+	// onto paths generated for documentation. This is useful for io/fs-based code
+	// generators, which typically only have knowledge of paths relative to the fs.FS
+	// root, typically an encapsulated subpath, but docs are easier to understand when
+	// paths are relative to a repository root.
+	//
+	// Note that all paths are normalized to use slashes, regardless of the
+	// OS running the code generator.
+	DocPathPrefix string
+}
+
+func (pt *PluginTree) GenerateGo(path string, cfg GoGenConfig) (WriteDiffer, error) {
+	t := (*pfs.Tree)(pt)
+	wd := NewWriteDiffer()
+
+	all := t.SubPlugins()
+	if all == nil {
+		all = make(map[string]pfs.PluginInfo)
+	}
+	all[""] = t.RootPlugin()
+	for subpath, plug := range all {
+		fullp := filepath.Join(path, subpath)
+		if cfg.Types {
+			gwd, err := genGoTypes(plug, path, subpath, cfg.DocPathPrefix)
+			if err != nil {
+				return nil, fmt.Errorf("error generating go types for %s: %w", fullp, err)
+			}
+			if err = wd.Merge(gwd); err != nil {
+				return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
+			}
+		}
+		if cfg.ThemaBindings {
+			twd, err := genThemaBindings(plug, path, subpath, cfg.DocPathPrefix)
+			if err != nil {
+				return nil, fmt.Errorf("error generating thema bindings for %s: %w", fullp, err)
+			}
+			if err = wd.Merge(twd); err != nil {
+				return nil, fmt.Errorf("error merging file set to generate for %s: %w", fullp, err)
+			}
+		}
+	}
+
+	return wd, nil
+}
+
+func genGoTypes(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
+	wd := NewWriteDiffer()
+	for slotname, lin := range plug.SlotImplementations() {
+		lowslot := strings.ToLower(slotname)
+		lib := lin.Library()
+		sch := thema.SchemaP(lin, thema.LatestVersion(lin))
+
+		// FIXME gotta hack this out of thema in order to deal with our custom imports :scream:
+		f, err := openapi.GenerateSchema(sch, nil)
+		if err != nil {
+			return nil, fmt.Errorf("thema openapi generation failed: %w", err)
+		}
+
+		str, err := yaml.Marshal(lib.Context().BuildFile(f))
+		if err != nil {
+			return nil, fmt.Errorf("cue-yaml marshaling failed: %w", err)
+		}
+
+		loader := openapi3.NewLoader()
+		oT, err := loader.LoadFromData([]byte(str))
+		if err != nil {
+			return nil, fmt.Errorf("loading generated openapi failed; %w", err)
+		}
+
+		buf := new(bytes.Buffer)
+		if err = tmpls.Lookup("autogen_header.tmpl").Execute(buf, tvars_autogen_header{
+			GeneratorPath:  "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			LineagePath:    filepath.ToSlash(filepath.Join(prefix, subpath, "models.cue")),
+			LineageCUEPath: slotname,
+			GenLicense:     true,
+		}); err != nil {
+			return nil, fmt.Errorf("error generating file header: %w", err)
+		}
+
+		cgopt := codegen.Options{
+			GenerateTypes: true,
+			SkipPrune:     true,
+			SkipFmt:       true,
+			UserTemplates: map[string]string{
+				"imports.tmpl": "package {{ .PackageName }}",
+				"typedef.tmpl": tmplTypedef,
+			},
+		}
+		if isGroupLineage(slotname) {
+			cgopt.ExcludeSchemas = []string{lin.Name()}
+		}
+
+		gostr, err := codegen.Generate(oT, lin.Name(), cgopt)
+		if err != nil {
+			return nil, fmt.Errorf("openapi generation failed: %w", err)
+		}
+		fmt.Fprint(buf, gostr)
+
+		finalpath := filepath.Join(path, subpath, fmt.Sprintf("types_%s_gen.go", lowslot))
+		byt, err := postprocessGoFile(genGoFile{
+			path:   finalpath,
+			walker: makePrefixDropper(strings.Title(lin.Name()), slotname),
+			in:     buf.Bytes(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		wd[finalpath] = byt
+	}
+
+	return wd, nil
+}
+
+func genThemaBindings(plug pfs.PluginInfo, path, subpath, prefix string) (WriteDiffer, error) {
+	wd := NewWriteDiffer()
+	bindings := make([]tvars_plugin_lineage_binding, 0)
+	for slotname, lin := range plug.SlotImplementations() {
+		lv := thema.LatestVersion(lin)
+		bindings = append(bindings, tvars_plugin_lineage_binding{
+			SlotName:   slotname,
+			LatestMajv: lv[0],
+			LatestMinv: lv[1],
+		})
+	}
+
+	buf := new(bytes.Buffer)
+	if err := tmpls.Lookup("plugin_lineage_file.tmpl").Execute(buf, tvars_plugin_lineage_file{
+		PackageName: sanitizePluginId(plug.Meta().Id),
+		PluginType:  string(plug.Meta().Type),
+		PluginID:    plug.Meta().Id,
+		SlotImpls:   bindings,
+		HasModels:   len(bindings) != 0,
+		Header: tvars_autogen_header{
+			GeneratorPath: "public/app/plugins/gen.go", // FIXME hardcoding is not OK
+			GenLicense:    true,
+			LineagePath:   filepath.Join(prefix, subpath),
+		},
+	}); err != nil {
+		return nil, fmt.Errorf("error executing plugin lineage file template: %w", err)
+	}
+
+	fullpath := filepath.Join(path, subpath, "pfs_gen.go")
+	if byt, err := postprocessGoFile(genGoFile{
+		path: fullpath,
+		in:   buf.Bytes(),
+	}); err != nil {
+		return nil, err
+	} else {
+		wd[fullpath] = byt
+	}
+
+	return wd, nil
+}
+
+// Plugin IDs are allowed to contain characters that aren't allowed in CUE
+// package names, Go package names, TS or Go type names, etc.
+// TODO expose this as standard
+func sanitizePluginId(s string) string {
+	return strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z':
+			fallthrough
+		case r >= 'A' && r <= 'Z':
+			fallthrough
+		case r >= '0' && r <= '9':
+			fallthrough
+		case r == '_':
+			return r
+		case r == '-':
+			return '_'
+		default:
+			return -1
+		}
+	}, s)
+}
+
+// FIXME unexport this and refactor, this is way too one-off to be in here
+func GenPluginTreeList(trees []TreeAndPath, prefix, target string, ref bool) (WriteDiffer, error) {
+	buf := new(bytes.Buffer)
+	vars := tvars_plugin_registry{
+		Header: tvars_autogen_header{
+			GenLicense: true,
+		},
+		Plugins: make([]struct {
+			PkgName, Path, ImportPath string
+			NoAlias                   bool
+		}, 0, len(trees)),
+	}
+
+	type tpl struct {
+		PkgName, Path, ImportPath string
+		NoAlias                   bool
+	}
+
+	// No sub-plugin support here. If we never allow subplugins in core, that's probably fine.
+	// But still worth noting.
+	for _, pt := range trees {
+		rp := (*pfs.Tree)(pt.Tree).RootPlugin()
+		vars.Plugins = append(vars.Plugins, tpl{
+			PkgName:    sanitizePluginId(rp.Meta().Id),
+			NoAlias:    sanitizePluginId(rp.Meta().Id) != filepath.Base(pt.Path),
+			ImportPath: filepath.ToSlash(filepath.Join(prefix, pt.Path)),
+			Path:       path.Join(append(strings.Split(prefix, "/")[3:], pt.Path)...),
+		})
+	}
+
+	tmplname := "plugin_registry.tmpl"
+	if ref {
+		tmplname = "plugin_registry_ref.tmpl"
+	}
+
+	if err := tmpls.Lookup(tmplname).Execute(buf, vars); err != nil {
+		return nil, fmt.Errorf("failed executing plugin registry template: %w", err)
+	}
+
+	byt, err := postprocessGoFile(genGoFile{
+		path: target,
+		in:   buf.Bytes(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error postprocessing plugin registry: %w", err)
+	}
+
+	wd := NewWriteDiffer()
+	wd[target] = byt
+	return wd, nil
+}
+
+// FIXME unexport this and refactor, this is way too one-off to be in here
+type TreeAndPath struct {
+	Tree *PluginTree
+	// path relative to path prefix UUUGHHH (basically {panel,datasource}/<dir>}
+	Path string
+}
+
 // TODO convert this to use cuetsy ts types, once import * form is supported
 func convertImport(im *ast.ImportSpec) *tsImport {
 	var err error
@@ -182,11 +442,6 @@ func convertImport(im *ast.ImportSpec) *tsImport {
 	return tsim
 }
 
-type tsFile struct {
-	Imports  []*tsImport
-	Sections []tsSection
-}
-
 type tsSection struct {
 	V         thema.SyntacticVersion
 	ModelName string
@@ -197,15 +452,3 @@ type tsImport struct {
 	Ident string
 	Pkg   string
 }
-
-var tsSectionTemplate = template.Must(template.New("cuetsymulti").Parse(`//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// This file is autogenerated. DO NOT EDIT.
-//
-// To regenerate, run "make gen-cue" from the repository root.
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-{{range .Imports}}
-import * as {{.Ident}} from '{{.Pkg}}';{{end}}
-{{range .Sections}}{{if ne .ModelName "" }}
-export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
-{{end}}
-{{.Body}}{{end}}`))

pkg/codegen/tmpl.go

@@ -0,0 +1,65 @@
+package codegen
+
+import (
+	"embed"
+	"text/template"
+	"time"
+)
+
+// All the parsed templates in the tmpl subdirectory
+var tmpls *template.Template
+
+func init() {
+	base := template.New("codegen").Funcs(template.FuncMap{
+		"now": time.Now,
+	})
+	tmpls = template.Must(base.ParseFS(tmplFS, "tmpl/*.tmpl"))
+}
+
+//go:embed tmpl/*.tmpl
+var tmplFS embed.FS
+
+// The following group of types, beginning with tvars_*, all contain the set
+// of variables expected by the corresponding named template file under tmpl/
+type (
+	tvars_autogen_header struct {
+		GeneratorPath  string
+		LineagePath    string
+		LineageCUEPath string
+		GenLicense     bool
+	}
+	tvars_coremodel_registry struct {
+		Header     tvars_autogen_header
+		Coremodels []tplVars
+	}
+	tvars_coremodel_imports struct {
+		PackageName string
+	}
+	tvars_plugin_lineage_binding struct {
+		SlotName               string
+		LatestMajv, LatestMinv uint
+	}
+	tvars_plugin_lineage_file struct {
+		PackageName string
+		PluginID    string
+		PluginType  string
+		HasModels   bool
+		RootCUE     bool
+		SlotImpls   []tvars_plugin_lineage_binding
+		Header      tvars_autogen_header
+	}
+	tvars_cuetsy_multi struct {
+		Header   tvars_autogen_header
+		Imports  []*tsImport
+		Sections []tsSection
+	}
+	tvars_plugin_registry struct {
+		Header  tvars_autogen_header
+		Plugins []struct {
+			PkgName    string
+			Path       string
+			ImportPath string
+			NoAlias    bool
+		}
+	}
+)

pkg/codegen/tmpl/addenda.tmpl

@@ -0,0 +1,64 @@
+
+
+//go:embed coremodel.cue
+var cueFS embed.FS
+
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
+var currentVersion = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
+
+// Lineage returns the Thema lineage representing a Grafana {{ .Name }}.
+//
+// The lineage is the canonical specification of the current {{ .Name }} schema,
+// all prior schema versions, and the mappings that allow migration between
+// schema versions.
+{{- if .IsComposed }}//
+// This is the base variant of the schema. It does not include any composed
+// plugin schemas.{{ end }}
+func Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("pkg", "coremodel", "{{ .Name }}"), cueFS, lib, opts...)
+}
+
+var _ thema.LineageFactory = Lineage
+var _ coremodel.Interface = &Coremodel{}
+
+// Coremodel contains the foundational schema declaration for {{ .Name }}s.
+// It implements coremodel.Interface.
+type Coremodel struct {
+	lin thema.Lineage
+}
+
+// Lineage returns the canonical {{ .Name }} Lineage.
+func (c *Coremodel) Lineage() thema.Lineage {
+	return c.lin
+}
+
+// CurrentSchema returns the current (latest) {{ .Name }} Thema schema.
+func (c *Coremodel) CurrentSchema() thema.Schema {
+	return thema.SchemaP(c.lin, currentVersion)
+}
+
+// GoType returns a pointer to an empty Go struct that corresponds to
+// the current Thema schema.
+func (c *Coremodel) GoType() interface{} {
+	return &Model{}
+}
+
+// New returns a new instance of the {{ .Name }} coremodel.
+//
+// Note that this function does not cache, and initially loading a Thema lineage
+// can be expensive. As such, the Grafana backend should prefer to access this
+// coremodel through a registry (pkg/framework/coremodel/registry), which does cache.
+func New(lib thema.Library) (*Coremodel, error) {
+	lin, err := Lineage(lib)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Coremodel{
+		lin: lin,
+	}, nil
+}

pkg/codegen/tmpl/autogen_header.tmpl

@@ -0,0 +1,28 @@
+{{ if .GenLicense -}}
+// Copyright {{ now.Year }} Grafana Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+{{ end -}}
+// This file is autogenerated. DO NOT EDIT.
+{{- if ne .GeneratorPath "" }}
+//
+// Generated by {{ .GeneratorPath }}
+{{- end }}
+{{- if ne .LineagePath "" }}
+//
+// Derived from the Thema lineage declared in {{ .LineagePath }}{{ if ne .LineageCUEPath "" }} at CUE path "{{ .LineageCUEPath }}"{{ end }}
+{{- end }}
+//
+// Run `make gen-cue` from repository root to regenerate.
+

pkg/codegen/tmpl/coremodel_imports.tmpl

@@ -0,0 +1,10 @@
+package {{ .PackageName }}
+
+import (
+	"embed"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+	"github.com/grafana/thema"
+)

pkg/codegen/tmpl/coremodel_registry.tmpl

@@ -0,0 +1,58 @@
+{{ template "autogen_header.tmpl" .Header }}
+package registry
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/google/wire"
+	{{range .Coremodels }}
+	"{{ .PkgPath }}"{{end}}
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/framework/coremodel"
+	"github.com/grafana/thema"
+)
+
+// Base is a registry of coremodel.Interface. It provides two modes for accessing
+// coremodels: individually via literal named methods, or as a slice returned from All().
+//
+// Prefer the individual named methods for use cases where the particular coremodel(s) that
+// are needed are known to the caller. For example, a dashboard linter can know that it
+// specifically wants the dashboard coremodel.
+//
+// Prefer All() when performing operations generically across all coremodels. For example,
+// a validation HTTP middleware for any coremodel-schematized object type.
+type Base struct {
+	all []coremodel.Interface
+	{{- range .Coremodels }}
+	{{ .Name }} *{{ .Name }}.Coremodel{{end}}
+}
+
+// type guards
+var (
+{{- range .Coremodels }}
+	_ coremodel.Interface = &{{ .Name }}.Coremodel{}{{end}}
+)
+
+{{range .Coremodels }}
+// {{ .TitleName }} returns the {{ .Name }} coremodel. The return value is guaranteed to
+// implement coremodel.Interface.
+func (b *Base) {{ .TitleName }}() *{{ .Name }}.Coremodel {
+	return b.{{ .Name }}
+}
+{{end}}
+
+func doProvideBase(lib thema.Library) *Base {
+	var err error
+	reg := &Base{}
+
+{{range .Coremodels }}
+	reg.{{ .Name }}, err = {{ .Name }}.New(lib)
+	if err != nil {
+		panic(fmt.Sprintf("error while initializing {{ .Name }} coremodel: %s", err))
+	}
+    reg.all = append(reg.all, reg.{{ .Name }})
+{{end}}
+
+	return reg
+}

pkg/codegen/tmpl/cuetsy_multi.tmpl

@@ -0,0 +1,7 @@
+{{ template "autogen_header.tmpl" .Header -}}
+{{range .Imports}}
+import * as {{.Ident}} from '{{.Pkg}}';{{end}}
+{{range .Sections}}{{if ne .ModelName "" }}
+export const {{.ModelName}}ModelVersion = Object.freeze([{{index .V 0}}, {{index .V 1}}]);
+{{end}}
+{{.Body}}{{end}}

pkg/codegen/tmpl/plugin_lineage_binding.tmpl

@@ -0,0 +1,12 @@
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
+var currentVersion{{ .SlotName }} = thema.SV({{ .LatestSeqv }}, {{ .LatestSchv }})
+
+// {{ .SlotName }}Lineage returns the Thema lineage for the {{ .PluginID }} {{ .PluginType }} plugin's
+// {{ .SlotName }} ["github.com/grafana/grafana/pkg/framework/coremodel".Slot] implementation.
+func {{ .SlotName }}Lineage(lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+	return cuectx.LoadGrafanaInstancesWithThema(filepath.Join("public", "app", "{{ .Name }}"), cueFS, lib, opts...)
+}

pkg/codegen/tmpl/plugin_lineage_file.tmpl

@@ -0,0 +1,58 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package {{ .PackageName }}
+
+import (
+	"embed"
+	"fmt"
+	"path/filepath"
+	"sync"
+
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+var parseOnce sync.Once
+var ptree *pfs.Tree
+
+//go:embed plugin.{{ if .RootCUE }}cue{{ else }}json{{ end }}{{ if .HasModels }} models.cue{{ end }}
+var plugFS embed.FS
+
+// PluginTree returns the plugin tree representing the statically analyzable contents of the {{ .PluginID }} plugin.
+func PluginTree(lib *thema.Library) *pfs.Tree {
+	var err error
+	if lib == nil {
+		parseOnce.Do(func() {
+			ptree, err = pfs.ParsePluginFS(plugFS, cuectx.ProvideThemaLibrary())
+		})
+	} else {
+		ptree, err = pfs.ParsePluginFS(plugFS, cuectx.ProvideThemaLibrary())
+	}
+
+	if err != nil {
+		// Even the most rudimentary testing in CI ensures this is unreachable
+		panic(fmt.Errorf("error parsing plugin fs tree: %w", err))
+	}
+
+	return ptree
+}
+
+{{ $pluginfo := . }}{{ range $slot := .SlotImpls }}
+// {{ .SlotName }}Lineage returns the Thema lineage for the {{ $pluginfo.PluginID }} {{ $pluginfo.PluginType }} plugin's
+// {{ .SlotName }} ["github.com/grafana/grafana/pkg/framework/coremodel".Slot] implementation.
+func {{ .SlotName }}Lineage(lib *thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
+  t := PluginTree(lib)
+  lin, has := t.RootPlugin().SlotImplementations()["{{ .SlotName }}"]
+  if !has {
+    panic("unreachable: lineage for {{ .SlotName }} does not exist, but code is only generated for existing lineages")
+  }
+  return lin, nil
+}
+
+// The current schema version of the {{ .SlotName }} slot implementation.
+//
+// Code generation ensures that this is always the version number for the latest schema
+// in the {{ .SlotName }} Thema lineage.
+var currentVersion{{ .SlotName }} = thema.SV({{ .LatestMajv }}, {{ .LatestMinv }})
+
+{{ end }}

pkg/codegen/tmpl/plugin_registry.tmpl

@@ -0,0 +1,31 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package corelist
+
+import (
+  "fmt"
+  "io/fs"
+  "sync"
+  "github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+func makeTreeOrPanic(path string, pkgname string, lib thema.Library) *pfs.Tree {
+  sub, err := fs.Sub(grafana.CueSchemaFS, path)
+  if err != nil {
+    panic("could not create fs sub to " + path)
+  }
+  tree, err := pfs.ParsePluginFS(sub, lib)
+  if err != nil {
+    panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
+  }
+  return tree
+}
+
+func coreTreeList(lib thema.Library) pfs.TreeList{
+  return pfs.TreeList{
+    {{- range .Plugins }}
+    makeTreeOrPanic("{{ .Path }}", "{{ .PkgName }}", lib),
+    {{- end }}
+  }
+}

pkg/codegen/tmpl/plugin_registry_ref.tmpl

@@ -0,0 +1,16 @@
+{{ template "autogen_header.tmpl" .Header -}}
+package registry
+
+import (
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+  {{ range .Plugins }}
+  {{ if .NoAlias }}{{ .PkgName }} {{end}}"{{ .Path }}"{{ end }}
+)
+
+func coreTreeLoaders() []func(*thema.Library) *pfs.Tree{
+  return []func(*thema.Library) *pfs.Tree{
+    {{- range .Plugins }}
+    {{ .PkgName }}.PluginTree,{{ end }}
+  }
+}

pkg/codegen/util_go.go

@@ -0,0 +1,67 @@
+package codegen
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/format"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/tools/imports"
+)
+
+type genGoFile struct {
+	path   string
+	walker ast.Visitor
+	in     []byte
+}
+
+func postprocessGoFile(cfg genGoFile) ([]byte, error) {
+	fname := filepath.Base(cfg.path)
+	buf := new(bytes.Buffer)
+	fset := token.NewFileSet()
+	gf, err := parser.ParseFile(fset, fname, string(cfg.in), parser.ParseComments)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing generated file: %w", err)
+	}
+
+	if cfg.walker != nil {
+		ast.Walk(cfg.walker, gf)
+
+		err = format.Node(buf, fset, gf)
+		if err != nil {
+			return nil, fmt.Errorf("error formatting Go AST: %w", err)
+		}
+	} else {
+		buf = bytes.NewBuffer(cfg.in)
+	}
+
+	byt, err := imports.Process(fname, buf.Bytes(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("goimports processing failed: %w", err)
+	}
+
+	// Compare imports before and after; warn about performance if some were added
+	gfa, _ := parser.ParseFile(fset, fname, string(byt), parser.ParseComments)
+	imap := make(map[string]bool)
+	for _, im := range gf.Imports {
+		imap[im.Path.Value] = true
+	}
+	var added []string
+	for _, im := range gfa.Imports {
+		if !imap[im.Path.Value] {
+			added = append(added, im.Path.Value)
+		}
+	}
+
+	if len(added) != 0 {
+		// TODO improve the guidance in this error if/when we better abstract over imports to generate
+		fmt.Fprintf(os.Stderr, "The following imports were added by goimports while generating %s: \n\t%s\nRelying on goimports to find imports significantly slows down code generation. Consider adding these to the relevant template.\n", cfg.path, strings.Join(added, "\n\t"))
+	}
+
+	return byt, nil
+}

pkg/coremodel/dashboard/dashboard_gen.go

@@ -1,8 +1,10 @@
 // This file is autogenerated. DO NOT EDIT.
 //
-// Run "make gen-cue" from repository root to regenerate.
+// Generated by pkg/framework/coremodel/gen.go
 //
-// Derived from the Thema lineage at pkg/coremodel/dashboard
+// Derived from the Thema lineage declared in pkg/coremodel/dashboard/coremodel.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
 
 package dashboard
 
@@ -15,7 +17,7 @@ import (
 	"github.com/grafana/thema"
 )
 
-// Defines values for DashboardGraphTooltip.
+// Defines values for GraphTooltip.
 const (
 	GraphTooltipN0 GraphTooltip = 0
 
@@ -24,14 +26,14 @@ const (
 	GraphTooltipN2 GraphTooltip = 2
 )
 
-// Defines values for DashboardStyle.
+// Defines values for Style.
 const (
 	StyleDark Style = "dark"
 
 	StyleLight Style = "light"
 )
 
-// Defines values for DashboardTimezone.
+// Defines values for Timezone.
 const (
 	TimezoneBrowser Timezone = "browser"
 
@@ -40,7 +42,7 @@ const (
 	TimezoneUtc Timezone = "utc"
 )
 
-// Defines values for DashboardDashboardCursorSync.
+// Defines values for DashboardCursorSync.
 const (
 	DashboardCursorSyncN0 DashboardCursorSync = 0
 
@@ -49,14 +51,14 @@ const (
 	DashboardCursorSyncN2 DashboardCursorSync = 2
 )
 
-// Defines values for DashboardDashboardLinkType.
+// Defines values for DashboardLinkType.
 const (
 	DashboardLinkTypeDashboards DashboardLinkType = "dashboards"
 
 	DashboardLinkTypeLink DashboardLinkType = "link"
 )
 
-// Defines values for DashboardFieldColorModeId.
+// Defines values for FieldColorModeId.
 const (
 	FieldColorModeIdContinuousGrYlRd FieldColorModeId = "continuous-GrYlRd"
 
@@ -69,7 +71,7 @@ const (
 	FieldColorModeIdThresholds FieldColorModeId = "thresholds"
 )
 
-// Defines values for DashboardFieldColorSeriesByMode.
+// Defines values for FieldColorSeriesByMode.
 const (
 	FieldColorSeriesByModeLast FieldColorSeriesByMode = "last"
 
@@ -78,43 +80,43 @@ const (
 	FieldColorSeriesByModeMin FieldColorSeriesByMode = "min"
 )
 
-// Defines values for DashboardGraphPanelType.
+// Defines values for GraphPanelType.
 const (
 	GraphPanelTypeGraph GraphPanelType = "graph"
 )
 
-// Defines values for DashboardHeatmapPanelType.
+// Defines values for HeatmapPanelType.
 const (
 	HeatmapPanelTypeHeatmap HeatmapPanelType = "heatmap"
 )
 
-// Defines values for DashboardPanelRepeatDirection.
+// Defines values for PanelRepeatDirection.
 const (
 	PanelRepeatDirectionH PanelRepeatDirection = "h"
 
 	PanelRepeatDirectionV PanelRepeatDirection = "v"
 )
 
-// Defines values for DashboardRowPanelType.
+// Defines values for RowPanelType.
 const (
 	RowPanelTypeRow RowPanelType = "row"
 )
 
-// Defines values for DashboardThresholdsConfigMode.
+// Defines values for ThresholdsConfigMode.
 const (
 	ThresholdsConfigModeAbsolute ThresholdsConfigMode = "absolute"
 
 	ThresholdsConfigModePercentage ThresholdsConfigMode = "percentage"
 )
 
-// Defines values for DashboardThresholdsMode.
+// Defines values for ThresholdsMode.
 const (
 	ThresholdsModeAbsolute ThresholdsMode = "absolute"
 
 	ThresholdsModePercentage ThresholdsMode = "percentage"
 )
 
-// Defines values for DashboardVariableModelType.
+// Defines values for VariableModelType.
 const (
 	VariableModelTypeAdhoc VariableModelType = "adhoc"
 
@@ -133,7 +135,7 @@ const (
 	VariableModelTypeTextbox VariableModelType = "textbox"
 )
 
-// Defines values for DashboardVariableType.
+// Defines values for VariableType.
 const (
 	VariableTypeAdhoc VariableType = "adhoc"
 
@@ -152,7 +154,7 @@ const (
 	VariableTypeTextbox VariableType = "textbox"
 )
 
-// Dashboard defines model for dashboard.
+// Model is the Go representation of a dashboard.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -243,7 +245,7 @@ type Model struct {
 	WeekStart *string `json:"weekStart,omitempty"`
 }
 
-// DashboardGraphTooltip defines model for Dashboard.GraphTooltip.
+// GraphTooltip is the Go representation of a Model.GraphTooltip.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -294,7 +296,7 @@ type AnnotationQuery struct {
 	// Schema for panel targets is specified by datasource
 	// plugins. We use a placeholder definition, which the Go
 	// schema loader either left open/as-is with the Base
-	// variant of the Dashboard and Panel families, or filled
+	// variant of the Model and Panel families, or filled
 	// with types derived from plugins in the Instance variant.
 	// When working directly from CUE, importers can extend this
 	// type directly to achieve the same effect.
@@ -310,7 +312,7 @@ type AnnotationQuery struct {
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type DashboardCursorSync int
 
-// FROM public/app/features/dashboard/state/DashboardModels.ts - ish
+// FROM public/app/features/dashboard/state/Models.ts - ish
 // TODO docs
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
@@ -328,7 +330,7 @@ type DashboardLink struct {
 	Url         *string           `json:"url,omitempty"`
 }
 
-// DashboardDashboardLinkType defines model for DashboardDashboardLink.Type.
+// DashboardLinkType is the Go representation of a DashboardLink.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -361,7 +363,7 @@ type FieldColorModeId string
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type FieldColorSeriesByMode string
 
-// DashboardGraphPanel defines model for dashboard.GraphPanel.
+// GraphPanel is the Go representation of a dashboard.GraphPanel.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -376,7 +378,7 @@ type GraphPanel struct {
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type GraphPanelType string
 
-// DashboardGridPos defines model for dashboard.GridPos.
+// GridPos is the Go representation of a dashboard.GridPos.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -397,7 +399,7 @@ type GridPos struct {
 	Y int `json:"y"`
 }
 
-// DashboardHeatmapPanel defines model for dashboard.HeatmapPanel.
+// HeatmapPanel is the Go representation of a dashboard.HeatmapPanel.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -405,13 +407,13 @@ type HeatmapPanel struct {
 	Type HeatmapPanelType `json:"type"`
 }
 
-// DashboardHeatmapPanelType defines model for DashboardHeatmapPanel.Type.
+// HeatmapPanelType is the Go representation of a HeatmapPanel.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type HeatmapPanelType string
 
-// Dashboard panels. Panels are canonically defined inline
+// Model panels. Panels are canonically defined inline
 // because they share a version timeline with the dashboard
 // schema; they do not evolve independently.
 //
@@ -588,7 +590,7 @@ type RowPanel struct {
 	Type   RowPanelType `json:"type"`
 }
 
-// DashboardRowPanelType defines model for DashboardRowPanel.Type.
+// RowPanelType is the Go representation of a RowPanel.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -597,7 +599,7 @@ type RowPanelType string
 // Schema for panel targets is specified by datasource
 // plugins. We use a placeholder definition, which the Go
 // schema loader either left open/as-is with the Base
-// variant of the Dashboard and Panel families, or filled
+// variant of the Model and Panel families, or filled
 // with types derived from plugins in the Instance variant.
 // When working directly from CUE, importers can extend this
 // type directly to achieve the same effect.
@@ -624,7 +626,7 @@ type Threshold struct {
 	Value *float32 `json:"value,omitempty"`
 }
 
-// DashboardThresholdsConfig defines model for dashboard.ThresholdsConfig.
+// ThresholdsConfig is the Go representation of a dashboard.ThresholdsConfig.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -647,13 +649,13 @@ type ThresholdsConfig struct {
 	} `json:"steps"`
 }
 
-// DashboardThresholdsConfigMode defines model for DashboardThresholdsConfig.Mode.
+// ThresholdsConfigMode is the Go representation of a ThresholdsConfig.Mode.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type ThresholdsConfigMode string
 
-// DashboardThresholdsMode defines model for dashboard.ThresholdsMode.
+// ThresholdsMode is the Go representation of a dashboard.ThresholdsMode.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -682,7 +684,7 @@ type VariableModel struct {
 	Type  VariableModelType `json:"type"`
 }
 
-// DashboardVariableModelType defines model for DashboardVariableModel.Type.
+// VariableModelType is the Go representation of a VariableModel.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -699,7 +701,11 @@ type VariableType string
 //go:embed coremodel.cue
 var cueFS embed.FS
 
-// codegen ensures that this is always the latest Thema schema version
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
 var currentVersion = thema.SV(0, 0)
 
 // Lineage returns the Thema lineage representing a Grafana dashboard.

pkg/coremodel/pluginmeta/pluginmeta_gen.go

@@ -1,8 +1,10 @@
 // This file is autogenerated. DO NOT EDIT.
 //
-// Run "make gen-cue" from repository root to regenerate.
+// Generated by pkg/framework/coremodel/gen.go
 //
-// Derived from the Thema lineage at pkg/coremodel/pluginmeta
+// Derived from the Thema lineage declared in pkg/coremodel/pluginmeta/coremodel.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
 
 package pluginmeta
 
@@ -15,7 +17,7 @@ import (
 	"github.com/grafana/thema"
 )
 
-// Defines values for PluginmetaCategory.
+// Defines values for Category.
 const (
 	CategoryCloud Category = "cloud"
 
@@ -32,7 +34,7 @@ const (
 	CategoryTsdb Category = "tsdb"
 )
 
-// Defines values for PluginmetaType.
+// Defines values for Type.
 const (
 	TypeApp Type = "app"
 
@@ -45,7 +47,7 @@ const (
 	TypeSecretsmanager Type = "secretsmanager"
 )
 
-// Defines values for PluginmetaDependencyType.
+// Defines values for DependencyType.
 const (
 	DependencyTypeApp DependencyType = "app"
 
@@ -54,7 +56,7 @@ const (
 	DependencyTypePanel DependencyType = "panel"
 )
 
-// Defines values for PluginmetaIncludeRole.
+// Defines values for IncludeRole.
 const (
 	IncludeRoleAdmin IncludeRole = "Admin"
 
@@ -63,7 +65,7 @@ const (
 	IncludeRoleViewer IncludeRole = "Viewer"
 )
 
-// Defines values for PluginmetaIncludeType.
+// Defines values for IncludeType.
 const (
 	IncludeTypeApp IncludeType = "app"
 
@@ -80,7 +82,7 @@ const (
 	IncludeTypeSecretsmanager IncludeType = "secretsmanager"
 )
 
-// Defines values for PluginmetaReleaseState.
+// Defines values for ReleaseState.
 const (
 	ReleaseStateAlpha ReleaseState = "alpha"
 
@@ -91,7 +93,7 @@ const (
 	ReleaseStateStable ReleaseState = "stable"
 )
 
-// Pluginmeta defines model for pluginmeta.
+// Model is the Go representation of a pluginmeta.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -287,7 +289,7 @@ type Category string
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type Type string
 
-// PluginmetaBuildInfo defines model for pluginmeta.BuildInfo.
+// BuildInfo is the Go representation of a pluginmeta.BuildInfo.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -307,7 +309,7 @@ type BuildInfo struct {
 	Time *int64 `json:"time,omitempty"`
 }
 
-// PluginmetaDependencies defines model for pluginmeta.Dependencies.
+// Dependencies is the Go representation of a pluginmeta.Dependencies.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -338,7 +340,7 @@ type Dependency struct {
 	Version string         `json:"version"`
 }
 
-// PluginmetaDependencyType defines model for PluginmetaDependency.Type.
+// DependencyType is the Go representation of a Dependency.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -383,13 +385,13 @@ type Include struct {
 	Uid *string `json:"uid,omitempty"`
 }
 
-// PluginmetaIncludeRole defines model for PluginmetaInclude.Role.
+// IncludeRole is the Go representation of a Include.Role.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
 type IncludeRole string
 
-// PluginmetaIncludeType defines model for PluginmetaInclude.Type.
+// IncludeType is the Go representation of a Include.Type.
 //
 // THIS TYPE IS INTENDED FOR INTERNAL USE BY THE GRAFANA BACKEND, AND IS SUBJECT TO BREAKING CHANGES.
 // Equivalent Go types at stable import paths are provided in https://github.com/grafana/grok.
@@ -546,7 +548,11 @@ type URLParam struct {
 //go:embed coremodel.cue
 var cueFS embed.FS
 
-// codegen ensures that this is always the latest Thema schema version
+// The current version of the coremodel schema, as declared in coremodel.cue.
+// This version determines what schema version is returned from [Coremodel.CurrentSchema],
+// and which schema version is used for code generation within the grafana/grafana repository.
+//
+// The code generator ensures that this is always the latest Thema schema version.
 var currentVersion = thema.SV(0, 0)
 
 // Lineage returns the Thema lineage representing a Grafana pluginmeta.

pkg/framework/coremodel/registry/provide.go

@@ -54,6 +54,6 @@ func provideBase(lib *thema.Library) *Base {
 //
 // The returned slice is sorted lexicographically by coremodel name. It should
 // not be modified.
-func (s *Base) All() []coremodel.Interface {
+func (b *Base) All() []coremodel.Interface {
-	return s.all
+	return b.all
 }

pkg/framework/coremodel/registry/registry_gen.go

@@ -1,7 +1,8 @@
 // This file is autogenerated. DO NOT EDIT.
 //
 // Generated by pkg/framework/coremodel/gen.go
-// Run "make gen-cue" from repository root to regenerate.
+//
+// Run `make gen-cue` from repository root to regenerate.
 
 package registry
 
@@ -37,14 +38,14 @@ var (
 
 // Dashboard returns the dashboard coremodel. The return value is guaranteed to
 // implement coremodel.Interface.
-func (s *Base) Dashboard() *dashboard.Coremodel {
+func (b *Base) Dashboard() *dashboard.Coremodel {
-	return s.dashboard
+	return b.dashboard
 }
 
 // Pluginmeta returns the pluginmeta coremodel. The return value is guaranteed to
 // implement coremodel.Interface.
-func (s *Base) Pluginmeta() *pluginmeta.Coremodel {
+func (b *Base) Pluginmeta() *pluginmeta.Coremodel {
-	return s.pluginmeta
+	return b.pluginmeta
 }
 
 func doProvideBase(lib thema.Library) *Base {

pkg/framework/coremodel/slot.go

@@ -42,6 +42,22 @@ func (s Slot) ForPluginType(plugintype string) (may, must bool) {
 	return
 }
 
+// IsGroup indicates whether the slot specifies a group lineage - one in which
+// each top-level key represents a distinct schema for objects that are expected
+// to exist in the wild, but objects corresponding to the root of the schema are not
+// expected to exist.
+func (s Slot) IsGroup() bool {
+	// TODO rely on first-class Thema properties for this, one they exist - https://github.com/grafana/thema/issues/62
+	switch s.name {
+	case "Panel", "DSOptions":
+		return true
+	case "Query":
+		return false
+	default:
+		panic("unreachable - unknown slot name " + s.name)
+	}
+}
+
 func AllSlots() map[string]*Slot {
 	fw := CUEFramework()
 	slots := make(map[string]*Slot)

pkg/plugins/pfs/corelist/loadlist_gen.go

@@ -0,0 +1,86 @@
+// Copyright 2022 Grafana Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This file is autogenerated. DO NOT EDIT.
+//
+// Run `make gen-cue` from repository root to regenerate.
+
+package corelist
+
+import (
+	"fmt"
+	"io/fs"
+
+	"github.com/grafana/grafana"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+func makeTreeOrPanic(path string, pkgname string, lib thema.Library) *pfs.Tree {
+	sub, err := fs.Sub(grafana.CueSchemaFS, path)
+	if err != nil {
+		panic("could not create fs sub to " + path)
+	}
+	tree, err := pfs.ParsePluginFS(sub, lib)
+	if err != nil {
+		panic(fmt.Sprintf("error parsing plugin metadata for %s: %s", pkgname, err))
+	}
+	return tree
+}
+
+func coreTreeList(lib thema.Library) pfs.TreeList {
+	return pfs.TreeList{
+		makeTreeOrPanic("public/app/plugins/datasource/alertmanager", "alertmanager", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/cloud-monitoring", "stackdriver", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/cloudwatch", "cloudwatch", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/dashboard", "dashboard", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/elasticsearch", "elasticsearch", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/grafana", "grafana", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/grafana-azure-monitor-datasource", "grafana_azure_monitor_datasource", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/graphite", "graphite", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/jaeger", "jaeger", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/loki", "loki", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/mssql", "mssql", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/mysql", "mysql", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/postgres", "postgres", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/prometheus", "prometheus", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/tempo", "tempo", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/testdata", "testdata", lib),
+		makeTreeOrPanic("public/app/plugins/datasource/zipkin", "zipkin", lib),
+		makeTreeOrPanic("public/app/plugins/panel/alertGroups", "alertGroups", lib),
+		makeTreeOrPanic("public/app/plugins/panel/alertlist", "alertlist", lib),
+		makeTreeOrPanic("public/app/plugins/panel/annolist", "annolist", lib),
+		makeTreeOrPanic("public/app/plugins/panel/barchart", "barchart", lib),
+		makeTreeOrPanic("public/app/plugins/panel/bargauge", "bargauge", lib),
+		makeTreeOrPanic("public/app/plugins/panel/dashlist", "dashlist", lib),
+		makeTreeOrPanic("public/app/plugins/panel/debug", "debug", lib),
+		makeTreeOrPanic("public/app/plugins/panel/gauge", "gauge", lib),
+		makeTreeOrPanic("public/app/plugins/panel/geomap", "geomap", lib),
+		makeTreeOrPanic("public/app/plugins/panel/gettingstarted", "gettingstarted", lib),
+		makeTreeOrPanic("public/app/plugins/panel/graph", "graph", lib),
+		makeTreeOrPanic("public/app/plugins/panel/histogram", "histogram", lib),
+		makeTreeOrPanic("public/app/plugins/panel/icon", "icon", lib),
+		makeTreeOrPanic("public/app/plugins/panel/live", "live", lib),
+		makeTreeOrPanic("public/app/plugins/panel/logs", "logs", lib),
+		makeTreeOrPanic("public/app/plugins/panel/news", "news", lib),
+		makeTreeOrPanic("public/app/plugins/panel/nodeGraph", "nodeGraph", lib),
+		makeTreeOrPanic("public/app/plugins/panel/piechart", "piechart", lib),
+		makeTreeOrPanic("public/app/plugins/panel/stat", "stat", lib),
+		makeTreeOrPanic("public/app/plugins/panel/table-old", "table_old", lib),
+		makeTreeOrPanic("public/app/plugins/panel/text", "text", lib),
+		makeTreeOrPanic("public/app/plugins/panel/traces", "traces", lib),
+		makeTreeOrPanic("public/app/plugins/panel/welcome", "welcome", lib),
+		makeTreeOrPanic("public/app/plugins/panel/xychart", "xychart", lib),
+	}
+}

pkg/plugins/pfs/corelist/new.go

@@ -0,0 +1,30 @@
+package corelist
+
+import (
+	"sync"
+
+	"github.com/grafana/grafana/pkg/cuectx"
+	"github.com/grafana/grafana/pkg/plugins/pfs"
+	"github.com/grafana/thema"
+)
+
+var coreTrees pfs.TreeList
+var coreOnce sync.Once
+
+// New returns a pfs.TreeList containing the plugin trees for all core plugins
+// in the current version of Grafana.
+//
+// Go code within the grafana codebase should only ever call this with nil.
+func New(lib *thema.Library) pfs.TreeList {
+	var tl pfs.TreeList
+	if lib == nil {
+		coreOnce.Do(func() {
+			coreTrees = coreTreeList(cuectx.ProvideThemaLibrary())
+		})
+		tl = make(pfs.TreeList, len(coreTrees))
+		copy(tl, coreTrees)
+	} else {
+		return coreTreeList(*lib)
+	}
+	return tl
+}

pkg/plugins/pfs/errors.go

@@ -17,6 +17,9 @@ var ErrInvalidRootFile = errors.New("plugin.json is invalid")
 //   - A required slot for its type is not implemented (e.g. panel plugin does not implemented Panel)
 var ErrImplementedSlots = errors.New("slot implementation not allowed for this plugin type")
 
+// ErrInvalidCUE indicates that a plugin's model.cue file contained invalid CUE.
+var ErrInvalidCUE = errors.New("CUE syntax error")
+
 // ErrInvalidLineage indicates that the plugin contains an invalid lineage
 // declaration, according to Thema's validation rules in
 // ["github.com/grafana/thema".BindLineage].

pkg/plugins/pfs/pfs.go

@@ -133,7 +133,28 @@ func (t *Tree) RootPlugin() PluginInfo {
 // SubPlugins returned a map of the PluginInfos for subplugins
 // within the tree, if any, keyed by subpath.
 func (t *Tree) SubPlugins() map[string]PluginInfo {
-	panic("TODO")
+	// TODO implement these once ParsePluginFS descends
+	return nil
+}
+
+// TreeList is a slice of validated plugin fs Trees with helper methods
+// for filtering to particular subsets of its members.
+type TreeList []*Tree
+
+// LineagesForSlot returns the set of plugin-defined lineages that implement a
+// particular named Grafana slot (See ["github.com/grafana/grafana/pkg/framework/coremodel".Slot]).
+func (tl TreeList) LineagesForSlot(slotname string) map[string]thema.Lineage {
+	m := make(map[string]thema.Lineage)
+	for _, tree := range tl {
+		rootp := tree.RootPlugin()
+		rid := rootp.Meta().Id
+
+		if lin, has := rootp.SlotImplementations()[slotname]; has {
+			m[rid] = lin
+		}
+	}
+
+	return m
 }
 
 // PluginInfo represents everything knowable about a single plugin from static
@@ -166,8 +187,8 @@ func (pi PluginInfo) Meta() pluginmeta.Model {
 // ParsePluginFS takes an fs.FS and checks that it represents exactly one valid
 // plugin fs tree, with the fs.FS root as the root of the tree.
 //
-// It does not descend into subdirectories to search for additional
+// It does not descend into subdirectories to search for additional plugin.json
-// plugin.json files.
+// files.
 // TODO no descent is ok for core plugins, but won't cut it in general
 func ParsePluginFS(f fs.FS, lib thema.Library) (*Tree, error) {
 	if f == nil {
@@ -234,6 +255,9 @@ func ParsePluginFS(f fs.FS, lib thema.Library) (*Tree, error) {
 		}
 
 		val := ctx.BuildInstance(bi)
+		if val.Err() != nil {
+			return nil, ewrap(fmt.Errorf("models.cue is invalid CUE: %w", val.Err()), ErrInvalidCUE)
+		}
 		for _, s := range allslots {
 			iv := val.LookupPath(cue.ParsePath(s.slot.Name()))
 			lin, err := bindSlotLineage(iv, s.slot, r.meta, lib)
@@ -249,7 +273,7 @@ func ParsePluginFS(f fs.FS, lib thema.Library) (*Tree, error) {
 	return tree, nil
 }
 
-func bindSlotLineage(v cue.Value, s *coremodel.Slot, meta pluginmeta.Model, lib thema.Library) (thema.Lineage, error) {
+func bindSlotLineage(v cue.Value, s *coremodel.Slot, meta pluginmeta.Model, lib thema.Library, opts ...thema.BindOption) (thema.Lineage, error) {
 	accept, required := s.ForPluginType(string(meta.Type))
 	exists := v.Exists()
 
@@ -269,7 +293,7 @@ func bindSlotLineage(v cue.Value, s *coremodel.Slot, meta pluginmeta.Model, lib
 
 	// TODO make this opt real in thema, then uncomment to enforce joinSchema
 	// lin, err := thema.BindLineage(iv, lib, thema.SatisfiesJoinSchema(s.MetaSchema()))
-	lin, err := thema.BindLineage(v, lib)
+	lin, err := thema.BindLineage(v, lib, opts...)
 	if err != nil {
 		return nil, ewrap(fmt.Errorf("%s: invalid thema lineage for slot %s: %w", meta.Id, s.Name(), err), ErrInvalidLineage)
 	}

public/app/plugins/gen.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/codegen"
 	"github.com/grafana/grafana/pkg/cuectx"
@@ -29,6 +30,8 @@ var skipPlugins = map[string]bool{
 	"opentsdb":       true, // plugin.json fails validation (defaultMatchFormat)
 }
 
+const sep = string(filepath.Separator)
+
 // Generate TypeScript for all plugin models.cue
 func main() {
 	if len(os.Args) > 1 {
@@ -41,15 +44,17 @@ func main() {
 		fmt.Fprintf(os.Stderr, "could not get working directory: %s", err)
 		os.Exit(1)
 	}
+	grootp := strings.Split(cwd, sep)
+	groot := filepath.Join(sep, filepath.Join(grootp[:len(grootp)-3]...))
 
 	wd := codegen.NewWriteDiffer()
 	lib := cuectx.ProvideThemaLibrary()
 
 	type ptreepath struct {
-		fullpath string
+		Path string
-		tree     *codegen.PluginTree
+		Tree *codegen.PluginTree
 	}
-	var ptrees []ptreepath
+	var ptrees []codegen.TreeAndPath
 	for _, typ := range []string{"datasource", "panel"} {
 		dir := filepath.Join(cwd, typ)
 		treeor, err := codegen.ExtractPluginTrees(os.DirFS(dir), lib)
@@ -64,9 +69,9 @@ func main() {
 			}
 
 			if option.Tree != nil {
-				ptrees = append(ptrees, ptreepath{
+				ptrees = append(ptrees, codegen.TreeAndPath{
-					fullpath: filepath.Join(typ, name),
+					Path: filepath.Join(typ, name),
-					tree:     option.Tree,
+					Tree: option.Tree,
 				})
 			} else if !errors.Is(option.Err, pfs.ErrNoRootFile) {
 				fmt.Fprintf(os.Stderr, "error parsing plugin directory %s: %s\n", filepath.Join(dir, name), option.Err)
@@ -79,18 +84,39 @@ func main() {
 	// having multiple core plugins with errors can cause confusing error
 	// flip-flopping
 	sort.Slice(ptrees, func(i, j int) bool {
-		return ptrees[i].fullpath < ptrees[j].fullpath
+		return ptrees[i].Path < ptrees[j].Path
 	})
 
+	var wdm codegen.WriteDiffer
 	for _, ptp := range ptrees {
-		twd, err := ptp.tree.GenerateTS(ptp.fullpath)
+		wdm, err = ptp.Tree.GenerateTS(ptp.Path)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "generating typescript failed for %s: %s\n", ptp.Path, err)
+			os.Exit(1)
+		}
+		wd.Merge(wdm)
+
+		relp, _ := filepath.Rel(groot, ptp.Path)
+		wdm, err = ptp.Tree.GenerateGo(ptp.Path, codegen.GoGenConfig{
+			Types: isDatasource(ptp.Tree),
+			// TODO false until we decide on a consistent codegen format for core and external plugins
+			ThemaBindings: false,
+			DocPathPrefix: relp,
+		})
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "generating typescript failed for %s: %s\n", ptp.fullpath, err)
+			fmt.Fprintf(os.Stderr, "generating Go failed for %s: %s\n", ptp.Path, err)
 			os.Exit(1)
 		}
-		wd.Merge(twd)
+		wd.Merge(wdm)
 	}
 
+	wdm, err = codegen.GenPluginTreeList(ptrees, "github.com/grafana/grafana/public/app/plugins", filepath.Join(groot, "pkg", "plugins", "pfs", "corelist", "loadlist_gen.go"), false)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "generating plugin loader registry failed: %s\n", err)
+		os.Exit(1)
+	}
+	wd.Merge(wdm)
+
 	if _, set := os.LookupEnv("CODEGEN_VERIFY"); set {
 		err = wd.Verify()
 		if err != nil {
@@ -105,3 +131,7 @@ func main() {
 		}
 	}
 }
+
+func isDatasource(pt *codegen.PluginTree) bool {
+	return string((*pfs.Tree)(pt).RootPlugin().Meta().Type) == "datasource"
+}

public/app/plugins/panel/annolist/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 
 export const PanelModelVersion = Object.freeze([0, 0]);
@@ -33,3 +36,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
   showUser: true,
   tags: [],
 };
+

public/app/plugins/panel/barchart/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -45,3 +48,4 @@ export const defaultPanelFieldConfig: Partial<PanelFieldConfig> = {
   gradientMode: ui.GraphGradientMode.None,
   lineWidth: 1,
 };
+

public/app/plugins/panel/bargauge/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -22,3 +25,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
   minVizWidth: 0,
   showUnfilled: true,
 };
+

public/app/plugins/panel/dashlist/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 
 export const PanelModelVersion = Object.freeze([0, 0]);
@@ -35,3 +38,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
   showStarred: true,
   tags: [],
 };
+

public/app/plugins/panel/gauge/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -18,3 +21,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
   showThresholdLabels: false,
   showThresholdMarkers: true,
 };
+

public/app/plugins/panel/histogram/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -30,3 +33,4 @@ export const defaultPanelFieldConfig: Partial<PanelFieldConfig> = {
   gradientMode: ui.GraphGradientMode.None,
   lineWidth: 1,
 };
+

public/app/plugins/panel/news/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 
 export const PanelModelVersion = Object.freeze([0, 0]);
@@ -16,3 +19,4 @@ export interface PanelOptions {
 export const defaultPanelOptions: Partial<PanelOptions> = {
   showImage: true,
 };
+

public/app/plugins/panel/piechart/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -44,3 +47,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
 };
 
 export interface PanelFieldConfig extends ui.HideableFieldConfig {}
+

public/app/plugins/panel/stat/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 import * as ui from '@grafana/schema';
 
@@ -22,3 +25,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
   justifyMode: ui.BigValueJustifyMode.Auto,
   textMode: ui.BigValueTextMode.Auto,
 };
+

public/app/plugins/panel/text/models.gen.ts

@@ -1,8 +1,11 @@
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // This file is autogenerated. DO NOT EDIT.
 //
-// To regenerate, run "make gen-cue" from the repository root.
+// Generated by public/app/plugins/gen.go
-//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// Derived from the Thema lineage declared in models.cue
+//
+// Run `make gen-cue` from repository root to regenerate.
+
 
 
 export const PanelModelVersion = Object.freeze([0, 0]);
@@ -52,3 +55,4 @@ export const defaultPanelOptions: Partial<PanelOptions> = {
 For markdown syntax help: [commonmark.org/help](https://commonmark.org/help/)`,
   mode: TextMode.Markdown,
 };
+