codegen: Refactor core jennies for reusability, add version-picking metajennies (#58995)

* Add each-major jenny, refactor TS jenny for it

* Introduce LatestJenny, refactor GoTypesJenny for it

kinds/gen.go

@@ -21,8 +21,6 @@ import (
 	"github.com/grafana/grafana/pkg/kindsys"
 )
 
-const sep = string(filepath.Separator)
-
 func main() {
 	if len(os.Args) > 1 {
 		fmt.Fprintf(os.Stderr, "plugin thema code generator does not currently accept any arguments\n, got %q", os.Args)
@@ -37,16 +35,11 @@ func main() {
 
 	// All the jennies that comprise the core kinds generator pipeline
 	coreKindsGen.Append(
-		codegen.GoTypesJenny(kindsys.GoCoreKindParentPath, nil),
+		codegen.LatestJenny(kindsys.GoCoreKindParentPath, codegen.GoTypesJenny{}),
 		codegen.CoreStructuredKindJenny(kindsys.GoCoreKindParentPath, nil),
 		codegen.RawKindJenny(kindsys.GoCoreKindParentPath, nil),
 		codegen.BaseCoreRegistryJenny(filepath.Join("pkg", "registry", "corekind"), kindsys.GoCoreKindParentPath),
-		codegen.TSTypesJenny(kindsys.TSCoreKindParentPath, &codegen.TSTypesGeneratorConfig{
+		codegen.LatestMajorsOrXJenny(kindsys.TSCoreKindParentPath, codegen.TSTypesJenny{}),
-			GenDirName: func(decl *codegen.DeclForGen) string {
-				// FIXME this hardcodes always generating to experimental dir. OK for now, but need generator fanout
-				return filepath.Join(decl.Meta.Common().MachineName, "x")
-			},
-		}),
 		codegen.TSVeneerIndexJenny(filepath.Join("packages", "grafana-schema", "src")),
 	)
 

packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts

@@ -4,6 +4,7 @@
 //     kinds/gen.go
 // Using jennies:
 //     TSTypesJenny
+//     LatestMajorsOrXJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

packages/grafana-schema/src/raw/playlist/x/playlist_types.gen.ts

@@ -4,6 +4,7 @@
 //     kinds/gen.go
 // Using jennies:
 //     TSTypesJenny
+//     LatestMajorsOrXJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/codegen/generators.go

@@ -57,7 +57,6 @@ func StructuredForGen(k kindsys.Structured) *DeclForGen {
 type DeclForGen struct {
 	*kindsys.SomeDecl
 	lin thema.Lineage
-	sch thema.Lineage
 }
 
 // Lineage returns the [thema.Lineage] for the underlying [kindsys.SomeDecl].
@@ -65,10 +64,15 @@ func (decl *DeclForGen) Lineage() thema.Lineage {
 	return decl.lin
 }
 
-// Schema returns the [thema.Schema] that a jenny should operate against, for those
+// ForLatestSchema returns a [SchemaForGen] for the latest schema in this
-// jennies that target a single schema.
+// DeclForGen's lineage.
-func (decl *DeclForGen) Schema() thema.Lineage {
+func (decl *DeclForGen) ForLatestSchema() SchemaForGen {
-	return decl.sch
+	comm := decl.Meta.Common()
+	return SchemaForGen{
+		Name:    comm.Name,
+		Schema:  decl.Lineage().Latest(),
+		IsGroup: comm.LineageIsGroup,
+	}
 }
 
 // SlashHeaderMapper produces a FileMapper that injects a comment header onto
@@ -82,22 +86,28 @@ func SlashHeaderMapper(maingen string) codejen.FileMapper {
 		case ".json", ".yml", ".yaml":
 			return f, nil
 		default:
-			b := new(bytes.Buffer)
+			buf := new(bytes.Buffer)
-			fmt.Fprintf(b, headerTmpl, filepath.ToSlash(maingen), f.FromString())
+			if err := tmpls.Lookup("gen_header.tmpl").Execute(buf, tvars_gen_header{
-			fmt.Fprint(b, string(f.Data))
+				MainGenerator: maingen,
-			f.Data = b.Bytes()
+				Using:         f.From,
+			}); err != nil {
+				return codejen.File{}, fmt.Errorf("failed executing gen header template: %w", err)
+			}
+			fmt.Fprint(buf, string(f.Data))
+			f.Data = buf.Bytes()
 		}
 		return f, nil
 	}
 }
 
-var headerTmpl = `// THIS FILE IS GENERATED. EDITING IS FUTILE.
+// SchemaForGen is an intermediate values type for jennies that holds both a thema.Schema,
-//
+// and values relevant to generating the schema that should properly, eventually, be in
-// Generated by:
+// thema itself.
-//     %s
+type SchemaForGen struct {
-// Using jennies:
+	// The PascalCase name of the schematized type.
-//     %s
+	Name string
-//
+	// The schema to be rendered for the type itself.
-// Run 'make gen-cue' from repository root to regenerate.
+	Schema thema.Schema
-
+	// Whether the schema is grouped. See https://github.com/grafana/thema/issues/62
-`
+	IsGroup bool
+}

pkg/codegen/jenny_eachmajor.go

@@ -0,0 +1,74 @@
+package codegen
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/grafana/codejen"
+	"github.com/grafana/grafana/pkg/kindsys"
+)
+
+// LatestMajorsOrXJenny returns a jenny that repeats the input for the latest in each major version,
+func LatestMajorsOrXJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToMany {
+	if inner == nil {
+		panic("inner jenny must not be nil")
+	}
+
+	return &lmox{
+		parentdir: parentdir,
+		inner:     inner,
+	}
+}
+
+type lmox struct {
+	parentdir string
+	inner     codejen.OneToOne[SchemaForGen]
+}
+
+func (j *lmox) JennyName() string {
+	return "LatestMajorsOrXJenny"
+}
+
+func (j *lmox) Generate(decl *DeclForGen) (codejen.Files, error) {
+	if decl.IsRaw() {
+		return nil, nil
+	}
+	comm := decl.Meta.Common()
+	sfg := SchemaForGen{
+		Name:    comm.Name,
+		IsGroup: comm.LineageIsGroup,
+	}
+
+	do := func(sfg SchemaForGen, infix string) (codejen.Files, error) {
+		f, err := j.inner.Generate(sfg)
+		if err != nil {
+			return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err)
+		}
+		if f == nil || !f.Exists() {
+			return nil, nil
+		}
+
+		f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, infix, f.RelativePath)
+		f.From = append(f.From, j)
+		return codejen.Files{*f}, nil
+	}
+
+	if comm.Maturity.Less(kindsys.MaturityStable) {
+		sfg.Schema = decl.Lineage().Latest()
+		return do(sfg, "x")
+	}
+
+	var fl codejen.Files
+	for sch := decl.Lineage().First(); sch != nil; sch.Successor() {
+		sfg.Schema = sch.LatestInMajor()
+		files, err := do(sfg, fmt.Sprintf("v%v", sch.Version()[0]))
+		if err != nil {
+			return nil, err
+		}
+		fl = append(fl, files...)
+	}
+	if fl.Validate() != nil {
+		return nil, fl.Validate()
+	}
+	return fl, nil
+}

pkg/codegen/jenny_gotypes.go

@@ -1,98 +1,29 @@
 package codegen
 
 import (
-	"fmt"
-	"path/filepath"
-
 	"github.com/grafana/codejen"
-	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/gocode"
 	"golang.org/x/tools/go/ast/astutil"
 )
 
-// GoTypesJenny creates a [OneToOne] that produces Go types for the latest
+// GoTypesJenny creates a [OneToOne] that produces Go types for the provided
-// Thema schema in a structured kind's lineage.
+// [thema.Schema].
-//
+type GoTypesJenny struct{}
-// At minimum, a gokindsdir must be provided. This should be the path to the parent
-// directory of the directory in which the types should be generated, relative
-// to the project root. For example, if the types for a kind named "foo"
-// should live at pkg/kind/foo/foo_gen.go, relpath should be "pkg/kind".
-//
-// This generator is a no-op for raw kinds.
-func GoTypesJenny(gokindsdir string, cfg *GoTypesGeneratorConfig) OneToOne {
-	if cfg == nil {
-		cfg = new(GoTypesGeneratorConfig)
-	}
-	if cfg.GenDirName == nil {
-		cfg.GenDirName = func(decl *DeclForGen) string {
-			return decl.Meta.Common().MachineName
-		}
-	}
-
-	return &genGoTypes{
-		gokindsdir: gokindsdir,
-		cfg:        cfg,
-	}
-}
-
-// GoTypesGeneratorConfig holds configuration options for [GoTypesJenny].
-type GoTypesGeneratorConfig struct {
-	// Apply is an optional AST manipulation func that, if provided, will be run
-	// against the generated Go file prior to running it through goimports.
-	Apply astutil.ApplyFunc
-
-	// GenDirName returns the name of the parent directory in which the type file
-	// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
-	GenDirName func(*DeclForGen) string
-
-	// Version of the schema to generate. If nil, latest is generated.
-	Version *thema.SyntacticVersion
-}
 
-type genGoTypes struct {
+func (j GoTypesJenny) JennyName() string {
-	gokindsdir string
-	cfg        *GoTypesGeneratorConfig
-}
-
-func (gen *genGoTypes) JennyName() string {
 	return "GoTypesJenny"
 }
 
-func (gen *genGoTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
+func (j GoTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
-	if decl.IsRaw() {
-		return nil, nil
-	}
-
-	var sch thema.Schema
-	var err error
-
-	lin := decl.Lineage()
-	if gen.cfg.Version == nil {
-		sch = lin.Latest()
-	} else {
-		sch, err = lin.Schema(*gen.cfg.Version)
-		if err != nil {
-			return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
-		}
-	}
-
-	// always drop prefixes.
-	var appf []astutil.ApplyFunc
-	if gen.cfg.Apply != nil {
-		appf = append(appf, gen.cfg.Apply)
-	}
-	appf = append(appf, PrefixDropper(decl.Meta.Common().Name))
-
-	pdir := gen.cfg.GenDirName(decl)
-	fpath := filepath.Join(gen.gokindsdir, pdir, lin.Name()+"_types_gen.go")
 	// TODO allow using name instead of machine name in thema generator
-	b, err := gocode.GenerateTypesOpenAPI(sch, &gocode.TypeConfigOpenAPI{
+	b, err := gocode.GenerateTypesOpenAPI(sfg.Schema, &gocode.TypeConfigOpenAPI{
-		PackageName: filepath.Base(pdir),
+		// TODO will need to account for sanitizing e.g. dashes here at some point
-		ApplyFuncs:  appf,
+		PackageName: sfg.Schema.Lineage().Name(),
+		ApplyFuncs:  []astutil.ApplyFunc{PrefixDropper(sfg.Name)},
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	return codejen.NewFile(fpath, b, gen), nil
+	return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types_gen.go", b, j), nil
 }

pkg/codegen/jenny_tstypes.go

@@ -1,85 +1,32 @@
 package codegen
 
 import (
-	"fmt"
-	"path/filepath"
-
 	"github.com/grafana/codejen"
-	"github.com/grafana/thema"
 	"github.com/grafana/thema/encoding/typescript"
 )
 
-// TSTypesJenny creates a [OneToOne] that produces TypeScript types and
+// TSTypesJenny is a [OneToOne] that produces TypeScript types and
-// defaults for the latest Thema schema in a structured kind's lineage.
+// defaults for a Thema schema.
-//
-// At minimum, a tskindsdir must be provided. This should be the path to the parent
-// directory of the directory in which the types should be generated, relative
-// to the project root. For example, if the types for a kind named "foo"
-// should live at packages/grafana-schema/src/raw/foo, relpath should be "pkg/kind".
 //
-// This generator is a no-op for raw kinds.
+// Thema's generic TS jenny will be able to replace this one once
-func TSTypesJenny(tskindsdir string, cfg *TSTypesGeneratorConfig) OneToOne {
+// https://github.com/grafana/thema/issues/89 is complete.
-	if cfg == nil {
+type TSTypesJenny struct{}
-		cfg = new(TSTypesGeneratorConfig)
-	}
-	if cfg.GenDirName == nil {
-		cfg.GenDirName = func(decl *DeclForGen) string {
-			return decl.Meta.Common().MachineName
-		}
-	}
-
-	return &genTSTypes{
-		tskindsdir: tskindsdir,
-		cfg:        cfg,
-	}
-}
-
-// TSTypesGeneratorConfig holds configuration options for [TSTypesJenny].
-type TSTypesGeneratorConfig struct {
-	// GenDirName returns the name of the parent directory in which the type file
-	// should be generated. If nil, the DeclForGen.Lineage().Name() will be used.
-	GenDirName func(*DeclForGen) string
 
-	// Version of the schema to generate. If nil, latest is generated.
+var _ codejen.OneToOne[SchemaForGen] = &TSTypesJenny{}
-	Version *thema.SyntacticVersion
-}
-
-type genTSTypes struct {
-	tskindsdir string
-	cfg        *TSTypesGeneratorConfig
-}
 
-func (gen *genTSTypes) JennyName() string {
+func (j TSTypesJenny) JennyName() string {
 	return "TSTypesJenny"
 }
 
-func (gen *genTSTypes) Generate(decl *DeclForGen) (*codejen.File, error) {
+func (j TSTypesJenny) Generate(sfg SchemaForGen) (*codejen.File, error) {
-	if decl.IsRaw() {
-		return nil, nil
-	}
-	var sch thema.Schema
-	var err error
-
-	lin := decl.Lineage()
-	if gen.cfg.Version == nil {
-		sch = lin.Latest()
-	} else {
-		sch, err = lin.Schema(*gen.cfg.Version)
-		if err != nil {
-			return nil, fmt.Errorf("error in configured version for %s generator: %w", *gen.cfg.Version, err)
-		}
-	}
-
 	// TODO allow using name instead of machine name in thema generator
-	f, err := typescript.GenerateTypes(sch, &typescript.TypeConfig{
+	f, err := typescript.GenerateTypes(sfg.Schema, &typescript.TypeConfig{
-		RootName: decl.Meta.Common().Name,
+		RootName: sfg.Name,
-		Group:    decl.Meta.Common().LineageIsGroup,
+		Group:    sfg.IsGroup,
 	})
 	if err != nil {
 		return nil, err
 	}
-	return codejen.NewFile(
+
-		filepath.Join(gen.tskindsdir, gen.cfg.GenDirName(decl), lin.Name()+"_types.gen.ts"),
+	return codejen.NewFile(sfg.Schema.Lineage().Name()+"_types.gen.ts", []byte(f.String()), j), nil
-		[]byte(f.String()),
-		gen), nil
 }

pkg/codegen/latest_jenny.go

@@ -0,0 +1,56 @@
+package codegen
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/grafana/codejen"
+)
+
+// LatestJenny returns a jenny that runs another jenny for only the latest
+// schema in a DeclForGen, and prefixes the resulting file with the provided
+// parentdir (e.g. "pkg/kinds/") and with a directory based on the kind's
+// machine name (e.g. "dashboard/").
+func LatestJenny(parentdir string, inner codejen.OneToOne[SchemaForGen]) OneToOne {
+	if inner == nil {
+		panic("inner jenny must not be nil")
+	}
+
+	return &latestj{
+		parentdir: parentdir,
+		inner:     inner,
+	}
+}
+
+type latestj struct {
+	parentdir string
+	inner     codejen.OneToOne[SchemaForGen]
+}
+
+func (j *latestj) JennyName() string {
+	return "LatestJenny"
+}
+
+func (j *latestj) Generate(decl *DeclForGen) (*codejen.File, error) {
+	if decl.IsRaw() {
+		return nil, nil
+	}
+	comm := decl.Meta.Common()
+	sfg := SchemaForGen{
+		Name:    comm.Name,
+		Schema:  decl.Lineage().Latest(),
+		IsGroup: comm.LineageIsGroup,
+	}
+
+	f, err := j.inner.Generate(sfg)
+	if err != nil {
+		return nil, fmt.Errorf("%s jenny failed on %s schema for %s: %w", j.inner.JennyName(), sfg.Schema.Version(), decl.Meta.Common().Name, err)
+	}
+	if f == nil || !f.Exists() {
+		return nil, nil
+	}
+
+	f.RelativePath = filepath.Join(j.parentdir, comm.MachineName, f.RelativePath)
+	f.From = append(f.From, j)
+	return f, nil
+}

pkg/codegen/tmpl.go

@@ -5,6 +5,8 @@ import (
 	"embed"
 	"text/template"
 	"time"
+
+	"github.com/grafana/codejen"
 )
 
 // All the parsed templates in the tmpl subdirectory
@@ -29,6 +31,11 @@ type (
 		LineageCUEPath string
 		GenLicense     bool
 	}
+	tvars_gen_header struct {
+		MainGenerator string
+		Using         []codejen.NamedJenny
+		From          string
+	}
 	tvars_kind_registry struct {
 		// Header tvars_autogen_header
 		NumRaw, NumStructured int

pkg/codegen/tmpl/gen_header.tmpl

@@ -0,0 +1,11 @@
+// THIS FILE IS GENERATED. EDITING IS FUTILE.
+//
+// Generated by:
+//     {{ .MainGenerator }}
+// Using jennies:
+{{- range .Using }}
+//     {{ .JennyName }}
+{{- end }}
+//
+// Run 'make gen-cue' from repository root to regenerate.
+

pkg/kinds/dashboard/dashboard_types_gen.go

@@ -4,6 +4,7 @@
 //     kinds/gen.go
 // Using jennies:
 //     GoTypesJenny
+//     LatestJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.
 

pkg/kinds/playlist/playlist_types_gen.go

@@ -4,6 +4,7 @@
 //     kinds/gen.go
 // Using jennies:
 //     GoTypesJenny
+//     LatestJenny
 //
 // Run 'make gen-cue' from repository root to regenerate.