refactory to move trim/apply default into schema.go (#33754)

* refactory to move trim/apply default into schema.go

* fix comments

* move schema test outside of load folder

pkg/schema/load/applydefault_test.go

@@ -1,115 +0,0 @@
-package load
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"testing"
-
-	"cuelang.org/go/cue"
-	"github.com/google/go-cmp/cmp"
-	"github.com/grafana/grafana/pkg/schema"
-	"golang.org/x/tools/txtar"
-)
-
-var CasesDir = filepath.Join("testdata", "artifacts", "dashboards", "trimdefault")
-
-type Case struct {
-	Name string
-
-	CUE    string
-	Full   string
-	Trimed string
-}
-
-func TestGenerate(t *testing.T) {
-	cases, err := loadCases(CasesDir)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, c := range cases {
-		t.Run(c.Name+" apply default value", func(t *testing.T) {
-			var r cue.Runtime
-			scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
-			if err != nil {
-				t.Fatal(err)
-			}
-			inputResource := schema.Resource{Value: c.Trimed}
-			scm := genericVersionedSchema{actual: scmInstance.Value()}
-			out, err := scm.ApplyDefaults(inputResource)
-			if err != nil {
-				t.Fatal(err)
-			}
-			b := []byte(out.Value.(string))
-
-			if s := cmp.Diff(string(b), c.Full); s != "" {
-				t.Fatal(s)
-			}
-		})
-	}
-
-	for _, c := range cases {
-		t.Run(c.Name+" trim default value", func(t *testing.T) {
-			var r cue.Runtime
-			scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
-			if err != nil {
-				t.Fatal(err)
-			}
-			inputResource := schema.Resource{Value: c.Full}
-			scm := genericVersionedSchema{actual: scmInstance.Value()}
-			out, err := scm.TrimDefaults(inputResource)
-			if err != nil {
-				t.Fatal(err)
-			}
-			b := []byte(out.Value.(string))
-			if s := cmp.Diff(string(b), c.Trimed); s != "" {
-				t.Fatal(s)
-			}
-		})
-	}
-}
-
-func loadCases(dir string) ([]Case, error) {
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-
-	var cases []Case
-
-	for _, fi := range files {
-		file := filepath.Join(dir, fi.Name())
-		a, err := txtar.ParseFile(file)
-		if err != nil {
-			return nil, err
-		}
-
-		if len(a.Files) != 3 {
-			return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files))
-		}
-
-		fullBuffer := new(bytes.Buffer)
-		fullJson := a.Files[1].Data
-		if err := json.Compact(fullBuffer, fullJson); err != nil {
-			return nil, err
-		}
-
-		trimBuffer := new(bytes.Buffer)
-		trimedJson := a.Files[2].Data
-		if err := json.Compact(trimBuffer, trimedJson); err != nil {
-			return nil, err
-		}
-
-		cases = append(cases, Case{
-			Name:   fi.Name(),
-			CUE:    string(a.Files[0].Data),
-			Full:   fullBuffer.String(),
-			Trimed: trimBuffer.String(),
-		})
-	}
-
-	return cases, nil
-}

pkg/schema/load/dashboard.go

@@ -129,20 +129,6 @@ func (cds *compositeDashboardSchema) Validate(r schema.Resource) error {
 	return cds.actual.Unify(rv.Value()).Validate(cue.Concrete(true))
 }
 
-// ApplyDefaults returns a new, concrete copy of the Resource with all paths
-// that are 1) missing in the Resource AND 2) specified by the schema,
-// filled with default values specified by the schema.
-func (cds *compositeDashboardSchema) ApplyDefaults(_ schema.Resource) (schema.Resource, error) {
-	panic("not implemented") // TODO: Implement
-}
-
-// TrimDefaults returns a new, concrete copy of the Resource where all paths
-// in the  where the values at those paths are the same as the default value
-// given in the schema.
-func (cds *compositeDashboardSchema) TrimDefaults(_ schema.Resource) (schema.Resource, error) {
-	panic("not implemented") // TODO: Implement
-}
-
 // CUE returns the cue.Value representing the actual schema.
 func (cds *compositeDashboardSchema) CUE() cue.Value {
 	return cds.actual

pkg/schema/load/generic.go

@@ -1,13 +1,8 @@
 package load
 
 import (
-	"bytes"
-	"fmt"
-	"strings"
-
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/load"
-	cuejson "cuelang.org/go/pkg/encoding/json"
 	"github.com/grafana/grafana/pkg/schema"
 )
 
@@ -108,142 +103,6 @@ func (gvs *genericVersionedSchema) Validate(r schema.Resource) error {
 	return gvs.actual.Unify(rv.Value()).Validate(cue.Concrete(true))
 }
 
-// ApplyDefaults returns a new, concrete copy of the Resource with all paths
-// that are 1) missing in the Resource AND 2) specified by the schema,
-// filled with default values specified by the schema.
-func (gvs *genericVersionedSchema) ApplyDefaults(r schema.Resource) (schema.Resource, error) {
-	rv, err := rt.Compile("resource", r.Value)
-	if err != nil {
-		return r, err
-	}
-	rvUnified := rv.Value().Unify(gvs.CUE())
-	re, err := convertCUEValueToString(rvUnified)
-	if err != nil {
-		return r, err
-	}
-	return schema.Resource{Value: re}, nil
-}
-
-func convertCUEValueToString(inputCUE cue.Value) (string, error) {
-	re, err := cuejson.Marshal(inputCUE)
-	if err != nil {
-		return re, err
-	}
-
-	result := []byte(re)
-	result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1)
-	result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1)
-	result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1)
-	return string(result), nil
-}
-
-// TrimDefaults returns a new, concrete copy of the Resource where all paths
-// in the  where the values at those paths are the same as the default value
-// given in the schema.
-func (gvs *genericVersionedSchema) TrimDefaults(r schema.Resource) (schema.Resource, error) {
-	rvInstance, err := rt.Compile("resource", r.Value)
-	if err != nil {
-		return r, err
-	}
-	rv, _, err := removeDefaultHelper(gvs.CUE(), rvInstance.Value())
-	if err != nil {
-		return r, err
-	}
-	re, err := convertCUEValueToString(rv)
-	fmt.Println("the trimed fields would be: ", re)
-	if err != nil {
-		return r, err
-	}
-	return schema.Resource{Value: re}, nil
-}
-
-func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
-	// Since for now, panel definition is open validation,
-	// we need to loop on the input CUE for trimming
-	rvInstance, err := rt.Compile("resource", []byte{})
-	if err != nil {
-		return input, false, err
-	}
-	rv := rvInstance.Value()
-
-	switch inputdef.IncompleteKind() {
-	case cue.StructKind:
-		// Get all fields including optional fields
-		iter, err := inputdef.Fields(cue.Optional(true))
-		if err != nil {
-			return rv, false, err
-		}
-		keySet := make(map[string]bool)
-		for iter.Next() {
-			lable, _ := iter.Value().Label()
-			keySet[lable] = true
-			lv := input.LookupPath(cue.MakePath(cue.Str(lable)))
-			if err != nil {
-				continue
-			}
-			if lv.Exists() {
-				re, isEqual, err := removeDefaultHelper(iter.Value(), lv)
-				if err == nil && !isEqual {
-					rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re)
-				}
-			}
-		}
-		// Get all the fields that are not defined in schema yet for panel
-		iter, err = input.Fields()
-		if err != nil {
-			return rv, false, err
-		}
-		for iter.Next() {
-			lable, _ := iter.Value().Label()
-			if exists := keySet[lable]; !exists {
-				rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value())
-			}
-		}
-		return rv, false, nil
-	case cue.ListKind:
-		val, _ := inputdef.Default()
-		err1 := input.Subsume(val)
-		err2 := val.Subsume(input)
-		if val.IsConcrete() && err1 == nil && err2 == nil {
-			return rv, true, nil
-		}
-		ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex))
-		if ele.IncompleteKind() == cue.BottomKind {
-			return rv, true, nil
-		}
-
-		iter, err := input.List()
-		if err != nil {
-			return rv, true, nil
-		}
-		var iterlist []string
-		for iter.Next() {
-			re, isEqual, err := removeDefaultHelper(ele, iter.Value())
-			if err == nil && !isEqual {
-				reString, err := convertCUEValueToString(re)
-				if err != nil {
-					return rv, true, nil
-				}
-				iterlist = append(iterlist, reString)
-			}
-		}
-		iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ","))
-		liInstance, err := rt.Compile("resource", []byte(iterlistContent))
-		if err != nil {
-			return rv, false, err
-		}
-		return liInstance.Value(), false, nil
-	default:
-		val, _ := inputdef.Default()
-		err1 := input.Subsume(val)
-		err2 := val.Subsume(input)
-		if val.IsConcrete() && err1 == nil && err2 == nil {
-			return input, true, nil
-		}
-		return input, false, nil
-	}
-}
-
 // CUE returns the cue.Value representing the actual schema.
 func (gvs *genericVersionedSchema) CUE() cue.Value {
 	return gvs.actual

pkg/schema/load/load_test.go

@@ -51,7 +51,7 @@ func TestDashboardValidity(t *testing.T) {
 	// TODO FIXME remove this once we actually have dashboard schema filled in
 	// enough that the tests pass, lol
 	t.Skip()
-	validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards", "basic"))
+	validdir := os.DirFS(filepath.Join("testdata", "artifacts", "dashboards"))
 
 	dash, err := BaseDashboardFamily(p)
 	require.NoError(t, err, "error while loading base dashboard scuemata")

pkg/schema/schema.go

@@ -1,13 +1,18 @@
 package schema
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"math/bits"
+	"strings"
 
 	"cuelang.org/go/cue"
+	cuejson "cuelang.org/go/pkg/encoding/json"
 )
 
+var rt = &cue.Runtime{}
+
 // CueSchema represents a single, complete CUE-based schema that can perform
 // operations on Resources.
 //
@@ -23,16 +28,6 @@ type CueSchema interface {
 	// Validate checks that the resource is correct with respect to the schema.
 	Validate(Resource) error
 
-	// ApplyDefaults returns a new, concrete copy of the Resource with all paths
-	// that are 1) missing in the Resource AND 2) specified by the schema,
-	// filled with default values specified by the schema.
-	ApplyDefaults(Resource) (Resource, error)
-
-	// TrimDefaults returns a new, concrete copy of the Resource where all paths
-	// in the  where the values at those paths are the same as the default value
-	// given in the schema.
-	TrimDefaults(Resource) (Resource, error)
-
 	// Migrate transforms a Resource into a new Resource that is correct with
 	// respect to its Successor schema. It returns the transformed resource,
 	// the schema to which the resource now conforms, and any errors that
@@ -257,6 +252,142 @@ func Exact(maj, min int) SearchOption {
 	}
 }
 
+// ApplyDefaults returns a new, concrete copy of the Resource with all paths
+// that are 1) missing in the Resource AND 2) specified by the schema,
+// filled with default values specified by the schema.
+func ApplyDefaults(r Resource, scue cue.Value) (Resource, error) {
+	rv, err := rt.Compile("resource", r.Value)
+	if err != nil {
+		return r, err
+	}
+	rvUnified := rv.Value().Unify(scue)
+	re, err := convertCUEValueToString(rvUnified)
+	if err != nil {
+		return r, err
+	}
+	return Resource{Value: re}, nil
+}
+
+func convertCUEValueToString(inputCUE cue.Value) (string, error) {
+	re, err := cuejson.Marshal(inputCUE)
+	if err != nil {
+		return re, err
+	}
+
+	result := []byte(re)
+	result = bytes.Replace(result, []byte("\\u003c"), []byte("<"), -1)
+	result = bytes.Replace(result, []byte("\\u003e"), []byte(">"), -1)
+	result = bytes.Replace(result, []byte("\\u0026"), []byte("&"), -1)
+	return string(result), nil
+}
+
+// TrimDefaults returns a new, concrete copy of the Resource where all paths
+// in the  where the values at those paths are the same as the default value
+// given in the schema.
+func TrimDefaults(r Resource, scue cue.Value) (Resource, error) {
+	rvInstance, err := rt.Compile("resource", r.Value)
+	if err != nil {
+		return r, err
+	}
+	rv, _, err := removeDefaultHelper(scue, rvInstance.Value())
+	if err != nil {
+		return r, err
+	}
+	re, err := convertCUEValueToString(rv)
+	if err != nil {
+		return r, err
+	}
+	return Resource{Value: re}, nil
+}
+
+func isCueValueEqual(inputdef cue.Value, input cue.Value) bool {
+	val, _ := inputdef.Default()
+	return input.Subsume(val) == nil && val.Subsume(input) == nil
+}
+
+func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool, error) {
+	// To include all optional fields, we need to use inputdef for iteration,
+	// since the lookuppath with optional field doesn't work very well
+	rvInstance, err := rt.Compile("resource", []byte{})
+	if err != nil {
+		return input, false, err
+	}
+	rv := rvInstance.Value()
+
+	switch inputdef.IncompleteKind() {
+	case cue.StructKind:
+		// Get all fields including optional fields
+		iter, err := inputdef.Fields(cue.Optional(true))
+		if err != nil {
+			return rv, false, err
+		}
+		keySet := make(map[string]bool)
+		for iter.Next() {
+			lable, _ := iter.Value().Label()
+			keySet[lable] = true
+			lv := input.LookupPath(cue.MakePath(cue.Str(lable)))
+			if err != nil {
+				continue
+			}
+			if lv.Exists() {
+				re, isEqual, err := removeDefaultHelper(iter.Value(), lv)
+				if err == nil && !isEqual {
+					rv = rv.FillPath(cue.MakePath(cue.Str(lable)), re)
+				}
+			}
+		}
+		// Get all the fields that are not defined in schema yet for panel
+		iter, err = input.Fields()
+		if err != nil {
+			return rv, false, err
+		}
+		for iter.Next() {
+			lable, _ := iter.Value().Label()
+			if exists := keySet[lable]; !exists {
+				rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value())
+			}
+		}
+		return rv, false, nil
+	case cue.ListKind:
+		if isCueValueEqual(inputdef, input) {
+			return rv, true, nil
+		}
+		ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex))
+		if ele.IncompleteKind() == cue.BottomKind {
+			return rv, true, nil
+		}
+
+		iter, err := input.List()
+		if err != nil {
+			return rv, true, nil
+		}
+
+		// The following code is workaround since today overwrite list element doesn't work
+		var iterlist []string
+		for iter.Next() {
+			re, isEqual, err := removeDefaultHelper(ele, iter.Value())
+			if err == nil && !isEqual {
+				reString, err := convertCUEValueToString(re)
+				if err != nil {
+					return rv, true, nil
+				}
+				iterlist = append(iterlist, reString)
+			}
+		}
+		iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ","))
+		liInstance, err := rt.Compile("resource", []byte(iterlistContent))
+		if err != nil {
+			return rv, false, err
+		}
+		return liInstance.Value(), false, nil
+	default:
+		if isCueValueEqual(inputdef, input) {
+			return input, true, nil
+		}
+		return input, false, nil
+	}
+}
+
 // A Resource represents a concrete data object - e.g., JSON
 // representing a dashboard.
 //

pkg/schema/schema_test.go

@@ -1,4 +1,108 @@
 package schema
 
-// TODO tests for this stuff! Everything in this package is totally generic,
+import (
-// nothing is specific to Grafana
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"cuelang.org/go/cue"
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/tools/txtar"
+)
+
+var CasesDir = filepath.Join("testdata", "trimapplydefaults")
+
+type Case struct {
+	Name    string
+	CUE     string
+	Full    string
+	Trimmed string
+}
+
+func TestGenerate(t *testing.T) {
+	cases, err := loadCases(CasesDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, c := range cases {
+		t.Run(c.Name+" apply default value", func(t *testing.T) {
+			var r cue.Runtime
+			scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
+			if err != nil {
+				t.Fatal(err)
+			}
+			inputResource := Resource{Value: c.Trimmed}
+			out, err := ApplyDefaults(inputResource, scmInstance.Value())
+			if err != nil {
+				t.Fatal(err)
+			}
+			b := []byte(out.Value.(string))
+
+			if s := cmp.Diff(string(b), c.Full); s != "" {
+				t.Fatal(s)
+			}
+		})
+	}
+
+	for _, c := range cases {
+		t.Run(c.Name+" trim default value", func(t *testing.T) {
+			var r cue.Runtime
+			scmInstance, err := r.Compile(c.Name+".cue", c.CUE)
+			if err != nil {
+				t.Fatal(err)
+			}
+			inputResource := Resource{Value: c.Full}
+			out, err := TrimDefaults(inputResource, scmInstance.Value())
+			if err != nil {
+				t.Fatal(err)
+			}
+			b := []byte(out.Value.(string))
+			if s := cmp.Diff(string(b), c.Trimmed); s != "" {
+				t.Fatal(s)
+			}
+		})
+	}
+}
+
+func loadCases(dir string) ([]Case, error) {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	var cases []Case
+	for _, fi := range files {
+		file := filepath.Join(dir, fi.Name())
+		a, err := txtar.ParseFile(file)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(a.Files) != 3 {
+			return nil, fmt.Errorf("Malformed test case '%s': Must contain exactly three files (CUE, Full and Trimed), but has %d", file, len(a.Files))
+		}
+
+		fullBuffer := new(bytes.Buffer)
+		fullJson := a.Files[1].Data
+		if err := json.Compact(fullBuffer, fullJson); err != nil {
+			return nil, err
+		}
+
+		trimBuffer := new(bytes.Buffer)
+		trimedJson := a.Files[2].Data
+		if err := json.Compact(trimBuffer, trimedJson); err != nil {
+			return nil, err
+		}
+
+		cases = append(cases, Case{
+			Name:    fi.Name(),
+			CUE:     string(a.Files[0].Data),
+			Full:    fullBuffer.String(),
+			Trimmed: trimBuffer.String(),
+		})
+	}
+	return cases, nil
+}

pkg/services/schemaloader/schemaloader.go

@@ -62,7 +62,7 @@ func (rs *SchemaLoaderService) DashboardApplyDefaults(input *simplejson.Json) (*
 	val = removeNils(val)
 	data, _ := json.Marshal(val)
 	dsSchema := schema.Find(rs.DashFamily, schema.Latest())
-	result, err := dsSchema.ApplyDefaults(schema.Resource{Value: data})
+	result, err := schema.ApplyDefaults(schema.Resource{Value: data}, dsSchema.CUE())
 	if err != nil {
 		return input, err
 	}
@@ -83,7 +83,7 @@ func (rs *SchemaLoaderService) DashboardTrimDefaults(input simplejson.Json) (sim
 		return input, err
 	}
 	// spew.Dump(dsSchema)
-	result, err := dsSchema.TrimDefaults(schema.Resource{Value: data})
+	result, err := schema.TrimDefaults(schema.Resource{Value: data}, dsSchema.CUE())
 	if err != nil {
 		return input, err
 	}