Storage: Switch from tenant to namespace & remove GRN (#79250)

* remove GRN and switch tenant to namespace

* clean up remaining references

* simplify and remove inconsistency in With* parameters

* parse listing keys so we can use db index

* bump the schema version

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
pull/78970/head^2
Dan Cech 2 years ago committed by GitHub
parent bb4aa16b13
commit 9f6144059a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      .github/CODEOWNERS
  2. 1
      Makefile
  3. 13
      pkg/infra/grn/doc.go
  4. 9
      pkg/infra/grn/errors.go
  5. 24
      pkg/infra/grn/generate.sh
  6. 87
      pkg/infra/grn/grn.go
  7. 185
      pkg/infra/grn/grn.pb.go
  8. 24
      pkg/infra/grn/grn.proto
  9. 84
      pkg/infra/grn/grn_test.go
  10. 18
      pkg/services/grafana-apiserver/storage/entity/storage.go
  11. 119
      pkg/services/grafana-apiserver/storage/entity/utils.go
  12. 16
      pkg/services/store/entity/client_wrapper.go
  13. 9
      pkg/services/store/entity/dummy/fake_store.go
  14. 1398
      pkg/services/store/entity/entity.pb.go
  15. 126
      pkg/services/store/entity/entity.proto
  16. 164
      pkg/services/store/entity/entity_grpc.pb.go
  17. 61
      pkg/services/store/entity/key.go
  18. 19
      pkg/services/store/entity/migrations/entity_store_mig.go
  19. 3
      pkg/services/store/entity/models.go
  20. 20
      pkg/services/store/entity/sqlstash/folder_support.go
  21. 656
      pkg/services/store/entity/sqlstash/sql_storage_server.go
  22. 198
      pkg/services/store/entity/tests/server_integration_test.go
  23. 9
      pkg/services/store/entity/utils.go
  24. 2
      pkg/services/store/resolver/service.go

@ -84,7 +84,6 @@
/pkg/ifaces/ @grafana/backend-platform
/pkg/infra/appcontext/ @grafana/backend-platform
/pkg/infra/db/ @grafana/backend-platform
/pkg/infra/grn/ @grafana/backend-platform
/pkg/infra/localcache/ @grafana/backend-platform
/pkg/infra/log/ @grafana/backend-platform
/pkg/infra/metrics/ @grafana/backend-platform

@ -308,7 +308,6 @@ protobuf: ## Compile protobuf definitions
bash pkg/plugins/backendplugin/pluginextensionv2/generate.sh
bash pkg/plugins/backendplugin/secretsmanagerplugin/generate.sh
bash pkg/services/store/entity/generate.sh
bash pkg/infra/grn/generate.sh
clean: ## Clean up intermediate build artifacts.
@echo "cleaning"

@ -1,13 +0,0 @@
// package GRN provides utilities for working with Grafana Resource Names
// (GRNs).
// A GRN is an identifier which encodes all data necessary to retrieve a given
// resource from its respective service.
// A GRN string is expressed in the format:
//
// grn:${tenant_id}:${group}/${kind}/${id}
//
// The format of the final id is defined by the owning service and not
// validated by the GRN parser. Prefer using UIDs where possible.
package grn

@ -1,9 +0,0 @@
package grn
import (
"github.com/grafana/grafana/pkg/util/errutil"
)
var (
ErrInvalidGRN = errutil.ValidationFailed("grn.InvalidGRN")
)

@ -1,24 +0,0 @@
#!/bin/bash
# To compile all protobuf files in this repository, run
# "make protobuf" at the top-level.
set -eu
DST_DIR=./
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
cd "$DIR"
protoc \
-I ./ \
-I ../../../ \
--go_out=${DST_DIR} \
--go_opt=paths=source_relative \
--go-grpc_out=${DST_DIR} \
--go-grpc_opt=paths=source_relative \
--go-grpc_opt=require_unimplemented_servers=false \
*.proto

@ -1,87 +0,0 @@
package grn
import (
"fmt"
"strconv"
"strings"
)
// ParseStr attempts to parse a string into a GRN. It returns an error if the
// given string does not match the GRN format, but does not validate the values.
// grn:<TenantID>:<ResourceGroup>/<ResourceKind>/<ResourceIdentifier>
// No component of the GRN may contain a colon
// The TenantID is optional, but must be an integer string if specified
// The ResourceGroup, ResourceKind and ResourceIdentifier must be non-empty strings
// The ResourceGroup and ResourceKind may not contain slashes
// The ResourceIdentifier may contain slashes
func ParseStr(str string) (*GRN, error) {
ret := &GRN{}
parts := strings.Split(str, ":")
if len(parts) != 3 {
return ret, ErrInvalidGRN.Errorf("%q is not a complete GRN", str)
}
if parts[0] != "grn" {
return ret, ErrInvalidGRN.Errorf("%q does not look like a GRN", str)
}
if parts[1] != "" {
tID, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return ret, ErrInvalidGRN.Errorf("Tenant ID segment cannot be converted to an integer")
}
ret.TenantID = tID
}
// split the rest of the GRN into Group, Kind and Identifier
parts = strings.SplitN(parts[2], "/", 3)
if len(parts) != 3 {
return ret, ErrInvalidGRN.Errorf("%q is not a complete GRN", str)
}
ret.ResourceGroup = parts[0]
ret.ResourceKind = parts[1]
ret.ResourceIdentifier = parts[2]
if ret.ResourceGroup == "" {
return ret, ErrInvalidGRN.Errorf("Cannot find resource group in GRN %q", str)
}
if ret.ResourceKind == "" {
return ret, ErrInvalidGRN.Errorf("Cannot find resource kind in GRN %q", str)
}
if ret.ResourceIdentifier == "" {
return ret, ErrInvalidGRN.Errorf("Cannot find resource identifier in GRN %q", str)
}
return ret, nil
}
// MustParseStr is a wrapper around ParseStr that panics if the given input is
// not a valid GRN. This is intended for use in tests.
func MustParseStr(str string) *GRN {
grn, err := ParseStr(str)
if err != nil {
panic("bad grn!")
}
return grn
}
// ToGRNString returns a string representation of a grn in the format
// grn:tenantID:kind/resourceIdentifier
func (g *GRN) ToGRNString() string {
return fmt.Sprintf("grn:%d:%s/%s/%s", g.TenantID, g.ResourceGroup, g.ResourceKind, g.ResourceIdentifier)
}
// Check if the two GRNs reference to the same object
// we can not use simple `*x == *b` because of the internal settings
func (g *GRN) Equal(b *GRN) bool {
if b == nil {
return false
}
return g == b || (g.TenantID == b.TenantID &&
g.ResourceGroup == b.ResourceGroup &&
g.ResourceKind == b.ResourceKind &&
g.ResourceIdentifier == b.ResourceIdentifier)
}

@ -1,185 +0,0 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.30.0
// protoc v4.23.4
// source: grn.proto
package grn
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type GRN struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// TenantID contains the ID of the tenant (in hosted grafana) or
// organization (in other environments) the resource belongs to. This field
// may be omitted for global Grafana resources which are not associated with
// an organization.
TenantID int64 `protobuf:"varint,1,opt,name=TenantID,proto3" json:"TenantID,omitempty"`
// The kind of resource being identified, for e.g. "dashboard" or "user".
// The caller is responsible for validating the value.
ResourceKind string `protobuf:"bytes,3,opt,name=ResourceKind,proto3" json:"ResourceKind,omitempty"`
// ResourceIdentifier is used by the underlying service to identify the
// resource.
ResourceIdentifier string `protobuf:"bytes,4,opt,name=ResourceIdentifier,proto3" json:"ResourceIdentifier,omitempty"`
// The group represents the specific resource group the resource belongs to.
// This is a unique value for each plugin and maps to the k8s Group
ResourceGroup string `protobuf:"bytes,5,opt,name=ResourceGroup,proto3" json:"ResourceGroup,omitempty"`
}
func (x *GRN) Reset() {
*x = GRN{}
if protoimpl.UnsafeEnabled {
mi := &file_grn_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *GRN) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GRN) ProtoMessage() {}
func (x *GRN) ProtoReflect() protoreflect.Message {
mi := &file_grn_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GRN.ProtoReflect.Descriptor instead.
func (*GRN) Descriptor() ([]byte, []int) {
return file_grn_proto_rawDescGZIP(), []int{0}
}
func (x *GRN) GetTenantID() int64 {
if x != nil {
return x.TenantID
}
return 0
}
func (x *GRN) GetResourceKind() string {
if x != nil {
return x.ResourceKind
}
return ""
}
func (x *GRN) GetResourceIdentifier() string {
if x != nil {
return x.ResourceIdentifier
}
return ""
}
func (x *GRN) GetResourceGroup() string {
if x != nil {
return x.ResourceGroup
}
return ""
}
var File_grn_proto protoreflect.FileDescriptor
var file_grn_proto_rawDesc = []byte{
0x0a, 0x09, 0x67, 0x72, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x67, 0x72, 0x6e,
0x22, 0x9b, 0x01, 0x0a, 0x03, 0x47, 0x52, 0x4e, 0x12, 0x1a, 0x0a, 0x08, 0x54, 0x65, 0x6e, 0x61,
0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x54, 0x65, 0x6e, 0x61,
0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
0x4b, 0x69, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x2e, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x18, 0x04,
0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x64,
0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x24, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x6f,
0x75, 0x72, 0x63, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0d, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x2a,
0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61,
0x66, 0x61, 0x6e, 0x61, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, 0x6b, 0x67,
0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x67, 0x72, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_grn_proto_rawDescOnce sync.Once
file_grn_proto_rawDescData = file_grn_proto_rawDesc
)
func file_grn_proto_rawDescGZIP() []byte {
file_grn_proto_rawDescOnce.Do(func() {
file_grn_proto_rawDescData = protoimpl.X.CompressGZIP(file_grn_proto_rawDescData)
})
return file_grn_proto_rawDescData
}
var file_grn_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_grn_proto_goTypes = []interface{}{
(*GRN)(nil), // 0: grn.GRN
}
var file_grn_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_grn_proto_init() }
func file_grn_proto_init() {
if File_grn_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_grn_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GRN); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_grn_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_grn_proto_goTypes,
DependencyIndexes: file_grn_proto_depIdxs,
MessageInfos: file_grn_proto_msgTypes,
}.Build()
File_grn_proto = out.File
file_grn_proto_rawDesc = nil
file_grn_proto_goTypes = nil
file_grn_proto_depIdxs = nil
}

@ -1,24 +0,0 @@
syntax = "proto3";
package grn;
option go_package = "github.com/grafana/grafana/pkg/infra/grn";
message GRN {
// TenantID contains the ID of the tenant (in hosted grafana) or
// organization (in other environments) the resource belongs to. This field
// may be omitted for global Grafana resources which are not associated with
// an organization.
int64 TenantID = 1;
// The kind of resource being identified, for e.g. "dashboard" or "user".
// The caller is responsible for validating the value.
string ResourceKind = 3;
// ResourceIdentifier is used by the underlying service to identify the
// resource.
string ResourceIdentifier = 4;
// The group represents the specific resource group the resource belongs to.
// This is a unique value for each plugin and maps to the k8s Group
string ResourceGroup = 5;
}

@ -1,84 +0,0 @@
package grn
import (
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestParseGRNStr(t *testing.T) {
tests := []struct {
input string
expect *GRN
expectErr bool
}{
{ // empty
"",
&GRN{},
true,
},
{ // too few parts
"grn:dashboards",
&GRN{},
true,
},
{ // too many parts
"grn::dashboards:user:orgs:otherthings:hello:stillgoing",
&GRN{},
true,
},
{ // Does not look like a GRN
"hrn:grafana::123:dashboards/foo",
&GRN{},
true,
},
{ // Missing Kind
"grn::foo",
&GRN{},
true,
},
{ // Missing Group
"grn::foo/Bar",
&GRN{},
true,
},
{ // good!
"grn::core.grafana.com/Role/Admin",
&GRN{TenantID: 0, ResourceGroup: "core.grafana.com", ResourceKind: "Role", ResourceIdentifier: "Admin"},
false,
},
{ // good!
"grn::core.grafana.com/Role/Admin/with/some/slashes",
&GRN{TenantID: 0, ResourceGroup: "core.grafana.com", ResourceKind: "Role", ResourceIdentifier: "Admin/with/some/slashes"},
false,
},
{ // good!
"grn:123456789:core.grafana.com/Role/Admin/with/some/slashes",
&GRN{TenantID: 123456789, ResourceGroup: "core.grafana.com", ResourceKind: "Role", ResourceIdentifier: "Admin/with/some/slashes"},
false,
},
{ // Weird, but valid.
"grn::core.grafana.com/Role///Admin/with/leading/slashes",
&GRN{TenantID: 0, ResourceGroup: "core.grafana.com", ResourceKind: "Role", ResourceIdentifier: "//Admin/with/leading/slashes"},
false,
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("ParseStr(%q)", test.input), func(t *testing.T) {
got, err := ParseStr(test.input)
if test.expectErr && err == nil {
t.Fatal("wrong result. Expected error, got success")
}
if err != nil && !test.expectErr {
t.Fatalf("wrong result. Expected success, got error %s", err.Error())
}
if !cmp.Equal(test.expect, got) {
t.Fatalf("wrong result. Wanted %s, got %s\n", test.expect.String(), got.String())
}
})
}
}

@ -89,7 +89,7 @@ func (s *Storage) Create(ctx context.Context, key string, obj runtime.Object, ou
// Replace the default name generation strategy
if metaAccessor.GetGenerateName() != "" {
k, err := ParseKey(key)
k, err := entityStore.ParseKey(key)
if err != nil {
return err
}
@ -138,18 +138,13 @@ func (s *Storage) Create(ctx context.Context, key string, obj runtime.Object, ou
// current version of the object to avoid read operation from storage to get it.
// However, the implementations have to retry in case suggestion is stale.
func (s *Storage) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object) error {
grn, err := keyToGRN(key)
if err != nil {
return apierrors.NewInternalError(err)
}
previousVersion := ""
if preconditions != nil && preconditions.ResourceVersion != nil {
previousVersion = *preconditions.ResourceVersion
}
rsp, err := s.store.Delete(ctx, &entityStore.DeleteEntityRequest{
GRN: grn,
Key: key,
PreviousVersion: previousVersion,
})
if err != nil {
@ -183,16 +178,14 @@ func (s *Storage) Watch(ctx context.Context, key string, opts storage.ListOption
func (s *Storage) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error {
rsp, err := s.store.Read(ctx, &entityStore.ReadEntityRequest{
Key: key,
WithMeta: true,
WithBody: true,
WithStatus: true,
WithSummary: true,
})
if err != nil {
return err
}
if rsp.GRN == nil {
if rsp.Key == "" {
if opts.IgnoreNotFound {
return nil
}
@ -227,8 +220,7 @@ func (s *Storage) GetList(ctx context.Context, key string, opts storage.ListOpti
rsp, err := s.store.List(ctx, &entityStore.EntityListRequest{
Key: []string{key},
WithBody: true,
WithLabels: true,
WithFields: true,
WithStatus: true,
NextPageToken: opts.Predicate.Continue,
Limit: opts.Predicate.Limit,
// TODO push label/field matching down to storage
@ -340,8 +332,6 @@ func (s *Storage) guaranteedUpdate(
return err
}
// e.GRN.ResourceKind = destination.GetObjectKind().GroupVersionKind().Kind
previousVersion := ""
if preconditions != nil && preconditions.ResourceVersion != nil {
previousVersion = *preconditions.ResourceVersion

@ -2,10 +2,7 @@ package entity
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"strings"
"time"
"k8s.io/apimachinery/pkg/api/meta"
@ -14,102 +11,10 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/endpoints/request"
"github.com/grafana/grafana/pkg/infra/grn"
"github.com/grafana/grafana/pkg/kinds"
entityStore "github.com/grafana/grafana/pkg/services/store/entity"
)
type Key struct {
Group string
Resource string
Namespace string
Name string
Subresource string
}
func ParseKey(key string) (*Key, error) {
// /<group>/<resource>/<namespace>/<name>(/<subresource>)
parts := strings.SplitN(key, "/", 6)
if len(parts) != 5 && len(parts) != 6 {
return nil, fmt.Errorf("invalid key (expecting 4 or 5 parts) " + key)
}
if parts[0] != "" {
return nil, fmt.Errorf("invalid key (expecting leading slash) " + key)
}
k := &Key{
Group: parts[1],
Resource: parts[2],
Namespace: parts[3],
Name: parts[4],
}
if len(parts) == 6 {
k.Subresource = parts[5]
}
return k, nil
}
func (k *Key) String() string {
if len(k.Subresource) > 0 {
return fmt.Sprintf("/%s/%s/%s/%s/%s", k.Group, k.Resource, k.Namespace, k.Name, k.Subresource)
}
return fmt.Sprintf("/%s/%s/%s/%s", k.Group, k.Resource, k.Namespace, k.Name)
}
func (k *Key) IsEqual(other *Key) bool {
return k.Group == other.Group &&
k.Resource == other.Resource &&
k.Namespace == other.Namespace &&
k.Name == other.Name &&
k.Subresource == other.Subresource
}
func (k *Key) TenantID() (int64, error) {
if k.Namespace == "default" {
return 1, nil
}
tid := strings.Split(k.Namespace, "-")
if len(tid) != 2 || !(tid[0] == "org" || tid[0] == "tenant") {
return 0, fmt.Errorf("invalid namespace, expected org|tenant-${#}")
}
intVar, err := strconv.ParseInt(tid[1], 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid namespace, expected number")
}
return intVar, nil
}
func (k *Key) ToGRN() (*grn.GRN, error) {
tid, err := k.TenantID()
if err != nil {
return nil, err
}
fullResource := k.Resource
if k.Subresource != "" {
fullResource = fmt.Sprintf("%s/%s", k.Resource, k.Subresource)
}
return &grn.GRN{
ResourceGroup: k.Group,
ResourceKind: fullResource,
ResourceIdentifier: k.Name,
TenantID: tid,
}, nil
}
// Convert an etcd key to GRN style
func keyToGRN(key string) (*grn.GRN, error) {
k, err := ParseKey(key)
if err != nil {
return nil, err
}
return k.ToGRN()
}
// this is terrible... but just making it work!!!!
func entityToResource(rsp *entityStore.Entity, res runtime.Object) error {
var err error
@ -119,10 +24,6 @@ func entityToResource(rsp *entityStore.Entity, res runtime.Object) error {
return err
}
if rsp.GRN == nil {
return fmt.Errorf("invalid entity, missing GRN")
}
if len(rsp.Meta) > 0 {
err = json.Unmarshal(rsp.Meta, res)
if err != nil {
@ -130,12 +31,8 @@ func entityToResource(rsp *entityStore.Entity, res runtime.Object) error {
}
}
metaAccessor.SetName(rsp.GRN.ResourceIdentifier)
if rsp.GRN.TenantID != 1 {
metaAccessor.SetNamespace(fmt.Sprintf("tenant-%d", rsp.GRN.TenantID))
} else {
metaAccessor.SetNamespace("default") // org 1
}
metaAccessor.SetName(rsp.Uid)
metaAccessor.SetNamespace(rsp.Namespace)
metaAccessor.SetUID(types.UID(rsp.Guid))
metaAccessor.SetResourceVersion(rsp.Version)
metaAccessor.SetCreationTimestamp(metav1.Unix(rsp.CreatedAt/1000, rsp.CreatedAt%1000*1000000))
@ -202,18 +99,16 @@ func resourceToEntity(key string, res runtime.Object, requestInfo *request.Reque
return nil, err
}
g, err := keyToGRN(key)
if err != nil {
return nil, err
}
grafanaAccessor := kinds.MetaAccessor(metaAccessor)
rsp := &entityStore.Entity{
GRN: g,
Group: requestInfo.APIGroup,
GroupVersion: requestInfo.APIVersion,
Resource: requestInfo.Resource,
Subresource: requestInfo.Subresource,
Namespace: metaAccessor.GetNamespace(),
Key: key,
Name: metaAccessor.GetName(),
Uid: metaAccessor.GetName(),
Guid: string(metaAccessor.GetUID()),
Version: metaAccessor.GetResourceVersion(),
Folder: grafanaAccessor.GetFolder(),

@ -33,13 +33,6 @@ func (c *entityStoreClientWrapper) BatchRead(ctx context.Context, in *BatchReadE
}
return c.EntityStoreClient.BatchRead(ctx, in)
}
func (c *entityStoreClientWrapper) Write(ctx context.Context, in *WriteEntityRequest) (*WriteEntityResponse, error) {
ctx, err := c.wrapContext(ctx)
if err != nil {
return nil, err
}
return c.EntityStoreClient.Write(ctx, in)
}
func (c *entityStoreClientWrapper) Create(ctx context.Context, in *CreateEntityRequest) (*CreateEntityResponse, error) {
ctx, err := c.wrapContext(ctx)
if err != nil {
@ -96,15 +89,6 @@ func (c *entityStoreClientWrapper) wrapContext(ctx context.Context) (context.Con
return ctx, nil
}
// TEMPORARY... while we split this into a new service (see below)
func (c *entityStoreClientWrapper) AdminWrite(ctx context.Context, in *AdminWriteEntityRequest) (*WriteEntityResponse, error) {
ctx, err := c.wrapContext(ctx)
if err != nil {
return nil, err
}
return c.EntityStoreClient.AdminWrite(ctx, in)
}
func NewEntityStoreClientWrapper(cc grpc.ClientConnInterface) EntityStoreServer {
return &entityStoreClientWrapper{&entityStoreClient{cc}}
}

@ -9,7 +9,6 @@ import (
// Make sure we implement both store + admin
var _ entity.EntityStoreServer = &fakeEntityStore{}
var _ entity.EntityStoreAdminServer = &fakeEntityStore{}
func ProvideFakeEntityServer() entity.EntityStoreServer {
return &fakeEntityStore{}
@ -17,14 +16,6 @@ func ProvideFakeEntityServer() entity.EntityStoreServer {
type fakeEntityStore struct{}
func (i fakeEntityStore) AdminWrite(ctx context.Context, r *entity.AdminWriteEntityRequest) (*entity.WriteEntityResponse, error) {
return nil, fmt.Errorf("unimplemented")
}
func (i fakeEntityStore) Write(ctx context.Context, r *entity.WriteEntityRequest) (*entity.WriteEntityResponse, error) {
return nil, fmt.Errorf("unimplemented")
}
func (i fakeEntityStore) Create(ctx context.Context, r *entity.CreateEntityRequest) (*entity.CreateEntityResponse, error) {
return nil, fmt.Errorf("unimplemented")
}

File diff suppressed because it is too large Load Diff

@ -3,8 +3,6 @@ package entity;
option go_package = "github.com/grafana/grafana/pkg/services/store/entity";
import "pkg/infra/grn/grn.proto";
// The canonical entity/document data -- this represents the raw bytes and storage level metadata
message Entity {
@ -14,10 +12,18 @@ message Entity {
// The resourceVersion -- it will change whenever anythign on the object is saved
string version = 2;
// Entity identifier -- tenant_id, kind, uid
grn.GRN GRN = 3;
// group version of the entity
// group
string group = 24;
// kind resource
string resource = 25;
// namespace
string namespace = 26;
// k8s name
string uid = 27;
// subresource
string subresource = 28;
// group version
string group_version = 23;
// k8s key value
@ -109,28 +115,20 @@ message EntityErrorInfo {
message ReadEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
string key = 7;
string key = 1;
// Fetch an explicit version (default is latest)
string version = 2;
// Include the full meta bytes
bool with_meta = 3;
// Include the full body bytes
// Include the full body
bool with_body = 4;
// Include the status bytes (ignored for now)
// Include the status
bool with_status = 5;
// Include derived summary metadata
bool with_summary = 6;
}
//------------------------------------------------------
// Make many read requests at once (by Kind+ID+version)
// Make many read requests at once (by key+version)
//------------------------------------------------------
message BatchReadEntityRequest {
@ -141,53 +139,6 @@ message BatchReadEntityResponse {
repeated Entity results = 1;
}
//-----------------------------------------------
// Write request/response
//-----------------------------------------------
message WriteEntityRequest {
// Entity details
Entity entity = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
string previous_version = 2;
}
// This operation is useful when syncing a resource from external sources
// that have more accurate metadata information (git, or an archive).
// This process can bypass the forced checks that
message AdminWriteEntityRequest {
// Entity details
Entity entity = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
// This may not be used along with an explicit version in the request
string previous_version = 2;
// Request that all previous versions are removed from the history
// This will make sense for systems that manage history explicitly externallay
bool clear_history = 3;
}
message WriteEntityResponse {
// Error info -- if exists, the save did not happen
EntityErrorInfo error = 1;
// Entity details
Entity entity = 2;
// Status code
Status status = 3;
// Status enumeration
enum Status {
ERROR = 0;
CREATED = 1;
UPDATED = 2;
UNCHANGED = 3;
}
}
//-----------------------------------------------
// Create request/response
//-----------------------------------------------
@ -250,9 +201,7 @@ message UpdateEntityResponse {
message DeleteEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
string key = 3;
string key = 1;
// Used for optimistic locking. If missing, the current version will be deleted regardless
string previous_version = 2;
@ -282,7 +231,7 @@ message DeleteEntityResponse {
message EntityHistoryRequest {
// Entity identifier
grn.GRN GRN = 1;
string key = 1;
// Maximum number of items to return
int64 limit = 3;
@ -293,7 +242,7 @@ message EntityHistoryRequest {
message EntityHistoryResponse {
// Entity identifier
grn.GRN GRN = 1;
string key = 1;
// Entity metadata without the raw bytes
repeated Entity versions = 2;
@ -317,8 +266,11 @@ message EntityListRequest {
// Free text query string -- mileage may vary :)
string query = 3;
// limit to a specific kind (empty is all)
repeated string kind = 4;
// limit to a specific group (empty is all)
repeated string group = 9;
// limit to a specific resource (empty is all)
repeated string resource = 4;
// limit to a specific key
repeated string key = 11;
@ -336,10 +288,7 @@ message EntityListRequest {
bool with_body = 8;
// Return the full body in each payload
bool with_labels = 9;
// Return the full body in each payload
bool with_fields = 10;
bool with_status = 10;
}
message ReferenceRequest {
@ -350,7 +299,7 @@ message ReferenceRequest {
int64 limit = 2;
// Free text query string -- mileage may vary :)
string kind = 3;
string resource = 3;
// Free text query string -- mileage may vary :)
string uid = 4;
@ -372,10 +321,10 @@ message EntityWatchRequest {
int64 since = 1;
// Watch sppecific entities
repeated grn.GRN GRN = 2;
repeated string key = 2;
// limit to a specific kind (empty is all)
repeated string kind = 3;
// limit to a specific resource (empty is all)
repeated string resource = 3;
// Limit results to items in a specific folder
string folder = 4;
@ -386,11 +335,8 @@ message EntityWatchRequest {
// Return the full body in each payload
bool with_body = 6;
// Return the full body in each payload
bool with_labels = 7;
// Return the full body in each payload
bool with_fields = 8;
// Return the full status in each payload
bool with_status = 7;
}
message EntityWatchResponse {
@ -465,20 +411,10 @@ message EntityExternalReference {
service EntityStore {
rpc Read(ReadEntityRequest) returns (Entity);
rpc BatchRead(BatchReadEntityRequest) returns (BatchReadEntityResponse);
rpc Write(WriteEntityRequest) returns (WriteEntityResponse);
rpc Create(CreateEntityRequest) returns (CreateEntityResponse);
rpc Update(UpdateEntityRequest) returns (UpdateEntityResponse);
rpc Delete(DeleteEntityRequest) returns (DeleteEntityResponse);
rpc History(EntityHistoryRequest) returns (EntityHistoryResponse);
rpc List(EntityListRequest) returns (EntityListResponse);
rpc Watch(EntityWatchRequest) returns (stream EntityWatchResponse);
// TEMPORARY... while we split this into a new service (see below)
rpc AdminWrite(AdminWriteEntityRequest) returns (WriteEntityResponse);
}
// The admin service extends the basic entity store interface, but provides
// more explicit control that can support bulk operations like efficient git sync
service EntityStoreAdmin {
rpc AdminWrite(AdminWriteEntityRequest) returns (WriteEntityResponse);
}

@ -21,14 +21,12 @@ const _ = grpc.SupportPackageIsVersion7
const (
EntityStore_Read_FullMethodName = "/entity.EntityStore/Read"
EntityStore_BatchRead_FullMethodName = "/entity.EntityStore/BatchRead"
EntityStore_Write_FullMethodName = "/entity.EntityStore/Write"
EntityStore_Create_FullMethodName = "/entity.EntityStore/Create"
EntityStore_Update_FullMethodName = "/entity.EntityStore/Update"
EntityStore_Delete_FullMethodName = "/entity.EntityStore/Delete"
EntityStore_History_FullMethodName = "/entity.EntityStore/History"
EntityStore_List_FullMethodName = "/entity.EntityStore/List"
EntityStore_Watch_FullMethodName = "/entity.EntityStore/Watch"
EntityStore_AdminWrite_FullMethodName = "/entity.EntityStore/AdminWrite"
)
// EntityStoreClient is the client API for EntityStore service.
@ -37,15 +35,12 @@ const (
type EntityStoreClient interface {
Read(ctx context.Context, in *ReadEntityRequest, opts ...grpc.CallOption) (*Entity, error)
BatchRead(ctx context.Context, in *BatchReadEntityRequest, opts ...grpc.CallOption) (*BatchReadEntityResponse, error)
Write(ctx context.Context, in *WriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error)
Create(ctx context.Context, in *CreateEntityRequest, opts ...grpc.CallOption) (*CreateEntityResponse, error)
Update(ctx context.Context, in *UpdateEntityRequest, opts ...grpc.CallOption) (*UpdateEntityResponse, error)
Delete(ctx context.Context, in *DeleteEntityRequest, opts ...grpc.CallOption) (*DeleteEntityResponse, error)
History(ctx context.Context, in *EntityHistoryRequest, opts ...grpc.CallOption) (*EntityHistoryResponse, error)
List(ctx context.Context, in *EntityListRequest, opts ...grpc.CallOption) (*EntityListResponse, error)
Watch(ctx context.Context, in *EntityWatchRequest, opts ...grpc.CallOption) (EntityStore_WatchClient, error)
// TEMPORARY... while we split this into a new service (see below)
AdminWrite(ctx context.Context, in *AdminWriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error)
}
type entityStoreClient struct {
@ -74,15 +69,6 @@ func (c *entityStoreClient) BatchRead(ctx context.Context, in *BatchReadEntityRe
return out, nil
}
func (c *entityStoreClient) Write(ctx context.Context, in *WriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error) {
out := new(WriteEntityResponse)
err := c.cc.Invoke(ctx, EntityStore_Write_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *entityStoreClient) Create(ctx context.Context, in *CreateEntityRequest, opts ...grpc.CallOption) (*CreateEntityResponse, error) {
out := new(CreateEntityResponse)
err := c.cc.Invoke(ctx, EntityStore_Create_FullMethodName, in, out, opts...)
@ -160,30 +146,18 @@ func (x *entityStoreWatchClient) Recv() (*EntityWatchResponse, error) {
return m, nil
}
func (c *entityStoreClient) AdminWrite(ctx context.Context, in *AdminWriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error) {
out := new(WriteEntityResponse)
err := c.cc.Invoke(ctx, EntityStore_AdminWrite_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// EntityStoreServer is the server API for EntityStore service.
// All implementations should embed UnimplementedEntityStoreServer
// for forward compatibility
type EntityStoreServer interface {
Read(context.Context, *ReadEntityRequest) (*Entity, error)
BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error)
Write(context.Context, *WriteEntityRequest) (*WriteEntityResponse, error)
Create(context.Context, *CreateEntityRequest) (*CreateEntityResponse, error)
Update(context.Context, *UpdateEntityRequest) (*UpdateEntityResponse, error)
Delete(context.Context, *DeleteEntityRequest) (*DeleteEntityResponse, error)
History(context.Context, *EntityHistoryRequest) (*EntityHistoryResponse, error)
List(context.Context, *EntityListRequest) (*EntityListResponse, error)
Watch(*EntityWatchRequest, EntityStore_WatchServer) error
// TEMPORARY... while we split this into a new service (see below)
AdminWrite(context.Context, *AdminWriteEntityRequest) (*WriteEntityResponse, error)
}
// UnimplementedEntityStoreServer should be embedded to have forward compatible implementations.
@ -196,9 +170,6 @@ func (UnimplementedEntityStoreServer) Read(context.Context, *ReadEntityRequest)
func (UnimplementedEntityStoreServer) BatchRead(context.Context, *BatchReadEntityRequest) (*BatchReadEntityResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method BatchRead not implemented")
}
func (UnimplementedEntityStoreServer) Write(context.Context, *WriteEntityRequest) (*WriteEntityResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Write not implemented")
}
func (UnimplementedEntityStoreServer) Create(context.Context, *CreateEntityRequest) (*CreateEntityResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Create not implemented")
}
@ -217,9 +188,6 @@ func (UnimplementedEntityStoreServer) List(context.Context, *EntityListRequest)
func (UnimplementedEntityStoreServer) Watch(*EntityWatchRequest, EntityStore_WatchServer) error {
return status.Errorf(codes.Unimplemented, "method Watch not implemented")
}
func (UnimplementedEntityStoreServer) AdminWrite(context.Context, *AdminWriteEntityRequest) (*WriteEntityResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method AdminWrite not implemented")
}
// UnsafeEntityStoreServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to EntityStoreServer will
@ -268,24 +236,6 @@ func _EntityStore_BatchRead_Handler(srv interface{}, ctx context.Context, dec fu
return interceptor(ctx, in, info, handler)
}
func _EntityStore_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(WriteEntityRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EntityStoreServer).Write(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: EntityStore_Write_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EntityStoreServer).Write(ctx, req.(*WriteEntityRequest))
}
return interceptor(ctx, in, info, handler)
}
func _EntityStore_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CreateEntityRequest)
if err := dec(in); err != nil {
@ -397,24 +347,6 @@ func (x *entityStoreWatchServer) Send(m *EntityWatchResponse) error {
return x.ServerStream.SendMsg(m)
}
func _EntityStore_AdminWrite_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AdminWriteEntityRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EntityStoreServer).AdminWrite(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: EntityStore_AdminWrite_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EntityStoreServer).AdminWrite(ctx, req.(*AdminWriteEntityRequest))
}
return interceptor(ctx, in, info, handler)
}
// EntityStore_ServiceDesc is the grpc.ServiceDesc for EntityStore service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -430,10 +362,6 @@ var EntityStore_ServiceDesc = grpc.ServiceDesc{
MethodName: "BatchRead",
Handler: _EntityStore_BatchRead_Handler,
},
{
MethodName: "Write",
Handler: _EntityStore_Write_Handler,
},
{
MethodName: "Create",
Handler: _EntityStore_Create_Handler,
@ -454,10 +382,6 @@ var EntityStore_ServiceDesc = grpc.ServiceDesc{
MethodName: "List",
Handler: _EntityStore_List_Handler,
},
{
MethodName: "AdminWrite",
Handler: _EntityStore_AdminWrite_Handler,
},
},
Streams: []grpc.StreamDesc{
{
@ -468,91 +392,3 @@ var EntityStore_ServiceDesc = grpc.ServiceDesc{
},
Metadata: "entity.proto",
}
const (
EntityStoreAdmin_AdminWrite_FullMethodName = "/entity.EntityStoreAdmin/AdminWrite"
)
// EntityStoreAdminClient is the client API for EntityStoreAdmin service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type EntityStoreAdminClient interface {
AdminWrite(ctx context.Context, in *AdminWriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error)
}
type entityStoreAdminClient struct {
cc grpc.ClientConnInterface
}
func NewEntityStoreAdminClient(cc grpc.ClientConnInterface) EntityStoreAdminClient {
return &entityStoreAdminClient{cc}
}
func (c *entityStoreAdminClient) AdminWrite(ctx context.Context, in *AdminWriteEntityRequest, opts ...grpc.CallOption) (*WriteEntityResponse, error) {
out := new(WriteEntityResponse)
err := c.cc.Invoke(ctx, EntityStoreAdmin_AdminWrite_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// EntityStoreAdminServer is the server API for EntityStoreAdmin service.
// All implementations should embed UnimplementedEntityStoreAdminServer
// for forward compatibility
type EntityStoreAdminServer interface {
AdminWrite(context.Context, *AdminWriteEntityRequest) (*WriteEntityResponse, error)
}
// UnimplementedEntityStoreAdminServer should be embedded to have forward compatible implementations.
type UnimplementedEntityStoreAdminServer struct {
}
func (UnimplementedEntityStoreAdminServer) AdminWrite(context.Context, *AdminWriteEntityRequest) (*WriteEntityResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method AdminWrite not implemented")
}
// UnsafeEntityStoreAdminServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to EntityStoreAdminServer will
// result in compilation errors.
type UnsafeEntityStoreAdminServer interface {
mustEmbedUnimplementedEntityStoreAdminServer()
}
func RegisterEntityStoreAdminServer(s grpc.ServiceRegistrar, srv EntityStoreAdminServer) {
s.RegisterService(&EntityStoreAdmin_ServiceDesc, srv)
}
func _EntityStoreAdmin_AdminWrite_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(AdminWriteEntityRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(EntityStoreAdminServer).AdminWrite(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: EntityStoreAdmin_AdminWrite_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(EntityStoreAdminServer).AdminWrite(ctx, req.(*AdminWriteEntityRequest))
}
return interceptor(ctx, in, info, handler)
}
// EntityStoreAdmin_ServiceDesc is the grpc.ServiceDesc for EntityStoreAdmin service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var EntityStoreAdmin_ServiceDesc = grpc.ServiceDesc{
ServiceName: "entity.EntityStoreAdmin",
HandlerType: (*EntityStoreAdminServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "AdminWrite",
Handler: _EntityStoreAdmin_AdminWrite_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "entity.proto",
}

@ -0,0 +1,61 @@
package entity
import (
"fmt"
"strings"
)
type Key struct {
Group string
Resource string
Namespace string
Name string
Subresource string
}
func ParseKey(key string) (*Key, error) {
// /<group>/<resource>/<namespace>(/<name>(/<subresource>))
parts := strings.SplitN(key, "/", 6)
if len(parts) < 4 {
return nil, fmt.Errorf("invalid key (expecting at least 3 parts): %s", key)
}
if parts[0] != "" {
return nil, fmt.Errorf("invalid key (expecting leading slash): %s", key)
}
k := &Key{
Group: parts[1],
Resource: parts[2],
Namespace: parts[3],
}
if len(parts) > 4 {
k.Name = parts[4]
}
if len(parts) > 5 {
k.Subresource = parts[5]
}
return k, nil
}
func (k *Key) String() string {
s := k.Group + "/" + k.Resource + "/" + k.Namespace
if len(k.Name) > 0 {
s += "/" + k.Name
if len(k.Subresource) > 0 {
s += "/" + k.Subresource
}
}
return s
}
func (k *Key) IsEqual(other *Key) bool {
return k.Group == other.Group &&
k.Resource == other.Resource &&
k.Namespace == other.Namespace &&
k.Name == other.Name &&
k.Subresource == other.Subresource
}

@ -7,7 +7,7 @@ import (
)
func initEntityTables(mg *migrator.Migrator) string {
marker := "Initialize entity tables (v005)" // changing this key wipe+rewrite everything
marker := "Initialize entity tables (v006)" // changing this key wipe+rewrite everything
mg.AddMigration(marker, &migrator.RawSQLMigration{})
tables := []migrator.Table{}
@ -19,11 +19,12 @@ func initEntityTables(mg *migrator.Migrator) string {
{Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
// The entity identifier
{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
{Name: "key", Type: migrator.DB_Text, Nullable: false},
// k8s namespace names must be RFC1123 label names, 63 characters or less
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "group_version", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "kind", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "uid", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "folder", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, // uid of folder
@ -59,7 +60,7 @@ func initEntityTables(mg *migrator.Migrator) string {
{Name: "errors", Type: migrator.DB_Text, Nullable: true}, // JSON object
},
Indices: []*migrator.Index{
{Cols: []string{"tenant_id", "kind", "uid"}, Type: migrator.UniqueIndex},
{Cols: []string{"namespace", "group", "resource", "uid"}, Type: migrator.UniqueIndex},
{Cols: []string{"folder"}, Type: migrator.IndexType},
},
})
@ -73,11 +74,12 @@ func initEntityTables(mg *migrator.Migrator) string {
{Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
// The entity identifier
{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
{Name: "key", Type: migrator.DB_Text, Nullable: false},
// k8s namespace names must be RFC1123 label names, 63 characters or less
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "group_version", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "kind", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "uid", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "folder", Type: migrator.DB_NVarchar, Length: 190, Nullable: false}, // uid of folder
@ -115,7 +117,7 @@ func initEntityTables(mg *migrator.Migrator) string {
},
Indices: []*migrator.Index{
{Cols: []string{"guid", "version"}, Type: migrator.UniqueIndex},
{Cols: []string{"tenant_id", "kind", "uid", "version"}, Type: migrator.UniqueIndex},
{Cols: []string{"namespace", "group", "resource", "uid", "version"}, Type: migrator.UniqueIndex},
},
})
@ -124,7 +126,8 @@ func initEntityTables(mg *migrator.Migrator) string {
Name: "entity_folder",
Columns: []*migrator.Column{
{Name: "guid", Type: migrator.DB_NVarchar, Length: 36, Nullable: false, IsPrimaryKey: true},
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
{Name: "uid", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
{Name: "slug_path", Type: migrator.DB_Text, Nullable: false}, // /slug/slug/slug/
{Name: "tree", Type: migrator.DB_Text, Nullable: false}, // JSON []{uid, title}
{Name: "depth", Type: migrator.DB_Int, Nullable: false}, // starts at 1

@ -7,6 +7,9 @@ import context "context"
//-----------------------------------------------------------------------------------------------------
const (
FolderGroupName = "folder.grafana.app"
FolderResourceName = "folders"
StandardKindDashboard = "dashboard"
StandardKindPlaylist = "playlist"
StandardKindFolder = "folder"

@ -34,17 +34,17 @@ type folderInfo struct {
// This will replace all entries in `entity_folder`
// This is pretty heavy weight, but it does give us a sorted folder list
// NOTE: this could be done async with a mutex/lock? reconciler pattern
func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenantId int64) error {
_, err := tx.Exec(ctx, "DELETE FROM entity_folder WHERE tenant_id=?", tenantId)
func updateFolderTree(ctx context.Context, tx *session.SessionTx, namespace string) error {
_, err := tx.Exec(ctx, "DELETE FROM entity_folder WHERE namespace=?", namespace)
if err != nil {
return err
}
query := "SELECT guid,uid,folder,name,slug" +
" FROM entity" +
" WHERE kind=? AND tenant_id=?" +
" WHERE group=? AND resource=? AND namespace=?" +
" ORDER BY slug asc"
args := []interface{}{entity.StandardKindFolder, tenantId}
args := []interface{}{entity.FolderGroupName, entity.FolderResourceName, namespace}
all := []*folderInfo{}
rows, err := tx.Query(ctx, query, args...)
@ -69,13 +69,13 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenantId int64
return err
}
err = insertFolderInfo(ctx, tx, tenantId, root, false)
err = insertFolderInfo(ctx, tx, namespace, root, false)
if err != nil {
return err
}
for _, folder := range lost {
err = insertFolderInfo(ctx, tx, tenantId, folder, true)
err = insertFolderInfo(ctx, tx, namespace, folder, true)
if err != nil {
return err
}
@ -137,14 +137,14 @@ func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, er
return folder.right, nil
}
func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenantId int64, folder *folderInfo, isDetached bool) error {
func insertFolderInfo(ctx context.Context, tx *session.SessionTx, namespace string, folder *folderInfo, isDetached bool) error {
js, _ := json.Marshal(folder.stack)
_, err := tx.Exec(ctx,
`INSERT INTO entity_folder `+
"(guid, tenant_id, uid, slug_path, tree, depth, lft, rgt, detached) "+
"(guid, namespace, uid, slug_path, tree, depth, lft, rgt, detached) "+
`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
folder.Guid,
tenantId,
namespace,
folder.UID,
folder.SlugPath,
string(js),
@ -158,7 +158,7 @@ func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenantId int64
}
for _, sub := range folder.children {
err := insertFolderInfo(ctx, tx, tenantId, sub, isDetached)
err := insertFolderInfo(ctx, tx, namespace, sub, isDetached)
if err != nil {
return err
}

File diff suppressed because it is too large Load Diff

@ -11,10 +11,8 @@ import (
"github.com/stretchr/testify/require"
"google.golang.org/grpc/metadata"
"github.com/grafana/grafana/pkg/infra/grn"
"github.com/grafana/grafana/pkg/services/store"
"github.com/grafana/grafana/pkg/services/store/entity"
"github.com/grafana/grafana/pkg/util"
)
var (
@ -25,7 +23,7 @@ var (
)
type rawEntityMatcher struct {
grn *grn.GRN
key string
createdRange []time.Time
updatedRange []time.Time
createdBy string
@ -53,16 +51,8 @@ func requireEntityMatch(t *testing.T, obj *entity.Entity, m rawEntityMatcher) {
require.NotNil(t, obj)
mismatches := ""
if m.grn != nil {
if m.grn.TenantID > 0 && m.grn.TenantID != obj.GRN.TenantID {
mismatches += fmt.Sprintf("expected tenant: %d, actual: %d\n", m.grn.TenantID, obj.GRN.TenantID)
}
if m.grn.ResourceKind != "" && m.grn.ResourceKind != obj.GRN.ResourceKind {
mismatches += fmt.Sprintf("expected ResourceKind: %s, actual: %s\n", m.grn.ResourceKind, obj.GRN.ResourceKind)
}
if m.grn.ResourceIdentifier != "" && m.grn.ResourceIdentifier != obj.GRN.ResourceIdentifier {
mismatches += fmt.Sprintf("expected ResourceIdentifier: %s, actual: %s\n", m.grn.ResourceIdentifier, obj.GRN.ResourceIdentifier)
}
if m.key != "" && m.key != obj.Key {
mismatches += fmt.Sprintf("expected key: %s, actual: %s\n", m.key, obj.Key)
}
if len(m.createdRange) == 2 && !timestampInRange(obj.CreatedAt, m.createdRange) {
@ -134,59 +124,63 @@ func TestIntegrationEntityServer(t *testing.T) {
fakeUser := store.GetUserIDString(testCtx.user)
firstVersion := "1"
kind := entity.StandardKindJSONObj
testGrn := &grn.GRN{
ResourceKind: kind,
ResourceIdentifier: "my-test-entity",
}
group := "test.grafana.app"
resource := "jsonobjs"
resource2 := "playlists"
namespace := "default"
uid := "my-test-entity"
testKey := "/" + group + "/" + resource + "/" + namespace + "/" + uid
body := []byte("{\"name\":\"John\"}")
t.Run("should not retrieve non-existent objects", func(t *testing.T) {
resp, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: testGrn,
Key: testKey,
})
require.NoError(t, err)
require.NotNil(t, resp)
require.Nil(t, resp.GRN)
require.Empty(t, resp.Key)
})
t.Run("should be able to read persisted objects", func(t *testing.T) {
before := time.Now()
writeReq := &entity.WriteEntityRequest{
createReq := &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: testGrn,
Key: testKey,
Group: group,
Resource: resource,
Namespace: namespace,
Uid: uid,
Body: body,
Message: "first entity!",
},
}
writeResp, err := testCtx.client.Write(ctx, writeReq)
createResp, err := testCtx.client.Create(ctx, createReq)
require.NoError(t, err)
versionMatcher := objectVersionMatcher{
updatedRange: []time.Time{before, time.Now()},
updatedBy: fakeUser,
version: &firstVersion,
comment: &writeReq.Entity.Message,
comment: &createReq.Entity.Message,
}
requireVersionMatch(t, writeResp.Entity, versionMatcher)
requireVersionMatch(t, createResp.Entity, versionMatcher)
readResp, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: testGrn,
Key: testKey,
Version: "",
WithBody: true,
})
require.NoError(t, err)
require.NotNil(t, readResp)
foundGRN := readResp.GRN
require.NotNil(t, foundGRN)
require.Equal(t, testCtx.user.OrgID, foundGRN.TenantID) // orgId becomes the tenant id when not set
require.Equal(t, testGrn.ResourceKind, foundGRN.ResourceKind)
require.Equal(t, testGrn.ResourceIdentifier, foundGRN.ResourceIdentifier)
require.Equal(t, testKey, readResp.Key)
require.Equal(t, namespace, readResp.Namespace) // orgId becomes the tenant id when not set
require.Equal(t, resource, readResp.Resource)
require.Equal(t, uid, readResp.Uid)
objectMatcher := rawEntityMatcher{
grn: testGrn,
key: testKey,
createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
@ -197,74 +191,74 @@ func TestIntegrationEntityServer(t *testing.T) {
requireEntityMatch(t, readResp, objectMatcher)
deleteResp, err := testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{
GRN: testGrn,
PreviousVersion: writeResp.Entity.Version,
Key: testKey,
PreviousVersion: readResp.Version,
})
require.NoError(t, err)
require.Equal(t, deleteResp.Status, entity.DeleteEntityResponse_DELETED)
readRespAfterDelete, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: testGrn,
Key: testKey,
Version: "",
WithBody: true,
})
require.NoError(t, err)
require.Nil(t, readRespAfterDelete.GRN)
require.Empty(t, readRespAfterDelete.Key)
})
t.Run("should be able to update an object", func(t *testing.T) {
before := time.Now()
testGrn := &grn.GRN{
ResourceKind: kind,
ResourceIdentifier: util.GenerateShortUID(),
}
writeReq1 := &entity.WriteEntityRequest{
createReq := &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: testGrn,
Key: testKey,
Group: group,
Resource: resource,
Namespace: namespace,
Uid: uid,
Body: body,
Message: "first entity!",
},
}
writeResp1, err := testCtx.client.Write(ctx, writeReq1)
createResp, err := testCtx.client.Create(ctx, createReq)
require.NoError(t, err)
require.Equal(t, entity.WriteEntityResponse_CREATED, writeResp1.Status)
require.Equal(t, entity.CreateEntityResponse_CREATED, createResp.Status)
body2 := []byte("{\"name\":\"John2\"}")
writeReq2 := &entity.WriteEntityRequest{
updateReq := &entity.UpdateEntityRequest{
Entity: &entity.Entity{
GRN: testGrn,
Key: testKey,
Body: body2,
Message: "update1",
},
}
writeResp2, err := testCtx.client.Write(ctx, writeReq2)
updateResp, err := testCtx.client.Update(ctx, updateReq)
require.NoError(t, err)
require.NotEqual(t, writeResp1.Entity.Version, writeResp2.Entity.Version)
require.NotEqual(t, createResp.Entity.Version, updateResp.Entity.Version)
// Duplicate write (no change)
writeDupRsp, err := testCtx.client.Write(ctx, writeReq2)
writeDupRsp, err := testCtx.client.Update(ctx, updateReq)
require.NoError(t, err)
require.Nil(t, writeDupRsp.Error)
require.Equal(t, entity.WriteEntityResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, writeResp2.Entity.Version, writeDupRsp.Entity.Version)
require.Equal(t, writeResp2.Entity.ETag, writeDupRsp.Entity.ETag)
require.Equal(t, entity.UpdateEntityResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, updateResp.Entity.Version, writeDupRsp.Entity.Version)
require.Equal(t, updateResp.Entity.ETag, writeDupRsp.Entity.ETag)
body3 := []byte("{\"name\":\"John3\"}")
writeReq3 := &entity.WriteEntityRequest{
writeReq3 := &entity.UpdateEntityRequest{
Entity: &entity.Entity{
GRN: testGrn,
Key: testKey,
Body: body3,
Message: "update3",
},
}
writeResp3, err := testCtx.client.Write(ctx, writeReq3)
writeResp3, err := testCtx.client.Update(ctx, writeReq3)
require.NoError(t, err)
require.NotEqual(t, writeResp3.Entity.Version, writeResp2.Entity.Version)
require.NotEqual(t, writeResp3.Entity.Version, updateResp.Entity.Version)
latestMatcher := rawEntityMatcher{
grn: testGrn,
key: testKey,
createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
@ -273,7 +267,7 @@ func TestIntegrationEntityServer(t *testing.T) {
version: &writeResp3.Entity.Version,
}
readRespLatest, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: testGrn,
Key: testKey,
Version: "", // latest
WithBody: true,
})
@ -281,15 +275,15 @@ func TestIntegrationEntityServer(t *testing.T) {
requireEntityMatch(t, readRespLatest, latestMatcher)
readRespFirstVer, err := testCtx.client.Read(ctx, &entity.ReadEntityRequest{
GRN: testGrn,
Version: writeResp1.Entity.Version,
Key: testKey,
Version: createResp.Entity.Version,
WithBody: true,
})
require.NoError(t, err)
require.NotNil(t, readRespFirstVer)
requireEntityMatch(t, readRespFirstVer, rawEntityMatcher{
grn: testGrn,
key: testKey,
createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
@ -299,17 +293,17 @@ func TestIntegrationEntityServer(t *testing.T) {
})
history, err := testCtx.client.History(ctx, &entity.EntityHistoryRequest{
GRN: testGrn,
Key: testKey,
})
require.NoError(t, err)
require.Equal(t, []*entity.Entity{
writeResp3.Entity,
writeResp2.Entity,
writeResp1.Entity,
updateResp.Entity,
createResp.Entity,
}, history.Versions)
deleteResp, err := testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{
GRN: testGrn,
Key: testKey,
PreviousVersion: writeResp3.Entity.Version,
})
require.NoError(t, err)
@ -317,53 +311,40 @@ func TestIntegrationEntityServer(t *testing.T) {
})
t.Run("should be able to list objects", func(t *testing.T) {
uid2 := "uid2"
uid3 := "uid3"
uid4 := "uid4"
kind2 := entity.StandardKindPlaylist
w1, err := testCtx.client.Write(ctx, &entity.WriteEntityRequest{
w1, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: testGrn,
Key: testKey + "1",
Body: body,
},
})
require.NoError(t, err)
w2, err := testCtx.client.Write(ctx, &entity.WriteEntityRequest{
w2, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: &grn.GRN{
ResourceIdentifier: uid2,
ResourceKind: kind,
},
Key: testKey + "2",
Body: body,
},
})
require.NoError(t, err)
w3, err := testCtx.client.Write(ctx, &entity.WriteEntityRequest{
w3, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: &grn.GRN{
ResourceIdentifier: uid3,
ResourceKind: kind2,
},
Key: testKey + "3",
Body: body,
},
})
require.NoError(t, err)
w4, err := testCtx.client.Write(ctx, &entity.WriteEntityRequest{
w4, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: &grn.GRN{
ResourceIdentifier: uid4,
ResourceKind: kind2,
},
Key: testKey + "4",
Body: body,
},
})
require.NoError(t, err)
resp, err := testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind, kind2},
Resource: []string{resource, resource2},
WithBody: false,
})
require.NoError(t, err)
@ -373,8 +354,8 @@ func TestIntegrationEntityServer(t *testing.T) {
kinds := make([]string, 0, len(resp.Results))
version := make([]string, 0, len(resp.Results))
for _, res := range resp.Results {
uids = append(uids, res.GRN.ResourceIdentifier)
kinds = append(kinds, res.GRN.ResourceKind)
uids = append(uids, res.Uid)
kinds = append(kinds, res.Resource)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2", "uid3", "uid4"}, uids)
@ -388,15 +369,15 @@ func TestIntegrationEntityServer(t *testing.T) {
// Again with only one kind
respKind1, err := testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Resource: []string{resource},
})
require.NoError(t, err)
uids = make([]string, 0, len(respKind1.Results))
kinds = make([]string, 0, len(respKind1.Results))
version = make([]string, 0, len(respKind1.Results))
for _, res := range respKind1.Results {
uids = append(uids, res.GRN.ResourceIdentifier)
kinds = append(kinds, res.GRN.ResourceKind)
uids = append(uids, res.Uid)
kinds = append(kinds, res.Resource)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2"}, uids)
@ -409,32 +390,25 @@ func TestIntegrationEntityServer(t *testing.T) {
t.Run("should be able to filter objects based on their labels", func(t *testing.T) {
kind := entity.StandardKindDashboard
_, err := testCtx.client.Write(ctx, &entity.WriteEntityRequest{
_, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: &grn.GRN{
ResourceKind: kind,
ResourceIdentifier: "blue-green",
},
Key: "/grafana/dashboards/blue-green",
Body: []byte(dashboardWithTagsBlueGreen),
},
})
require.NoError(t, err)
_, err = testCtx.client.Write(ctx, &entity.WriteEntityRequest{
_, err = testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
GRN: &grn.GRN{
ResourceKind: kind,
ResourceIdentifier: "red-green",
},
Key: "/grafana/dashboards/red-green",
Body: []byte(dashboardWithTagsRedGreen),
},
})
require.NoError(t, err)
resp, err := testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Key: []string{kind},
WithBody: false,
WithLabels: true,
Labels: map[string]string{
"red": "",
},
@ -442,12 +416,11 @@ func TestIntegrationEntityServer(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, resp)
require.Len(t, resp.Results, 1)
require.Equal(t, resp.Results[0].GRN.ResourceIdentifier, "red-green")
require.Equal(t, resp.Results[0].Uid, "red-green")
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Key: []string{kind},
WithBody: false,
WithLabels: true,
Labels: map[string]string{
"red": "",
"green": "",
@ -456,12 +429,11 @@ func TestIntegrationEntityServer(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, resp)
require.Len(t, resp.Results, 1)
require.Equal(t, resp.Results[0].GRN.ResourceIdentifier, "red-green")
require.Equal(t, resp.Results[0].Uid, "red-green")
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Key: []string{kind},
WithBody: false,
WithLabels: true,
Labels: map[string]string{
"red": "invalid",
},
@ -471,9 +443,8 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Len(t, resp.Results, 0)
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Key: []string{kind},
WithBody: false,
WithLabels: true,
Labels: map[string]string{
"green": "",
},
@ -483,9 +454,8 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Len(t, resp.Results, 2)
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Kind: []string{kind},
Key: []string{kind},
WithBody: false,
WithLabels: true,
Labels: map[string]string{
"yellow": "",
},

@ -1,9 +0,0 @@
package entity
// The admin request is a superset of write request features
func ToAdminWriteEntityRequest(req *WriteEntityRequest) *AdminWriteEntityRequest {
return &AdminWriteEntityRequest{
Entity: req.Entity,
PreviousVersion: req.PreviousVersion,
}
}

@ -23,7 +23,7 @@ var getNow = func() time.Time { return time.Now() }
type ResolutionInfo struct {
OK bool `json:"ok"`
Key string `json:"key,omitempty"` // GRN? UID?
Key string `json:"key,omitempty"` // k8s key
Warning string `json:"kind,omitempty"` // old syntax? (name>uid) references a renamed object?
Timestamp time.Time `json:"timestamp,omitempty"`
}

Loading…
Cancel
Save