apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Storage: Cleanup object history API (#56215)

Browse Source
pull/56204/head^2
Ryan McKinley 3 years ago committed by GitHub
parent 7715672fb3
commit 4fc9b9aa35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 658 additions and 411 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      .betterer.results
  2. 100
      pkg/services/store/object/dummy/dummy_server.go
  3. 12
      pkg/services/store/object/http.go
  4. 101
      pkg/services/store/object/json.go
  5. 675
      pkg/services/store/object/object.pb.go
  6. 47
      pkg/services/store/object/object.proto
  7. 2
      pkg/services/store/object/object_grpc.pb.go
  8. 124
      pkg/services/store/object/tests/server_integration_test.go

8
.betterer.results
Unescape View File

@ -6030,11 +6030,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "11"],
[0, 0, 0, "Unexpected any. Specify a different type.", "12"],
[0, 0, 0, "Unexpected any. Specify a different type.", "13"],
[0, 0, 0, "Unexpected any. Specify a different type.", "14"],
[0, 0, 0, "Do not use any type assertions.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "17"],
[0, 0, 0, "Unexpected any. Specify a different type.", "18"]
[0, 0, 0, "Do not use any type assertions.", "14"],
[0, 0, 0, "Unexpected any. Specify a different type.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"]
],
"public/app/plugins/datasource/elasticsearch/components/AddRemove.tsx:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]

100
pkg/services/store/object/dummy/dummy_server.go
Unescape View File

@ -17,9 +17,15 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
type ObjectVersionWithBody struct {
*object.ObjectVersionInfo `json:"info,omitempty"`
Body []byte `json:"body,omitempty"`
}
type RawObjectWithHistory struct {
*object.RawObject `json:"rawObject,omitempty"`
History []*object.RawObject `json:"history,omitempty"`
History []*ObjectVersionWithBody `json:"history,omitempty"`
}
var (
@ -74,13 +80,26 @@ func (i dummyObjectServer) findObject(ctx context.Context, uid string, kind stri
getLatestVersion := version == ""
if getLatestVersion {
objVersion := obj.History[len(obj.History)-1]
return obj, objVersion, nil
return obj, obj.RawObject, nil
}
for _, objVersion := range obj.History {
if objVersion.Version == version {
return obj, objVersion, nil
copy := &object.RawObject{
UID: obj.UID,
Kind: obj.Kind,
Created: obj.Created,
CreatedBy: obj.CreatedBy,
Modified: objVersion.Modified,
ModifiedBy: objVersion.ModifiedBy,
ETag: objVersion.ETag,
Version: objVersion.Version,
// Body is added from the dummy server cache (it does not exist in ObjectVersionInfo)
Body: objVersion.Body,
}
return obj, copy, nil
}
}
@ -125,7 +144,7 @@ func createContentsHash(contents []byte) string {
}
func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequest, namespace string) (*object.WriteObjectResponse, error) {
var updated *object.RawObject
rsp := &object.WriteObjectResponse{}
updatedCount, err := i.collection.Update(ctx, namespace, func(i *RawObjectWithHistory) (bool, *RawObjectWithHistory, error) {
match := i.UID == r.UID && i.Kind == r.Kind
@ -144,7 +163,7 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
modifier := userFromContext(ctx)
updated = &object.RawObject{
updated := &object.RawObject{
UID: r.UID,
Kind: r.Kind,
Created: i.Created,
@ -158,12 +177,32 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", prevVersion+1),
Comment: r.Comment,
}
versionInfo := &ObjectVersionWithBody{
Body: r.Body,
ObjectVersionInfo: &object.ObjectVersionInfo{
Version: updated.Version,
Modified: updated.Modified,
ModifiedBy: updated.ModifiedBy,
Size: updated.Size,
ETag: updated.ETag,
Comment: r.Comment,
},
}
rsp.Object = versionInfo.ObjectVersionInfo
rsp.Status = object.WriteObjectResponse_MODIFIED
// When saving, it must be different than the head version
if i.ETag == updated.ETag {
versionInfo.ObjectVersionInfo.Version = i.Version
rsp.Status = object.WriteObjectResponse_UNCHANGED
return false, nil, nil
}
return true, &RawObjectWithHistory{
RawObject: updated,
History: append(i.History, updated),
History: append(i.History, versionInfo),
}, nil
})
@ -171,14 +210,11 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
return nil, err
}
if updatedCount == 0 {
if updatedCount == 0 && rsp.Object == nil {
return nil, fmt.Errorf("could not find object with uid %s and kind %s", r.UID, r.Kind)
}
return &object.WriteObjectResponse{
Error: nil,
Object: updated,
}, nil
return rsp, nil
}
func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequest, namespace string) (*object.WriteObjectResponse, error) {
@ -200,11 +236,23 @@ func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequ
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", 1),
Comment: r.Comment,
}
info := &object.ObjectVersionInfo{
Version: rawObj.Version,
Modified: rawObj.Modified,
ModifiedBy: rawObj.ModifiedBy,
Size: rawObj.Size,
ETag: rawObj.ETag,
Comment: r.Comment,
}
newObj := &RawObjectWithHistory{
RawObject: rawObj,
History: []*object.RawObject{rawObj},
History: []*ObjectVersionWithBody{{
ObjectVersionInfo: info,
Body: r.Body,
}},
}
err := i.collection.Insert(ctx, namespace, newObj)
@ -214,13 +262,17 @@ func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequ
return &object.WriteObjectResponse{
Error: nil,
Object: newObj.RawObject,
Object: info,
Status: object.WriteObjectResponse_CREATED,
}, nil
}
func (i dummyObjectServer) Write(ctx context.Context, r *object.WriteObjectRequest) (*object.WriteObjectResponse, error) {
namespace := namespaceFromUID(r.UID)
obj, err := i.collection.FindFirst(ctx, namespace, func(i *RawObjectWithHistory) (bool, error) {
if i == nil || r == nil {
return false, nil
}
return i.UID == r.UID, nil
})
if err != nil {
@ -263,15 +315,15 @@ func (i dummyObjectServer) History(ctx context.Context, r *object.ObjectHistoryR
return nil, err
}
if obj == nil {
return &object.ObjectHistoryResponse{
Object: nil,
}, nil
rsp := &object.ObjectHistoryResponse{}
if obj != nil {
// Return the most recent versions first
// Better? save them in this order?
for i := len(obj.History) - 1; i >= 0; i-- {
rsp.Versions = append(rsp.Versions, obj.History[i].ObjectVersionInfo)
}
}
return &object.ObjectHistoryResponse{
Object: obj.History,
}, nil
return rsp, nil
}
func (i dummyObjectServer) Search(ctx context.Context, r *object.ObjectSearchRequest) (*object.ObjectSearchResponse, error) {

12
pkg/services/store/object/http.go
Unescape View File

@ -60,6 +60,14 @@ func parseRequestParams(req *http.Request) (uid string, kind string, params map[
uid = path
kind = "?"
}
// Read parameters that are encoded in the URL
vals := req.URL.Query()
for k, v := range vals {
if len(v) > 0 {
params[k] = v[0]
}
}
return
}
@ -151,7 +159,7 @@ func (s *httpObjectStore) doWriteObject(c *models.ReqContext) response.Response
Kind: kind,
Body: b,
Comment: params["comment"],
PreviousVersion: params["previous"],
PreviousVersion: params["previousVersion"],
})
if err != nil {
return response.Error(500, "?", err)
@ -164,7 +172,7 @@ func (s *httpObjectStore) doDeleteObject(c *models.ReqContext) response.Response
rsp, err := s.store.Delete(c.Req.Context(), &DeleteObjectRequest{
UID: uid,
Kind: kind,
PreviousVersion: params["previous"],
PreviousVersion: params["previousVersion"],
})
if err != nil {
return response.Error(500, "?", err)

101
pkg/services/store/object/json.go
Unescape View File

@ -1,101 +0,0 @@
package object
import (
"encoding/json"
"unsafe"
jsoniter "github.com/json-iterator/go"
)
func init() { //nolint:gochecknoinits
//jsoniter.RegisterTypeEncoder("object.ReadObjectResponse", &readObjectResponseCodec{})
jsoniter.RegisterTypeEncoder("object.RawObject", &rawObjectCodec{})
}
// Unlike the standard JSON marshal, this will write bytes as JSON when it can
type rawObjectCodec struct{}
// Custom marshal for RawObject (if JSON body)
func (obj *RawObject) MarshalJSON() ([]byte, error) {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
return json.Marshal(obj)
}
func (codec *rawObjectCodec) IsEmpty(ptr unsafe.Pointer) bool {
f := (*RawObject)(ptr)
return f.UID == "" && f.Body == nil
}
func (codec *rawObjectCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
obj := (*RawObject)(ptr)
stream.WriteObjectStart()
stream.WriteObjectField("UID")
stream.WriteString(obj.UID)
if obj.Kind != "" {
stream.WriteMore()
stream.WriteObjectField("kind")
stream.WriteString(obj.Kind)
}
if obj.Created > 0 {
stream.WriteMore()
stream.WriteObjectField("created")
stream.WriteInt64(obj.Created)
}
if obj.CreatedBy != nil {
stream.WriteMore()
stream.WriteObjectField("createdBy")
stream.WriteVal(obj.CreatedBy)
}
if obj.Modified > 0 {
stream.WriteMore()
stream.WriteObjectField("modified")
stream.WriteInt64(obj.Modified)
}
if obj.ModifiedBy != nil {
stream.WriteMore()
stream.WriteObjectField("modifiedBy")
stream.WriteVal(obj.ModifiedBy)
}
if obj.Size > 0 {
stream.WriteMore()
stream.WriteObjectField("size")
stream.WriteInt64(obj.Size)
}
if obj.ETag != "" {
stream.WriteMore()
stream.WriteObjectField("etag")
stream.WriteString(obj.ETag)
}
if obj.Version != "" {
stream.WriteMore()
stream.WriteObjectField("version")
stream.WriteString(obj.Version)
}
// The one real difference (encodes JSON things directly)
if obj.Body != nil {
stream.WriteMore()
stream.WriteObjectField("body")
if json.Valid(obj.Body) {
stream.WriteRaw(string(obj.Body)) // works for strings
} else {
stream.WriteString("// link to raw bytes //")
//stream.WriteVal(obj.Body)
}
}
if obj.SyncSrc != "" {
stream.WriteMore()
stream.WriteObjectField("syncSrc")
stream.WriteString(obj.SyncSrc)
}
if obj.SyncTime > 0 {
stream.WriteMore()
stream.WriteObjectField("syncTime")
stream.WriteInt64(obj.SyncTime)
}
stream.WriteObjectEnd()
}

675
pkg/services/store/object/object.pb.go
View File

File diff suppressed because it is too large Load Diff

47
pkg/services/store/object/object.proto
Unescape View File

@ -47,20 +47,15 @@ message RawObject {
// NOTE: currently managed by the dashboard+dashboard_version tables
string version = 10;
// optional "save" or "commit" message
//
// NOTE: currently managed by the dashboard_version table, and will be returned from a "history" command
string comment = 11;
// Location (path/repo/etc) that defines the canonocal form
//
// NOTE: currently managed by the dashboard_provisioning table
string sync_src = 12;
string sync_src = 11;
// Time in epoch milliseconds that the object was last synced with an external system (provisioning/git)
//
// NOTE: currently managed by the dashboard_provisioning table
int64 sync_time = 13;
int64 sync_time = 12;
}
// Report error while working with objects
@ -76,6 +71,29 @@ message ObjectErrorInfo {
bytes details_json = 3;
}
// This is a subset of RawObject that does not include body or sync info
message ObjectVersionInfo {
// The version will change when the object is saved. It is not necessarily sortable
string version = 1;
// Time in epoch milliseconds that the object was modified
int64 modified = 2;
// Who modified the object
UserInfo modified_by = 3;
// Content Length
int64 size = 4;
// MD5 digest of the body
string ETag = 5;
// optional "save" or "commit" message
//
// NOTE: currently managed by the dashboard_version table, and will be returned from a "history" command
string comment = 6;
}
//-----------------------------------------------
// Get request/response
//-----------------------------------------------
@ -143,10 +161,21 @@ message WriteObjectResponse {
ObjectErrorInfo error = 1;
// Object details with the body removed
RawObject object = 2;
ObjectVersionInfo object = 2;
// Object summary as JSON
bytes summary_json = 3;
// Status code
Status status = 4;
// Status enumeration
enum Status {
ERROR = 0;
CREATED = 1;
MODIFIED = 2;
UNCHANGED = 3;
}
}
//-----------------------------------------------
@ -188,7 +217,7 @@ message ObjectHistoryRequest {
message ObjectHistoryResponse {
// Object metadata without the raw bytes
repeated RawObject object = 1;
repeated ObjectVersionInfo versions = 1;
// More results exist... pass this in the next request
string next_page_token = 2;

2
pkg/services/store/object/object_grpc.pb.go
Unescape View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.5
// - protoc v3.21.7
// source: object.proto
package object

124
pkg/services/store/object/tests/server_integration_test.go
Unescape View File

@ -29,12 +29,23 @@ type rawObjectMatcher struct {
modifiedBy *object.UserInfo
body []byte
version *string
}
type objectVersionMatcher struct {
modifiedRange []time.Time
modifiedBy *object.UserInfo
version *string
etag *string
comment *string
}
func userInfoMatches(expected *object.UserInfo, actual *object.UserInfo) (bool, string) {
var mismatches []string
if actual == nil && expected != nil {
return true, "Missing user info"
}
if expected.Id != actual.Id {
mismatches = append(mismatches, fmt.Sprintf("expected ID %d, actual ID: %d", expected.Id, actual.Id))
}
@ -52,6 +63,8 @@ func timestampInRange(ts int64, tsRange []time.Time) bool {
func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher) {
t.Helper()
require.NotNil(t, obj)
mismatches := ""
if m.uid != nil && *m.uid != obj.UID {
mismatches += fmt.Sprintf("expected UID: %s, actual UID: %s\n", *m.uid, obj.UID)
@ -66,7 +79,7 @@ func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher)
}
if len(m.modifiedRange) == 2 && !timestampInRange(obj.Modified, m.modifiedRange) {
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.createdRange[0], m.createdRange[1], time.Unix(obj.Created, 0))
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.modifiedRange[0], m.modifiedRange[1], time.Unix(obj.Modified, 0))
}
if m.createdBy != nil {
@ -97,8 +110,30 @@ func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher)
mismatches += fmt.Sprintf("expected version: %s, actual version: %s\n", *m.version, obj.Version)
}
if m.comment != nil && *m.comment != obj.Comment {
mismatches += fmt.Sprintf("expected comment: %s, actual comment: %s\n", *m.comment, obj.Comment)
require.True(t, len(mismatches) == 0, mismatches)
}
func requireVersionMatch(t *testing.T, obj *object.ObjectVersionInfo, m objectVersionMatcher) {
t.Helper()
mismatches := ""
if m.etag != nil && *m.etag != obj.ETag {
mismatches += fmt.Sprintf("expected etag: %s, actual etag: %s\n", *m.etag, obj.ETag)
}
if len(m.modifiedRange) == 2 && !timestampInRange(obj.Modified, m.modifiedRange) {
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.modifiedRange[0], m.modifiedRange[1], time.Unix(obj.Modified, 0))
}
if m.modifiedBy != nil {
userInfoMatches, msg := userInfoMatches(m.modifiedBy, obj.ModifiedBy)
if !userInfoMatches {
mismatches += fmt.Sprintf("modifiedBy: %s\n", msg)
}
}
if m.version != nil && *m.version != obj.Version {
mismatches += fmt.Sprintf("expected version: %s, actual version: %s\n", *m.version, obj.Version)
}
require.True(t, len(mismatches) == 0, mismatches)
@ -144,18 +179,13 @@ func TestObjectServer(t *testing.T) {
writeResp, err := testCtx.client.Write(ctx, writeReq)
require.NoError(t, err)
objectMatcher := rawObjectMatcher{
uid: &uid,
kind: &kind,
createdRange: []time.Time{before, time.Now()},
versionMatcher := objectVersionMatcher{
modifiedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
comment: &writeReq.Comment,
}
requireObjectMatch(t, writeResp.Object, objectMatcher)
requireVersionMatch(t, writeResp.Object, versionMatcher)
readResp, err := testCtx.client.Read(ctx, &object.ReadObjectRequest{
UID: uid,
@ -165,7 +195,18 @@ func TestObjectServer(t *testing.T) {
})
require.NoError(t, err)
require.Nil(t, readResp.SummaryJson)
requireObjectMatch(t, writeResp.Object, objectMatcher)
objectMatcher := rawObjectMatcher{
uid: &uid,
kind: &kind,
createdRange: []time.Time{before, time.Now()},
modifiedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
}
requireObjectMatch(t, readResp.Object, objectMatcher)
deleteResp, err := testCtx.client.Delete(ctx, &object.DeleteObjectRequest{
UID: uid,
@ -195,6 +236,7 @@ func TestObjectServer(t *testing.T) {
}
writeResp1, err := testCtx.client.Write(ctx, writeReq1)
require.NoError(t, err)
require.Equal(t, object.WriteObjectResponse_CREATED, writeResp1.Status)
body2 := []byte("{\"name\":\"John2\"}")
@ -208,6 +250,14 @@ func TestObjectServer(t *testing.T) {
require.NoError(t, err)
require.NotEqual(t, writeResp1.Object.Version, writeResp2.Object.Version)
// Duplicate write (no change)
writeDupRsp, err := testCtx.client.Write(ctx, writeReq2)
require.NoError(t, err)
require.Nil(t, writeDupRsp.Error)
require.Equal(t, object.WriteObjectResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, writeResp2.Object.Version, writeDupRsp.Object.Version)
require.Equal(t, writeResp2.Object.ETag, writeDupRsp.Object.ETag)
body3 := []byte("{\"name\":\"John3\"}")
writeReq3 := &object.WriteObjectRequest{
UID: uid,
@ -228,7 +278,6 @@ func TestObjectServer(t *testing.T) {
modifiedBy: fakeUser,
body: body3,
version: &writeResp3.Object.Version,
comment: &writeReq3.Comment,
}
readRespLatest, err := testCtx.client.Read(ctx, &object.ReadObjectRequest{
UID: uid,
@ -259,7 +308,6 @@ func TestObjectServer(t *testing.T) {
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
comment: &writeReq1.Comment,
})
history, err := testCtx.client.History(ctx, &object.ObjectHistoryRequest{
@ -267,11 +315,11 @@ func TestObjectServer(t *testing.T) {
Kind: kind,
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
writeResp1.Object,
writeResp2.Object,
require.Equal(t, []*object.ObjectVersionInfo{
writeResp3.Object,
}, history.Object)
writeResp2.Object,
writeResp1.Object,
}, history.Versions)
deleteResp, err := testCtx.client.Delete(ctx, &object.DeleteObjectRequest{
UID: uid,
@ -316,19 +364,47 @@ func TestObjectServer(t *testing.T) {
require.NoError(t, err)
search, err := testCtx.client.Search(ctx, &object.ObjectSearchRequest{
Kind: []string{kind, kind2},
Kind: []string{kind, kind2},
WithBody: false,
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
w1.Object, w2.Object, w3.Object, w4.Object,
}, search.Results)
require.NotNil(t, search)
uids := make([]string, 0, len(search.Results))
kinds := make([]string, 0, len(search.Results))
version := make([]string, 0, len(search.Results))
for _, res := range search.Results {
uids = append(uids, res.UID)
kinds = append(kinds, res.Kind)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2", "uid3", "uid4"}, uids)
require.Equal(t, []string{"dashboard", "dashboard", "kind2", "kind2"}, kinds)
require.Equal(t, []string{
w1.Object.Version,
w2.Object.Version,
w3.Object.Version,
w4.Object.Version,
}, version)
// Again with only one kind
searchKind1, err := testCtx.client.Search(ctx, &object.ObjectSearchRequest{
Kind: []string{kind},
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
w1.Object, w2.Object,
}, searchKind1.Results)
uids = make([]string, 0, len(searchKind1.Results))
kinds = make([]string, 0, len(searchKind1.Results))
version = make([]string, 0, len(searchKind1.Results))
for _, res := range searchKind1.Results {
uids = append(uids, res.UID)
kinds = append(kinds, res.Kind)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2"}, uids)
require.Equal(t, []string{"dashboard", "dashboard"}, kinds)
require.Equal(t, []string{
w1.Object.Version,
w2.Object.Version,
}, version)
})
}

Write Preview
Loading…
Cancel
Save