apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix(unified-storage): add pagination support for folder list (#105444)

Browse Source
98279-update-grafana-versionrelease-link-on-login-page-to-the-whats-new-doc
Mustafa Sencer Özcan 2 months ago committed by GitHub
parent d4d1514ecb
commit 167b201525
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 281 additions and 10 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      pkg/services/folder/folderimpl/folder_unifiedstorage.go
  2. 49
      pkg/services/folder/folderimpl/unifiedstore.go
  3. 241
      pkg/services/folder/folderimpl/unifiedstore_test.go

1
pkg/services/folder/folderimpl/folder_unifiedstorage.go
Unescape View File

@ -34,6 +34,7 @@ import (
)
const folderSearchLimit = 100000
const folderListLimit = 100
func (s *Service) getFoldersFromApiServer(ctx context.Context, q folder.GetFoldersQuery) ([]*folder.Folder, error) {
ctx, span := s.tracer.Start(ctx, "folder.getFoldersFromApiServer")

49
pkg/services/folder/folderimpl/unifiedstore.go
Unescape View File

@ -317,16 +317,14 @@ func (ss *FolderUnifiedStoreImpl) GetHeight(ctx context.Context, foldrUID string
// The full path UIDs of B is "uid1/uid2".
// The full path UIDs of A is "uid1".
func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFoldersFromStoreQuery) ([]*folder.Folder, error) {
opts := v1.ListOptions{
Limit: folderSearchLimit,
}
opts := v1.ListOptions{}
if q.WithFullpath || q.WithFullpathUIDs {
// only supported in modes 0-2, to keep the alerting queries from causing tons of get folder requests
// to retrieve the parent for all folders in grafana
opts.LabelSelector = utils.LabelGetFullpath + "=true"
}
out, err := ss.k8sclient.List(ctx, q.OrgID, opts)
out, err := ss.list(ctx, q.OrgID, opts)
if err != nil {
return nil, err
}
@ -371,7 +369,7 @@ func (ss *FolderUnifiedStoreImpl) GetFolders(ctx context.Context, q folder.GetFo
}
func (ss *FolderUnifiedStoreImpl) GetDescendants(ctx context.Context, orgID int64, ancestor_uid string) ([]*folder.Folder, error) {
out, err := ss.k8sclient.List(ctx, orgID, v1.ListOptions{})
out, err := ss.list(ctx, orgID, v1.ListOptions{})
if err != nil {
return nil, err
}
@ -447,6 +445,47 @@ func (ss *FolderUnifiedStoreImpl) CountInOrg(ctx context.Context, orgID int64) (
return resp.Stats[0].Count, nil
}
func (ss *FolderUnifiedStoreImpl) list(ctx context.Context, orgID int64, opts v1.ListOptions) (*unstructured.UnstructuredList, error) {
var allItems []unstructured.Unstructured
listOpts := opts.DeepCopy()
if listOpts.Limit == 0 {
listOpts.Limit = folderListLimit
}
for {
out, err := ss.k8sclient.List(ctx, orgID, *listOpts)
if err != nil {
return nil, err
}
if out == nil {
return nil, fmt.Errorf("k8s folder list returned nil")
}
if len(out.Items) > 0 {
allItems = append(allItems, out.Items...)
}
if out.GetContinue() == "" || (opts.Limit > 0 && int64(len(allItems)) >= opts.Limit) {
break
}
listOpts.Continue = out.GetContinue()
}
result := &unstructured.UnstructuredList{
Items: allItems,
}
if opts.Limit > 0 && int64(len(allItems)) > opts.Limit {
result.Items = allItems[:opts.Limit]
}
return result, nil
}
func toFolderLegacyCounts(u *unstructured.Unstructured) (*folder.DescendantCounts, error) {
ds, err := folderv1.UnstructuredToDescendantCounts(u)
if err != nil {

241
pkg/services/folder/folderimpl/unifiedstore_test.go
Unescape View File

@ -465,7 +465,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
@ -521,7 +521,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
@ -588,7 +588,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
LabelSelector: "grafana.app/fullpath=true",
}).Return(&unstructured.UnstructuredList{
@ -650,7 +650,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
LabelSelector: "grafana.app/fullpath=true",
}).Return(&unstructured.UnstructuredList{
@ -724,7 +724,7 @@ func TestGetFolders(t *testing.T) {
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, orgID, metav1.ListOptions{
Limit: 100000,
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: "folders.folder.grafana.app", Resource: "folder"}, "folder1")).Once()
},
@ -880,3 +880,234 @@ func TestBuildFolderFullPaths(t *testing.T) {
})
}
}
func TestList(t *testing.T) {
type args struct {
ctx context.Context
orgID int64
opts metav1.ListOptions
}
tests := []struct {
name string
args args
mock func(mockCli *client.MockK8sHandler)
want *unstructured.UnstructuredList
wantErr bool
}{
{
name: "should return all folders",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 0,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
},
},
{
name: "should return folders with limit",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder2",
"uid": "folder2",
},
"spec": map[string]interface{}{
"title": "folder2",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
},
wantErr: false,
},
{
name: "should return folders with continue token",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: 1,
TypeMeta: metav1.TypeMeta{},
}).Return(&unstructured.UnstructuredList{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"continue": "continue-token",
},
},
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
}, nil).Once()
},
want: &unstructured.UnstructuredList{
Items: []unstructured.Unstructured{
{
Object: map[string]interface{}{
"metadata": map[string]interface{}{
"name": "folder1",
"uid": "folder1",
},
"spec": map[string]interface{}{
"title": "folder1",
},
},
},
},
},
wantErr: false,
},
{
name: "should return error if k8s returns error",
args: args{
ctx: context.Background(),
orgID: orgID,
opts: metav1.ListOptions{
Limit: 0,
TypeMeta: metav1.TypeMeta{},
},
},
mock: func(mockCli *client.MockK8sHandler) {
mockCli.On("List", mock.Anything, int64(1), metav1.ListOptions{
Limit: folderListLimit,
TypeMeta: metav1.TypeMeta{},
}).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: "folders.folder.grafana.app", Resource: "folder"}, "folder1")).Once()
},
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockCLI := new(client.MockK8sHandler)
tt.mock(mockCLI)
ss := &FolderUnifiedStoreImpl{
k8sclient: mockCLI,
userService: usertest.NewUserServiceFake(),
}
got, err := ss.list(tt.args.ctx, tt.args.orgID, tt.args.opts)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tt.want, got)
})
}
}

Write Preview
Loading…
Cancel
Save