Build: Fix Redis/Memcached integration tests (#64298)

* Build: Fix integration cache tests

* Allow REDIS_URL with scheme

* Reduce cache integration tests timeout to 5m

* Apply suggestion from code review

* Run redis/memcached integration tests in OSS pipelines

* Change redis image

.drone.yml

@@ -797,6 +797,12 @@ services:
   volumes:
   - name: mysql
     path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
   - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST"
@@ -892,6 +898,26 @@ steps:
     MYSQL_HOST: mysql
   image: grafana/build-container:1.7.2
   name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
   event:
   - pull_request
@@ -1797,6 +1823,12 @@ services:
   volumes:
   - name: mysql
     path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
   - mkdir -p bin
@@ -1870,6 +1902,26 @@ steps:
     MYSQL_HOST: mysql
   image: grafana/build-container:1.7.2
   name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
   branch: main
   event:
@@ -4956,6 +5008,12 @@ services:
   volumes:
   - name: mysql
     path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
   - mkdir -p bin
@@ -5022,6 +5080,26 @@ steps:
     MYSQL_HOST: mysql
   image: grafana/build-container:1.7.2
   name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
   ref:
   - refs/heads/v[0-9]*
@@ -5617,7 +5695,7 @@ services:
   - name: mysql
     path: /var/lib/mysql
 - environment: {}
-  image: redis:6.2.1-alpine
+  image: redis:6.2.11-alpine
   name: redis
 - environment: {}
   image: memcached:1.6.9-alpine
@@ -5718,8 +5796,7 @@ steps:
 - commands:
   - dockerize -wait tcp://redis:6379/0 -timeout 120s
   - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
   depends_on:
   - wire-install
   environment:
@@ -5729,8 +5806,7 @@ steps:
 - commands:
   - dockerize -wait tcp://memcached:11211 -timeout 120s
   - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
   depends_on:
   - wire-install
   environment:
@@ -6112,6 +6188,12 @@ services:
   volumes:
   - name: mysql
     path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
   - mkdir -p bin
@@ -6178,6 +6260,26 @@ steps:
     MYSQL_HOST: mysql
   image: grafana/build-container:1.7.2
   name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
   event:
   - promote
@@ -6230,7 +6332,7 @@ services:
   - name: mysql
     path: /var/lib/mysql
 - environment: {}
-  image: redis:6.2.1-alpine
+  image: redis:6.2.11-alpine
   name: redis
 - environment: {}
   image: memcached:1.6.9-alpine
@@ -6332,8 +6434,7 @@ steps:
 - commands:
   - dockerize -wait tcp://redis:6379/0 -timeout 120s
   - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
   depends_on:
   - wire-install
   environment:
@@ -6343,8 +6444,7 @@ steps:
 - commands:
   - dockerize -wait tcp://memcached:11211 -timeout 120s
   - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
   depends_on:
   - wire-install
   environment:
@@ -6653,6 +6753,6 @@ kind: secret
 name: enterprise2_security_prefix
 ---
 kind: signature
-hmac: 675c90070f9b68e15f349d079fbafa5288c3e5c30c345a4c9678d69afd1cc5a7
+hmac: 54bded4dc7c3ffbbf3859a4f8f6364f59f3b6e0696152a0b835211eca7119e50
 
 ...

Makefile

@@ -143,6 +143,18 @@ test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backe
 	$(GO) clean -testcache
 	$(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=mysql go test -run Integration -covermode=atomic -timeout=2m {}'
 
+.PHONY: test-go-integration-redis
+test-go-integration-redis: ## Run integration tests for redis cache.
+	@echo "test backend integration redis tests"
+	$(GO) clean -testcache
+	REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+
+.PHONY: test-go-integration-memcached
+test-go-integration-memcached: ## Run integration tests for memcached cache.
+	@echo "test backend integration memcached tests"
+	$(GO) clean -testcache
+	MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+
 test-js: ## Run tests for frontend.
 	@echo "test frontend"
 	yarn test

pkg/infra/remotecache/memcached_storage_integration_test.go

@@ -1,16 +1,23 @@
-//go:build memcached
-// +build memcached
-
 package remotecache
 
 import (
+	"os"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func TestMemcachedCacheStorage(t *testing.T) {
-	opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: "localhost:11211"}
+func TestIntegrationMemcachedCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	u, ok := os.LookupEnv("MEMCACHED_HOSTS")
+	if !ok || u == "" {
+		t.Skip("No Memcached hosts provided")
+	}
+
+	opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: u}
 	client := createTestClient(t, opts, nil)
 	runTestsForClient(t, client)
 	runCountTestsForClient(t, opts, nil)

pkg/infra/remotecache/redis_storage_integration_test.go

@@ -1,17 +1,40 @@
-//go:build redis
-// +build redis
-
 package remotecache
 
 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"
 
+	"github.com/go-redis/redis/v8"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func TestRedisCacheStorage(t *testing.T) {
+func TestIntegrationRedisCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	u, ok := os.LookupEnv("REDIS_URL")
+	if !ok || u == "" {
+		t.Skip("No redis URL supplied")
+	}
+
+	addr := u
+	db := 0
+	parsed, err := redis.ParseURL(u)
+	if err == nil {
+		addr = parsed.Addr
+		db = parsed.DB
+	}
+
+	b := strings.Builder{}
+	b.WriteString(fmt.Sprintf("addr=%s", addr))
+	if db != 0 {
+		b.WriteString(fmt.Sprintf(",db=%d", db))
+	}
 
-	opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: "addr=localhost:6379"}
+	opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: b.String()}
 	client := createTestClient(t, opts, nil)
 	runTestsForClient(t, client)
 	runCountTestsForClient(t, opts, nil)

pkg/infra/remotecache/remotecache_test.go

@@ -95,7 +95,8 @@ func canPutGetAndDeleteCachedObjects(t *testing.T, client CacheStorage) {
 	assert.Equal(t, err, nil)
 
 	_, err = client.Get(context.Background(), "key1")
-	assert.Equal(t, err, ErrCacheItemNotFound)
+	// redis client returns redis.Nil error when key does not exist.
+	assert.Error(t, err)
 }
 
 func canNotFetchExpiredItems(t *testing.T, client CacheStorage) {
@@ -109,7 +110,8 @@ func canNotFetchExpiredItems(t *testing.T, client CacheStorage) {
 
 	// should not be able to read that value since its expired
 	_, err = client.Get(context.Background(), "key1")
-	assert.Equal(t, err, ErrCacheItemNotFound)
+	// redis client returns redis.Nil error when key does not exist.
+	assert.Error(t, err)
 }
 
 func TestCollectUsageStats(t *testing.T) {

pkg/services/live/managedstream/cache_redis_test.go

@@ -1,18 +1,34 @@
-//go:build redis
-// +build redis
-
 package managedstream
 
 import (
+	"os"
 	"testing"
 
 	"github.com/go-redis/redis/v8"
 	"github.com/stretchr/testify/require"
 )
 
-func TestRedisCacheStorage(t *testing.T) {
+func TestIntegrationRedisCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	u, ok := os.LookupEnv("REDIS_URL")
+	if !ok || u == "" {
+		t.Skip("No redis URL supplied")
+	}
+
+	addr := u
+	db := 0
+	parsed, err := redis.ParseURL(u)
+	if err == nil {
+		addr = parsed.Addr
+		db = parsed.DB
+	}
+
 	redisClient := redis.NewClient(&redis.Options{
-		Addr: "localhost:6379",
+		Addr: addr,
+		DB:   db,
 	})
 	c := NewRedisFrameCache(redisClient)
 	require.NotNil(t, c)

scripts/drone/events/release.star

@@ -141,7 +141,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
 
     environment = {"EDITION": "oss"}
 
-    services = integration_test_services(edition = "oss")
+    services = integration_test_services()
     volumes = integration_test_services_volumes()
 
     init_steps = [
@@ -203,6 +203,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
     integration_test_steps = [
         postgres_integration_tests_step(),
         mysql_integration_tests_step(),
+        redis_integration_tests_step(),
+        memcached_integration_tests_step(),
     ]
 
     # We don't need to run integration tests at release time since they have
@@ -282,7 +284,7 @@ def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):
 
     environment = {"EDITION": "enterprise"}
 
-    services = integration_test_services(edition = "enterprise")
+    services = integration_test_services()
     volumes = integration_test_services_volumes()
 
     init_steps = [
@@ -694,11 +696,9 @@ def integration_test_pipelines():
     }
     pipelines = []
     volumes = integration_test_services_volumes()
-    oss_integration_test_steps = [
+    integration_test_steps = [
         postgres_integration_tests_step(),
         mysql_integration_tests_step(),
-    ]
-    enterprise_integration_test_steps = oss_integration_test_steps + [
         redis_integration_tests_step(),
         memcached_integration_tests_step(),
     ]
@@ -708,7 +708,7 @@ def integration_test_pipelines():
         name = "integration-tests-oss",
         edition = "oss",
         trigger = trigger,
-        services = integration_test_services(edition = "oss"),
+        services = integration_test_services(),
         steps = [
                     download_grabpl_step(),
                     identify_runner_step(),
@@ -716,7 +716,7 @@ def integration_test_pipelines():
                     verify_gen_jsonnet_step(),
                     wire_install_step(),
                 ] +
-                oss_integration_test_steps,
+                integration_test_steps,
         environment = {"EDITION": "oss"},
         volumes = volumes,
     ))
@@ -725,7 +725,7 @@ def integration_test_pipelines():
         name = "integration-tests-enterprise",
         edition = "enterprise",
         trigger = trigger,
-        services = integration_test_services(edition = "enterprise"),
+        services = integration_test_services(),
         steps = [
                     download_grabpl_step(),
                     identify_runner_step(),
@@ -744,7 +744,7 @@ def integration_test_pipelines():
                 [
                     wire_install_step(),
                 ] +
-                enterprise_integration_test_steps,
+                integration_test_steps,
         environment = {"EDITION": "enterprise"},
         volumes = volumes,
     ))

scripts/drone/pipelines/integration_tests.star

@@ -8,8 +8,10 @@ load(
     "download_grabpl_step",
     "enterprise_setup_step",
     "identify_runner_step",
+    "memcached_integration_tests_step",
     "mysql_integration_tests_step",
     "postgres_integration_tests_step",
+    "redis_integration_tests_step",
     "verify_gen_cue_step",
     "verify_gen_jsonnet_step",
     "wire_install_step",
@@ -37,7 +39,7 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
     """
     environment = {"EDITION": "oss"}
 
-    services = integration_test_services(edition = "oss")
+    services = integration_test_services()
     volumes = integration_test_services_volumes()
 
     init_steps = []
@@ -66,6 +68,8 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
     test_steps = [
         postgres_integration_tests_step(),
         mysql_integration_tests_step(),
+        redis_integration_tests_step(),
+        memcached_integration_tests_step(),
     ]
 
     return pipeline(

scripts/drone/services/services.star

@@ -8,7 +8,7 @@ def integration_test_services_volumes():
         {"name": "mysql", "temp": {"medium": "memory"}},
     ]
 
-def integration_test_services(edition):
+def integration_test_services():
     services = [
         {
             "name": "postgres",
@@ -34,24 +34,18 @@ def integration_test_services(edition):
             },
             "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}],
         },
+        {
+            "name": "redis",
+            "image": "redis:6.2.11-alpine",
+            "environment": {},
+        },
+        {
+            "name": "memcached",
+            "image": "memcached:1.6.9-alpine",
+            "environment": {},
+        },
     ]
 
-    if edition in ("enterprise", "enterprise2"):
-        services.extend(
-            [
-                {
-                    "name": "redis",
-                    "image": "redis:6.2.1-alpine",
-                    "environment": {},
-                },
-                {
-                    "name": "memcached",
-                    "image": "memcached:1.6.9-alpine",
-                    "environment": {},
-                },
-            ],
-        )
-
     return services
 
 def ldap_service():

scripts/drone/steps/lib.star

@@ -1144,7 +1144,7 @@ def redis_integration_tests_step():
         "commands": [
             "dockerize -wait tcp://redis:6379/0 -timeout 120s",
             "go clean -testcache",
-            "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+            "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...",
         ],
     }
 
@@ -1159,7 +1159,7 @@ def memcached_integration_tests_step():
         "commands": [
             "dockerize -wait tcp://memcached:11211 -timeout 120s",
             "go clean -testcache",
-            "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+            "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...",
         ],
     }