chore: Change local setup under dev to use tsdb and v13 schema (#12057)

Co-authored-by: Christian Haudum <christian.haudum@gmail.com>

Makefile

@@ -816,7 +816,7 @@ validate-example-configs: loki
 	for f in $$(grep -rL $(EXAMPLES_SKIP_VALIDATION_FLAG) $(EXAMPLES_YAML_PATH)/*.yaml); do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done
 
 validate-dev-cluster-config: loki
-	./cmd/loki/loki -config.file=./tools/dev/loki-boltdb-storage-s3/config/loki.yaml -verify-config
+	./cmd/loki/loki -config.file=./tools/dev/loki-tsdb-storage-s3/config/loki.yaml -verify-config
 
 # Dynamically generate ./docs/sources/configure/examples.md using the example configs that we provide.
 # This target should be run if any of our example configs change.

pkg/ingester/index/bitprefix_test.go

@@ -90,9 +90,9 @@ func Test_BitPrefixDeleteAddLoopkup(t *testing.T) {
 
 func Test_BitPrefix_hash_mapping(t *testing.T) {
 	lbs := labels.Labels{
-		labels.Label{Name: "compose_project", Value: "loki-boltdb-storage-s3"},
+		labels.Label{Name: "compose_project", Value: "loki-tsdb-storage-s3"},
 		labels.Label{Name: "compose_service", Value: "ingester-2"},
-		labels.Label{Name: "container_name", Value: "loki-boltdb-storage-s3_ingester-2_1"},
+		labels.Label{Name: "container_name", Value: "loki-tsdb-storage-s3_ingester-2_1"},
 		labels.Label{Name: "filename", Value: "/var/log/docker/790fef4c6a587c3b386fe85c07e03f3a1613f4929ca3abaa4880e14caadb5ad1/json.log"},
 		labels.Label{Name: "host", Value: "docker-desktop"},
 		labels.Label{Name: "source", Value: "stderr"},
@@ -115,7 +115,7 @@ func Test_BitPrefix_hash_mapping(t *testing.T) {
 			res, err := ii.Lookup(
 				[]*labels.Matcher{{Type: labels.MatchEqual,
 					Name:  "compose_project",
-					Value: "loki-boltdb-storage-s3"}},
+					Value: "loki-tsdb-storage-s3"}},
 				&astmapper.ShardAnnotation{
 					Shard: int(expShard),
 					Of:    requestedFactor,

pkg/ingester/index/index_test.go

@@ -95,9 +95,9 @@ func TestDeleteAddLoopkup(t *testing.T) {
 
 func Test_hash_mapping(t *testing.T) {
 	lbs := labels.Labels{
-		labels.Label{Name: "compose_project", Value: "loki-boltdb-storage-s3"},
+		labels.Label{Name: "compose_project", Value: "loki-tsdb-storage-s3"},
 		labels.Label{Name: "compose_service", Value: "ingester-2"},
-		labels.Label{Name: "container_name", Value: "loki-boltdb-storage-s3_ingester-2_1"},
+		labels.Label{Name: "container_name", Value: "loki-tsdb-storage-s3_ingester-2_1"},
 		labels.Label{Name: "filename", Value: "/var/log/docker/790fef4c6a587c3b386fe85c07e03f3a1613f4929ca3abaa4880e14caadb5ad1/json.log"},
 		labels.Label{Name: "host", Value: "docker-desktop"},
 		labels.Label{Name: "source", Value: "stderr"},
@@ -108,7 +108,7 @@ func Test_hash_mapping(t *testing.T) {
 			ii := NewWithShards(shard)
 			ii.Add(logproto.FromLabelsToLabelAdapters(lbs), 1)
 
-			res, err := ii.Lookup([]*labels.Matcher{{Type: labels.MatchEqual, Name: "compose_project", Value: "loki-boltdb-storage-s3"}}, &astmapper.ShardAnnotation{Shard: int(labelsSeriesIDHash(lbs) % 16), Of: 16})
+			res, err := ii.Lookup([]*labels.Matcher{{Type: labels.MatchEqual, Name: "compose_project", Value: "loki-tsdb-storage-s3"}}, &astmapper.ShardAnnotation{Shard: int(labelsSeriesIDHash(lbs) % 16), Of: 16})
 			require.NoError(t, err)
 			require.Len(t, res, 1)
 			require.Equal(t, model.Fingerprint(1), res[0])

tools/dev/loki-tsdb-storage-s3/README.md

@@ -7,7 +7,7 @@ It runs the current code base in the repository this means you can debug new fea
 To start the stack simply run:
 
 ```bash
-./tools/dev/loki-boltdb-storage-s3/compose-up.sh
+./tools/dev/loki-tsdb-storage-s3/compose-up.sh
 ```
 
 You can then access grafana locally with http://localhost:3000 (default account admin/admin). The grafana container should already have the datasource to correctly query the frontend.
@@ -15,7 +15,7 @@ You can then access grafana locally with http://localhost:3000 (default account
 To tear it down use:
 
 ```bash
-./tools/dev/loki-boltdb-storage-s3/compose-down.sh
+./tools/dev/loki-tsdb-storage-s3/compose-down.sh
 ```
 
 > On MacOS :apple: docker can get stuck when restarting the stack, you can restart docker to workaround the problem :shrug:
@@ -58,7 +58,7 @@ If you use vs-code, you can add this snippet bellow in your [`launch.json`](http
     ],
     "port": 18002,
     "host": "127.0.0.1",
-    "cwd": "${workspaceFolder}/tools/dev/loki-boltdb-storage-s3/loki",
+    "cwd": "${workspaceFolder}/tools/dev/loki-tsdb-storage-s3/loki",
     "remotePath": "/loki/loki",
     "showLog": true,
     "trace": "log",
@@ -67,3 +67,7 @@ If you use vs-code, you can add this snippet bellow in your [`launch.json`](http
 ```
 
 Then you can debug `ingester-1` with the `Launch Loki remote` configuration within the debugging tab.
+
+### GoLand IDE
+
+If you use the [GoLand](https://www.jetbrains.com/go/) IDE, just create a Go remote debug configuration and use the appropriate port with the process you wish to debug.

tools/dev/loki-tsdb-storage-s3/config/loki.yaml

@@ -1,6 +1,6 @@
 auth_enabled: true 
 common:
-    compactor_address: compactor:8006
+    compactor_address: http://compactor:8006
 chunk_store_config:
     chunk_cache_config:
         memcached:
@@ -29,6 +29,7 @@ frontend:
     log_queries_longer_than: 5s
     max_outstanding_per_tenant: 512
     encoding: protobuf
+    scheduler_address: query-scheduler:9009
 frontend_worker:
     grpc_client_config:
       max_send_msg_size: 1.048576e+08
@@ -103,10 +104,10 @@ schema_config:
       - from: "2020-07-30"
         index:
             period: 24h
-            prefix: loki_boltdb_shipper_index_
+            prefix: index_
         object_store: s3
-        schema: v11
-        store: boltdb-shipper
+        schema: v13
+        store: tsdb
         row_shards: 4
 server:
     graceful_shutdown_timeout: 5s
@@ -134,6 +135,11 @@ storage_config:
             parallelism: 10
         memcached_client:
             addresses: memcached:11211
+    tsdb_shipper:
+        active_index_directory: /data/tsdb-index
+        cache_location: /data/tsdb-cache
+        index_gateway_client:
+            server_address: index-gateway:9008
 table_manager:
     creation_grace_period: 3h
     poll_interval: 10m

tools/dev/loki-tsdb-storage-s3/dev.dockerfile

@@ -1,8 +1,8 @@
-FROM golang:1.20.4
+FROM golang:1.21.3
 ENV CGO_ENABLED=0
-RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.1
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
 
-FROM alpine:3.18.5
+FROM alpine:3.19.1
 
 RUN     mkdir /loki
 WORKDIR /loki

tools/dev/loki-tsdb-storage-s3/docker-compose.yml

@@ -15,9 +15,9 @@ services:
   minio:
     logging:
       <<: *logging
-    image: minio/minio:RELEASE.2022-03-11T23-57-45Z
+    image: minio/minio:RELEASE.2024-02-24T17-11-14Z
     entrypoint: sh
-    command: -c 'mkdir -p /data/loki && /opt/bin/minio server --console-address :9001 /data'
+    command: -c 'mkdir -p /data/loki && /usr/bin/minio server --console-address :9001 /data'
     environment:
       - MINIO_ACCESS_KEY=loki
       - MINIO_SECRET_KEY=supersecret
@@ -30,7 +30,7 @@ services:
   memcached:
     logging:
       <<: *logging
-    image: memcached:1.6
+    image: memcached:1.6.17-alpine
 
   jaeger:
     logging: