chore(bench): Restructure data directory of logql benchmarks (#19198)

Reconfigure benchmark storage setup so that the resulting data can be read by a local Loki instance. An example config file for running Loki against the benchmark data directory is included.


Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

pkg/logql/bench/.gitignore

@@ -1,8 +1,8 @@
 # Ignore generated data directory
 data/
 *.prof
+*.log
 bench.test
 trace.out
 result.new.txt
 result.old.txt
-debug.log

pkg/logql/bench/bench_test.go

@@ -2,6 +2,7 @@ package bench
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -184,6 +185,9 @@ func TestStorageEquality(t *testing.T) {
 					humanize.Bytes(uint64(actual.Statistics.Summary.BytesProcessedPerSecond)),
 				)
 
+				dataobjStats, _ := json.Marshal(&actual.Statistics.Querier.Store.Dataobj)
+				t.Log("Dataobj stats:", string(dataobjStats))
+
 				expected, err := baseStore.Engine.Query(params).Exec(ctx)
 				require.NoError(t, err)
 				t.Logf(`Summary stats: store=%s lines_processed=%d, entries_returned=%d, bytes_processed=%s, execution_time_in_secs=%d, bytes_processed_per_sec=%s`,

pkg/logql/bench/config.yaml

@@ -0,0 +1,93 @@
+# Loki configuration file for reading data from the logql benchmark dataset.
+# Run Loki from the project root 
+#   $ make loki
+#   $ ./cmd/loki/loki -config.file=./pkg/logql/bench/config.yaml -config.expand-env=true
+#
+# Folder layout generated by the benchmark tool:
+#
+# data
+# ├── storage
+# │   ├── dataobj
+# │   │   ├── index
+# │   │   │   └── v0
+# │   │   │       ├── indexes
+# │   │   │       │   └── 33
+# │   │   │       └── tocs
+# │   │   ├── objects
+# │   │   │   ├── 10
+# │   │   │   ├── ..
+# │   │   │   └── ce
+# │   │   └── tocs
+# │   ├── index
+# │   │   └── index_19723
+# │   └── test-tenant
+# │       ├── 1167f1bd2bcde088
+# │       ├── ...
+# │       └── ff9868d673bb8ef1
+# └── workingdir
+#     ├── cache
+#     └── tsdb-shipper-active
+#         ├── multitenant
+#         │   └── index_19723
+#         ├── per_tenant
+#         ├── scratch
+#         │   └── filesystem_-292275055-05-16
+#         ├── uploader
+#         └── wal
+#             └── filesystem_-292275055-05-16
+
+auth_enabled: true
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+  log_level: info
+
+common:
+  compactor_address: 127.0.0.1
+  instance_addr: 127.0.0.1
+  path_prefix: ${PWD}/pkg/logql/bench/data/workingdir
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+limits_config:
+  tsdb_max_query_parallelism: 1
+  tsdb_sharding_strategy: power_of_two
+  tsdb_max_bytes_per_shard: 128MB
+  allow_structured_metadata: true
+  query_timeout: 1h
+  split_queries_by_interval: 1h
+  ingestion_rate_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        path_prefix: index/
+        prefix: index_
+        period: 24h
+
+storage_config:
+  use_thanos_objstore: true
+  object_store:
+    filesystem:
+      dir: ${PWD}/pkg/logql/bench/data/storage
+
+querier:
+  engine:
+    enable_v2_engine: true
+    batch_size: 8192
+
+dataobj:
+  storage_bucket_prefix: "dataobj"
+
+tracing:
+  enabled: false
+
+analytics:
+ reporting_enabled: false

pkg/logql/bench/store.go

@@ -7,6 +7,11 @@ import (
 	"github.com/grafana/loki/v3/pkg/logproto"
 )
 
+var (
+	storageDir = "storage"
+	workingDir = "workingdir"
+)
+
 // Store represents a storage backend for log data
 type Store interface {
 	// Write writes a batch of streams to the store

pkg/logql/bench/store_chunk.go

@@ -41,24 +41,29 @@ type ChunkStore struct {
 }
 
 func NewChunkStore(dir, tenantID string) (*ChunkStore, error) {
-	// NOTE(rfratto): ChunkStore should use dir as the base directory to imitate
-	// production setup: dataobjs are a subdirectory of the directory used for
-	// chunks.
-	indexshipperDir := filepath.Join(dir, "indexshipper")
-	if err := os.MkdirAll(indexshipperDir, 0o755); err != nil {
-		return nil, fmt.Errorf("failed to create index directory %s: %w", indexshipperDir, err)
+	storageDir := filepath.Join(dir, storageDir)
+	workingDir := filepath.Join(dir, workingDir)
+
+	tsdbShipperDir := filepath.Join(workingDir, "tsdb-shipper-active")
+	if err := os.MkdirAll(tsdbShipperDir, 0o755); err != nil {
+		return nil, fmt.Errorf("failed to create index directory %s: %w", tsdbShipperDir, err)
+	}
+	cacheDir := filepath.Join(workingDir, "cache")
+	if err := os.MkdirAll(cacheDir, 0o755); err != nil {
+		return nil, fmt.Errorf("failed to create index directory %s: %w", cacheDir, err)
 	}
+
 	storeConfig := storage.Config{
 		MaxChunkBatchSize: 50,
 		TSDBShipperConfig: indexshipper.Config{
-			ActiveIndexDirectory: indexshipperDir,
+			ActiveIndexDirectory: tsdbShipperDir,
 			Mode:                 indexshipper.ModeReadWrite,
 			IngesterName:         "test",
-			CacheLocation:        filepath.Join(dir, "index_cache"),
+			CacheLocation:        cacheDir,
 			ResyncInterval:       5 * time.Minute,
 			CacheTTL:             24 * time.Hour,
 		},
-		FSConfig: local.FSConfig{Directory: dir},
+		FSConfig: local.FSConfig{Directory: storageDir},
 	}
 	period := config.PeriodConfig{
 		From:       config.DayTime{Time: model.Earliest},

pkg/logql/bench/store_dataobj.go

@@ -42,11 +42,10 @@ type DataObjStore struct {
 }
 
 // NewDataObjStore creates a new DataObjStore
-func NewDataObjStore(path, tenant string) (*DataObjStore, error) {
-	// NOTE(rfratto): DataObjStore should use a dataobj subdirectory to imitate
-	// production setup: a dataobj subdirectory in the location used for chunks.
-	basePath := filepath.Join(path, "dataobj")
+func NewDataObjStore(dir, tenant string) (*DataObjStore, error) {
+	storageDir := filepath.Join(dir, storageDir)
 
+	basePath := filepath.Join(storageDir, "dataobj")
 	objectsPath := filepath.Join(basePath, "objects")
 	if err := os.MkdirAll(objectsPath, 0o755); err != nil {
 		return nil, fmt.Errorf("failed to create object path: %w", err)
@@ -75,7 +74,8 @@ func NewDataObjStore(path, tenant string) (*DataObjStore, error) {
 	}
 
 	builder, err := logsobj.NewBuilder(logsobj.BuilderConfig{
-		TargetPageSize:    2 * 1024 * 1024,   // 2MB
+		TargetPageSize:    2 * 1024 * 1024, // 2MB
+		MaxPageRows:       1000,
 		TargetObjectSize:  128 * 1024 * 1024, // 128MB
 		TargetSectionSize: 16 * 1024 * 1024,  // 16MB
 		BufferSize:        16 * 1024 * 1024,  // 16MB