Add DroneCI pipeline stage to validate loki example configs; create example configuration files (#4358)

* Add DroneCI pipeline stage to validate loki example configs; create example configuration files

* Actually run loki in the validate-example-configurations pipeline stage

* Change method for running loki binary

* Add step dependency for config validation pipeline stage

* Actually loop over example configs and verify them!

* Push bad config to test failing builds

* Create remaining example configs

* Fix provided example loki configs

* Use make target to validate example configs

* Add make target to generate a new example configuration doc; add make target for CI to verify doc is re-generated upon changes to example configs

* Update CI to check the generated example config doc

* Clean up Makefile

* Mark new example config validation make targets as phony; fix conditional for config example doc checking

* Fix conditional

* Remove redundant step in generate-example-config-doc phony make target

* Add a correction to PR 4358

* Improve grammar and add newlines in example configuration descriptions

Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com>

* Regenerate examples.md

Co-authored-by: Karen Miller <karen.miller@grafana.com>
Co-authored-by: Karen Miller <84039272+KMiller-Grafana@users.noreply.github.com>

.drone/drone.jsonnet

@@ -284,6 +284,9 @@ local manifest(apps) = pipeline('manifest') {
         image: 'koalaman/shellcheck-alpine:stable',
         commands: ['apk add make bash && make lint-scripts'],
       },
+      make('loki', container=false) { depends_on: ['clone'] },
+      make('validate-example-configs', container=false) { depends_on: ['loki'] },
+      make('check-example-config-doc', container=false) { depends_on: ['clone'] },
     ],
   },
 ] + [

.drone/drone.yml

@@ -46,6 +46,27 @@ steps:
   commands:
   - apk add make bash && make lint-scripts
 
+- name: loki
+  image: grafana/loki-build-image:0.17.0
+  commands:
+  - make BUILD_IN_CONTAINER=false loki
+  depends_on:
+  - clone
+
+- name: validate-example-configs
+  image: grafana/loki-build-image:0.17.0
+  commands:
+  - make BUILD_IN_CONTAINER=false validate-example-configs
+  depends_on:
+  - loki
+
+- name: check-example-config-doc
+  image: grafana/loki-build-image:0.17.0
+  commands:
+  - make BUILD_IN_CONTAINER=false check-example-config-doc
+  depends_on:
+  - clone
+
 ---
 kind: pipeline
 name: docker-amd64
@@ -973,6 +994,6 @@ get:
 
 ---
 kind: signature
-hmac: b70be41d1a7f91c11af945a34bdbdc7a4f7613cf830c13f5438dba0bf33a1ec5
+hmac: f1681daaa15376eb15eae5dd623a3ec19544b0447506aba2d59a92f0fd8fee49
 
 ...

Makefile

@@ -7,6 +7,7 @@
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-mod
 .PHONY: migrate migrate-image lint-markdown ragel
+.PHONY: validate-example-configs generate-example-config-doc check-example-config-doc
 
 SHELL = /usr/bin/env bash
 
@@ -635,3 +636,30 @@ format:
 		-type f -name '*.go' -exec gofmt -w -s {} \;
 	find . $(DONT_FIND) -name '*.pb.go' -prune -o -name '*.y.go' -prune -o -name '*.rl.go' -prune -o \
 		-type f -name '*.go' -exec goimports -w -local github.com/grafana/loki {} \;
+
+###################
+# Example Configs #
+###################
+
+# Validate the example configurations that we provide in ./docs/sources/configuration/examples
+validate-example-configs: loki
+	for f in ./docs/sources/configuration/examples/*.yaml; do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done
+
+# Dynamically generate ./docs/sources/configuration/examples.md using the example configs that we provide.
+# This target should be run if any of our example configs change.
+generate-example-config-doc:
+	echo "Removing existing doc at loki/docs/configuration/examples.md and re-generating. . ."
+	# Title and Heading
+	echo -e "---\ntitle: Examples\n---\n # Loki Configuration Examples" > ./docs/sources/configuration/examples.md
+	# Append each configuration and its file name to examples.md
+	for f in ./docs/sources/configuration/examples/*.yaml; do echo -e "\n## $$(basename $$f)\n\n\`\`\`yaml\n$$(cat $$f)\n\`\`\`\n" >> ./docs/sources/configuration/examples.md; done
+
+# Fail our CI build if changes are made to example configurations but our doc is not updated
+check-example-config-doc: generate-example-config-doc
+	@if ! (git diff --exit-code ./docs/sources/configuration/examples.md); then \
+		echo -e "\nChanges found in generated example configuration doc"; \
+		echo "Run 'make generate-example-config-doc' and commit the changes to fix this error."; \
+		echo "If you are actively developing these files you can ignore this error"; \
+		echo -e "(Don't forget to check in the generated files when finished)\n"; \
+		exit 1; \
+	fi

docs/sources/configuration/examples.md

@@ -1,109 +1,112 @@
 ---
 title: Examples
 ---
-# Grafana Loki Configuration Examples
+ # Loki Configuration Examples
 
-## Complete Local configuration
+## almost-zero-dependency.yaml
 
 ```yaml
+# This is a configuration to deploy Loki depending only on a storage solution 
+# for example, an S3-compatible API like MinIO. 
+# The ring configuration is based on the gossip memberlist and the index is shipped to storage 
+# via Single Store (boltdb-shipper)
+
 auth_enabled: false
 
 server:
   http_listen_port: 3100
 
+distributor:
+  ring:
+    kvstore:
+      store: memberlist
+
 ingester:
   lifecycler:
-    address: 127.0.0.1
     ring:
       kvstore:
-        store: inmemory
+        store: memberlist
       replication_factor: 1
     final_sleep: 0s
   chunk_idle_period: 5m
   chunk_retain_period: 30s
 
+memberlist:
+  abort_if_cluster_join_fails: false
+
+  # Expose this port on all distributor, ingester
+  # and querier replicas.
+  bind_port: 7946
+
+  # You can use a headless k8s service for all distributor,
+  # ingester and querier components.
+  join_members:
+  - loki-gossip-ring.loki.svc.cluster.local:7946
+
+  max_join_backoff: 1m
+  max_join_retries: 10
+  min_join_backoff: 1s
+
 schema_config:
   configs:
   - from: 2020-05-15
-    store: boltdb
-    object_store: filesystem
+    store: boltdb-shipper
+    object_store: s3
     schema: v11
     index:
       prefix: index_
-      period: 168h
+      period: 24h
 
 storage_config:
-  boltdb:
-    directory: /tmp/loki/index
+ boltdb_shipper:
+   active_index_directory: /loki/index
+   cache_location: /loki/index_cache
+   shared_store: s3
 
-  filesystem:
-    directory: /tmp/loki/chunks
+ aws:
+   s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
+   s3forcepathstyle: true
 
 limits_config:
   enforce_metric_name: false
   reject_old_samples: true
   reject_old_samples_max_age: 168h
 
+compactor:
+  working_directory: /data/compactor
+  shared_store: s3
+  compaction_interval: 5m
 ```
 
-## Google Cloud Storage
 
-This is partial config that uses GCS and Bigtable for the chunk and index
-stores, respectively.
+## aws-basic-config-no-creds.yaml
 
 ```yaml
-schema_config:
-  configs:
-  - from: 2020-05-15
-    store: bigtable
-    object_store: gcs
-    schema: v11
-    index:
-      prefix: loki_index_
-      period: 168h
-
-storage_config:
-  bigtable:
-    instance: BIGTABLE_INSTANCE
-    project: BIGTABLE_PROJECT
-  gcs:
-    bucket_name: GCS_BUCKET_NAME
-```
-
-## Cassandra Index
+# If you don't wish to hard-code S3 credentials you can also configure an EC2
+# instance role by changing the `storage_config` section
 
-This is a partial config that uses the local filesystem for chunk storage and
-Cassandra for the index storage:
-
-```yaml
 schema_config:
   configs:
   - from: 2020-05-15
-    store: cassandra
-    object_store: filesystem
+    store: aws
+    object_store: s3
     schema: v11
     index:
-      prefix: cassandra_table
-      period: 168h
-
+      prefix: loki_
 storage_config:
-  cassandra:
-    username: cassandra
-    password: cassandra
-    addresses: 127.0.0.1
-    auth: true
-    keyspace: lokiindex
-
-  filesystem:
-    directory: /tmp/loki/chunks
+  aws:
+    s3: s3://region/bucket_name
+    dynamodb:
+      dynamodb_url: dynamodb://region
+      
 ```
 
-## AWS
 
-This is a partial config that uses S3 for chunk storage and DynamoDB for the
-index storage:
+## aws-basic-config.yaml
 
 ```yaml
+# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage
+
 schema_config:
   configs:
   - from: 2020-05-15
@@ -117,59 +120,40 @@ storage_config:
     s3: s3://access_key:secret_access_key@region/bucket_name
     dynamodb:
       dynamodb_url: dynamodb://access_key:secret_access_key@region
+      
 ```
 
-If you don't wish to hard-code S3 credentials, you can also configure an EC2
-instance role by changing the `storage_config` section:
-
-```yaml
-storage_config:
-  aws:
-    s3: s3://region/bucket_name
-    dynamodb:
-      dynamodb_url: dynamodb://region
-```
 
-### S3-compatible APIs
-
-S3-compatible APIs (e.g. Ceph Object Storage with an S3-compatible API) can be
-used. If the API supports path-style URL rather than virtual hosted bucket
-addressing, configure the URL in `storage_config` with the custom endpoint:
+## cassandra-index.yaml
 
 ```yaml
-storage_config:
-  aws:
-    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
-    s3forcepathstyle: true
-```
-
-### S3 Expanded Config
+# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage
 
-S3 config now supports expanded config. Either `s3` endpoint URL can be used
-or expanded config can be used.
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: cassandra
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: cassandra_table
+      period: 168h
 
-```yaml
 storage_config:
-  aws:
-    bucketnames: bucket_name1, bucket_name2
-    endpoint: s3.endpoint.com
-    region: s3_region
-    access_key_id: s3_access_key_id
-    secret_access_key: s3_secret_access_key
-    insecure: false
-    sse_encryption: false
-    http_config:
-      idle_conn_timeout: 90s
-      response_header_timeout: 0s
-      insecure_skip_verify: false
-    s3forcepathstyle: true
+  cassandra:
+    username: cassandra
+    password: cassandra
+    addresses: 127.0.0.1
+    auth: true
+    keyspace: lokiindex
+
+  filesystem:
+    directory: /tmp/loki/chunks
+    
 ```
 
-## Almost zero dependencies setup
 
-This is a configuration to deploy Loki depending only on storage solution, e.g. an
-S3-compatible API like minio. The ring configuration is based on the gossip memberlist
-and the index is shipped to storage via [Single Store (boltdb-shipper)](../../operations/storage/boltdb-shipper/).
+## complete-local-config.yaml
 
 ```yaml
 auth_enabled: false
@@ -177,75 +161,51 @@ auth_enabled: false
 server:
   http_listen_port: 3100
 
-distributor:
-  ring:
-    kvstore:
-      store: memberlist
-
 ingester:
   lifecycler:
+    address: 127.0.0.1
     ring:
       kvstore:
-        store: memberlist
+        store: inmemory
       replication_factor: 1
     final_sleep: 0s
   chunk_idle_period: 5m
   chunk_retain_period: 30s
 
-memberlist:
-  abort_if_cluster_join_fails: false
-
-  # Expose this port on all distributor, ingester
-  # and querier replicas.
-  bind_port: 7946
-
-  # You can use a headless k8s service for all distributor,
-  # ingester and querier components.
-  join_members:
-  - loki-gossip-ring.loki.svc.cluster.local:7946
-
-  max_join_backoff: 1m
-  max_join_retries: 10
-  min_join_backoff: 1s
-
 schema_config:
   configs:
   - from: 2020-05-15
-    store: boltdb-shipper
-    object_store: s3
+    store: boltdb
+    object_store: filesystem
     schema: v11
     index:
       prefix: index_
-      period: 24h
+      period: 168h
 
 storage_config:
- boltdb_shipper:
-   active_index_directory: /loki/index
-   cache_location: /loki/index_cache
-   shared_store: s3
+  boltdb:
+    directory: /tmp/loki/index
 
- aws:
-   s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
-   s3forcepathstyle: true
+  filesystem:
+    directory: /tmp/loki/chunks
 
 limits_config:
   enforce_metric_name: false
   reject_old_samples: true
   reject_old_samples_max_age: 168h
-
-compactor:
-  working_directory: /data/compactor
-  shared_store: s3
-  compaction_interval: 5m
 ```
 
-## schema_config
+
+## example-schema-config.yaml
 
 ```yaml
-configs:
-  # Starting from 2018-04-15 Loki should store indexes on Cassandra
-  # using weekly periodic tables and chunks on filesystem.
-  # The index tables will be prefixed with "index_".
+# Additional example schema configuration for Cassandra
+
+schema_config:
+  configs:
+    # Starting from 2018-04-15 Loki should store indexes on Cassandra
+    # using weekly periodic tables and chunks on filesystem.
+    # The index tables will be prefixed with "index_".
   - from: "2018-04-15"
     store: cassandra
     object_store: filesystem
@@ -262,4 +222,87 @@ configs:
     index:
         period: 168h
         prefix: index_
+        
 ```
+
+
+## google-cloud-storage-config.yaml
+
+```yaml
+# This partial configuration uses GCS for chunk storage and uses BigTable for index storage
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: bigtable
+    object_store: gcs
+    schema: v11
+    index:
+      prefix: loki_index_
+      period: 168h
+
+storage_config:
+  bigtable:
+    instance: BIGTABLE_INSTANCE
+    project: BIGTABLE_PROJECT
+  gcs:
+    bucket_name: GCS_BUCKET_NAME
+    
+```
+
+
+## s3-compatible-apis.yaml
+
+```yaml
+# S3-compatible APIs such as Ceph Object Storage with an S3-compatible API, can be used.
+# If the API supports path-style URLs rather than virtual hosted bucket addressing,
+# configure the URL in `storage_config` with the custom endpoint
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    s3: s3://access_key:secret_access_key@region/bucket_name
+    dynamodb:
+      dynamodb_url: dynamodb://access_key:secret_access_key@region
+      
+```
+
+
+## s3-expanded-config.yaml
+
+```yaml
+# S3 configuration supports an expanded configuration. 
+# Either an `s3` endpoint URL can be used, or an expanded configuration can be used.
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    bucketnames: bucket_name1, bucket_name2
+    endpoint: s3.endpoint.com
+    region: s3_region
+    access_key_id: s3_access_key_id
+    secret_access_key: s3_secret_access_key
+    insecure: false
+    sse_encryption: false
+    http_config:
+      idle_conn_timeout: 90s
+      response_header_timeout: 0s
+      insecure_skip_verify: false
+    s3forcepathstyle: true
+    
+```
+

docs/sources/configuration/examples/almost-zero-dependency.yaml

@@ -0,0 +1,70 @@
+# This is a configuration to deploy Loki depending only on a storage solution 
+# for example, an S3-compatible API like MinIO. 
+# The ring configuration is based on the gossip memberlist and the index is shipped to storage 
+# via Single Store (boltdb-shipper)
+
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+distributor:
+  ring:
+    kvstore:
+      store: memberlist
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: memberlist
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+memberlist:
+  abort_if_cluster_join_fails: false
+
+  # Expose this port on all distributor, ingester
+  # and querier replicas.
+  bind_port: 7946
+
+  # You can use a headless k8s service for all distributor,
+  # ingester and querier components.
+  join_members:
+  - loki-gossip-ring.loki.svc.cluster.local:7946
+
+  max_join_backoff: 1m
+  max_join_retries: 10
+  min_join_backoff: 1s
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb-shipper
+    object_store: s3
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+ boltdb_shipper:
+   active_index_directory: /loki/index
+   cache_location: /loki/index_cache
+   shared_store: s3
+
+ aws:
+   s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
+   s3forcepathstyle: true
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+compactor:
+  working_directory: /data/compactor
+  shared_store: s3
+  compaction_interval: 5m

docs/sources/configuration/examples/aws-basic-config-no-creds.yaml

@@ -0,0 +1,17 @@
+# If you don't wish to hard-code S3 credentials you can also configure an EC2
+# instance role by changing the `storage_config` section
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    s3: s3://region/bucket_name
+    dynamodb:
+      dynamodb_url: dynamodb://region
+      

docs/sources/configuration/examples/aws-basic-config.yaml

@@ -0,0 +1,16 @@
+# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    s3: s3://access_key:secret_access_key@region/bucket_name
+    dynamodb:
+      dynamodb_url: dynamodb://access_key:secret_access_key@region
+      

docs/sources/configuration/examples/cassandra-index.yaml

@@ -0,0 +1,23 @@
+# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: cassandra
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: cassandra_table
+      period: 168h
+
+storage_config:
+  cassandra:
+    username: cassandra
+    password: cassandra
+    addresses: 127.0.0.1
+    auth: true
+    keyspace: lokiindex
+
+  filesystem:
+    directory: /tmp/loki/chunks
+    

docs/sources/configuration/examples/complete-local-config.yaml

@@ -0,0 +1,37 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /tmp/loki/index
+
+  filesystem:
+    directory: /tmp/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h

docs/sources/configuration/examples/example-schema-config.yaml

@@ -0,0 +1,24 @@
+# Additional example schema configuration for Cassandra
+
+schema_config:
+  configs:
+    # Starting from 2018-04-15 Loki should store indexes on Cassandra
+    # using weekly periodic tables and chunks on filesystem.
+    # The index tables will be prefixed with "index_".
+  - from: "2018-04-15"
+    store: cassandra
+    object_store: filesystem
+    schema: v11
+    index:
+        period: 168h
+        prefix: index_
+
+  # Starting from 2020-6-15 we moved from filesystem to AWS S3 for storing the chunks.
+  - from: "2020-06-15"
+    store: cassandra
+    object_store: s3
+    schema: v11
+    index:
+        period: 168h
+        prefix: index_
+        

docs/sources/configuration/examples/google-cloud-storage-config.yaml

@@ -0,0 +1,19 @@
+# This partial configuration uses GCS for chunk storage and uses BigTable for index storage
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: bigtable
+    object_store: gcs
+    schema: v11
+    index:
+      prefix: loki_index_
+      period: 168h
+
+storage_config:
+  bigtable:
+    instance: BIGTABLE_INSTANCE
+    project: BIGTABLE_PROJECT
+  gcs:
+    bucket_name: GCS_BUCKET_NAME
+    

docs/sources/configuration/examples/s3-compatible-apis.yaml

@@ -0,0 +1,18 @@
+# S3-compatible APIs such as Ceph Object Storage with an S3-compatible API, can be used.
+# If the API supports path-style URLs rather than virtual hosted bucket addressing,
+# configure the URL in `storage_config` with the custom endpoint
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    s3: s3://access_key:secret_access_key@region/bucket_name
+    dynamodb:
+      dynamodb_url: dynamodb://access_key:secret_access_key@region
+      

docs/sources/configuration/examples/s3-expanded-config.yaml

@@ -0,0 +1,26 @@
+# S3 configuration supports an expanded configuration. 
+# Either an `s3` endpoint URL can be used, or an expanded configuration can be used.
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: aws
+    object_store: s3
+    schema: v11
+    index:
+      prefix: loki_
+storage_config:
+  aws:
+    bucketnames: bucket_name1, bucket_name2
+    endpoint: s3.endpoint.com
+    region: s3_region
+    access_key_id: s3_access_key_id
+    secret_access_key: s3_secret_access_key
+    insecure: false
+    sse_encryption: false
+    http_config:
+      idle_conn_timeout: 90s
+      response_header_timeout: 0s
+      insecure_skip_verify: false
+    s3forcepathstyle: true
+