Introduce coverage to PR pipelines (#5357)

* Add code coverage diff to CI

* remove test package from coverage diff

* review feedback

* lint

* Env variable test

* faster experiment

* drone vars

* drone vars

* how does the shell work?

* add pr comment

* escape json

* report diff in comment

* properly format json

* quote post body

* review feedback

* add querier/queryrange to coverage

.drone/drone.jsonnet

@@ -46,15 +46,19 @@ local github_secret = secret('github_token', 'infra/data/ci/github/grafanabot',
 // Injected in a secret because this is a public repository and having the config here would leak our environment names
 local deploy_configuration = secret('deploy_config', 'secret/data/common/loki_ci_autodeploy', 'config.json');
 
-
-local run(name, commands) = {
+local run(name, commands, env={}) = {
   name: name,
   image: 'grafana/loki-build-image:%s' % build_image_version,
   commands: commands,
+  environment: env,
 };
 
-local make(target, container=true) = run(target, [
-  'make ' + (if !container then 'BUILD_IN_CONTAINER=false ' else '') + target,
+local make(target, container=true, args=[]) = run(target, [
+  std.join(' ', [
+    'make',
+    'BUILD_IN_CONTAINER=' + container,
+    target,
+  ] + args),
 ]);
 
 local docker(arch, app) = {
@@ -369,7 +373,23 @@ local manifest(apps) = pipeline('manifest') {
     steps: [
       make('check-drone-drift', container=false) { depends_on: ['clone'] },
       make('check-generated-files', container=false) { depends_on: ['clone'] },
-      make('test', container=false) { depends_on: ['clone', 'check-generated-files'] },
+      make('test', container=false) { depends_on: ['clone'] },
+      run('clone-main', commands=['cd ..', 'git clone $CI_REPO_REMOTE loki-main', 'cd -']),
+      run('test-main', commands=['cd ../loki-main', 'BUILD_IN_CONTAINER=false make test']) { depends_on: ['clone-main'] },
+      make('compare-coverage', container=false, args=[
+        'old=../loki-main/test_results.txt',
+        'new=test_results.txt',
+        'packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki',
+        '> diff.txt',
+      ]) { depends_on: ['test', 'test-main'] },
+      run('report-coverage', commands=[
+        "pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')",
+        "body=$(jq -Rs '{body: . }' diff.txt)",
+        'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments -d "$body" > /dev/null',
+      ], env={
+        USER: 'grafanabot',
+        TOKEN: { from_secret: github_secret.name },
+      }) { depends_on: ['compare-coverage'] },
       make('lint', container=false) { depends_on: ['clone', 'check-generated-files'] },
       make('check-mod', container=false) { depends_on: ['clone', 'test', 'lint'] },
       {

.drone/drone.yml

@@ -40,26 +40,67 @@ steps:
   - make BUILD_IN_CONTAINER=false check-drone-drift
   depends_on:
   - clone
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: check-drone-drift
 - commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: check-generated-files
 - commands:
   - make BUILD_IN_CONTAINER=false test
   depends_on:
   - clone
-  - check-generated-files
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: test
+- commands:
+  - cd ..
+  - git clone $CI_REPO_REMOTE loki-main
+  - cd -
+  environment: {}
+  image: grafana/loki-build-image:0.20.4
+  name: clone-main
+- commands:
+  - cd ../loki-main
+  - BUILD_IN_CONTAINER=false make test
+  depends_on:
+  - clone-main
+  environment: {}
+  image: grafana/loki-build-image:0.20.4
+  name: test-main
+- commands:
+  - make BUILD_IN_CONTAINER=false compare-coverage old=../loki-main/test_results.txt
+    new=test_results.txt packages=ingester,distributor,querier,querier/queryrange,iter,storage,chunkenc,logql,loki
+    > diff.txt
+  depends_on:
+  - test
+  - test-main
+  environment: {}
+  image: grafana/loki-build-image:0.20.4
+  name: compare-coverage
+- commands:
+  - pull=$(echo $CI_COMMIT_REF | awk -F '/' '{print $3}')
+  - 'body=$(jq -Rs ''{body: . }'' diff.txt)'
+  - 'curl -X POST -u $USER:$TOKEN -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/grafana/loki/issues/$pull/comments
+    -d "$body" > /dev/null'
+  depends_on:
+  - compare-coverage
+  environment:
+    TOKEN:
+      from_secret: github_token
+    USER: grafanabot
+  image: grafana/loki-build-image:0.20.4
+  name: report-coverage
 - commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
   - clone
   - check-generated-files
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: lint
 - commands:
@@ -68,6 +109,7 @@ steps:
   - clone
   - test
   - lint
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: check-mod
 - commands:
@@ -79,18 +121,21 @@ steps:
   depends_on:
   - clone
   - check-generated-files
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: loki
 - commands:
   - make BUILD_IN_CONTAINER=false validate-example-configs
   depends_on:
   - loki
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: validate-example-configs
 - commands:
   - make BUILD_IN_CONTAINER=false check-example-config-doc
   depends_on:
   - clone
+  environment: {}
   image: grafana/loki-build-image:0.20.4
   name: check-example-config-doc
 trigger:
@@ -109,6 +154,7 @@ steps:
   - make BUILD_IN_CONTAINER=false lint-jsonnet
   depends_on:
   - clone
+  environment: {}
   image: grafana/jsonnet-build:c8b75df
   name: lint-jsonnet
 trigger:
@@ -1118,6 +1164,6 @@ kind: secret
 name: deploy_config
 ---
 kind: signature
-hmac: 96966b3eec7f8976408f2c2f2c36a29b87a694c8a32afb2fdb16209e1f9e7521
+hmac: 4596e741ac788d461b3bbb2429c1f61efabaf943aeec6b3cd59eeff8d769de5e
 
 ...

.gitignore

@@ -27,6 +27,7 @@ dlv
 rootfs/
 dist
 coverage.txt
+test_results.txt
 .DS_Store
 .aws-sam
 .idea
@@ -40,4 +41,4 @@ coverage.txt
 *.tfvars
 
 # vscode
-.vscode
+.vscode

Makefile

@@ -10,7 +10,7 @@
 .PHONY: validate-example-configs generate-example-config-doc check-example-config-doc
 .PHONY: clean clean-protos
 
-SHELL = /usr/bin/env bash
+SHELL = /usr/bin/env bash -o pipefail
 
 GOTEST ?= go test
 
@@ -260,7 +260,10 @@ lint:
 ########
 
 test: all
-	$(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./...
+	$(GOTEST) -covermode=atomic -coverprofile=coverage.txt -p=4 ./... | tee test_results.txt
+
+compare-coverage:
+	./tools/diff_coverage.sh $(old) $(new) $(packages)
 
 #########
 # Clean #

loki-build-image/Dockerfile

@@ -68,7 +68,7 @@ RUN apt-get update && \
     musl gnupg ragel \
     file zip unzip jq gettext\
     protobuf-compiler libprotobuf-dev \
-    libsystemd-dev && \
+    libsystemd-dev jq && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY --from=docker /usr/bin/docker /usr/bin/docker

tools/diff_coverage.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ ! -f "$1" ]] || [[ ! -f "$2" ]]; then
+    echo "unable to compare test coverage: both old and new files must exist"
+    exit 0
+fi
+
+echo '```diff'
+for pkg in ${3//,/ }; do
+  old=$(grep "pkg/${pkg}\s" "$1" | sed s/%// | awk '{print $5}')
+  new=$(grep "pkg/${pkg}\s" "$2" | sed s/%// | awk '{print $5}')
+  echo | awk -v pkg="${pkg}" -v old="${old:-0}" -v new="${new:-0}" \
+  '{
+      sign=new - old < 0 ? "-" : "+"
+      printf ("%s %11s\t%s\n", sign, pkg, new - old)
+  }'
+done
+echo '```'