Fix compiled logproto protobuf (#1183)

* Enforce recompile of protobuf sources on 'make check-generated-files'

* Pinned version of protobuf in loki-build-image

* Pinned version of goyacc in loki-build-image

* Added 'make build-image-push' to Makefile

* Documented how to release loki-build-image

.circleci/config.yml

@@ -47,7 +47,7 @@ workflows:
 # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 .defaults: &defaults
   docker:
-    - image: grafana/loki-build-image:0.7.3
+    - image: grafana/loki-build-image:0.7.4
   working_directory: /src/loki
 
 jobs:

.drone/drone.yml

@@ -5,19 +5,19 @@ steps:
   - make BUILD_IN_CONTAINER=false test
   depends_on:
   - clone
-  image: grafana/loki-build-image:0.7.3
+  image: grafana/loki-build-image:0.7.4
   name: test
 - commands:
   - make BUILD_IN_CONTAINER=false lint
   depends_on:
   - clone
-  image: grafana/loki-build-image:0.7.3
+  image: grafana/loki-build-image:0.7.4
   name: lint
 - commands:
   - make BUILD_IN_CONTAINER=false check-generated-files
   depends_on:
   - clone
-  image: grafana/loki-build-image:0.7.3
+  image: grafana/loki-build-image:0.7.4
   name: check-generated-files
 - commands:
   - make BUILD_IN_CONTAINER=false check-mod
@@ -25,7 +25,7 @@ steps:
   - clone
   - test
   - lint
-  image: grafana/loki-build-image:0.7.3
+  image: grafana/loki-build-image:0.7.4
   name: check-mod
 workspace:
   base: /src
@@ -492,7 +492,7 @@ steps:
   environment:
     CIRCLE_TOKEN:
       from_secret: circle_token
-  image: grafana/loki-build-image:0.7.3
+  image: grafana/loki-build-image:0.7.4
   name: trigger
 trigger:
   ref:

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := all
-.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos
+.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources
 .PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean
 .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push
 .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test
@@ -20,7 +20,7 @@ IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,
 # make BUILD_IN_CONTAINER=false target
 # or you can override this with an environment variable
 BUILD_IN_CONTAINER ?= true
-BUILD_IMAGE_VERSION := 0.7.3
+BUILD_IMAGE_VERSION := 0.7.4
 
 # Docker image info
 IMAGE_PREFIX ?= grafana
@@ -106,15 +106,22 @@ binfmt:
 all: promtail logcli loki loki-canary check-generated-files
 
 # This is really a check for the CI to make sure generated files are built and checked in manually
-check-generated-files: yacc protos pkg/promtail/server/ui/assets_vfsdata.go
+check-generated-files: touch-protobuf-sources yacc protos pkg/promtail/server/ui/assets_vfsdata.go
 	@if ! (git diff --exit-code $(YACC_GOS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \
 		echo "\nChanges found in generated files"; \
-		echo "Run 'make all' and commit the changes to fix this error."; \
+		echo "Run 'make check-generated-files' and commit the changes to fix this error."; \
 		echo "If you are actively developing these files you can ignore this error"; \
 		echo "(Don't forget to check in the generated files when finished)\n"; \
 		exit 1; \
 	fi
 
+# Trick used to ensure that protobuf files are always compiled even if not changed, because the
+# tooling may have been upgraded and the compiled output may be different. We're not using a
+# PHONY target so that we can control where we want to touch it.
+touch-protobuf-sources:
+	for def in $(PROTO_DEFS); do \
+		touch $$def; \
+	done
 
 ##########
 # Logcli #
@@ -463,6 +470,10 @@ loki-canary-push: loki-canary-image-cross
 build-image: OCI_PLATFORMS=
 build-image:
 	$(SUDO) $(BUILD_OCI) -t $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) ./loki-build-image
+build-image-push: build-image
+	$(call push,loki-build-image,$(BUILD_IMAGE_VERSION))
+	$(call push,loki-build-image,latest)
+
 
 ########
 # Misc #

cmd/docker-driver/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .

cmd/loki-canary/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/promtail -f cmd/promtail/Dockerfile .

cmd/loki/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/loki -f cmd/loki/Dockerfile .

cmd/promtail/Dockerfile.cross

@@ -1,4 +1,4 @@
-ARG BUILD_IMAGE=grafana/loki-build-image:0.7.3
+ARG BUILD_IMAGE=grafana/loki-build-image:0.7.4
 # Directories in this file are referenced from the root of the project not this folder
 # This file is intented to be called from the root like so:
 # docker build -t grafana/promtail -f cmd/promtail/Dockerfile .

docs/maintaining/README.md

@@ -3,3 +3,4 @@
 This section details information for maintainers of Loki.
 
 1. [Releasing Loki](./release.md)
+2. [Releasing `loki-build-image`](./release-loki-build-image.md)

docs/maintaining/release-loki-build-image.md

@@ -0,0 +1,8 @@
+# Releasing `loki-build-image`
+
+The [`loki-build-image`](../../loki-build-image/) is the Docker image used to run tests and build Loki binaries in CI.
+
+## How To Perform a Release
+
+1. Update `BUILD_IMAGE_VERSION` in the `Makefile`
+2. Run `make build-image-push`

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/fluent/fluent-bit-go v0.0.0-20190925192703-ea13c021720c
 	github.com/go-kit/kit v0.9.0
 	github.com/gocql/gocql v0.0.0-20181124151448-70385f88b28b // indirect
-	github.com/gogo/protobuf v1.3.0
+	github.com/gogo/protobuf v1.3.0 // remember to update loki-build-image/Dockerfile too
 	github.com/golang/snappy v0.0.1
 	github.com/gorilla/mux v1.6.2
 	github.com/gorilla/websocket v1.4.0

loki-build-image/Dockerfile

@@ -26,12 +26,17 @@ RUN apt-get update && \
 COPY --from=docker /usr/bin/docker /usr/bin/docker
 COPY --from=helm /usr/bin/helm /usr/bin/helm
 COPY --from=golangci /bin/golangci-lint /usr/local/bin
-RUN go get \
-    github.com/golang/protobuf/protoc-gen-go \
-    github.com/gogo/protobuf/protoc-gen-gogoslick \
-    github.com/gogo/protobuf/gogoproto \
+
+# Enable go 1.11 modules to be able to install pinned version of dependencies
+# even if we're installing them in the GOPATH
+RUN GO111MODULE=on go get \
+    github.com/golang/protobuf/protoc-gen-go@v1.3.0 \
+    github.com/gogo/protobuf/protoc-gen-gogoslick@v1.3.0 \
+    github.com/gogo/protobuf/gogoproto@v1.3.0 \
     github.com/go-delve/delve/cmd/dlv \
-    golang.org/x/tools/cmd/goyacc \
+    # Due to the lack of a proper release tag, we use the commit hash of
+    # https://github.com/golang/tools/releases v0.1.7
+    golang.org/x/tools/cmd/goyacc@58d531046acdc757f177387bc1725bfa79895d69 \
     github.com/mitchellh/gox \
     github.com/tcnksm/ghr && \
     rm -rf /go/pkg /go/src