deb and rpm packaging for all binaries (#6456)

* deb and rpm packaging for all binaries
There is currently deb and rpm packages being published for logcli amd64 but they aren't signed

In this PR:
- Add RPM and deb packages for `logcli`, `loki-canary` `loki` and `promtail` for arm, arm64 and amd64
- Sign them with the Grafana GPG key (packages.grafana.com/gpg.key). This will be the same key for all Grafana products
- Add CI steps to test the RPM and deb packages. This launches a docker image with systemd installed, then it installs both loki and promtail and does a `logcli` query on the Loki instance

* Remove specific version in test scripts

* Bump build image to 0.22.0

* Add restart policy for promtail's service

.drone/drone.jsonnet

@@ -39,6 +39,8 @@ local ecr_key = secret('ecr_key', 'infra/data/ci/loki/aws-credentials', 'access_
 local ecr_secret_key = secret('ecr_secret_key', 'infra/data/ci/loki/aws-credentials', 'secret_access_key');
 local pull_secret = secret('dockerconfigjson', 'secret/data/common/gcr', '.dockerconfigjson');
 local github_secret = secret('github_token', 'infra/data/ci/github/grafanabot', 'pat');
+local gpg_passphrase = secret('gpg_passphrase', 'infra/data/ci/packages-publish/gpg', 'passphrase');
+local gpg_private_key = secret('gpg_private_key', 'infra/data/ci/packages-publish/gpg', 'private-key');
 
 // Injected in a secret because this is a public repository and having the config here would leak our environment names
 local deploy_configuration = secret('deploy_config', 'secret/data/common/loki_ci_autodeploy', 'config.json');
@@ -580,18 +582,92 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
       event: ['pull_request', 'tag'],
     },
     image_pull_secrets: [pull_secret.name],
+    volumes+: [
+      {
+        name: 'cgroup',
+        host: {
+          path: '/sys/fs/cgroup',
+        },
+      },
+      {
+        name: 'docker',
+        host: {
+          path: '/var/run/docker.sock',
+        },
+      },
+    ],
+    // Launch docker images with systemd
+    services: [
+      {
+        name: 'systemd-debian',
+        image: 'jrei/systemd-debian:12',
+        volumes: [
+          {
+            name: 'cgroup',
+            path: '/sys/fs/cgroup',
+          },
+        ],
+        privileged: true,
+      },
+      {
+        name: 'systemd-centos',
+        image: 'jrei/systemd-centos:8',
+        volumes: [
+          {
+            name: 'cgroup',
+            path: '/sys/fs/cgroup',
+          },
+        ],
+        privileged: true,
+      },
+    ],
+    // Package and test the packages
     steps: [
-      run(
+      run('write-key',
-        'test packaging',
+          commands=['printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE'],
-        commands=['make BUILD_IN_CONTAINER=false packages']
+          env={
-      ) { when: { event: ['pull_request'] } },
+            NFPM_SIGNING_KEY: { from_secret: gpg_private_key.name },
-      run(
+            NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
-        'publish',
+          }),
-        commands=['make BUILD_IN_CONTAINER=false publish'],
+      run('test packaging',
-        env={
+          commands=[
-          GITHUB_TOKEN: { from_secret: github_secret.name },
+            'make BUILD_IN_CONTAINER=false packages',
-        }
+          ],
-      ) { when: { event: ['tag'] } },
+          env={
+            NFPM_PASSPHRASE: { from_secret: gpg_passphrase.name },
+            NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
+          }),
+      {
+        name: 'test deb package',
+        image: 'docker',
+        commands: ['./tools/packaging/verify-deb-install.sh'],
+        volumes: [
+          {
+            name: 'docker',
+            path: '/var/run/docker.sock',
+          },
+        ],
+        privileged: true,
+      },
+      {
+        name: 'test rpm package',
+        image: 'docker',
+        commands: ['./tools/packaging/verify-rpm-install.sh'],
+        volumes: [
+          {
+            name: 'docker',
+            path: '/var/run/docker.sock',
+          },
+        ],
+        privileged: true,
+      },
+      run('publish',
+          commands=['make BUILD_IN_CONTAINER=false publish'],
+          env={
+            GITHUB_TOKEN: { from_secret: github_secret.name },
+            NFPM_PASSPHRASE: { from_secret: gpg_passphrase.name },
+            NFPM_SIGNING_KEY_FILE: '/drone/src/private-key.key',
+          }) { when: { event: ['tag'] } },
     ],
   },
 ]
@@ -610,4 +686,6 @@ local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
   ecr_key,
   ecr_secret_key,
   deploy_configuration,
+  gpg_passphrase,
+  gpg_private_key,
 ]

.drone/drone.yml

@@ -1183,20 +1183,60 @@ image_pull_secrets:
 - dockerconfigjson
 kind: pipeline
 name: release
+services:
+- image: jrei/systemd-debian:12
+  name: systemd-debian
+  privileged: true
+  volumes:
+  - name: cgroup
+    path: /sys/fs/cgroup
+- image: jrei/systemd-centos:8
+  name: systemd-centos
+  privileged: true
+  volumes:
+  - name: cgroup
+    path: /sys/fs/cgroup
 steps:
+- commands:
+  - printf "%s" "$NFPM_SIGNING_KEY" > $NFPM_SIGNING_KEY_FILE
+  environment:
+    NFPM_SIGNING_KEY:
+      from_secret: gpg_private_key
+    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
+  image: grafana/loki-build-image:0.22.0
+  name: write-key
 - commands:
   - make BUILD_IN_CONTAINER=false packages
-  environment: {}
+  environment:
+    NFPM_PASSPHRASE:
+      from_secret: gpg_passphrase
+    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
   image: grafana/loki-build-image:0.22.0
   name: test packaging
-  when:
+- commands:
-    event:
+  - ./tools/packaging/verify-deb-install.sh
-    - pull_request
+  image: docker
+  name: test deb package
+  privileged: true
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+- commands:
+  - ./tools/packaging/verify-rpm-install.sh
+  image: docker
+  name: test rpm package
+  privileged: true
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
 - commands:
   - make BUILD_IN_CONTAINER=false publish
   environment:
     GITHUB_TOKEN:
       from_secret: github_token
+    NFPM_PASSPHRASE:
+      from_secret: gpg_passphrase
+    NFPM_SIGNING_KEY_FILE: /drone/src/private-key.key
   image: grafana/loki-build-image:0.22.0
   name: publish
   when:
@@ -1211,6 +1251,13 @@ trigger:
   - refs/heads/k???
   - refs/tags/v*
   - refs/pull/*/head
+volumes:
+- host:
+    path: /sys/fs/cgroup
+  name: cgroup
+- host:
+    path: /var/run/docker.sock
+  name: docker
 ---
 depends_on:
 - check
@@ -1416,7 +1463,19 @@ get:
 kind: secret
 name: deploy_config
 ---
+get:
+  name: passphrase
+  path: infra/data/ci/packages-publish/gpg
+kind: secret
+name: gpg_passphrase
+---
+get:
+  name: private-key
+  path: infra/data/ci/packages-publish/gpg
+kind: secret
+name: gpg_private_key
+---
 kind: signature
-hmac: 4fa63210e8bdd946f1732051b6b841eba0ae1b0313436301c743341d29893c8f
+hmac: 5211ba463ba49d82d8930928f2ea05232999d07d37f0a51dc8b535bfcd10727d
 
 ...

Makefile

@@ -263,11 +263,7 @@ dist: clean
 	pushd dist && sha256sum * > SHA256SUMS && popd
 
 packages: dist
-	mkdir -p dist/tmp
+	@tools/packaging/nfpm.sh
-	unzip dist/logcli-linux-amd64.zip -d dist/tmp
-	nfpm package -f tools/nfpm.yaml -p rpm -t dist/
-	nfpm package -f tools/nfpm.yaml -p deb -t dist/
-	rm -rf dist/tmp
 
 publish: packages
 	./tools/release

tools/nfpm.yaml

@@ -1,18 +0,0 @@
----
-name: "logcli"
-arch: "amd64"
-platform: "linux"
-version: ${DRONE_TAG}
-section: "default"
-provides:
-  - logcli
-maintainer: "Grafana Labs <support@grafana.com>"
-description: |
-  LogCLI is the command-line interface to Loki.
-  It facilitates running LogQL queries against a Loki instance.
-vendor: "Grafana Labs Inc"
-homepage: "https://grafana.com/loki"
-license: "AGPL-3.0"
-contents:
-  - src: ./dist/tmp/logcli-linux-amd64
-    dst: /usr/local/bin/logcli

tools/packaging/loki-postinstall.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+# Based on https://nfpm.goreleaser.com/tips/
+
+if ! command -V systemctl >/dev/null 2>&1; then
+  echo "Could not find systemd. Skipping system installation." && exit 0
+else
+    systemd_version=$(systemctl --version | head -1 | sed 's/systemd //g')
+fi
+
+cleanInstall() {
+    printf "\033[32m Post Install of a clean install\033[0m\n"
+
+    # Create the user
+    if ! id loki > /dev/null 2>&1 ; then
+        adduser --system --shell /bin/false "loki"
+    fi
+
+    # rhel/centos7 cannot use ExecStartPre=+ to specify the pre start should be run as root
+    # even if you want your service to run as non root.
+    if [ "${systemd_version}" -lt 231 ]; then
+        printf "\033[31m systemd version %s is less then 231, fixing the service file \033[0m\n" "${systemd_version}"
+        sed -i "s/=+/=/g" /etc/systemd/system/loki.service
+    fi
+    printf "\033[32m Reload the service unit from disk\033[0m\n"
+    systemctl daemon-reload ||:
+    printf "\033[32m Unmask the service\033[0m\n"
+    systemctl unmask loki ||:
+    printf "\033[32m Set the preset flag for the service unit\033[0m\n"
+    systemctl preset loki ||:
+    printf "\033[32m Set the enabled flag for the service unit\033[0m\n"
+    systemctl enable loki ||:
+    systemctl restart loki ||:
+}
+
+upgrade() {
+    :
+    # printf "\033[32m Post Install of an upgrade\033[0m\n"
+}
+
+action="$1"
+if  [ "$1" = "configure" ] && [ -z "$2" ]; then
+  # Alpine linux does not pass args, and deb passes $1=configure
+  action="install"
+elif [ "$1" = "configure" ] && [ -n "$2" ]; then
+    # deb passes $1=configure $2=<current version>
+    action="upgrade"
+fi
+
+case "${action}" in
+  "1" | "install")
+    cleanInstall
+    ;;
+  "2" | "upgrade")
+    upgrade
+    ;;
+  *)
+    # $1 == version being installed
+    printf "\033[32m Alpine\033[0m"
+    cleanInstall
+    ;;
+esac

tools/packaging/loki.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Loki service
+After=network.target
+
+[Service]
+Type=simple
+User=loki
+ExecStart=/usr/bin/loki -config.file /etc/loki/config.yml
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec = 120
+Restart = on-failure
+RestartSec = 2
+
+[Install]
+WantedBy=multi-user.target

tools/packaging/nfpm.jsonnet

@@ -0,0 +1,90 @@
+local overrides = {
+  logcli: {
+    description:
+      |||
+        LogCLI is the command-line interface to Loki.
+        It facilitates running LogQL queries against a Loki instance.
+      |||,
+  },
+
+  'loki-canary': {
+    description: 'Loki Canary is a standalone app that audits the log-capturing performance of a Grafana Loki cluster.',
+  },
+
+  loki: {
+    description: |||
+      Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. 
+      It is designed to be very cost effective and easy to operate. 
+      It does not index the contents of the logs, but rather a set of labels for each log stream.
+    |||,
+    contents+: [
+      {
+        src: './tools/packaging/loki.service',
+        dst: '/etc/systemd/system/loki.service',
+      },
+      {
+        src: './cmd/loki/loki-local-config.yaml',
+        dst: '/etc/loki/config.yml',
+        type: 'config|noreplace',
+      },
+    ],
+    scripts: {
+      postinstall: './tools/packaging/loki-postinstall.sh',
+    },
+  },
+
+  promtail: {
+    description: |||
+      Promtail is an agent which ships the contents of local logs to a private Grafana Loki instance or Grafana Cloud. 
+      It is usually deployed to every machine that has applications needed to be monitored.
+    |||,
+    license: 'Apache-2.0',
+    contents+: [
+      {
+        src: './tools/packaging/promtail.service',
+        dst: '/etc/systemd/system/promtail.service',
+      },
+      {
+        src: './clients/cmd/promtail/promtail-local-config.yaml',
+        dst: '/etc/promtail/config.yml',
+        type: 'config|noreplace',
+      },
+    ],
+    scripts: {
+      postinstall: './tools/packaging/promtail-postinstall.sh',
+    },
+  },
+};
+
+local name = std.extVar('name');
+local arch = std.extVar('arch');
+
+{
+  name: name,
+  arch: arch,
+  platform: 'linux',
+  version: '${DRONE_TAG}',
+  section: 'default',
+  provides: [name],
+  maintainer: 'Grafana Labs <support@grafana.com>',
+  vendor: 'Grafana Labs Inc',
+  homepage: 'https://grafana.com/loki',
+  license: 'AGPL-3.0',
+  contents: [{
+    src: './dist/tmp/packages/%s-linux-%s' % [name, arch],
+    dst: '/usr/bin/%s' % name,
+  }],
+
+  deb: {
+    signature: {
+      // Also set ${NFPM_PASSPHRASE}
+      key_file: '${NFPM_SIGNING_KEY_FILE}',
+    },
+  },
+  rpm: {
+    signature: {
+      // Also set ${NFPM_PASSPHRASE}
+      key_file: '${NFPM_SIGNING_KEY_FILE}',
+    },
+  },
+} + overrides[name]

tools/packaging/nfpm.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+if [[ -z "${NFPM_SIGNING_KEY_FILE}" ]]; then
+    echo "NFPM_SIGNING_KEY_FILE is not set"
+    exit 1
+fi
+if [[ -z "${NFPM_PASSPHRASE}" ]]; then
+    echo "NFPM_PASSPHRASE is not set"
+    exit 1
+fi
+
+rm -rf dist/tmp && mkdir -p dist/tmp/packages
+unzip dist/\*.zip -d dist/tmp/packages
+
+for name in loki loki-canary logcli promtail; do
+    for arch in amd64 arm64 arm; do
+        config_path="dist/tmp/config-${name}-${arch}.json"
+        jsonnet -V "name=${name}" -V "arch=${arch}" "tools/packaging/nfpm.jsonnet" > "${config_path}"
+        nfpm package -f "${config_path}" -p rpm -t dist/
+        nfpm package -f "${config_path}" -p deb -t dist/
+    done
+done
+
+rm -rf dist/tmp

tools/packaging/promtail-postinstall.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+# Based on https://nfpm.goreleaser.com/tips/
+
+if ! command -V systemctl >/dev/null 2>&1; then
+  echo "Could not find systemd. Skipping system installation." && exit 0
+else
+    systemd_version=$(systemctl --version | head -1 | sed 's/systemd //g')
+fi
+
+cleanInstall() {
+    printf "\033[32m Post Install of a clean install\033[0m\n"
+
+    # Create the user
+    if ! id promtail > /dev/null 2>&1 ; then
+        adduser --system --shell /bin/false "promtail"
+    fi
+
+    # rhel/centos7 cannot use ExecStartPre=+ to specify the pre start should be run as root
+    # even if you want your service to run as non root.
+    if [ "${systemd_version}" -lt 231 ]; then
+        printf "\033[31m systemd version %s is less then 231, fixing the service file \033[0m\n" "${systemd_version}"
+        sed -i "s/=+/=/g" /etc/systemd/system/promtail.service
+    fi
+    printf "\033[32m Reload the service unit from disk\033[0m\n"
+    systemctl daemon-reload ||:
+    printf "\033[32m Unmask the service\033[0m\n"
+    systemctl unmask promtail ||:
+    printf "\033[32m Set the preset flag for the service unit\033[0m\n"
+    systemctl preset promtail ||:
+    printf "\033[32m Set the enabled flag for the service unit\033[0m\n"
+    systemctl enable promtail ||:
+    systemctl restart promtail ||:
+}
+
+upgrade() {
+    :
+    # printf "\033[32m Post Install of an upgrade\033[0m\n"
+}
+
+action="$1"
+if  [ "$1" = "configure" ] && [ -z "$2" ]; then
+  # Alpine linux does not pass args, and deb passes $1=configure
+  action="install"
+elif [ "$1" = "configure" ] && [ -n "$2" ]; then
+    # deb passes $1=configure $2=<current version>
+    action="upgrade"
+fi
+
+case "${action}" in
+  "1" | "install")
+    cleanInstall
+    ;;
+  "2" | "upgrade")
+    upgrade
+    ;;
+  *)
+    # $1 == version being installed
+    printf "\033[32m Alpine\033[0m"
+    cleanInstall
+    ;;
+esac

tools/packaging/promtail.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Promtail service
+After=network.target
+
+[Service]
+Type=simple
+User=promtail
+ExecStart=/usr/bin/promtail -config.file /etc/promtail/config.yml
+# Give a reasonable amount of time for promtail to start up/shut down
+TimeoutSec = 60
+Restart = on-failure
+RestartSec = 2
+
+[Install]
+WantedBy=multi-user.target

tools/packaging/verify-deb-install.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+docker ps
+image="$(docker ps --filter ancestor=jrei/systemd-debian:12 --latest --format "{{.ID}}")"
+echo "Running on container: ${image}"
+
+dir="."
+if [ -n "${CI}" ]; then
+    dir="/drone/src"
+fi
+echo "Running on directory: ${dir}"
+
+cat <<EOF | docker exec --interactive "${image}" sh
+    # Install loki and check it's running
+    dpkg -i ${dir}/dist/loki_*_amd64.deb
+    [ "\$(systemctl is-active loki)" = "active" ] || (echo "loki is inactive" && exit 1)
+
+    # Install promtail and check it's running
+    dpkg -i ${dir}/dist/promtail_*_amd64.deb
+    [ "\$(systemctl is-active promtail)" = "active" ] || (echo "promtail is inactive" && exit 1)
+
+    # Write some logs
+    mkdir -p /var/log/
+    echo "blablabla" > /var/log/test.log
+
+    # Install logcli
+    dpkg -i ${dir}/dist/logcli_*_amd64.deb
+
+    # Check that there are labels
+    sleep 5
+    labels_found=\$(logcli labels)
+    echo "Found labels: \$labels_found"
+    [ "\$labels_found" != "" ] || (echo "no logs found with logcli" && exit 1)
+EOF

tools/packaging/verify-rpm-install.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+docker ps
+image="$(docker ps --filter ancestor=jrei/systemd-centos:8 --latest --format "{{.ID}}")"
+echo "Running on container: ${image}"
+
+dir="."
+if [ -n "${CI}" ]; then
+    dir="/drone/src"
+fi
+echo "Running on directory: ${dir}"
+
+cat <<EOF | docker exec --interactive "${image}" sh
+    # Import the Grafana GPG key
+    rpm --import https://packages.grafana.com/gpg.key
+
+    # Install loki and check it's running
+    rpm -i ${dir}/dist/loki-*.x86_64.rpm
+    [ "\$(systemctl is-active loki)" = "active" ] || (echo "loki is inactive" && exit 1)
+
+    # Install promtail and check it's running
+    rpm -i ${dir}/dist/promtail-*.x86_64.rpm
+    [ "\$(systemctl is-active promtail)" = "active" ] || (echo "promtail is inactive" && exit 1)
+
+    # Write some logs
+    mkdir -p /var/log/
+    echo "blablabla" > /var/log/test.log
+
+    # Install logcli
+    rpm -i ${dir}/dist/logcli-*.x86_64.rpm
+
+    # Check that there are labels
+    sleep 5
+    labels_found=\$(logcli labels)
+    echo "Found labels: \$labels_found"
+    [ "\$labels_found" != "" ] || (echo "no labels found with logcli" && exit 1)
+EOF