Add support for ARM64 to lambda-promtail drone build job (#5354)

* Add support for ARM64 to lambda-promtail drone build job * update drone.yml based on tomas' changes Signed-off-by: Callum Styan <callumstyan@gmail.com> * Add entry to CHANGELOG.md * Change Docker image for building Promtail Lambda * reverse previous commit Signed-off-by: Callum Styan <callumstyan@gmail.com> * Add manifest for ECR Public images * Minor change to drone.yml for volume mounts + generate drone.jsonnet again. Signed-off-by: Callum Styan <callumstyan@gmail.com> * temp, test drone changes without having to push to main branch Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix linting issues Signed-off-by: Callum Styan <callumstyan@gmail.com> * temp, push the lambda-promtail images always so we can test the manifest task Signed-off-by: Callum Styan <callumstyan@gmail.com> * We don't build a base arm image, only arm64 Signed-off-by: Callum Styan <callumstyan@gmail.com> * Add volumes definition to ECR manifest pipeline * Regenerate drone.yml with Tomas' most recent commit Signed-off-by: Callum Styan <callumstyan@gmail.com> * Test some some drone changes. Signed-off-by: Callum Styan <callumstyan@gmail.com> * fix lint error in drone.jsonnet Signed-off-by: Callum Styan <callumstyan@gmail.com> * Drop out one of my more recent test changes. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Regenerate drone config. Signed-off-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: Callum Styan <callumstyan@gmail.com>
3 years ago · 35cb40af96
parent d6f50ca5e1
commit 35cb40af96
4 changed files with 209 additions and 15 deletions
--- a/.drone/docker-manifest-ecr.tmpl
+++ b/.drone/docker-manifest-ecr.tmpl
@ -0,0 +1,21 @@
+image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}
+tags:
+  - main
+{{#if build.tag}}
+  - latest
+{{/if}}
+{{#if build.tags}}
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
--- a/.drone/drone.jsonnet
+++ b/.drone/drone.jsonnet
@ -257,17 +257,9 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) {
  depends_on: ['check'],
 };

-local lambda_promtail(tags='') = pipeline('lambda-promtail') {
+local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(arch) {
  steps+: [
-    {
-      name: 'image-tag',
-      image: 'alpine',
-      commands: [
-        'apk add --no-cache bash git',
-        'git fetch origin --tags',
-        'echo $(./tools/image-tag)-amd64 > .tags',
-      ] + if tags != '' then ['echo ",%s" >> .tags' % tags] else [],
-    },
+    // dry run for everything that is not tag or main
    lambda_promtail_ecr('lambda-promtail') {
      depends_on: ['image-tag'],
      when: condition('exclude').tagMain,
@ -342,6 +334,58 @@ local manifest(apps) = pipeline('manifest') {
  ],
 };

+local manifest_ecr(apps, archs) = pipeline('manifest-ecr') {
+  steps: std.foldl(
+    function(acc, app) acc + [{
+      name: 'manifest-' + app,
+      image: 'plugins/manifest',
+      volumes: [{
+        name: 'dockerconf',
+        path: '/.docker',
+      }],
+      settings: {
+        // the target parameter is abused for the app's name,
+        // as it is unused in spec mode. See docker-manifest-ecr.tmpl
+        target: app,
+        spec: '.drone/docker-manifest-ecr.tmpl',
+        ignore_missing: true,
+      },
+      depends_on: ['clone'] + (
+        // Depend on the previous app, if any.
+        if std.length(acc) > 0
+        then [acc[std.length(acc) - 1].name]
+        else []
+      ),
+    }],
+    apps,
+    [{
+      name: 'ecr-login',
+      image: 'docker:dind',
+      volumes: [{
+        name: 'dockerconf',
+        path: '/root/.docker',
+      }],
+      environment: {
+        AWS_ACCESS_KEY_ID: { from_secret: ecr_key.name },
+        AWS_SECRET_ACCESS_KEY: { from_secret: ecr_secret_key.name },
+      },
+      commands: [
+        'apk add --no-cache aws-cli',
+        'docker login --username AWS --password $(aws ecr-public get-login-password --region us-east-1) public.ecr.aws',
+      ],
+      depends_on: ['clone'],
+    }],
+  ),
+  volumes: [{
+    name: 'dockerconf',
+    temp: {},
+  }],
+  depends_on: [
+    'lambda-promtail-%s' % arch
+    for arch in archs
+  ],
+};
+
 [
  pipeline('loki-build-image') {
    workspace: {
@ -472,6 +516,7 @@ local manifest(apps) = pipeline('manifest') {
        commands: [
          'apk add --no-cache bash git',
          'git fetch origin --tags',
+          'echo $(./tools/image-tag)',
          'echo $(./tools/image-tag) > .tag',
        ],
        depends_on: ['clone'],
@ -489,5 +534,21 @@ local manifest(apps) = pipeline('manifest') {
    ],
  },
 ] + [promtail_win()]
-+ [lambda_promtail('main')]
-+ [github_secret, pull_secret, docker_username_secret, docker_password_secret, ecr_key, ecr_secret_key, deploy_configuration]
+ [
+  lambda_promtail(arch)
+  for arch in ['amd64', 'arm64']
+] + [
+  manifest_ecr(['lambda-promtail'], ['amd64', 'arm64']) {
+    trigger: condition('include').tagMain {
+      event: ['push'],
+    },
+  },
+] + [
+  github_secret,
+  pull_secret,
+  docker_username_secret,
+  docker_password_secret,
+  ecr_key,
+  ecr_secret_key,
+  deploy_configuration,
+]
--- a/.drone/drone.yml
+++ b/.drone/drone.yml
@ -1024,6 +1024,7 @@ steps:
 - commands:
  - apk add --no-cache bash git
  - git fetch origin --tags
+  - echo $(./tools/image-tag)
  - echo $(./tools/image-tag) > .tag
  depends_on:
  - clone
@ -1074,13 +1075,15 @@ trigger:
 depends_on:
 - check
 kind: pipeline
-name: lambda-promtail
+name: lambda-promtail-amd64
+platform:
+  arch: amd64
+  os: linux
 steps:
 - commands:
  - apk add --no-cache bash git
  - git fetch origin --tags
  - echo $(./tools/image-tag)-amd64 > .tags
-  - echo ",main" >> .tags
  image: alpine
  name: image-tag
 - depends_on:
@ -1131,6 +1134,114 @@ trigger:
  - pull_request
  - tag
 ---
+depends_on:
+- check
+kind: pipeline
+name: lambda-promtail-arm64
+platform:
+  arch: arm64
+  os: linux
+steps:
+- commands:
+  - apk add --no-cache bash git
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-arm64 > .tags
+  image: alpine
+  name: image-tag
+- depends_on:
+  - image-tag
+  image: cstyan/ecr
+  name: build-lambda-promtail-image
+  privileged: true
+  settings:
+    access_key:
+      from_secret: ecr_key
+    dockerfile: tools/lambda-promtail/Dockerfile
+    dry_run: true
+    region: us-east-1
+    registry: public.ecr.aws/grafana
+    repo: public.ecr.aws/grafana/lambda-promtail
+    secret_key:
+      from_secret: ecr_secret_key
+  when:
+    ref:
+      exclude:
+      - refs/heads/main
+      - refs/heads/k???
+      - refs/tags/v*
+- depends_on:
+  - image-tag
+  image: cstyan/ecr
+  name: publish-lambda-promtail-image
+  privileged: true
+  settings:
+    access_key:
+      from_secret: ecr_key
+    dockerfile: tools/lambda-promtail/Dockerfile
+    dry_run: false
+    region: us-east-1
+    registry: public.ecr.aws/grafana
+    repo: public.ecr.aws/grafana/lambda-promtail
+    secret_key:
+      from_secret: ecr_secret_key
+  when:
+    ref:
+      include:
+      - refs/heads/main
+      - refs/heads/k???
+      - refs/tags/v*
+trigger:
+  event:
+  - push
+  - pull_request
+  - tag
+---
+depends_on:
+- lambda-promtail-amd64
+- lambda-promtail-arm64
+kind: pipeline
+name: manifest-ecr
+steps:
+- commands:
+  - apk add --no-cache aws-cli
+  - docker login --username AWS --password $(aws ecr-public get-login-password --region
+    us-east-1) public.ecr.aws
+  depends_on:
+  - clone
+  environment:
+    AWS_ACCESS_KEY_ID:
+      from_secret: ecr_key
+    AWS_SECRET_ACCESS_KEY:
+      from_secret: ecr_secret_key
+  image: docker:dind
+  name: ecr-login
+  volumes:
+  - name: dockerconf
+    path: /root/.docker
+- depends_on:
+  - clone
+  - ecr-login
+  image: plugins/manifest
+  name: manifest-lambda-promtail
+  settings:
+    ignore_missing: true
+    spec: .drone/docker-manifest-ecr.tmpl
+    target: lambda-promtail
+  volumes:
+  - name: dockerconf
+    path: /.docker
+trigger:
+  event:
+  - push
+  ref:
+    include:
+    - refs/heads/main
+    - refs/heads/k???
+    - refs/tags/v*
+volumes:
+- name: dockerconf
+  temp: {}
+---
 get:
  name: pat
  path: infra/data/ci/github/grafanabot
@ -1174,6 +1285,6 @@ kind: secret
 name: deploy_config
 ---
 kind: signature
-hmac: 81be9fce60976703815abf1fd2f1c9255a142d615b90d4b5278ff4a64fa25e92
+hmac: 8335495d285498f513bb265cf90079175c25f5b6c5717deab7abfae65341726a

 ...
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -179,6 +179,7 @@ to include only the most relevant.
 * [5409](https://github.com/grafana/loki/pull/5409) **ldb**: Enable best effort parsing for Syslog messages
 * [5392](https://github.com/grafana/loki/pull/5392) **MichelHollands**: Etcd credentials are parsed as secrets instead of plain text now.
 * [5361](https://github.com/grafana/loki/pull/5361) **ctovena**: Add usage report to grafana.com.
+* [5354](https://github.com/grafana/loki/pull/5354) **tlinhart**: Add support for ARM64 to lambda-promtail drone build job.
 * [5289](https://github.com/grafana/loki/pull/5289) **ctovena**: Fix deduplication bug in queries when mutating labels.
 * [5302](https://github.com/grafana/loki/pull/5302) **MasslessParticle** Update azure blobstore client to use new sdk.
 * [5243](https://github.com/grafana/loki/pull/5290) **ssncferreira**: Update Promtail to support duration string formats.