Rework Docker CI pipeline to be granular (#1092)

- build and test each platform separately on CI - rework `Makefile` commands - renew Docker tags description in README Additionally: - show output of failed tests - ensure Docker images are not pulled in tests - remove usage of deprecated `::set-output` GitHub Actions feature
3 years ago · a999df65ac
parent 95373d3e2a
commit a999df65ac
8 changed files with 440 additions and 242 deletions
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@ -1,8 +1,11 @@
 name: Docker CI

 on:
-  pull_request:
  push:
+    branches: ["master"]
+    tags: ["docker/*"]
+  pull_request:
+    branches: ["master"]
  schedule:
    - cron: "13 13 * * 3"

@ -11,29 +14,28 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  buildx:
+
+  ############
+  # Building #
+  ############
+
+  build:
    strategy:
+      fail-fast: false
      matrix:
-        include:
-          - dockerfile: debian
-            cache: ${{ github.ref != 'refs/heads/master'
-                    && !startsWith(github.ref, 'refs/tags/docker/') }}
-            publish: ${{ github.event_name == 'push'
-                      && github.repository_owner == 'coturn'
-                      && (startsWith(github.ref, 'refs/tags/docker/')
-                          || github.ref == 'refs/heads/master') }}
-          - dockerfile: alpine
-            cache: ${{ github.ref != 'refs/heads/master'
-                    && !startsWith(github.ref, 'refs/tags/docker/') }}
-            publish: ${{ github.event_name == 'push'
-                      && github.repository_owner == 'coturn'
-                      && (startsWith(github.ref, 'refs/tags/docker/')
-                          || github.ref == 'refs/heads/master') }}
+        dist: ["alpine", "debian"]
+        arch:
+          - amd64
+          - arm32v6
+          - arm32v7
+          - arm64v8
+          - ppc64le
+          - s390x
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
-          fetch-depth: 0
+          fetch-depth: 0  # for correct image labeling via `git describe --tags`
      - uses: docker/setup-qemu-action@v2
      - uses: docker/setup-buildx-action@v2

@ -52,100 +54,204 @@ jobs:
        run: |
          test "${{ fromJSON(steps.git.outputs.result).ref }}" \
            == "$(grep -m1 'COTURN_VER ?=' Makefile | cut -d'=' -f2 | tr -d ' ')"
-        working-directory: ./docker/coturn
-        if: ${{ matrix.publish
-             && github.ref != 'refs/heads/master' }}
+        working-directory: docker/coturn/
+        if: ${{ github.event_name == 'push'
+             && startsWith(github.ref, 'refs/tags/docker/') }}

-      - uses: satackey/action-docker-layer-caching@v0.0.11
-        with:
-          key: docker-${{ matrix.dockerfile }}-buildx-{hash}
-          restore-keys: docker-${{ matrix.dockerfile }}-buildx-
-        continue-on-error: true
-        timeout-minutes: 10
-        if: ${{ matrix.cache }}
-      - name: Pre-build Docker images cache
-        run: make docker.build.cache DOCKERFILE=${{ matrix.dockerfile }}
-                  no-cache=${{ (matrix.cache && 'no') || 'yes' }}
+      - run: make docker.image no-cache=yes
+                  dockerfile=${{ matrix.dist }}
+                  platform=linux/${{ matrix.arch }}
                  ref=${{ fromJSON(steps.git.outputs.result).ref }}
-        working-directory: ./docker/coturn
+                  tag=build-${{ github.run_number }}-${{ matrix.dist }}-${{ matrix.arch }}
+        working-directory: docker/coturn/

-      - name: Test Docker images
-        run: |
-          # Enable experimental features of Docker Daemon to run multi-arch images.
-          echo "$(cat /etc/docker/daemon.json)" '{"experimental": true}' \
-          | jq --slurp 'reduce .[] as $item ({}; . * $item)' \
-          | sudo tee /etc/docker/daemon.json
-          sudo systemctl restart docker
-
-          make npm.install
-          make test.docker DOCKERFILE=${{ matrix.dockerfile }} \
-               platforms=@all build=yes \
-               ref=${{ fromJSON(steps.git.outputs.result).ref }}
+      - run: make docker.tar to-file=.cache/image.tar
+                  tags=build-${{ github.run_number }}-${{ matrix.dist }}-${{ matrix.arch }}
+        working-directory: docker/coturn/
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.dist }}-${{ matrix.arch }}-${{ github.run_number }}
+          path: docker/coturn/.cache/image.tar
+          retention-days: 1
+
+
+
+
+  ###########
+  # Testing #
+  ###########
+
+  test:
+    needs: ["build"]
+    strategy:
+      fail-fast: false
+      matrix:
+        dist: ["alpine", "debian"]
+        arch:
+          - amd64
+          - arm32v6
+          - arm32v7
+          - arm64v8
+          - ppc64le
+          - s390x
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: docker/setup-qemu-action@v2
+      - run: make npm.install
+        working-directory: docker/coturn/
+
+      - name: Detect correct Git version for image tests
+        id: git
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let out = {ref: 'HEAD', ver: ''};
+            if ('${{ github.ref }}'.startsWith('refs/tags/docker/')) {
+              out.ref = '${{ github.ref }}'.substring(17).split('-')[0];
+              out.ver = out.ref;
+            }
+            return out;
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ matrix.dist }}-${{ matrix.arch }}-${{ github.run_number }}
+          path: docker/coturn/.cache/
+      - run: make docker.untar from-file=.cache/image.tar
+        working-directory: docker/coturn/
+
+      - run: make test.docker
+                  platform=linux/${{ matrix.arch }}
+                  tag=build-${{ github.run_number }}-${{ matrix.dist }}-${{ matrix.arch }}
        env:
          COTURN_VERSION: ${{ fromJSON(steps.git.outputs.result).ver }}
-        working-directory: ./docker/coturn
+        working-directory: docker/coturn/

-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+
+
+
+  #############
+  # Releasing #
+  #############
+
+  push:
+    if: ${{ github.event_name == 'push'
+         && github.repository_owner == 'coturn'
+         && (startsWith(github.ref, 'refs/tags/')
+             || github.ref == 'refs/heads/master') }}
+    needs: ["build", "test"]
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        registry: ["docker.io", "ghcr.io", "quay.io"]
+        dist: ["alpine", "debian"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Parse Docker image name from Git repository name
+        id: image
+        uses: actions-ecosystem/action-regex-match@v2
        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-        if: ${{ matrix.publish }}
-      - name: Login to Quay.io
-        uses: docker/login-action@v2
+          text: ${{ github.repository }}
+          regex: '^${{ github.repository_owner }}/(.+)$'
+      - name: Parse semver versions from Git tag
+        id: semver
+        uses: actions-ecosystem/action-regex-match@v2
+        with:
+          text: ${{ github.ref }}
+          regex: '^refs/tags/docker/(((([0-9]+)\.[0-9]+)\.[0-9]+)-(.+))$'
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      - name: Form main Docker image tag
+        id: docker
+        run: echo "tag=${{ (startsWith(github.ref, 'refs/tags/')
+                            && steps.semver.outputs.group1)
+                        ||     'edge' }}-${{ matrix.dist }}"
+             >> $GITHUB_OUTPUT
+
+      - uses: actions/download-artifact@v3
        with:
-          registry: quay.io
-          username: ${{ secrets.QUAYIO_ROBOT_USERNAME }}
-          password: ${{ secrets.QUAYIO_ROBOT_TOKEN }}
-        if: ${{ matrix.publish }}
-      - name: Login to Docker Hub
+          path: docker/coturn/.cache/
+
+      - name: Login to ${{ matrix.registry }} container registry
        uses: docker/login-action@v2
        with:
-          username: ${{ secrets.DOCKERHUB_BOT_USER }}
-          password: ${{ secrets.DOCKERHUB_BOT_PASS }}
-        if: ${{ matrix.publish }}
+          registry: ${{ matrix.registry }}
+          username: ${{ (matrix.registry == 'docker.io'
+                         && secrets.DOCKERHUB_BOT_USER)
+                     || (matrix.registry == 'quay.io'
+                         && secrets.QUAYIO_ROBOT_USER)
+                     || github.repository_owner }}
+          password: ${{ (matrix.registry == 'docker.io'
+                         && secrets.DOCKERHUB_BOT_PASS)
+                     || (matrix.registry == 'quay.io'
+                         && secrets.QUAYIO_ROBOT_TOKEN)
+                     || secrets.GITHUB_TOKEN }}

-      - name: Publish version Docker tags
-        run: make docker.push DOCKERFILE=${{ matrix.dockerfile }}
-                  ref=${{ fromJSON(steps.git.outputs.result).ref }}
-        working-directory: ./docker/coturn
-        if: ${{ matrix.publish
-             && github.ref != 'refs/heads/master' }}
-      - name: Publish edge Docker tags
-        run: make docker.push DOCKERFILE=${{ matrix.dockerfile }}
-                  tags=edge-${{ matrix.dockerfile }}
-                  ref=${{ fromJSON(steps.git.outputs.result).ref }}
-        working-directory: ./docker/coturn
-        if: ${{ matrix.publish
-             && github.ref == 'refs/heads/master' }}
+      - name: Tag and push single-platform images
+        run: |
+          for arch in amd64 \
+                      arm32v6 \
+                      arm32v7 \
+                      arm64v8 \
+                      ppc64le \
+                      s390x
+          do
+            make docker.untar \
+                 from-file=.cache/${{ matrix.dist }}-$arch-${{ github.run_number }}/image.tar
+            make docker.tags \
+                 of=build-${{ github.run_number }}-${{ matrix.dist }}-$arch \
+                 tags=${{ steps.docker.outputs.tag }}-$arch \
+                 registries=${{ matrix.registry }}
+            make docker.push \
+                 tags=${{ steps.docker.outputs.tag }}-$arch \
+                 registries=${{ matrix.registry }}
+          done
+        working-directory: docker/coturn/
+      - name: Tag and push multi-platform images
+        run: make docker.manifest push=yes
+                  registries=${{ matrix.registry }}
+                  of='${{ steps.docker.outputs.tag }}-amd64
+                      ${{ steps.docker.outputs.tag }}-arm32v6
+                      ${{ steps.docker.outputs.tag }}-arm32v7
+                      ${{ steps.docker.outputs.tag }}-arm64v8
+                      ${{ steps.docker.outputs.tag }}-ppc64le
+                      ${{ steps.docker.outputs.tag }}-s390x'
+                 tags=${{ (startsWith(github.ref, 'refs/tags/')
+                           && '')
+                       ||     steps.docker.outputs.tag }}
+        env:
+          DOCKERFILE: ${{ matrix.dist }}  # for correct `tags` auto-detection
+        working-directory: docker/coturn/

      # On GitHub Container Registry README is automatically updated on pushes.
-      - name: Update README on Quay.io
+      - name: Update README on Docker Hub
        uses: christian-korneck/update-container-description-action@v1
-        env:
-          DOCKER_APIKEY: ${{ secrets.QUAYIO_API_TOKEN }}
        with:
-          provider: quay
-          destination_container_repo: quay.io/coturn/coturn
+          provider: dockerhub
+          destination_container_repo: ${{ github.repository_owner }}/${{ steps.image.outputs.group1 }}
          readme_file: docker/coturn/README.md
-        if: ${{ matrix.publish }}
-      - name: Update README on Docker Hub
-        uses: christian-korneck/update-container-description-action@v1
        env:
          DOCKER_USER: ${{ secrets.DOCKERHUB_BOT_USER }}
          DOCKER_PASS: ${{ secrets.DOCKERHUB_BOT_PASS }}
+        if: ${{ matrix.registry == 'docker.io' }}
+      - name: Update README on Quay.io
+        uses: christian-korneck/update-container-description-action@v1
        with:
-          provider: dockerhub
-          destination_container_repo: coturn/coturn
+          provider: quay
+          destination_container_repo: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ steps.image.outputs.group1 }}
          readme_file: docker/coturn/README.md
-        if: ${{ matrix.publish }}
+        env:
+          DOCKER_APIKEY: ${{ secrets.QUAYIO_API_TOKEN }}
+        if: ${{ matrix.registry == 'quay.io' }}

-  release:
-    needs: ["buildx"]
+  release-github:
+    name: release (GitHub)
    if: ${{ github.event_name == 'push'
         && github.repository_owner == 'coturn'
-         && startsWith(github.ref, 'refs/tags/docker/') }}
+         && startsWith(github.ref, 'refs/tags/') }}
+    needs: ["push"]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@ -156,9 +262,11 @@ jobs:
        with:
          text: ${{ github.ref }}
          regex: '^refs/tags/docker/(((([0-9]+)\.[0-9]+)\.[0-9]+)-(.+))$'
+
      - name: Parse CHANGELOG link
        id: changelog
-        run: echo ::set-output name=link::${{ github.server_url }}/${{ github.repository }}/blob/docker/${{ steps.semver.outputs.group1 }}/docker/coturn/CHANGELOG.md#$(sed -n '/^## \[${{ steps.semver.outputs.group1 }}\]/{s/^## \[\(.*\)\][^0-9]*\([0-9].*\)/\1--\2/;s/[^0-9a-z-]*//g;p;}' CHANGELOG.md)
+        run: echo "link=${{ github.server_url }}/${{ github.repository }}/blob/docker/${{ steps.semver.outputs.group1 }}/docker/coturn/CHANGELOG.md#$(sed -n '/^## \[${{ steps.semver.outputs.group1 }}\]/{s/^## \[\(.*\)\][^0-9]*\([0-9].*\)/\1--\2/;s/[^0-9a-z-]*//g;p;}' CHANGELOG.md)"
+             >> $GITHUB_OUTPUT
        working-directory: ./docker/coturn

      - name: Create GitHub release
--- a/docker/coturn/.gitignore
+++ b/docker/coturn/.gitignore
@ -1,3 +1,4 @@
+/.cache/
 /node_modules/
 /package-lock.json
 /yarn.lock
--- a/docker/coturn/CHANGELOG.md
+++ b/docker/coturn/CHANGELOG.md
@ -199,7 +199,6 @@ Coturn TURN server Docker image changelog



-
 [Alpine Linux]: https://www.alpinelinux.org
 [Coturn]: https://haraka.github.io
 [Debian Linux]: https://www.debian.org
--- a/docker/coturn/CONTRIBUTING.md
+++ b/docker/coturn/CONTRIBUTING.md
@ -23,7 +23,7 @@ Contribution Guide
 At the moment `coturn/coturn` Docker image's [workflow is automated][1] via [GitHub Actions] in the following manner:

 - On each push the image is built and tested.  
-  This helps to track image regressions due to changes in codebase.
+  This helps to track image regressions due to changes in the codebase.

 - Image is built and tested automatically from `master` branch on weekly basis.  
  This helps to track image regressions due to changes in parent OS images (`debian`, `alpine`), their system packages, and other dependencies.
@ -54,7 +54,6 @@ To produce a new release (version tag) of `coturn/coturn` Docker image, perform



-
 [CHANGELOG]: https://github.com/coturn/coturn/blob/master/docker/coturn/CHANGELOG.md
 [GitHub Actions]: https://docs.github.com/actions
 [GitHub Release]: https://github.com/coturn/coturn/releases
--- a/docker/coturn/Makefile
+++ b/docker/coturn/Makefile
@ -10,6 +10,13 @@ space := $(empty) $(empty)
 eq = $(if $(or $(1),$(2)),$(and $(findstring $(1),$(2)),\
                                $(findstring $(2),$(1))),1)

+# Maps platform identifier to the one accepted by Docker CLI.
+dockerify = $(strip $(if $(call eq,$(1),linux/arm32v6),linux/arm/v6,\
+                    $(if $(call eq,$(1),linux/arm32v7),linux/arm/v7,\
+                    $(if $(call eq,$(1),linux/arm64v8),linux/arm64/v8,\
+                    $(if $(call eq,$(1),linux/i386),   linux/386,\
+                                                       $(platform))))))
+



@ -23,10 +30,11 @@ COTURN_MAJ_VER = $(strip $(shell echo $(COTURN_VER) | cut -d '.' -f1))

 BUILD_REV ?= 0

-NAMESPACES := coturn \
-              ghcr.io/coturn \
-              quay.io/coturn
 NAME := coturn
+OWNER := $(or $(GITHUB_REPOSITORY_OWNER),coturn)
+REGISTRIES := $(strip $(subst $(comma), ,\
+	$(shell grep -m1 'registry: \["' ../../.github/workflows/docker.yml \
+	        | cut -d':' -f2 | tr -d '"][')))
 ALL_IMAGES := \
 	debian:$(COTURN_VER)-r$(BUILD_REV)-debian,$(COTURN_VER)-debian,$(COTURN_MIN_VER)-debian,$(COTURN_MAJ_VER)-debian,debian,$(COTURN_VER)-r$(BUILD_REV),$(COTURN_VER),$(COTURN_MIN_VER),$(COTURN_MAJ_VER),latest \
 	alpine:$(COTURN_VER)-r$(BUILD_REV)-alpine,$(COTURN_VER)-alpine,$(COTURN_MIN_VER)-alpine,$(COTURN_MAJ_VER)-alpine,alpine
@ -39,14 +47,6 @@ TAGS ?= $(word 1,$(subst |, ,\
 VERSION ?= $(word 1,$(subst -, ,$(TAGS)))-$(word 2,$(strip \
 	$(subst -, ,$(subst $(comma), ,$(TAGS)))))

-PLATFORMS ?= linux/amd64 \
-             linux/arm64 \
-             linux/arm/v6 \
-             linux/arm/v7 \
-             linux/ppc64le \
-             linux/s390x
-MAIN_PLATFORM ?= $(word 1,$(subst $(comma), ,$(PLATFORMS)))
-



@ -56,10 +56,14 @@ MAIN_PLATFORM ?= $(word 1,$(subst $(comma), ,$(PLATFORMS)))

 image: docker.image

+manifest: docker.manifest
+
 push: docker.push

 release: git.release

+tags: docker.tags
+
 test: test.docker


@ -69,29 +73,28 @@ test: test.docker
 # Docker commands #
 ###################

-docker-namespaces = $(strip $(if $(call eq,$(namespaces),),\
-                            $(NAMESPACES),$(subst $(comma), ,$(namespaces))))
-docker-tags = $(subst $(comma), ,$(or $(tags),$(TAGS)))
-docker-platforms = $(strip $(if $(call eq,$(platforms),),\
-                           $(PLATFORMS),$(subst $(comma), ,$(platforms))))
-
-# Runs `docker buildx build` command allowing to customize it for the purpose of
-# re-tagging or pushing.
-define docker.buildx
-	$(eval dockerfile := $(strip $(1)))
-	$(eval namespace := $(strip $(2)))
-	$(eval tag := $(strip $(3)))
-	$(eval git-ref := $(strip $(4)))
-	$(eval platform := $(strip $(5)))
-	$(eval no-cache := $(strip $(6)))
-	$(eval args := $(strip $(7)))
-	$(eval github_url := $(strip $(or $(GITHUB_SERVER_URL),https://github.com)))
-	$(eval github_repo := $(strip $(or $(GITHUB_REPOSITORY),coturn/coturn)))
+docker-registries = $(strip \
+	$(or $(subst $(comma), ,$(registries)),$(REGISTRIES)))
+docker-tags = $(strip $(or $(subst $(comma), ,$(tags)),$(TAGS)))
+
+
+# Build single-platform Docker image with the given tag.
+#
+# Usage:
+#	make docker.image [dockerfile=(debian|alpine)]
+#	                  [tag=($(VERSION)|<docker-tag>)]] [no-cache=(no|yes)]
+#	                  [platform=<os>/<arch>]
+#	                  [ref=<git-ref>]
+
+github_url := $(strip $(or $(GITHUB_SERVER_URL),https://github.com))
+github_repo := $(strip $(or $(GITHUB_REPOSITORY),$(OWNER)/$(NAME)))
+
+docker.image:
 	cd ../../ && \
-	docker buildx build --force-rm $(args) \
-		--platform $(platform) \
+	docker buildx build --force-rm \
+		$(if $(call eq,$(platform),),,--platform $(call dockerify,$(platform)))\
 		$(if $(call eq,$(no-cache),yes),--no-cache --pull,) \
-		$(if $(call eq,$(git-ref),),,--build-arg coturn_git_ref=$(git-ref)) \
+		$(if $(call eq,$(ref),),,--build-arg coturn_git_ref=$(ref)) \
 		--build-arg coturn_github_url=$(github_url) \
 		--build-arg coturn_github_repo=$(github_repo) \
 		--label org.opencontainers.image.source=$(github_url)/$(github_repo) \
@ -99,71 +102,108 @@ define docker.buildx
 			$(shell git show --pretty=format:%H --no-patch)) \
 		--label org.opencontainers.image.version=$(subst docker/,,$(strip \
 			$(shell git describe --tags --dirty --match='docker/*'))) \
-		-f docker/coturn/$(dockerfile)/Dockerfile \
-		-t $(namespace)/$(NAME):$(tag) ./
+		-f docker/coturn/$(or $(dockerfile),$(DOCKERFILE))/Dockerfile \
+		--load -t $(OWNER)/$(NAME):$(or $(tag),$(VERSION)) ./
+
+
+# Unite multiple single-platform Docker images as a multi-platform Docker image.
+#
+# WARNING: All the single-platform Docker images should be present on their
+#          remote registry. This is the limitation imposed by `docker manifest`
+#          command.
+#
+#	make docker.manifest [amend=(yes|no)] [push=(no|yes)]
+#	                     [of=($(VERSION)|<docker-tag-1>[,<docker-tag-2>...])]
+#	                     [tags=($(TAGS)|<docker-tag-1>[,<docker-tag-2>...])]
+#	                     [registries=($(REGISTRIES)|<prefix-1>[,<prefix-2>...])]
+
+docker.manifest:
+	$(foreach tag,$(subst $(comma), ,$(docker-tags)),\
+		$(foreach registry,$(subst $(comma), ,$(docker-registries)),\
+			$(call docker.manifest.create.do,$(or $(of),$(VERSION)),\
+			                                 $(registry),$(tag))))
+ifeq ($(push),yes)
+	$(foreach tag,$(subst $(comma), ,$(docker-tags)),\
+		$(foreach registry,$(subst $(comma), ,$(docker-registries)),\
+			$(call docker.manifest.push.do,$(registry),$(tag))))
+endif
+define docker.manifest.create.do
+	$(eval froms := $(strip $(1)))
+	$(eval repo := $(strip $(2)))
+	$(eval tag := $(strip $(3)))
+	docker manifest create $(if $(call eq,$(amend),no),,--amend) \
+		$(repo)/$(OWNER)/$(NAME):$(tag) \
+		$(foreach from,$(subst $(comma), ,$(froms)),\
+			$(repo)/$(OWNER)/$(NAME):$(from))
+endef
+define docker.manifest.push.do
+	$(eval repo := $(strip $(1)))
+	$(eval tag := $(strip $(2)))
+	docker manifest push $(repo)/$(OWNER)/$(NAME):$(tag)
 endef


-# Pre-build cache for Docker image builds.
+# Manually push single-platform Docker images to container registries.
 #
-# WARNING: This command doesn't apply tag to the built Docker image, just
-#          creates a build cache. To produce a Docker image with a tag, use
-#          `docker.tag` command right after running this one.
+# Usage:
+#	make docker.push [tags=($(TAGS)|<docker-tag-1>[,<docker-tag-2>...])]
+#	                 [registries=($(REGISTRIES)|<prefix-1>[,<prefix-2>...])]
+
+docker.push:
+	$(foreach tag,$(subst $(comma), ,$(docker-tags)),\
+		$(foreach registry,$(subst $(comma), ,$(docker-registries)),\
+			$(call docker.push.do,$(registry),$(tag))))
+define docker.push.do
+	$(eval repo := $(strip $(1)))
+	$(eval tag := $(strip $(2)))
+	docker push $(repo)/$(OWNER)/$(NAME):$(tag)
+endef
+
+
+# Tag single-platform Docker image with the given tags.
 #
 # Usage:
-#	make docker.build.cache [DOCKERFILE=(debian|alpine)]
-#		[platforms=($(PLATFORMS)|<platform-1>[,<platform-2>...])]
-#		[no-cache=(no|yes)]
-#		[ref=<git-ref>]
-
-docker.build.cache:
-	$(call docker.buildx,$(DOCKERFILE),\
-		coturn,\
-		build-cache,\
-		$(ref),\
-		$(shell echo "$(docker-platforms)" | tr -s '[:blank:]' ','),\
-		$(no-cache),\
-		--output 'type=image$(comma)push=false')
-
-
-# Build Docker image on the given platform with the given tag.
+#	make docker.tags [of=($(VERSION)|<docker-tag>)]
+#	                 [tags=($(TAGS)|<docker-tag-1>[,<docker-tag-2>...])]
+#	                 [registries=($(REGISTRIES)|<prefix-1>[,<prefix-2>...])]
+
+docker.tags:
+	$(foreach tag,$(subst $(comma), ,$(docker-tags)),\
+		$(foreach registry,$(subst $(comma), ,$(docker-registries)),\
+			$(call docker.tags.do,$(or $(of),$(VERSION)),$(registry),$(tag))))
+define docker.tags.do
+	$(eval from := $(strip $(1)))
+	$(eval repo := $(strip $(2)))
+	$(eval to := $(strip $(3)))
+	docker tag $(OWNER)/$(NAME):$(from) $(repo)/$(OWNER)/$(NAME):$(to)
+endef
+
+
+# Save single-platform Docker images to a tarball file.
 #
 # Usage:
-#	make docker.image [DOCKERFILE=(debian|alpine)]
-#		[tag=($(VERSION)|<tag>)]
-#		[platform=($(MAIN_PLATFORM)|<platform>)]
-#		[no-cache=(no|yes)]
-#		[ref=<git-ref>]
+#	make docker.tar [to-file=(.cache/image.tar|<file-path>)]
+#	                [tags=($(VERSION)|<docker-tag-1>[,<docker-tag-2>...])]
+
+docker-tar-file = $(or $(to-file),.cache/image.tar)
+
+docker.tar:
+	@mkdir -p $(dir $(docker-tar-file))
+	docker save -o $(docker-tar-file) \
+		$(foreach tag,$(subst $(comma), ,$(or $(tags),$(VERSION))),\
+			$(OWNER)/$(NAME):$(tag))

-docker.image:
-	$(call docker.buildx,$(DOCKERFILE),\
-		coturn,\
-		$(or $(tag),$(VERSION)),\
-		$(ref),\
-		$(or $(platform),$(MAIN_PLATFORM)),\
-		$(no-cache),\
-		--load)

+docker.test: test.docker

-# Push Docker images to their repositories (container registries),
-# along with the required multi-arch manifests.
+
+# Load single-platform Docker images from a tarball file.
 #
 # Usage:
-#	make docker.push [DOCKERFILE=(debian|alpine)]
-#		[namespaces=($(NAMESPACES)|<prefix-1>[,<prefix-2>...])]
-#		[tags=($(TAGS)|<tag-1>[,<tag-2>...])]
-#		[platforms=($(PLATFORMS)|<platform-1>[,<platform-2>...])]
-#		[ref=<git-ref>]
+#	make docker.untar [from-file=(.cache/image.tar|<file-path>)]

-docker.push:
-	$(foreach namespace,$(docker-namespaces),\
-		$(foreach tag,$(docker-tags),\
-			$(call docker.buildx,$(DOCKERFILE),\
-				$(namespace),\
-				$(tag),\
-				$(ref),\
-				$(shell echo "$(docker-platforms)" | tr -s '[:blank:]' ','),,\
-				--push)))
+docker.untar:
+	docker load -i $(or $(from-file),.cache/image.tar)



@ -178,35 +218,21 @@ docker.push:
 #	https://github.com/bats-core/bats-core
 #
 # Usage:
-#	make test.docker
-#		[tag=($(VERSION)|<tag>)]
-#		[platforms=($(MAIN_PLATFORM)|@all|<platform-1>[,<platform-2>...])]
-#		[( [build=no]
-#		 | build=yes [DOCKERFILE=(debian|alpine)]
-#		             [ref=<git-ref>] )]
-#		[with=ipv6]
-
-test-docker-platforms = $(strip $(if $(call eq,$(platforms),),$(MAIN_PLATFORM),\
-                                $(if $(call eq,$(platforms),@all),$(PLATFORMS),\
-                                $(docker-platforms))))
+#	make test.docker [tag=($(VERSION)|<docker-tag>)]
+#	                 [platform=(linux/amd64|<os>/<arch>)]
+#	                 [with=ipv6]
+
 test.docker:
 ifeq ($(wildcard node_modules/.bin/bats),)
 	@make npm.install
 endif
-	$(foreach platform,$(test-docker-platforms),\
-		$(call test.docker.do,$(or $(tag),$(VERSION)),$(platform)))
-define test.docker.do
-	$(eval tag := $(strip $(1)))
-	$(eval platform := $(strip $(2)))
-	$(if $(call eq,$(build),yes),\
-		@make docker.image DOCKERFILE=$(DOCKERFILE) \
-			no-cache=no tag=$(tag) platform=$(platform) ref=$(ref) ,)
-	IMAGE=coturn/$(NAME):$(tag) PLATFORM=$(platform) \
+	IMAGE=$(OWNER)/$(NAME):$(or $(tag),$(VERSION)) \
+	PLATFORM=$(or $(call dockerify,$(platform)),linux/amd64) \
 	$(if $(call eq,$(with),ipv6),TEST_IPV6=1,) \
 	node_modules/.bin/bats \
 		--timing $(if $(call eq,$(CI),),--pretty,--formatter tap) \
+		--print-output-on-failure \
 		tests/main.bats
-endef



@ -257,8 +283,9 @@ endif
 # .PHONY section #
 ##################

-.PHONY: image push release test \
-        docker.build.cache docker.image docker.push \
+.PHONY: image manifest push release test \
+        docker.image docker.manifest docker.push docker.tags docker.tar \
+        docker.test docker.untar \
        git.release \
        npm.install \
        test.docker
--- a/docker/coturn/README.md
+++ b/docker/coturn/README.md
@ -23,12 +23,7 @@ Coturn TURN server Docker image

 ## Supported platforms

- `linux/amd64`
- `linux/arm64`
- `linux/arm/v6`
- `linux/arm/v7`
- `linux/ppc64le`
- `linux/s390x`
+- `linux`: `amd64`, `arm32v6`, `arm32v7`, `arm64v8`, `ppc64le`, `s390x`



@ -134,38 +129,73 @@ docker run -d --network=host --mount type=tmpfs,destination=/var/lib/coturn cotu
 ## Image versions


-### `X`
+### `alpine`
+
+This image is based on the popular [Alpine Linux project][1], available in [the alpine official image][2]. [Alpine Linux][1] is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
+
+This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use [musl libc][4] instead of [glibc and friends][5], so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread][6] for more discussion of the issues that might arise and some pro/con comparisons of using [Alpine][1]-based images.
+
+
+### `<X>`
+
+Latest tag of the latest major `X` Coturn version.
+
+This is a multi-platform image.
+
+
+### `<X.Y>`
+
+Latest tag of the latest minor `X.Y` Coturn version.
+
+This is a multi-platform image.

-Latest tag of `X` Coturn's major version.

+### `<X.Y.Z>`/`<X.Y.Z.W>`

-### `X.Y`
+Latest tag of the concrete `X.Y.Z` (or `X.Y.Z.W`) Coturn version.

-Latest tag of `X` Coturn's minor version.
+This is a multi-platform image.


-### `X.Y.Z` or `X.Y.Z.W`
+### `<X.Y.Z>-r<N>`/`<X.Y.Z.W>-r<N>`

-Latest tag version of a concrete `X.Y.Z` or `X.Y.Z.W` version of Coturn.
+Concrete `N` image revision tag of the concrete `X.Y.Z` (or `X.Y.Z.W`) Coturn version.

+Once built, it's never updated.

-### `X.Y.Z-rN` or `X.Y.Z.W-rN`
+This is a multi-platform image.

-Concrete `N` image revision tag of a Coturn's concrete `X.Y.Z` or `X.Y.Z.W` version.
+
+### `<X.Y.Z>-r<N>-<dist>`/`<X.Y.Z.W>-r<N>-<dist>`
+
+Concrete `N` image revision tag of the concrete `X.Y.Z` (or `X.Y.Z.W`) Coturn version on the concrete `dist` (`alpine` or `debian`).
+
+Once built, it's never updated.
+
+This is a multi-platform image.
+
+
+### `<X.Y.Z>-r<N>-<dist>-<arch>`/`<X.Y.Z.W>-r<N>-<dist>-<arch>`
+
+Concrete `N` image revision tag of the concrete `X.Y.Z` (or `X.Y.Z.W`) Coturn version on the concrete `dist` (`alpine` or `debian`) and `arch`.

 Once build, it's never updated.

+This is a single-platform image.

-### `alpine`

-This image is based on the popular [Alpine Linux project][1], available in [the alpine official image][2]. Alpine Linux is much smaller than most distribution base images (~5MB), and thus leads to much slimmer images in general.
+### `edge-<dist>`
+
+Latest tag of the latest `master` branch of Coturn on the concrete `dist` (`alpine` or `debian`).
+
+This is a multi-platform image.

-This variant is highly recommended when final image size being as small as possible is desired. The main caveat to note is that it does use [musl libc][4] instead of [glibc and friends][5], so certain software might run into issues depending on the depth of their libc requirements. However, most software doesn't have an issue with this, so this variant is usually a very safe choice. See [this Hacker News comment thread][6] for more discussion of the issues that might arise and some pro/con comparisons of using Alpine-based images.

+### `edge-<dist>-<arch>`

-### `edge`
+Latest tag of the latest `master` branch of Coturn on the concrete `dist` (`alpine` or `debian`) and `arch`.

-Contains build of Coturn's latest `master` branch.
+This is a single-platform image.



--- a/docker/coturn/package.json
+++ b/docker/coturn/package.json
@ -1,5 +1,5 @@
 {
  "devDependencies": {
-    "bats": "^1.1"
+    "bats": "^1.8"
  }
 }
--- a/docker/coturn/tests/main.bats
+++ b/docker/coturn/tests/main.bats
@ -2,17 +2,18 @@


@test "Built on correct arch" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'uname -m'
  [ "$status" -eq 0 ]
  if [ "$PLATFORM" = "linux/amd64" ]; then
    [ "$output" = "x86_64" ]
-  elif [ "$PLATFORM" = "linux/arm64" ]; then
-    [ "$output" = "aarch64" ]
  elif [ "$PLATFORM" = "linux/arm/v6" ]; then
    [ "$output" = "armv7l" ]
  elif [ "$PLATFORM" = "linux/arm/v7" ]; then
    [ "$output" = "armv7l" ]
+  elif [ "$PLATFORM" = "linux/arm64/v8" ]; then
+    [ "$output" = "aarch64" ]
  else
    [ "$output" = "$(echo $PLATFORM | cut -d '/' -f2-)" ]
  fi
@ -20,13 +21,15 @@


@test "Coturn is installed" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'which turnserver'
  [ "$status" -eq 0 ]
 }

@test "Coturn runs ok" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'turnserver -h'
  [ "$status" -eq 0 ]
 }
@ -34,7 +37,8 @@
@test "Coturn has correct version" {
  [ -z "$COTURN_VERSION" ] && skip

-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep -m 1 'Version Coturn' \
                                     | cut -d ' ' -f2 \
                                     | cut -d '-' -f2"
@ -46,36 +50,55 @@
 }


-@test "TLS supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+@test "TLS supported" { # TODO: Remove on next Coturn version release.
+  [ ! "$COTURN_VERSION" = '4.6.0' ] && skip
+
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'TLS supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

-@test "DTLS supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+@test "TLS 1.3 supported" {
+  [ "$COTURN_VERSION" = '4.6.0' ] && skip
+
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
+    "turnserver -o --log-file=stdout | grep 'TLS 1.3 supported'"
+  [ "$status" -eq 0 ]
+  [ ! "$output" = '' ]
+}
+
+@test "DTLS supported" { # TODO: Remove on next Coturn version release.
+  [ ! "$COTURN_VERSION" = '4.6.0' ] && skip
+
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'DTLS supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "DTLS 1.2 supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'DTLS 1.2 supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "TURN/STUN ALPN supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'TURN/STUN ALPN supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "oAuth supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep '(oAuth) supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
@ -83,35 +106,40 @@


@test "SQLite supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'SQLite supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "Redis supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'Redis supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "PostgreSQL supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'PostgreSQL supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "MySQL supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'MySQL supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
 }

@test "MongoDB supported" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout | grep 'MongoDB supported'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
@ -120,7 +148,8 @@
@test "Prometheus supported" {
  # Support of Prometheus is not displayed in the output,
  # but using --prometheus flag does the job.
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    "turnserver -o --log-file=stdout --prometheus | grep 'Version Coturn'"
  [ "$status" -eq 0 ]
  [ ! "$output" = '' ]
@ -128,19 +157,22 @@


@test "detect-external-ip is present" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'which detect-external-ip'
  [ "$status" -eq 0 ]
 }

@test "detect-external-ip runs ok" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'detect-external-ip'
  [ "$status" -eq 0 ]
 }

@test "detect-external-ip returns valid IPv4" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'detect-external-ip --ipv4'
  [ "$status" -eq 0 ]

@ -151,7 +183,8 @@
@test "detect-external-ip returns valid IPv6" {
  [ -z "$TEST_IPV6" ] && skip

-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'detect-external-ip --ipv6'
  [ "$status" -eq 0 ]

@ -160,7 +193,8 @@
 }

@test "detect-external-ip returns IPv4 by default" {
-  run docker run --rm --platform $PLATFORM --entrypoint sh $IMAGE -c \
+  run docker run --rm --pull never --platform $PLATFORM \
+                 --entrypoint sh $IMAGE -c \
    'detect-external-ip --ipv4'
  [ "$status" -eq 0 ]