New alpine based plugin ci image (#23533)

* New alpine based plugin ci image - smaller, based on alpine (downloads in 7 to 10 seconds) - Has updated gget that uses tar.gz - Unpacks to /opt/grafana - Compatible cp with toolkit * needed build tools for go lint * added built vm to test tool * add circleci plugin config * added openssh to image * fix for gget & renamed to ginstall
5 years ago · ae95287d1e
parent a70dec6c88
commit ae95287d1e
12 changed files with 448 additions and 0 deletions
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/Dockerfile
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/Dockerfile
@ -0,0 +1,6 @@
+FROM alpine 
+USER root 
+ADD scripts scripts
+ADD install /usr/local/bin
+WORKDIR scripts
+RUN ./deploy.sh
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/README.md
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/README.md
@ -0,0 +1,61 @@
+# Using this docker image
+
+Currently tagged and uploaded to dockerhub as srclosson/integrations-ci-build 
+
+Based off of `circleci/node:12-browsers` 
+
+## User
+The user will be `circleci`
+The home directory will be `/home/circleci`
+
+## Node
+- node 12 is installed
+- yarn is installed globally
+- npm is installed globally
+
+## Go
+- Go 1.14 is installed in `/usr/local/bin/go`
+- golangci-lint 1.23.7 is installed in `/usr/local/bin/golangci-lint`
+- mage is installed in `/home/circleci/go/bin/mage`
+
+All of the above directories are in the path, so there is no need to specify fully qualified paths.
+
+## Grafana
+- Installed in `/home/circleci/src/grafana`
+- `yarn install` has been run
+
+## Integration/Release Testing
+There are 4 previous versions pre-downloaded to /usr/local/grafana. These versions are:
+1. 6.6.2
+2. 6.5.3
+3. 6.4.5
+4. 6.3.7
+
+To test, your CircleCI config will need a run section with something similar to the following
+```
+- run:
+        name: Setup Grafana (local install)
+        command: |
+          sudo dpkg -i /usr/local/grafana/deb/grafana_6.6.2_amd64.deb
+          sudo cp ci/grafana-test-env/custom.ini /usr/share/grafana/conf/custom.ini
+          sudo cp ci/grafana-test-env/custom.ini /etc/grafana/grafana.ini
+          sudo service grafana-server start
+          grafana-cli --version
+```
+
+
+# Building
+To build, cd to `<srcroot>/packages/grafana-toolkit/docker/grafana-plugin-ci`
+```
+./build.sh
+```
+
+# Developing/Testing
+To test, you should have docker-compose installed.
+```
+cd test
+./start.sh
+```
+
+You will be in /home/circleci/test with the buildscripts installed to the local directory.
+Do your edits/run tests. When saving, your edits will be available in the container immediately.
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/build.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/build.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+source ./common.sh
+
+output=$(docker build . | tee /dev/tty)
+hash=$(echo "$output" | tail -1 | sed -ne "s/^Successfully built \(.*\)/\1/p")
+docker tag "$hash" $DOCKER_IMAGE_NAME:latest
+docker push $DOCKER_IMAGE_NAME:latest
+
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/common.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/common.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+##
+## Common variable declarations
+##
+
+DOCKER_IMAGE_NAME="srclosson/grafana-plugin-ci-alpine"
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/install/cp
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/install/cp
@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ "$1" == "-rn" ]; then
+  false | busybox cp -i -r "$2" "$3" 2>/dev/null
+else
+  busybox cp $*
+fi
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/install/ginstall
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/install/ginstall
@ -0,0 +1,71 @@
+#!/bin/sh
+##
+# gget
+# A script to get and install grafana versions
+# for usage information see "show_help" below.
+#
+
+latest=$(wget -O - 'https://raw.githubusercontent.com/grafana/grafana/master/latest.json' | jq -r '.stable')
+canary=$(wget -O - "https://grafana.com/api/grafana/versions" | jq ".items[0].version" | tr -d '"')
+
+show_help() {
+    echo "Usage: gget <version>"
+    echo ""
+    echo "where <version> can be:"
+    echo "  1) A version from https://grafana.com/grafana/download (ex x.y.z)"
+    echo "  2) latest (currently $latest)"
+    echo "  3) canary (currently $canary)"
+	echo ""
+	echo "  -h, --help: Display this help message"
+    echo ""
+	exit 0
+}
+
+opts=$(getopt -o h --long help -n 'gget' -- "$@")
+[ $? -eq 0 ] || {
+	show_help
+}
+
+eval set -- "$opts"
+while true; do
+	case "$1" in
+	-h | --help)
+        show_help
+		;;
+	--)
+		shift
+		break
+		;;
+	*)
+		break
+		;;
+	esac
+	shift
+done
+
+[ -z "$1" ] && show_help
+
+# Make sure the script is being run as root
+if [ $EUID -ne 0 ]; then
+   echo "This script must be run as root" 
+   exit 1
+fi
+
+##
+# MAIN
+#
+# Enough setup, let's actually do something
+#
+version=$1
+if [ "$version" == "latest" ]; then
+  version="$latest"
+  wget -O - "https://dl.grafana.com/oss/release/grafana-${version}.linux-amd64.tar.gz" | tar -C /opt -zxf 
+ln -s /opt/grafana-${version} /opt/grafana 
+elif [ "$version" == "canary" ]; then
+  version="$canary"
+  wget -O - "https://dl.grafana.com/oss/master/grafana-${version}.linux-amd64.tar.gz" | tar -C /opt -zxf -
+fi
+ln -s /opt/grafana-${version} /opt/grafana
+
+# nohup /opt/grafana/bin/grafana-server -config /opt/grafana/conf/defaults.ini -homepath /opt/grafana >/dev/null 2>&1 &
+
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-common.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-common.sh
@ -0,0 +1,38 @@
+#!/bin/sh
+
+##
+# Script to deploy a docker image. Must return exit code 0
+#
+do_exit() {
+    message="$1"
+    exit_code="$2"
+
+    echo "$message"
+    exit $exit_code
+}
+
+
+##
+# Get file, get's a file, validates the SHA
+# @param filename
+# @param expected sha value
+# @returns 0 if successful, -1 of checksum validation failed.
+#
+get_file () {
+    [ -n "$1" ] && url=$1 || do_exit "url required" 1
+    [ -n "$2" ] && dest=$2 || do_exit "destination required" 2
+    sha=$3
+    file=$(basename $dest)
+    
+    wget "$url" -O "$dest"
+    if [ -n "$sha" ]; then
+        echo "$sha  $dest" | sha256sum || do_exit "Checksum validation failed for $file. Exiting" 1
+    fi
+}
+
+untar_file () {
+    [ -n "$1" ] && src=$1 || do_exit "src required" 1
+    [ -n "$2" ] && dest=$2 || dest="/usr/local"
+
+    tar -C "$dest" -xf "$src" && /bin/rm -rf "$src"
+}
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-user.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy-user.sh
@ -0,0 +1,3 @@
+#!/bin/sh
+source "./deploy-common.sh"
+
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/scripts/deploy.sh
@ -0,0 +1,48 @@
+#!/bin/sh
+source "./deploy-common.sh"
+
+# Make libgcc compatible
+mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
+
+# Replace cp with something that mocks the one that ci-package needs
+rm /bin/cp
+mv /usr/local/bin/cp /bin/cp
+
+sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories
+apk add nodejs npm yarn build-base openssh
+
+# Install Go
+filename="go1.14.linux-amd64.tar.gz"
+get_file "https://dl.google.com/go/$filename" "/tmp/$filename" "08df79b46b0adf498ea9f320a0f23d6ec59e9003660b4c9c1ce8e5e2c6f823ca"
+untar_file "/tmp/$filename"
+
+# Install golangci-lint
+filename="golangci-lint-1.23.7-linux-amd64.tar.gz"
+get_file "https://github.com/golangci/golangci-lint/releases/download/v1.23.7/$filename" \
+    "/tmp/$filename" \
+    "34df1794a2ea8e168b3c98eed3cc0f3e13ed4cba735e4e40ef141df5c41bc086"
+untar_file "/tmp/$filename"
+chmod 755 /usr/local/bin/golangci-lint
+ln -s /usr/local/golangci-lint-1.23.7-linux-amd64/golangci-lint /usr/local/bin/golangci-lint
+ln -s /usr/local/go/bin/go /usr/local/bin/go
+ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt
+
+# Install dependencies
+apk add fontconfig zip jq
+
+# Install code climate
+get_file "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64" \
+    "/usr/local/bin/cc-test-reporter" \
+    "38f2442892027f61a07f52c845818750261b2ba58bffb043a582495339d37c05"
+chmod +x /usr/local/bin/cc-test-reporter
+
+apk add git
+# Install Mage
+mkdir -pv /tmp/mage $HOME/go/bin
+git clone https://github.com/magefile/mage.git /tmp/mage
+cd /tmp/mage && go run bootstrap.go
+mv $HOME/go/bin/mage /usr/local/bin
+# Cleanup after yourself
+/bin/rm -rf /tmp/mage 
+/bin/rm -rf $HOME/go
+/bin/rm -rf /var/cache/apk/*
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/test/docker-compose.yml
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/test/docker-compose.yml
@ -0,0 +1,18 @@
+version: '3'
+services:
+  citest:
+    image: "amd64/alpine"
+    user: root
+    volumes:
+      - ../scripts:/home/circleci/scripts
+      - ../install:/home/circleci/install
+      - ${HOME}/.ssh:/root/.ssh
+      - ../../..:/home/circleci/grafana-toolkit
+  cibuilt:
+    image: "srclosson/grafana-plugin-ci-alpine"
+    user: root
+    volumes:
+      - ../scripts:/home/circleci/scripts
+      - ../install:/home/circleci/install
+      - ${HOME}/.ssh:/root/.ssh
+      - ../../..:/home/circleci/grafana-toolkit
--- a/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/test/start.sh
+++ b/packages/grafana-toolkit/docker/grafana-plugin-ci-alpine/test/start.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+function finish {
+  echo "Exiting and cleaning up docker image"
+  docker-compose down
+}
+trap finish EXIT
+
+# Enter the docker container
+if [ "$1" = "built" ]; then
+  docker-compose run cibuilt sh -c "cd /home/circleci; exec sh --login -i"
+else 
+  docker-compose run citest sh -c "cd /home/circleci; exec sh --login -i"
+fi
--- a/packages/grafana-toolkit/src/config/circleci/config.yml
+++ b/packages/grafana-toolkit/src/config/circleci/config.yml
@ -0,0 +1,167 @@
+version: 2.1
+
+aliases:
+  # Workflow filters
+  - &filter-only-master
+    branches:
+      only: master
+
+workflows:
+  version: 2
+  plugin_workflow:
+    jobs:
+      - compile_dependencies
+      - build_plugin:
+          requires: 
+            - compile_dependencies
+      - package_and_report:
+          requires:
+            - build_plugin
+      - test_integration:
+          requires:
+            - package_and_report
+      - approve_release:
+          type: approval
+          requires:
+            - test_integration
+          filters: *filter-only-master
+      - publish_github_release:
+          requires:
+            - approve_release
+          filters: *filter-only-master
+
+executors:
+  default_exec: # declares a reusable executor
+    docker:
+      - image: srclosson/grafana-plugin-ci-alpine:latest
+
+jobs:
+  compile_dependencies:
+    executor: default_exec
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+      - run:
+          name: Install dependencies
+          command: |
+            mkdir ci
+            [ -L ~/project/node_modules/.bin/grafana-toolkit ] || yarn install --frozen-lockfile
+      - save_cache:
+          paths:
+            - ~/project/node_modules
+          key: build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+      
+  build_plugin:
+    executor: default_exec
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+      - run:
+          name: Build and test frontend and docs
+          command:  |
+            npx grafana-toolkit plugin:ci-docs
+            npx grafana-toolkit plugin:ci-build
+      - run:
+          name: Build and test Backend
+          command: |
+            if [ -f "Magefile.go" ]; then
+              mage -v buildAll
+              mage -v lint
+              mage -v coverage
+            fi
+      - run:
+          name: Move results to ci folder
+          command:  |
+            npx grafana-toolkit plugin:ci-build --finish
+      - persist_to_workspace:
+          root: .
+          paths:
+            - ci
+
+  package_and_report:
+    executor: default_exec
+    steps:
+    - checkout
+    - restore_cache:
+        keys:
+        - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+    - attach_workspace:
+        at: .
+    - run:
+        name: Package distribution
+        command: |
+          npx grafana-toolkit plugin:ci-package
+    - run:
+        name: Toolkit report
+        command: |
+          npx grafana-toolkit plugin:ci-report
+    - persist_to_workspace:
+        root: .
+        paths:
+        - ci/jobs/package
+        - ci/packages
+        - ci/dist
+        - ci/grafana-test-env
+    - store_artifacts:
+        path: ci
+
+  test_integration:
+    executor: default_exec
+    steps:
+    - checkout
+    - attach_workspace:
+        at: .
+    - restore_cache:
+        keys:
+        - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+    - run:
+        name: Setup Grafana (local install)
+        command: |
+          ginstall latest
+          /opt/grafana/bin/grafana-server -config ci/grafana-test-env/custom.ini -homepath /opt/grafana &
+          /opt/grafana/bin/grafana-cli --version
+    - run:
+        name: Run e2e tests
+        command: |
+          [ -d cypress ] && npx grafana-e2e run || echo "skipping e2e"
+    - run:
+        name: Prepare task output dir
+        command: |
+          # TODO: probably move all of this to `@grafana/toolkit plugin:ci-test`
+          mkdir -m 0755 -p ci/jobs/test_integration
+          # only copy if they exist
+          if [ -d cypress ]; then 
+            [ -d cypress/screenshots ] && cp cypress/screenshots/ ci/jobs/test_integration 
+            [ -d cypress/videos ] && cp cypress/videos/ ci/jobs/test_integration
+          fi
+    - persist_to_workspace:
+        root: .
+        paths:
+        - ci/jobs/test_integration
+    - store_test_results:
+        path: ci/jobs/test_integration
+    - store_artifacts:
+        path: ci/jobs/test_integration
+
+
+  publish_github_release:
+    executor: default_exec
+    steps:
+      - checkout
+      - add_ssh_keys:
+          fingerprints:
+            - "dc:60:ab:c7:2d:8c:82:50:2a:2a:97:1a:c0:66:83:14"
+      - attach_workspace:
+          at: .
+      - restore_cache:
+          keys:
+          - build-cache-{{ .Environment.CACHE_VERSION }}-{{ checksum "yarn.lock" }}
+      - run:
+          name: "Publish Release on GitHub"
+          command: |
+            npx grafana-toolkit plugin:github-publish
+