From 9123e0fca8befc314570a5ca99d2e857cf046b95 Mon Sep 17 00:00:00 2001 From: Leonard Gram Date: Tue, 8 May 2018 09:42:20 +0200 Subject: [PATCH] build: crossplatform build with packages. Big thanks to @fg2it who created a POC as well as the build container that this work is based on. --- .circleci/config.yml | 5 ++- Gruntfile.js | 4 +++ build.go | 61 ++++++++++++++++++++++++----------- scripts/build/build.sh | 42 ++++++++++++++++++++---- scripts/grunt/release_task.js | 11 +++++-- 5 files changed, 95 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bad5a7c1cd0..1a0be0575e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,10 +70,13 @@ jobs: build: docker: - - image: grafana/build-container:v0.1 + - image: grafana/build-container:crosscompile working_directory: /go/src/github.com/grafana/grafana steps: - checkout + - run: + name: prepare build tools + command: '/tmp/bootstrap.sh' - run: name: build and package grafana command: './scripts/build/build.sh' diff --git a/Gruntfile.js b/Gruntfile.js index a0607ef49dc..23276e8a122 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -12,6 +12,10 @@ module.exports = function (grunt) { platform: process.platform.replace('win32', 'windows'), }; + if (grunt.option('platform')) { + config.platform = grunt.option('platform'); + } + if (grunt.option('arch')) { config.arch = grunt.option('arch'); } else { diff --git a/build.go b/build.go index 7e7183b8b83..f9611c99a47 100644 --- a/build.go +++ b/build.go @@ -27,8 +27,7 @@ var ( goarch string goos string gocc string - gocxx string - cgo string + cgo bool pkgArch string version string = "v1" // deb & rpm does not support semver so have to handle their version a little differently @@ -53,8 +52,7 @@ func main() { flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH") flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS") flag.StringVar(&gocc, "cc", "", "CC") - flag.StringVar(&gocxx, "cxx", "", "CXX") - flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED") + flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo") flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH") flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary") flag.BoolVar(&race, "race", race, "Use race detector") @@ -93,20 +91,24 @@ func main() { build("grafana-server", "./pkg/cmd/grafana-server", []string{}) case "build": - clean() + //clean() for _, binary := range binaries { build(binary, "./pkg/cmd/"+binary, []string{}) } + case "build-frontend": + grunt(gruntBuildArg("build")...) + case "test": test("./pkg/...") grunt("test") case "package": - grunt(gruntBuildArg("release")...) - if runtime.GOOS != "windows" { - createLinuxPackages() - } + grunt(gruntBuildArg("build")...) + packageGrafana() + + case "package-only": + packageGrafana() case "pkg-rpm": grunt(gruntBuildArg("release")...) @@ -131,6 +133,22 @@ func main() { } } +func packageGrafana() { + platformArg := fmt.Sprintf("--platform=%v", goos) + previousPkgArch := pkgArch + if pkgArch == "" { + pkgArch = goarch + } + postProcessArgs := gruntBuildArg("package") + postProcessArgs = append(postProcessArgs, platformArg) + grunt(postProcessArgs...) + pkgArch = previousPkgArch + + if goos == "linux" && goarch == "amd64"{ + createLinuxPackages() + } +} + func makeLatestDistCopies() { files, err := ioutil.ReadDir("dist") if err != nil { @@ -138,9 +156,9 @@ func makeLatestDistCopies() { } latestMapping := map[string]string{ - ".deb": "dist/grafana_latest_amd64.deb", - ".rpm": "dist/grafana-latest-1.x86_64.rpm", - ".tar.gz": "dist/grafana-latest.linux-x64.tar.gz", + "_amd64.deb": "dist/grafana_latest_amd64.deb", + ".x86_64.rpm": "dist/grafana-latest-1.x86_64.rpm", + ".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz", } for _, file := range files { @@ -386,7 +404,8 @@ func test(pkg string) { } func build(binaryName, pkg string, tags []string) { - binary := "./bin/" + binaryName + binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName) + if goos == "windows" { binary += ".exe" } @@ -408,6 +427,7 @@ func build(binaryName, pkg string, tags []string) { if !isDev { setBuildEnv() runPrint("go", "version") + fmt.Printf("Targeting %s/%s\n", goos, goarch) } runPrint("go", args...) @@ -451,6 +471,14 @@ func clean() { func setBuildEnv() { os.Setenv("GOOS", goos) + if goos == "windows" { + // require windows >=7 + os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601") + } + if goarch != "amd64" || goos != "linux" { + // needed for all other archs + cgo = true + } if strings.HasPrefix(goarch, "armv") { os.Setenv("GOARCH", "arm") os.Setenv("GOARM", goarch[4:]) @@ -460,15 +488,12 @@ func setBuildEnv() { if goarch == "386" { os.Setenv("GO386", "387") } - if cgo != "" { - os.Setenv("CGO_ENABLED", cgo) + if cgo { + os.Setenv("CGO_ENABLED", "1") } if gocc != "" { os.Setenv("CC", gocc) } - if gocxx != "" { - os.Setenv("CXX", gocxx) - } } func getGitSha() string { diff --git a/scripts/build/build.sh b/scripts/build/build.sh index 369d434d4f5..ab021d57692 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -4,6 +4,12 @@ # This script is executed from within the container. # +CCARMV7=arm-linux-gnueabihf-gcc +CCARM64=aarch64-linux-gnu-gcc +CCOSX64=/tmp/osxcross/target/bin/o64-clang +CCWIN64=x86_64-w64-mingw32-gcc +CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc + GOPATH=/go REPO_PATH=$GOPATH/src/github.com/grafana/grafana @@ -11,23 +17,45 @@ cd /go/src/github.com/grafana/grafana echo "current dir: $(pwd)" if [ "$CIRCLE_TAG" != "" ]; then - echo "Building a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false build + echo "Building releases from tag $CIRCLE_TAG" + go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build + go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build + go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build + go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build + CC=${CCX64} go run build.go -includeBuildNumber=false build else echo "Building incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build + go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build + CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build fi yarn install --pure-lockfile --no-progress -source /etc/profile.d/rvm.sh - echo "current dir: $(pwd)" +if [ -d "dist" ]; then + rm -rf dist +fi + if [ "$CIRCLE_TAG" != "" ]; then + echo "Building frontend from tag $CIRCLE_TAG" + go run build.go -includeBuildNumber=false build-frontend echo "Packaging a release from tag $CIRCLE_TAG" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false package latest + go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest + go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only + go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only + go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only + go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only else + echo "Building frontend for $CIRCLE_BRANCH" + go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend echo "Packaging incremental build for $CIRCLE_BRANCH" - go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest + go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest + go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only + go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only fi diff --git a/scripts/grunt/release_task.js b/scripts/grunt/release_task.js index 4fbc41cfb08..f24ec64203a 100644 --- a/scripts/grunt/release_task.js +++ b/scripts/grunt/release_task.js @@ -3,13 +3,20 @@ var path = require('path'); module.exports = function(grunt) { "use strict"; - // build, then zip and upload to s3 + // build then zip grunt.registerTask('release', [ 'build', 'build-post-process', 'compress:release' ]); + // package into archives + grunt.registerTask('package', [ + 'clean:temp', + 'build-post-process', + 'compress:release' + ]); + grunt.registerTask('build-post-process', function() { grunt.config('copy.public_to_temp', { expand: true, @@ -18,7 +25,7 @@ module.exports = function(grunt) { dest: '<%= tempDir %>/public/', }); grunt.config('copy.backend_bin', { - cwd: 'bin', + cwd: 'bin/<%= platform %>-<%= arch %>', expand: true, src: ['*'], options: { mode: true},