diff --git a/go.mod b/go.mod index efe22d55747..78270aecc63 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/fatih/color v1.13.0 github.com/gchaincl/sqlhooks v1.3.0 github.com/getsentry/sentry-go v0.13.0 + github.com/go-git/go-git/v5 v5.4.2 github.com/go-kit/kit v0.11.0 github.com/go-openapi/strfmt v0.20.2 github.com/go-redis/redis/v8 v8.11.4 @@ -213,7 +214,7 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/encoding v0.3.2 github.com/sercand/kuberesolver v2.4.0+incompatible // indirect - github.com/sergi/go-diff v1.0.0 // indirect + github.com/sergi/go-diff v1.1.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect github.com/sirupsen/logrus v1.8.1 // indirect @@ -273,7 +274,9 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.2.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect github.com/RoaringBitmap/roaring v0.9.4 // indirect + github.com/acomagu/bufpipe v1.0.3 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect @@ -288,17 +291,23 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect github.com/elazarl/goproxy v0.0.0-20220115173737-adb46da277ac // indirect + github.com/emirpasic/gods v1.12.0 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect + github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect github.com/klauspost/compress v1.15.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/labstack/echo/v4 v4.7.2 // indirect github.com/labstack/gommon v0.3.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect @@ -307,11 +316,13 @@ require ( github.com/smartystreets/goconvey v1.7.2 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect github.com/wk8/go-ordered-map v1.0.0 + github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xlab/treeprint v1.1.0 // indirect github.com/yudai/pp v2.0.1+incompatible // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 // indirect go.opentelemetry.io/proto/otlp v0.15.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect k8s.io/api v0.22.5 // indirect k8s.io/apimachinery v0.22.5 // indirect k8s.io/klog/v2 v2.30.0 // indirect diff --git a/go.sum b/go.sum index 1bb9a7c48e9..df4a323117f 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OneOfOne/xxhash v1.2.6/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -275,6 +277,8 @@ github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrU github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f h1:HR5nRmUQgXrwqZOwZ2DAc/aCi3Bu3xENpspW935vxu0= github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= @@ -294,6 +298,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antonmedv/expr v1.8.9/go.mod h1:5qsM3oLGDND7sDmQGDXHkYfkjYMUX14qsgqmHhwGEk8= @@ -316,6 +322,7 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -783,6 +790,8 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emicklei/proto v1.6.15/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/emicklei/proto v1.10.0 h1:pDGyFRVV5RvV+nkBK9iy3q67FBy9Xa7vwrOTE+g5aGw= github.com/emicklei/proto v1.10.0/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -816,6 +825,7 @@ github.com/felixge/fgprof v0.9.1/go.mod h1:7/HK6JFtFaARhIljgP2IV8rJLIoHDoOYoUphs github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o= github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -851,6 +861,8 @@ github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ER github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -863,6 +875,15 @@ github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0 github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -1557,6 +1578,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jaegertracing/jaeger v1.24.0/go.mod h1:mqdtFDA447va5j0UewDaAWyNlGreGQyhGxXVhbF58gQ= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= @@ -1618,6 +1641,8 @@ github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -1729,6 +1754,7 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -2226,8 +2252,9 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sercand/kuberesolver v2.4.0+incompatible h1:WE2OlRf6wjLxHwNkkFLQGaZcVLEXjMjBPjjEU5vksH8= github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -2426,6 +2453,8 @@ github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr github.com/wk8/go-ordered-map v1.0.0 h1:BV7z+2PaK8LTSd/mWgY12HyMAo5CEgkHqbkVq2thqr8= github.com/wk8/go-ordered-map v1.0.0/go.mod h1:9ZIbRunKbuvfPKyBP1SIKLcXNlv74YCOZ3t3VTS6gRk= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= @@ -2609,6 +2638,7 @@ golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -2644,6 +2674,7 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -2785,6 +2816,7 @@ golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= @@ -2984,6 +3016,7 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -3459,6 +3492,7 @@ gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/services/datasources/models.go b/pkg/services/datasources/models.go index 616074ce8cd..68b938da257 100644 --- a/pkg/services/datasources/models.go +++ b/pkg/services/datasources/models.go @@ -31,9 +31,9 @@ const ( type DsAccess string type DataSource struct { - Id int64 `json:"id"` - OrgId int64 `json:"orgId"` - Version int `json:"version"` + Id int64 `json:"id,omitempty"` + OrgId int64 `json:"orgId,omitempty"` + Version int `json:"version,omitempty"` Name string `json:"name"` Type string `json:"type"` @@ -54,8 +54,8 @@ type DataSource struct { ReadOnly bool `json:"readOnly"` Uid string `json:"uid"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Created time.Time `json:"created,omitempty"` + Updated time.Time `json:"updated,omitempty"` } // AllowedCookies parses the jsondata.keepCookies and returns a list of diff --git a/pkg/services/export/commit_helper.go b/pkg/services/export/commit_helper.go new file mode 100644 index 00000000000..b19749c6a03 --- /dev/null +++ b/pkg/services/export/commit_helper.go @@ -0,0 +1,153 @@ +package export + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "strings" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/services/sqlstore" + jsoniter "github.com/json-iterator/go" +) + +type commitHelper struct { + ctx context.Context + repo *git.Repository + work *git.Worktree + orgDir string // includes the orgID + workDir string // same as the worktree root + orgID int64 + users map[int64]*userInfo +} + +type commitBody struct { + fpath string // absolute + body []byte + frame *data.Frame +} + +type commitOptions struct { + body []commitBody + when time.Time + userID int64 + comment string +} + +func (ch *commitHelper) initOrg(sql *sqlstore.SQLStore, orgID int64) error { + return sql.WithDbSession(ch.ctx, func(sess *sqlstore.DBSession) error { + sess.Table("user"). + Join("inner", "org_user", "user.id = org_user.user_id"). + Cols("user.*", "org_user.role"). + Where("org_user.org_id = ?", orgID). + Asc("user.id") + + rows := make([]*userInfo, 0) + err := sess.Find(&rows) + if err != nil { + return err + } + + lookup := make(map[int64]*userInfo, len(rows)) + for _, row := range rows { + lookup[row.ID] = row + } + ch.users = lookup + ch.orgID = orgID + return err + }) +} + +func (ch *commitHelper) add(opts commitOptions) error { + for _, b := range opts.body { + if !strings.HasPrefix(b.fpath, ch.orgDir) { + return fmt.Errorf("invalid path, must be within the root folder") + } + + // make sure the parent exists + err := os.MkdirAll(path.Dir(b.fpath), 0750) + if err != nil { + return err + } + + body := b.body + if b.frame != nil { + body, err = jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(b.frame, "", " ") + if err != nil { + return err + } + } + + err = ioutil.WriteFile(b.fpath, body, 0644) + if err != nil { + return err + } + + sub := b.fpath[len(ch.workDir)+1:] + _, err = ch.work.Add(sub) + if err != nil { + status, e2 := ch.work.Status() + if e2 != nil { + return fmt.Errorf("error adding: %s (invalud work status: %s)", sub, e2.Error()) + } + fmt.Printf("STATUS: %+v\n", status) + return fmt.Errorf("unable to add file: %s (%d)", sub, len(b.body)) + } + } + + user, ok := ch.users[opts.userID] + if !ok { + user = &userInfo{ + Name: "admin", + Email: "admin@unknown.org", + } + } + sig := user.getAuthor() + if opts.when.Unix() > 10 { + sig.When = opts.when + } + + copts := &git.CommitOptions{ + Author: &sig, + } + + _, err := ch.work.Commit(opts.comment, copts) + return err +} + +type userInfo struct { + ID int64 `json:"-" xorm:"id"` + Login string `json:"login"` + Email string `json:"email"` + Name string `json:"name"` + Password string `json:"password"` + Salt string `json:"salt"` + Role string `json:"org_role"` // org role + Theme string `json:"-"` // managed in preferences + Created time.Time `json:"-"` // managed in git or external source + Updated time.Time `json:"-"` // managed in git or external source + IsDisabled bool `json:"disabled" xorm:"is_disabled"` + IsServiceAccount bool `json:"serviceAccount" xorm:"is_service_account"` + LastSeenAt time.Time `json:"-" xorm:"last_seen_at"` +} + +func (u *userInfo) getAuthor() object.Signature { + return object.Signature{ + Name: firstRealStringX(u.Name, u.Login, u.Email, "?"), + Email: firstRealStringX(u.Email, u.Login, u.Name, "?"), + } +} + +func firstRealStringX(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "?" +} diff --git a/pkg/services/export/dummy_job.go b/pkg/services/export/dummy_job.go index 029bc8edabb..f892b1cea6b 100644 --- a/pkg/services/export/dummy_job.go +++ b/pkg/services/export/dummy_job.go @@ -1,7 +1,6 @@ package export import ( - "errors" "fmt" "math" "math/rand" @@ -23,10 +22,6 @@ type dummyExportJob struct { } func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) { - if cfg.Format != "git" { - return nil, errors.New("only git format is supported") - } - job := &dummyExportJob{ logger: log.New("dummy_export_job"), cfg: cfg, diff --git a/pkg/services/export/export_anno.go b/pkg/services/export/export_anno.go new file mode 100644 index 00000000000..4f3d0f96684 --- /dev/null +++ b/pkg/services/export/export_anno.go @@ -0,0 +1,129 @@ +package export + +import ( + "encoding/json" + "fmt" + "path/filepath" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/services/sqlstore" + jsoniter "github.com/json-iterator/go" +) + +func exportAnnotations(helper *commitHelper, job *gitExportJob) error { + return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + type annoResult struct { + ID int64 `xorm:"id"` + DashboardID int64 `xorm:"dashboard_id"` + PanelID int64 `xorm:"panel_id"` + UserID int64 `xorm:"user_id"` + Text string `xorm:"text"` + Epoch int64 `xorm:"epoch"` + EpochEnd int64 `xorm:"epoch_end"` + Created int64 `xorm:"created"` // not used + Tags string `xorm:"tags"` // JSON Array + } + + type annoEvent struct { + PanelID int64 `json:"panel"` + Text string `json:"text"` + Epoch int64 `json:"epoch"` // dashboard/start+end is really the UID + EpochEnd int64 `json:"epoch_end,omitempty"` + Tags []string + } + + rows := make([]*annoResult, 0) + + sess.Table("annotation"). + Where("org_id = ? AND alert_id = 0", helper.orgID).Asc("epoch") + + err := sess.Find(&rows) + if err != nil { + return err + } + + count := len(rows) + f_ID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) + f_DashboardID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) + f_PanelID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) + f_Epoch := data.NewFieldFromFieldType(data.FieldTypeTime, count) + f_EpochEnd := data.NewFieldFromFieldType(data.FieldTypeNullableTime, count) + f_Text := data.NewFieldFromFieldType(data.FieldTypeString, count) + f_Tags := data.NewFieldFromFieldType(data.FieldTypeJSON, count) + + f_ID.Name = "ID" + f_DashboardID.Name = "DashboardID" + f_PanelID.Name = "PanelID" + f_Epoch.Name = "Epoch" + f_EpochEnd.Name = "EpochEnd" + f_Text.Name = "Text" + f_Tags.Name = "Tags" + + for id, row := range rows { + f_ID.Set(id, row.ID) + f_DashboardID.Set(id, row.DashboardID) + f_PanelID.Set(id, row.PanelID) + f_Epoch.Set(id, time.UnixMilli(row.Epoch)) + if row.Epoch != row.EpochEnd { + f_EpochEnd.SetConcrete(id, time.UnixMilli(row.EpochEnd)) + } + f_Text.Set(id, row.Text) + f_Tags.Set(id, json.RawMessage(row.Tags)) + + // Save a file for each + event := &annoEvent{ + PanelID: row.PanelID, + Text: row.Text, + } + err = json.Unmarshal([]byte(row.Tags), &event.Tags) + if err != nil { + return err + } + fname := fmt.Sprintf("%d", row.Epoch) + if row.Epoch != row.EpochEnd { + fname += "-" + fmt.Sprintf("%d", row.EpochEnd) + } + + err = helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(helper.orgDir, + "annotations", + "dashboard", + fmt.Sprintf("id-%d", row.DashboardID), + fname+".json"), + body: prettyJSON(event), + }, + }, + when: time.UnixMilli(row.Epoch), + comment: fmt.Sprintf("Added annotation (%d)", row.ID), + userID: row.UserID, + }) + if err != nil { + return err + } + } + + frame := data.NewFrame("", f_ID, f_DashboardID, f_PanelID, f_Epoch, f_EpochEnd, f_Text, f_Tags) + js, err := jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(frame, "", " ") + if err != nil { + return err + } + + err = helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(helper.orgDir, "annotations", "annotations.json"), + body: js, // TODO, pretty? + }, + }, + when: time.Now(), + comment: "Exported annotations", + }) + if err != nil { + return err + } + return err + }) +} diff --git a/pkg/services/export/export_auth.go b/pkg/services/export/export_auth.go new file mode 100644 index 00000000000..37da1a92d63 --- /dev/null +++ b/pkg/services/export/export_auth.go @@ -0,0 +1,73 @@ +package export + +import ( + "fmt" + "path" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { + return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + commit := commitOptions{ + comment: "auth tables dump", + } + + tables := []string{ + "user", // joined with "org_user" to get the role + "user_role", + "builtin_role", + "api_key", + "team", "team_group", "team_role", "team_member", + "role", + "temp_user", + "user_auth_token", // no org_id... is it temporary? + "permission", + } + + for _, table := range tables { + switch table { + case "permission": + sess.Table(table). + Join("left", "role", "permission.role_id = role.id"). + Cols("permission.*"). + Where("org_id = ?", helper.orgID). + Asc("permission.id") + case "user": + sess.Table(table). + Join("inner", "org_user", "user.id = org_user.user_id"). + Cols("user.*", "org_user.role"). + Where("org_user.org_id = ?", helper.orgID). + Asc("user.id") + case "user_auth_token": + sess.Table(table). + Join("inner", "org_user", "user_auth_token.id = org_user.user_id"). + Cols("user_auth_token.*"). + Where("org_user.org_id = ?", helper.orgID). + Asc("user_auth_token.id") + default: + sess.Table(table).Where("org_id = ?", helper.orgID).Asc("id") + } + + raw, err := sess.QueryInterface() + if err != nil { + return fmt.Errorf("unable to read: %s // %s", table, err.Error()) + } + if len(raw) < 1 { + continue // don't write empty files + } + frame, err := queryResultToDataFrame(raw, frameOpts{ + skip: []string{"org_id", "version", "help_flags1", "theme"}, + }) + if err != nil { + return err + } + frame.Name = table + commit.body = append(commit.body, commitBody{ + fpath: path.Join(helper.orgDir, "auth", "sql.dump", table+".json"), + frame: frame, + }) + } + return helper.add(commit) + }) +} diff --git a/pkg/services/export/export_dashboards.go b/pkg/services/export/export_dashboards.go new file mode 100644 index 00000000000..212c31c1a38 --- /dev/null +++ b/pkg/services/export/export_dashboards.go @@ -0,0 +1,229 @@ +package export + +import ( + "bytes" + "encoding/json" + "fmt" + "path" + "path/filepath" + "strings" + "time" + + "github.com/google/uuid" + "github.com/grafana/grafana/pkg/infra/filestorage" + "github.com/grafana/grafana/pkg/services/searchV2/extract" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error { + alias := make(map[string]string, 100) + ids := make(map[int64]string, 100) + folders := make(map[int64]string, 100) + + // Should root files be at the root or in a subfolder called "general"? + if true { + folders[0] = "general" + } + + rootDir := path.Join(helper.orgDir, "root") + folderStructure := commitOptions{ + when: time.Now(), + comment: "Exported folder structure", + } + + err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + type dashDataQueryResult struct { + Id int64 + UID string `xorm:"uid"` + IsFolder bool `xorm:"is_folder"` + FolderID int64 `xorm:"folder_id"` + Slug string `xorm:"slug"` + Data []byte + Created time.Time + Updated time.Time + } + + rows := make([]*dashDataQueryResult, 0) + + sess.Table("dashboard"). + Where("org_id = ?", helper.orgID). + Cols("id", "is_folder", "folder_id", "data", "slug", "created", "updated", "uid") + + err := sess.Find(&rows) + if err != nil { + return err + } + + // Process all folders + for _, row := range rows { + if !row.IsFolder { + continue + } + dash, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup) + if err != nil { + return err + } + + dash.UID = row.UID + slug := cleanFileName(dash.Title) + folder := map[string]string{ + "title": dash.Title, + } + + folderStructure.body = append(folderStructure.body, commitBody{ + fpath: path.Join(rootDir, slug, "__folder.json"), + body: prettyJSON(folder), + }) + + alias[dash.UID] = slug + folders[row.Id] = slug + + if row.Created.Before(folderStructure.when) { + folderStructure.when = row.Created + } + } + + // Now process the dashboards in each folder + for _, row := range rows { + if row.IsFolder { + continue + } + fname := row.Slug + "-dash.json" + fpath, ok := folders[row.FolderID] + if ok { + fpath = path.Join(fpath, fname) + } else { + fpath = fname + } + + alias[row.UID] = fpath + ids[row.Id] = fpath + } + + return err + }) + if err != nil { + return err + } + + err = helper.add(folderStructure) + if err != nil { + return err + } + + err = helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(helper.orgDir, "root-alias.json"), + body: prettyJSON(alias), + }, + { + fpath: filepath.Join(helper.orgDir, "root-ids.json"), + body: prettyJSON(ids), + }, + }, + when: folderStructure.when, + comment: "adding UID alias structure", + }) + if err != nil { + return err + } + + // Now walk the history + err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + type dashVersionResult struct { + DashId int64 `xorm:"dashboard_id"` + Version int64 `xorm:"version"` + Created time.Time `xorm:"created"` + CreatedBy int64 `xorm:"created_by"` + Message string `xorm:"message"` + Data []byte + } + + rows := make([]*dashVersionResult, 0, len(ids)) + + sess.Table("dashboard_version"). + Join("INNER", "dashboard", "dashboard.id = dashboard_version.dashboard_id"). + Where("org_id = ?", job.orgID). + Cols("dashboard_version.dashboard_id", + "dashboard_version.version", + "dashboard_version.created", + "dashboard_version.created_by", + "dashboard_version.message", + "dashboard_version.data"). + Asc("dashboard_version.created") + + err := sess.Find(&rows) + if err != nil { + return err + } + + count := int64(0) + + // Process all folders (only one level deep!!!) + for _, row := range rows { + fpath, ok := ids[row.DashId] + if !ok { + continue + } + + msg := row.Message + if msg == "" { + msg = fmt.Sprintf("Version: %d", row.Version) + } + + err = helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(rootDir, fpath), + body: cleanDashboardJSON(row.Data), + }, + }, + userID: row.CreatedBy, + when: row.Created, + comment: msg, + }) + if err != nil { + return err + } + + count++ + fmt.Printf("COMMIT: %d // %s (%d)\n", count, fpath, row.Version) + + job.status.Current = count + job.status.Last = fpath + job.status.Changed = time.Now().UnixMilli() + job.broadcaster(job.status) + } + + return nil + }) + + return err +} + +func cleanDashboardJSON(data []byte) []byte { + var dash map[string]interface{} + err := json.Unmarshal(data, &dash) + if err != nil { + return nil + } + delete(dash, "id") + delete(dash, "uid") + delete(dash, "version") + + clean, _ := json.MarshalIndent(dash, "", " ") + return clean +} + +// replace any unsafe file name characters... TODO, but be a standard way to do this cleanly!!! +func cleanFileName(name string) string { + name = strings.ReplaceAll(name, "/", "-") + name = strings.ReplaceAll(name, "\\", "-") + name = strings.ReplaceAll(name, ":", "-") + if err := filestorage.ValidatePath(filestorage.Delimiter + name); err != nil { + randomName, _ := uuid.NewRandom() + return randomName.String() + } + return name +} diff --git a/pkg/services/export/export_ds.go b/pkg/services/export/export_ds.go new file mode 100644 index 00000000000..4dbe53c5553 --- /dev/null +++ b/pkg/services/export/export_ds.go @@ -0,0 +1,72 @@ +package export + +import ( + "fmt" + "path/filepath" + "sort" + + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/searchV2/extract" +) + +type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef + +func exportDataSources(helper *commitHelper, job *gitExportJob) (dsLookup, error) { + cmd := &datasources.GetDataSourcesQuery{ + OrgId: job.orgID, + } + err := job.sql.GetDataSources(helper.ctx, cmd) + if err != nil { + return nil, err + } + + sort.SliceStable(cmd.Result, func(i, j int) bool { + return cmd.Result[i].Created.After(cmd.Result[j].Created) + }) + + byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result)) + byName := make(map[string]*extract.DataSourceRef, len(cmd.Result)) + for _, ds := range cmd.Result { + ref := &extract.DataSourceRef{ + UID: ds.Uid, + Type: ds.Type, + } + byUID[ds.Uid] = ref + byName[ds.Name] = ref + ds.OrgId = 0 + ds.Version = 0 + + err := helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(helper.orgDir, "datasources", fmt.Sprintf("%s-ds.json", ds.Uid)), + body: prettyJSON(ds), + }, + }, + when: ds.Created, + comment: fmt.Sprintf("Add datasource: %s", ds.Name), + }) + if err != nil { + return nil, err + } + } + + // Return the lookup function + return func(ref *extract.DataSourceRef) *extract.DataSourceRef { + if ref == nil || ref.UID == "" { + return &extract.DataSourceRef{ + UID: "default.uid", + Type: "default.type", + } + } + v, ok := byUID[ref.UID] + if ok { + return v + } + v, ok = byName[ref.UID] + if ok { + return v + } + return nil + }, nil +} diff --git a/pkg/services/export/export_snapshots.go b/pkg/services/export/export_snapshots.go new file mode 100644 index 00000000000..e9390cc8b79 --- /dev/null +++ b/pkg/services/export/export_snapshots.go @@ -0,0 +1,43 @@ +package export + +import ( + "fmt" + "path/filepath" + "time" + + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" +) + +func exportSnapshots(helper *commitHelper, job *gitExportJob) error { + cmd := &dashboardsnapshots.GetDashboardSnapshotsQuery{ + OrgId: job.orgID, + Limit: 500000, + SignedInUser: nil, + } + if cmd.SignedInUser == nil { + return fmt.Errorf("snapshots requires an admin user") + } + + err := job.dashboardsnapshotsService.SearchDashboardSnapshots(helper.ctx, cmd) + if err != nil { + return err + } + + if len(cmd.Result) < 1 { + return nil // nothing + } + + gitcmd := commitOptions{ + when: time.Now(), + comment: "Export playlists", + } + + for _, snapshot := range cmd.Result { + gitcmd.body = append(gitcmd.body, commitBody{ + fpath: filepath.Join(helper.orgDir, "snapshot", fmt.Sprintf("%d-snapshot.json", snapshot.Id)), + body: prettyJSON(snapshot), + }) + } + + return helper.add(gitcmd) +} diff --git a/pkg/services/export/export_sys_playlists.go b/pkg/services/export/export_sys_playlists.go new file mode 100644 index 00000000000..83943db9563 --- /dev/null +++ b/pkg/services/export/export_sys_playlists.go @@ -0,0 +1,40 @@ +package export + +import ( + "fmt" + "path/filepath" + "time" + + "github.com/grafana/grafana/pkg/models" +) + +func exportSystemPlaylists(helper *commitHelper, job *gitExportJob) error { + cmd := &models.GetPlaylistsQuery{ + OrgId: job.orgID, + Limit: 500000, + } + err := job.sql.SearchPlaylists(helper.ctx, cmd) + if err != nil { + return err + } + + if len(cmd.Result) < 1 { + return nil // nothing + } + + gitcmd := commitOptions{ + when: time.Now(), + comment: "Export playlists", + } + + for _, playlist := range cmd.Result { + // TODO: fix the playlist API so it returns the json we need :) + + gitcmd.body = append(gitcmd.body, commitBody{ + fpath: filepath.Join(helper.orgDir, "system", "playlists", fmt.Sprintf("%s-playlist.json", playlist.UID)), + body: prettyJSON(playlist), + }) + } + + return helper.add(gitcmd) +} diff --git a/pkg/services/export/export_sys_preferences.go b/pkg/services/export/export_sys_preferences.go new file mode 100644 index 00000000000..926240acc04 --- /dev/null +++ b/pkg/services/export/export_sys_preferences.go @@ -0,0 +1,128 @@ +package export + +import ( + "fmt" + "path" + "path/filepath" + "time" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func exportSystemPreferences(helper *commitHelper, job *gitExportJob) error { + type preferences struct { + UserID int64 `json:"-" xorm:"user_id"` + TeamID int64 `json:"-" xorm:"team_id"` + HomeDashboardID int64 `json:"-" xorm:"home_dashboard_id"` + Updated time.Time `json:"-" xorm:"updated"` + JSONData map[string]interface{} `json:"-" xorm:"json_data"` + + Theme string `json:"theme"` + Locale string `json:"locale"` + Timezone string `json:"timezone"` + WeekStart string `json:"week_start,omitempty"` + HomeDashboard string `json:"home,omitempty" xorm:"uid"` // dashboard + NavBar interface{} `json:"navbar,omitempty"` + QueryHistory interface{} `json:"queryHistory,omitempty"` + } + + prefsDir := path.Join(helper.orgDir, "system", "preferences") + users := make(map[int64]*userInfo, len(helper.users)) + for _, user := range helper.users { + users[user.ID] = user + } + + return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + rows := make([]*preferences, 0) + + sess.Table("preferences"). + Join("LEFT", "dashboard", "dashboard.id = preferences.home_dashboard_id"). + Cols("preferences.*", "dashboard.uid"). + Where("preferences.org_id = ?", helper.orgID) + + err := sess.Find(&rows) + if err != nil { + return err + } + + var comment string + var fpath string + for _, row := range rows { + if row.TeamID > 0 { + fpath = filepath.Join(prefsDir, "team", fmt.Sprintf("%d.json", row.TeamID)) + comment = fmt.Sprintf("Team preferences: %d", row.TeamID) + } else if row.UserID == 0 { + fpath = filepath.Join(prefsDir, "default.json") + comment = "Default preferences" + } else { + user, ok := users[row.UserID] + if ok { + delete(users, row.UserID) + } else { + user = &userInfo{ + Login: fmt.Sprintf("__%d__", row.UserID), + } + } + fpath = filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login)) + comment = fmt.Sprintf("User preferences: %s", user.getAuthor().Name) + } + + if row.JSONData != nil { + v, ok := row.JSONData["locale"] + if ok && row.Locale == "" { + s, ok := v.(string) + if ok { + row.Locale = s + } + } + + v, ok = row.JSONData["navbar"] + if ok && row.NavBar == nil { + row.NavBar = v + } + + v, ok = row.JSONData["queryHistory"] + if ok && row.QueryHistory == nil { + row.QueryHistory = v + } + } + + err := helper.add(commitOptions{ + body: []commitBody{ + { + fpath: fpath, + body: prettyJSON(row), + }, + }, + when: row.Updated, + comment: comment, + userID: row.UserID, + }) + if err != nil { + return err + } + } + + // add a file for all useres that may not be in the system + for _, user := range users { + row := preferences{ + Theme: user.Theme, // never set? + } + err := helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login)), + body: prettyJSON(row), + }, + }, + when: user.Updated, + comment: "user preferences", + userID: row.UserID, + }) + if err != nil { + return err + } + } + return err + }) +} diff --git a/pkg/services/export/export_sys_stars.go b/pkg/services/export/export_sys_stars.go new file mode 100644 index 00000000000..53fd6fea0b9 --- /dev/null +++ b/pkg/services/export/export_sys_stars.go @@ -0,0 +1,65 @@ +package export + +import ( + "fmt" + "path/filepath" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func exportSystemStars(helper *commitHelper, job *gitExportJob) error { + byUser := make(map[int64][]string, 50) + + err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + type starResult struct { + User int64 `xorm:"user_id"` + UID string `xorm:"uid"` + } + + rows := make([]*starResult, 0) + + sess.Table("star"). + Join("INNER", "dashboard", "dashboard.id = star.dashboard_id"). + Cols("star.user_id", "dashboard.uid"). + Where("dashboard.org_id = ?", helper.orgID) + + err := sess.Find(&rows) + if err != nil { + return err + } + + for _, row := range rows { + stars := append(byUser[row.User], fmt.Sprintf("dashboard/%s", row.UID)) + byUser[row.User] = stars + } + return err + }) + if err != nil { + return err + } + + for userID, stars := range byUser { + user, ok := helper.users[userID] + if !ok { + user = &userInfo{ + Login: fmt.Sprintf("__unknown_%d", userID), + } + } + + err := helper.add(commitOptions{ + body: []commitBody{ + { + fpath: filepath.Join(helper.orgDir, "system", "stars", fmt.Sprintf("%s.json", user.Login)), + body: prettyJSON(stars), + }, + }, + when: user.Updated, + comment: "user preferences", + userID: userID, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/services/export/frame_helper.go b/pkg/services/export/frame_helper.go new file mode 100644 index 00000000000..1f7a291d8e5 --- /dev/null +++ b/pkg/services/export/frame_helper.go @@ -0,0 +1,138 @@ +package export + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/grafana/grafana-plugin-sdk-go/data" +) + +type fieldInfo struct { + Name string + Conv data.FieldConverter +} + +type frameOpts struct { + schema []fieldInfo + skip []string +} + +func prettyJSON(v interface{}) []byte { + b, _ := json.MarshalIndent(v, "", " ") + return b +} + +func queryResultToDataFrame(rows []map[string]interface{}, opts frameOpts) (*data.Frame, error) { + count := len(rows) + if count < 1 { + return nil, nil // empty frame + } + + schema := opts.schema + if len(schema) < 1 { + skip := make(map[string]bool, len(opts.skip)) + for _, k := range opts.skip { + skip[k] = true + } + + for k, v := range rows[0] { + if skip[k] { + continue + } + field := fieldInfo{ + Name: k, + Conv: data.FieldConverter{ + OutputFieldType: data.FieldTypeFor(v), + }, + } + if field.Conv.OutputFieldType == data.FieldTypeUnknown { + fmt.Printf("UNKNOWN type: %s / %v\n", k, v) + continue + } + + // Don't write passwords to disk for now!!!! + if k == "password" || k == "salt" { + field.Conv.Converter = func(v interface{}) (interface{}, error) { + return fmt.Sprintf("<%s>", k), nil + } + } + + schema = append(schema, field) + } + } + + fields := make([]*data.Field, len(schema)) + for i, s := range schema { + fields[i] = data.NewFieldFromFieldType(s.Conv.OutputFieldType, count) + fields[i].Name = s.Name + } + + var err error + for i, row := range rows { + for j, s := range schema { + v, ok := row[s.Name] + if ok && v != nil { + if s.Conv.Converter != nil { + v, err = s.Conv.Converter(v) + if err != nil { + return nil, fmt.Errorf("converting field: %s // %s", s.Name, err.Error()) + } + } + fields[j].Set(i, v) + } + } + } + + // Fields are in random order + if len(opts.schema) < 1 { + last := []*data.Field{} + frame := data.NewFrame("") + lookup := make(map[string]*data.Field, len(fields)) + for _, f := range fields { + if f.Name == "id" { + frame.Fields = append(frame.Fields, f) // first + continue + } + lookup[f.Name] = f + } + + // First items + for _, name := range []string{"name", "login", "email", "role", "description", "uid"} { + f, ok := lookup[name] + if ok { + frame.Fields = append(frame.Fields, f) // first + delete(lookup, name) + } + } + + // IDs + for k, f := range lookup { + if strings.HasSuffix(k, "_id") { + frame.Fields = append(frame.Fields, f) // first + delete(lookup, k) + } else if strings.HasPrefix(k, "is_") { + last = append(last, f) // first + delete(lookup, k) + } + } + + // Last items + for _, name := range []string{"created", "updated"} { + f, ok := lookup[name] + if ok { + last = append(last, f) // first + delete(lookup, name) + } + } + + // Rest + for _, f := range lookup { + frame.Fields = append(frame.Fields, f) + } + + frame.Fields = append(frame.Fields, last...) + return frame, nil + } + return data.NewFrame("", fields...), nil +} diff --git a/pkg/services/export/git_export_job.go b/pkg/services/export/git_export_job.go new file mode 100644 index 00000000000..21243e05c52 --- /dev/null +++ b/pkg/services/export/git_export_job.go @@ -0,0 +1,205 @@ +package export + +import ( + "context" + "fmt" + "path" + "sync" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +var _ Job = new(gitExportJob) + +type gitExportJob struct { + logger log.Logger + sql *sqlstore.SQLStore + dashboardsnapshotsService dashboardsnapshots.Service + orgID int64 + rootDir string + + statusMu sync.Mutex + status ExportStatus + cfg ExportConfig + broadcaster statusBroadcaster +} + +type simpleExporter = func(helper *commitHelper, job *gitExportJob) error + +func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster) (Job, error) { + job := &gitExportJob{ + logger: log.New("git_export_job"), + cfg: cfg, + sql: sql, + dashboardsnapshotsService: dashboardsnapshotsService, + orgID: orgID, + rootDir: rootDir, + broadcaster: broadcaster, + status: ExportStatus{ + Running: true, + Target: "git export", + Started: time.Now().UnixMilli(), + Current: 0, + }, + } + + broadcaster(job.status) + go job.start() + return job, nil +} + +func (e *gitExportJob) getStatus() ExportStatus { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.status +} + +func (e *gitExportJob) getConfig() ExportConfig { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.cfg +} + +// Utility function to export dashboards +func (e *gitExportJob) start() { + defer func() { + e.logger.Info("Finished git export job") + + e.statusMu.Lock() + defer e.statusMu.Unlock() + s := e.status + if err := recover(); err != nil { + e.logger.Error("export panic", "error", err) + s.Status = fmt.Sprintf("ERROR: %v", err) + } + // Make sure it finishes OK + if s.Finished < 10 { + s.Finished = time.Now().UnixMilli() + } + s.Running = false + if s.Status == "" { + s.Status = "done" + } + s.Target = e.rootDir + e.status = s + e.broadcaster(s) + }() + + err := e.doExportWithHistory() + if err != nil { + e.logger.Error("ERROR", "e", err) + e.status.Status = "ERROR" + e.status.Last = err.Error() + e.broadcaster(e.status) + } +} + +func (e *gitExportJob) doExportWithHistory() error { + r, err := git.PlainInit(e.rootDir, false) + if err != nil { + return err + } + // default to "main" branch + h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) + err = r.Storer.SetReference(h) + if err != nil { + return err + } + + w, err := r.Worktree() + if err != nil { + return err + } + helper := &commitHelper{ + repo: r, + work: w, + ctx: context.Background(), + workDir: e.rootDir, + orgDir: e.rootDir, + } + + cmd := &models.SearchOrgsQuery{} + err = e.sql.SearchOrgs(helper.ctx, cmd) + if err != nil { + return err + } + + // Export each org + for _, org := range cmd.Result { + if len(cmd.Result) > 1 { + helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id)) + } + err = helper.initOrg(e.sql, org.Id) + if err != nil { + return err + } + + err = e.doOrgExportWithHistory(helper) + if err != nil { + return err + } + } + + // cleanup the folder + e.status.Target = "pruning..." + e.broadcaster(e.status) + err = r.Prune(git.PruneOptions{}) + + // TODO + // git gc --prune=now --aggressive + + return err +} + +func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error { + lookup, err := exportDataSources(helper, e) + if err != nil { + return err + } + + if true { + err = exportDashboards(helper, e, lookup) + if err != nil { + return err + } + } + + // Run all the simple exporters + exporters := []simpleExporter{ + dumpAuthTables, + exportSystemPreferences, + exportSystemStars, + exportSystemPlaylists, + exportAnnotations, + } + + // This needs a real admin user to use the interfaces (and decrypt) + if false { + exporters = append(exporters, exportSnapshots) + } + + for _, fn := range exporters { + err = fn(helper, e) + if err != nil { + return err + } + } + return err +} + +/** + +git remote add origin git@github.com:ryantxu/test-dash-repo.git +git branch -M main +git push -u origin main + +**/ diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go index ee4f1b040ea..6551775a065 100644 --- a/pkg/services/export/service.go +++ b/pkg/services/export/service.go @@ -2,15 +2,21 @@ package export import ( "encoding/json" + "fmt" "net/http" + "os" + "path/filepath" "sync" + "time" "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/dashboardsnapshots" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" ) type ExportService interface { @@ -22,25 +28,31 @@ type ExportService interface { } type StandardExport struct { - logger log.Logger - sql *sqlstore.SQLStore - glive *live.GrafanaLive - mutex sync.Mutex + logger log.Logger + glive *live.GrafanaLive + mutex sync.Mutex + dataDir string + + // Services + sql *sqlstore.SQLStore + dashboardsnapshotsService dashboardsnapshots.Service // updated with mutex exportJob Job } -func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService { +func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive, cfg *setting.Cfg, dashboardsnapshotsService dashboardsnapshots.Service) ExportService { if !features.IsEnabled(featuremgmt.FlagExport) { return &StubExport{} } return &StandardExport{ - sql: sql, - glive: gl, - logger: log.New("export_service"), - exportJob: &stoppedJob{}, + sql: sql, + glive: gl, + logger: log.New("export_service"), + dashboardsnapshotsService: dashboardsnapshotsService, + exportJob: &stoppedJob{}, + dataDir: cfg.DataPath, } } @@ -67,9 +79,23 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res return response.Error(http.StatusLocked, "export already running", nil) } - job, err := startDummyExportJob(cfg, func(s ExportStatus) { + var job Job + broadcast := func(s ExportStatus) { ex.broadcastStatus(c.OrgId, s) - }) + } + switch cfg.Format { + case "dummy": + job, err = startDummyExportJob(cfg, broadcast) + case "git": + dir := filepath.Join(ex.dataDir, "export_git", fmt.Sprintf("git_%d", time.Now().Unix())) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return response.Error(http.StatusBadRequest, "Error creating export folder", nil) + } + job, err = startGitExportJob(cfg, ex.sql, ex.dashboardsnapshotsService, dir, c.OrgId, broadcast) + default: + return response.Error(http.StatusBadRequest, "Unsupported job format", nil) + } + if err != nil { ex.logger.Error("failed to start export job", "err", err) return response.Error(http.StatusBadRequest, "failed to start export job", err) diff --git a/pkg/services/searchV2/extract/types.go b/pkg/services/searchV2/extract/types.go index 71ff5545bb1..9b084cd2409 100644 --- a/pkg/services/searchV2/extract/types.go +++ b/pkg/services/searchV2/extract/types.go @@ -22,6 +22,7 @@ type PanelInfo struct { } type DashboardInfo struct { + UID string `json:"uid,omitempty"` ID int64 `json:"id,omitempty"` // internal ID Title string `json:"title"` Description string `json:"description,omitempty"`