Plugins: Support templated urls in routes (#16599)

This adds support for using templated/dynamic urls in routes.
* refactor interpolateString into utils and add interpolation support for app plugin routes.
* cleanup and add error check for url parse failure
* add docs for interpolated route urls

Closes #16835

docs/sources/plugins/developing/auth-for-datasources.md

@@ -51,6 +51,36 @@ then the Grafana proxy will transform it into "https://management.azure.com/foo/
 
 The `method` parameter is optional. It can be set to any HTTP verb to provide more fine-grained control.
 
+### Dynamic Routes
+
+When using routes, you can also reference a variable stored in JsonData or SecureJsonData which will be interpolated when connecting to the datasource.
+
+With JsonData:
+```json
+"routes": [
+  {
+      "path": "custom/api/v5/*",
+      "method": "*",
+      "url": "{{.JsonData.dynamicUrl}}",
+      ...
+  },
+]
+```
+
+With SecureJsonData:
+```json
+"routes": [{
+      "path": "custom/api/v5/*",
+      "method": "*",
+      "url": "{{.SecureJsonData.dynamicUrl}}",
+  ...
+}]
+```
+
+In the above example, the app is able to set the value for `dynamicUrl` in JsonData or SecureJsonData and it will be replaced on-demand.
+
+An app using this feature can be found [here](https://github.com/grafana/kentik-app).
+
 ## Encrypting Sensitive Data
 
 When a user saves a password or secret with your datasource plugin's Config page, then you can save data to a column in the datasource table called `secureJsonData` that is an encrypted blob. Any data saved in the blob is encrypted by Grafana and can only be decrypted by the Grafana server on the backend. This means once a password is saved, no sensitive data is sent to the browser. If the password is saved in the `jsonData` blob or the `password` field then it is unencrypted and anyone with Admin access (with the help of Chrome Developer Tools) can read it.

pkg/api/pluginproxy/access_token_provider.go

@@ -67,14 +67,14 @@ func (provider *accessTokenProvider) getAccessToken(data templateData) (string,
 		}
 	}
 
-	urlInterpolated, err := interpolateString(provider.route.TokenAuth.Url, data)
+	urlInterpolated, err := InterpolateString(provider.route.TokenAuth.Url, data)
 	if err != nil {
 		return "", err
 	}
 
 	params := make(url.Values)
 	for key, value := range provider.route.TokenAuth.Params {
-		interpolatedParam, err := interpolateString(value, data)
+		interpolatedParam, err := InterpolateString(value, data)
 		if err != nil {
 			return "", err
 		}
@@ -119,7 +119,7 @@ func (provider *accessTokenProvider) getJwtAccessToken(ctx context.Context, data
 	conf := &jwt.Config{}
 
 	if val, ok := provider.route.JwtTokenAuth.Params["client_email"]; ok {
-		interpolatedVal, err := interpolateString(val, data)
+		interpolatedVal, err := InterpolateString(val, data)
 		if err != nil {
 			return "", err
 		}
@@ -127,7 +127,7 @@ func (provider *accessTokenProvider) getJwtAccessToken(ctx context.Context, data
 	}
 
 	if val, ok := provider.route.JwtTokenAuth.Params["private_key"]; ok {
-		interpolatedVal, err := interpolateString(val, data)
+		interpolatedVal, err := InterpolateString(val, data)
 		if err != nil {
 			return "", err
 		}
@@ -135,7 +135,7 @@ func (provider *accessTokenProvider) getJwtAccessToken(ctx context.Context, data
 	}
 
 	if val, ok := provider.route.JwtTokenAuth.Params["token_uri"]; ok {
-		interpolatedVal, err := interpolateString(val, data)
+		interpolatedVal, err := InterpolateString(val, data)
 		if err != nil {
 			return "", err
 		}

pkg/api/pluginproxy/ds_auth_provider.go

@@ -1,13 +1,11 @@
 package pluginproxy
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
-	"text/template"
 
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -24,7 +22,7 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route
 		SecureJsonData: ds.SecureJsonData.Decrypt(),
 	}
 
-	interpolatedURL, err := interpolateString(route.Url, data)
+	interpolatedURL, err := InterpolateString(route.Url, data)
 	if err != nil {
 		logger.Error("Error interpolating proxy url", "error", err)
 		return
@@ -81,24 +79,9 @@ func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route
 	logger.Info("Requesting", "url", req.URL.String())
 }
 
-func interpolateString(text string, data templateData) (string, error) {
-	t, err := template.New("content").Parse(text)
-	if err != nil {
-		return "", fmt.Errorf("could not parse template %s", text)
-	}
-
-	var contentBuf bytes.Buffer
-	err = t.Execute(&contentBuf, data)
-	if err != nil {
-		return "", fmt.Errorf("failed to execute template %s", text)
-	}
-
-	return contentBuf.String(), nil
-}
-
 func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error {
 	for _, header := range route.Headers {
-		interpolated, err := interpolateString(header.Content, data)
+		interpolated, err := InterpolateString(header.Content, data)
 		if err != nil {
 			return err
 		}

pkg/api/pluginproxy/ds_auth_provider_test.go

@@ -14,7 +14,7 @@ func TestDsAuthProvider(t *testing.T) {
 			},
 		}
 
-		interpolated, err := interpolateString("{{.SecureJsonData.Test}}", data)
+		interpolated, err := InterpolateString("{{.SecureJsonData.Test}}", data)
 		So(err, ShouldBeNil)
 		So(interpolated, ShouldEqual, "0asd+asd")
 	})

pkg/api/pluginproxy/pluginproxy.go

@@ -2,12 +2,13 @@ package pluginproxy
 
 import (
 	"encoding/json"
-	"github.com/grafana/grafana/pkg/setting"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 
+	"github.com/grafana/grafana/pkg/setting"
+
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
@@ -38,6 +39,24 @@ func getHeaders(route *plugins.AppPluginRoute, orgId int64, appID string) (http.
 	return result, err
 }
 
+func updateURL(route *plugins.AppPluginRoute, orgId int64, appID string) (string, error) {
+	query := m.GetPluginSettingByIdQuery{OrgId: orgId, PluginId: appID}
+	if err := bus.Dispatch(&query); err != nil {
+		return "", err
+	}
+
+	data := templateData{
+		JsonData:       query.Result.JsonData,
+		SecureJsonData: query.Result.SecureJsonData.Decrypt(),
+	}
+	interpolated, err := InterpolateString(route.Url, data)
+	if err != nil {
+		return "", err
+	}
+	return interpolated, err
+}
+
+// NewApiPluginProxy create a plugin proxy
 func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPluginRoute, appID string, cfg *setting.Cfg) *httputil.ReverseProxy {
 	targetURL, _ := url.Parse(route.Url)
 
@@ -48,7 +67,6 @@ func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPl
 		req.Host = targetURL.Host
 
 		req.URL.Path = util.JoinURLFragments(targetURL.Path, proxyPath)
-
 		// clear cookie headers
 		req.Header.Del("Cookie")
 		req.Header.Del("Set-Cookie")
@@ -72,13 +90,13 @@ func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPl
 		}
 
 		// Create a HTTP header with the context in it.
-		ctxJson, err := json.Marshal(ctx.SignedInUser)
+		ctxJSON, err := json.Marshal(ctx.SignedInUser)
 		if err != nil {
 			ctx.JsonApiErr(500, "failed to marshal context to json.", err)
 			return
 		}
 
-		req.Header.Add("X-Grafana-Context", string(ctxJson))
+		req.Header.Add("X-Grafana-Context", string(ctxJSON))
 
 		if cfg.SendUserHeader && !ctx.SignedInUser.IsAnonymous {
 			req.Header.Add("X-Grafana-User", ctx.SignedInUser.Login)
@@ -97,6 +115,27 @@ func NewApiPluginProxy(ctx *m.ReqContext, proxyPath string, route *plugins.AppPl
 			}
 		}
 
+		if len(route.Url) > 0 {
+			interpolatedURL, err := updateURL(route, ctx.OrgId, appID)
+			if err != nil {
+				ctx.JsonApiErr(500, "Could not interpolate plugin route url", err)
+			}
+			targetURL, err := url.Parse(interpolatedURL)
+			if err != nil {
+				ctx.JsonApiErr(500, "Could not parse custom url: %v", err)
+				return
+			}
+			req.URL.Scheme = targetURL.Scheme
+			req.URL.Host = targetURL.Host
+			req.Host = targetURL.Host
+			req.URL.Path = util.JoinURLFragments(targetURL.Path, proxyPath)
+
+			if err != nil {
+				ctx.JsonApiErr(500, "Could not interpolate plugin route url", err)
+				return
+			}
+		}
+
 		// reqBytes, _ := httputil.DumpRequestOut(req, true);
 		// log.Trace("Proxying plugin request: %s", string(reqBytes))
 	}

pkg/api/pluginproxy/pluginproxy_test.go

@@ -53,6 +53,7 @@ func TestPluginProxy(t *testing.T) {
 				},
 			},
 			&setting.Cfg{SendUserHeader: true},
+			nil,
 		)
 
 		Convey("Should add header with username", func() {
@@ -69,6 +70,7 @@ func TestPluginProxy(t *testing.T) {
 				},
 			},
 			&setting.Cfg{SendUserHeader: false},
+			nil,
 		)
 		Convey("Should not add header with username", func() {
 			// Get will return empty string even if header is not set
@@ -82,6 +84,7 @@ func TestPluginProxy(t *testing.T) {
 				SignedInUser: &m.SignedInUser{IsAnonymous: true},
 			},
 			&setting.Cfg{SendUserHeader: true},
+			nil,
 		)
 
 		Convey("Should not add header with username", func() {
@@ -89,14 +92,59 @@ func TestPluginProxy(t *testing.T) {
 			So(req.Header.Get("X-Grafana-User"), ShouldEqual, "")
 		})
 	})
+
+	Convey("When getting templated url", t, func() {
+		route := &plugins.AppPluginRoute{
+			Url:    "{{.JsonData.dynamicUrl}}",
+			Method: "GET",
+		}
+
+		bus.AddHandler("test", func(query *m.GetPluginSettingByIdQuery) error {
+			query.Result = &m.PluginSetting{
+				JsonData: map[string]interface{}{
+					"dynamicUrl": "https://dynamic.grafana.com",
+				},
+			}
+			return nil
+		})
+
+		req := getPluginProxiedRequest(
+			&m.ReqContext{
+				SignedInUser: &m.SignedInUser{
+					Login: "test_user",
+				},
+			},
+			&setting.Cfg{SendUserHeader: true},
+			route,
+		)
+		Convey("Headers should be updated", func() {
+			header, err := getHeaders(route, 1, "my-app")
+			So(err, ShouldBeNil)
+			So(header.Get("X-Grafana-User"), ShouldEqual, "")
+		})
+		Convey("Should set req.URL to be interpolated value from jsonData", func() {
+			So(req.URL.String(), ShouldEqual, "https://dynamic.grafana.com")
+		})
+		Convey("Route url should not be modified", func() {
+			So(route.Url, ShouldEqual, "{{.JsonData.dynamicUrl}}")
+		})
+	})
+
 }
 
 // getPluginProxiedRequest is a helper for easier setup of tests based on global config and ReqContext.
-func getPluginProxiedRequest(ctx *m.ReqContext, cfg *setting.Cfg) *http.Request {
-	route := &plugins.AppPluginRoute{}
+func getPluginProxiedRequest(ctx *m.ReqContext, cfg *setting.Cfg, route *plugins.AppPluginRoute) *http.Request {
+	// insert dummy route if none is specified
+	if route == nil {
+		route = &plugins.AppPluginRoute{
+			Path:    "api/v4/",
+			Url:     "https://www.google.com",
+			ReqRole: m.ROLE_EDITOR,
+		}
+	}
 	proxy := NewApiPluginProxy(ctx, "", route, "", cfg)
 
-	req, err := http.NewRequest(http.MethodGet, "http://grafana.com/sub", nil)
+	req, err := http.NewRequest(http.MethodGet, route.Url, nil)
 	So(err, ShouldBeNil)
 	proxy.Director(req)
 	return req

pkg/api/pluginproxy/utils.go

@@ -0,0 +1,49 @@
+package pluginproxy
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"text/template"
+
+	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/plugins"
+)
+
+// InterpolateString accepts template data and return a string with substitutions
+func InterpolateString(text string, data templateData) (string, error) {
+	t, err := template.New("content").Parse(text)
+	if err != nil {
+		return "", fmt.Errorf("could not parse template %s", text)
+	}
+
+	var contentBuf bytes.Buffer
+	err = t.Execute(&contentBuf, data)
+	if err != nil {
+		return "", fmt.Errorf("failed to execute template %s", text)
+	}
+
+	return contentBuf.String(), nil
+}
+
+// InterpolateURL accepts template data and return a string with substitutions
+func InterpolateURL(anURL *url.URL, route *plugins.AppPluginRoute, orgID int64, appID string) (*url.URL, error) {
+	query := m.GetPluginSettingByIdQuery{OrgId: orgID, PluginId: appID}
+	result, err := url.Parse(anURL.String())
+	if query.Result != nil {
+		if len(query.Result.JsonData) > 0 {
+			data := templateData{
+				JsonData: query.Result.JsonData,
+			}
+			interpolatedResult, err := InterpolateString(anURL.String(), data)
+			if err == nil {
+				result, err = url.Parse(interpolatedResult)
+				if err != nil {
+					return nil, fmt.Errorf("Error parsing plugin route url %v", err)
+				}
+			}
+		}
+	}
+
+	return result, err
+}

pkg/api/pluginproxy/utils_test.go

@@ -0,0 +1,21 @@
+package pluginproxy
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestInterpolateString(t *testing.T) {
+	Convey("When interpolating string", t, func() {
+		data := templateData{
+			SecureJsonData: map[string]string{
+				"Test": "0asd+asd",
+			},
+		}
+
+		interpolated, err := InterpolateString("{{.SecureJsonData.Test}}", data)
+		So(err, ShouldBeNil)
+		So(interpolated, ShouldEqual, "0asd+asd")
+	})
+}