Image Rendering: New setting to control render request concurrency (#23950)

Fixes #23806

Co-Authored-By: Torkel Ödegaard <torkel@grafana.com>
Co-Authored-By: Arve Knudsen <arve.knudsen@gmail.com>

conf/defaults.ini

@@ -683,6 +683,9 @@ container_name =
 server_url =
 # If the remote HTTP image renderer service runs on a different server than the Grafana server you may have to configure this to a URL where Grafana is reachable, e.g. http://grafana.domain/.
 callback_url =
+# Concurrent render request limit affects when the /render HTTP endpoint is used. Rendering many images at the same time can overload the server,
+# which this setting can help protect against by only allowing a certain amount of concurrent requests.
+concurrent_render_request_limit = 30
 
 [panels]
 # here for to support old env variables, can remove after a few months

conf/sample.ini

@@ -673,6 +673,9 @@
 ;server_url =
 # If the remote HTTP image renderer service runs on a different server than the Grafana server you may have to configure this to a URL where Grafana is reachable, e.g. http://grafana.domain/.
 ;callback_url =
+# Concurrent render request limit affects when the /render HTTP endpoint is used. Rendering many images at the same time can overload the server,
+# which this setting can help protect against by only allowing a certain amount of concurrent requests.
+;concurrent_render_request_limit = 30
 
 [panels]
 # If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.

docs/sources/installation/configuration.md

@@ -822,6 +822,11 @@ URL to a remote HTTP image renderer service, e.g. http://localhost:8081/render,
 
 If the remote HTTP image renderer service runs on a different server than the Grafana server you may have to configure this to a URL where Grafana is reachable, e.g. http://grafana.domain/.
 
+### concurrent_render_request_limit
+
+Concurrent render request limit affects when the /render HTTP endpoint is used. Rendering many images at the same time can overload the server,
+which this setting can help protect against by only allowing a certain amount of concurrent requests.
+
 ## [panels]
 
 ### disable_sanitize_html

pkg/api/render.go

@@ -52,7 +52,6 @@ func (hs *HTTPServer) RenderToPng(c *models.ReqContext) {
 		headers["Accept-Language"] = acceptLanguageHeader
 	}
 
-	maxConcurrentLimitForApiCalls := 30
 	result, err := hs.RenderService.Render(c.Req.Context(), rendering.Opts{
 		Width:             width,
 		Height:            height,
@@ -63,7 +62,7 @@ func (hs *HTTPServer) RenderToPng(c *models.ReqContext) {
 		Path:              c.Params("*") + queryParams,
 		Timezone:          queryReader.Get("tz", ""),
 		Encoding:          queryReader.Get("encoding", ""),
-		ConcurrentLimit:   maxConcurrentLimitForApiCalls,
+		ConcurrentLimit:   hs.Cfg.RendererConcurrentRequestLimit,
 		DeviceScaleFactor: scale,
 		Headers:           headers,
 	})

pkg/setting/setting.go

@@ -238,11 +238,10 @@ type Cfg struct {
 	Smtp SmtpSettings
 
 	// Rendering
-	ImagesDir             string
-	RendererUrl           string
-	RendererCallbackUrl   string
-	RendererLimit         int
-	RendererLimitAlerting int
+	ImagesDir                      string
+	RendererUrl                    string
+	RendererCallbackUrl            string
+	RendererConcurrentRequestLimit int
 
 	// Security
 	DisableInitAdminCreation         bool
@@ -938,6 +937,8 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 			log.Fatal(4, "Invalid callback_url(%s): %s", cfg.RendererCallbackUrl, err)
 		}
 	}
+	cfg.RendererConcurrentRequestLimit = renderSec.Key("concurrent_render_request_limit").MustInt(30)
+
 	cfg.ImagesDir = filepath.Join(cfg.DataPath, "png")
 	cfg.TempDataLifetime = iniFile.Section("paths").Key("temp_data_lifetime").MustDuration(time.Second * 3600 * 24)
 	cfg.MetricsEndpointEnabled = iniFile.Section("metrics").Key("enabled").MustBool(true)