Chore: Change so we cache loading plugins by its version (#41367)

* making it possible to cache plugins based on the version.

* feat(plugincache): introduce function to invalidate entries

* removed todo's

* added tests for the cache buster.

* fixed tests.

* fixed failing tests.

Co-authored-by: Jack Westbrook <jack.westbrook@gmail.com>

packages/grafana-data/src/types/config.ts

@@ -80,6 +80,16 @@ export interface SentryConfig {
   sampleRate: number;
 }
 
+/**
+ * Describes the plugins that should be preloaded prior to start Grafana.
+ *
+ * @public
+ */
+export type PreloadPlugin = {
+  path: string;
+  version: string;
+};
+
 /**
  * Describes all the different Grafana configuration values available for an instance.
  *
@@ -123,7 +133,7 @@ export interface GrafanaConfig {
   liveEnabled: boolean;
   theme: GrafanaTheme;
   theme2: GrafanaTheme2;
-  pluginsToPreload: string[];
+  pluginsToPreload: PreloadPlugin[];
   featureToggles: FeatureToggles;
   licenseInfo: LicenseInfo;
   http2Enabled: boolean;

packages/grafana-data/src/types/index.ts

@@ -36,6 +36,6 @@ export * from './live';
 export * from './variables';
 export * from './geometry';
 export { isUnsignedPluginSignature } from './pluginSignature';
-export { GrafanaConfig, BuildInfo, FeatureToggles, LicenseInfo } from './config';
+export { GrafanaConfig, BuildInfo, FeatureToggles, LicenseInfo, PreloadPlugin } from './config';
 export * from './alerts';
 export * from './slider';

packages/grafana-runtime/src/config.ts

@@ -10,6 +10,7 @@ import {
   LicenseInfo,
   MapLayerOptions,
   PanelPluginMeta,
+  PreloadPlugin,
   systemDateFormats,
   SystemDateFormatSettings,
 } from '@grafana/data';
@@ -59,7 +60,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
   liveEnabled = true;
   theme: GrafanaTheme;
   theme2: GrafanaTheme2;
-  pluginsToPreload: string[] = [];
+  pluginsToPreload: PreloadPlugin[] = [];
   featureToggles: FeatureToggles = {
     accesscontrol: false,
     trimDefaults: false,

pkg/api/frontendsettings.go

@@ -15,6 +15,11 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )
 
+type PreloadPlugin struct {
+	Path    string `json:"path"`
+	Version string `json:"version"`
+}
+
 func (hs *HTTPServer) getFSDataSources(c *models.ReqContext, enabledPlugins EnabledPlugins) (map[string]interface{}, error) {
 	orgDataSources := make([]*models.DataSource, 0)
 
@@ -150,10 +155,13 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 		return nil, err
 	}
 
-	pluginsToPreload := []string{}
+	pluginsToPreload := []*PreloadPlugin{}
 	for _, app := range enabledPlugins[plugins.App] {
 		if app.Preload {
-			pluginsToPreload = append(pluginsToPreload, app.Module)
+			pluginsToPreload = append(pluginsToPreload, &PreloadPlugin{
+				Path:    app.Module,
+				Version: app.Info.Version,
+			})
 		}
 	}
 
@@ -171,7 +179,10 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 
 		module, _ := dsM["module"].(string)
 		if preload, _ := dsM["preload"].(bool); preload && module != "" {
-			pluginsToPreload = append(pluginsToPreload, module)
+			pluginsToPreload = append(pluginsToPreload, &PreloadPlugin{
+				Path:    module,
+				Version: dsM["info"].(map[string]interface{})["version"].(string),
+			})
 		}
 	}
 
@@ -182,7 +193,10 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 		}
 
 		if panel.Preload {
-			pluginsToPreload = append(pluginsToPreload, panel.Module)
+			pluginsToPreload = append(pluginsToPreload, &PreloadPlugin{
+				Path:    panel.Module,
+				Version: panel.Info.Version,
+			})
 		}
 
 		panels[panel.ID] = map[string]interface{}{

public/app/app.ts

@@ -126,8 +126,8 @@ export class GrafanaApp {
 
       // Preload selected app plugins
       const promises: Array<Promise<any>> = [];
-      for (const modulePath of config.pluginsToPreload) {
-        promises.push(importPluginModule(modulePath));
+      for (const plugin of config.pluginsToPreload) {
+        promises.push(importPluginModule(plugin.path, plugin.version));
       }
 
       await Promise.all(promises);

public/app/features/plugins/admin/state/actions.ts

@@ -14,6 +14,7 @@ import {
 import { STATE_PREFIX } from '../constants';
 import { mergeLocalsAndRemotes, updatePanels } from '../helpers';
 import { CatalogPlugin, RemotePlugin } from '../types';
+import { invalidatePluginInCache } from '../../pluginCacheBuster';
 
 export const fetchAll = createAsyncThunk(`${STATE_PREFIX}/fetchAll`, async (_, thunkApi) => {
   try {
@@ -64,6 +65,10 @@ export const install = createAsyncThunk(
       await installPlugin(id, version);
       await updatePanels();
 
+      if (isUpdating) {
+        invalidatePluginInCache(id);
+      }
+
       return { id, changes } as Update<CatalogPlugin>;
     } catch (e) {
       return thunkApi.rejectWithValue('Unknown error.');
@@ -76,6 +81,8 @@ export const uninstall = createAsyncThunk(`${STATE_PREFIX}/uninstall`, async (id
     await uninstallPlugin(id);
     await updatePanels();
 
+    invalidatePluginInCache(id);
+
     return {
       id,
       changes: { isInstalled: false },

public/app/features/plugins/importPanelPlugin.ts

@@ -2,7 +2,6 @@ import config from 'app/core/config';
 import * as grafanaData from '@grafana/data';
 import { getPanelPluginLoadError } from '../panel/components/PanelPluginError';
 import { importPluginModule } from './plugin_loader';
-
 interface PanelCache {
   [key: string]: Promise<grafanaData.PanelPlugin>;
 }
@@ -30,7 +29,7 @@ export function importPanelPluginFromMeta(meta: grafanaData.PanelPluginMeta): Pr
 }
 
 function getPanelPlugin(meta: grafanaData.PanelPluginMeta): Promise<grafanaData.PanelPlugin> {
-  return importPluginModule(meta.module)
+  return importPluginModule(meta.module, meta.info?.version)
     .then((pluginExports) => {
       if (pluginExports.plugin) {
         return pluginExports.plugin as grafanaData.PanelPlugin;

public/app/features/plugins/pluginCacheBuster.test.ts

@@ -0,0 +1,42 @@
+import { invalidatePluginInCache, locateWithCache, registerPluginInCache } from './pluginCacheBuster';
+
+describe('PluginCacheBuster', () => {
+  const now = 12345;
+
+  it('should append plugin version as cache flag if plugin is registered in buster', () => {
+    const slug = 'bubble-chart-1';
+    const version = 'v1.0.0';
+    const path = resolvePath(slug);
+    const address = `http://localhost:3000/public/${path}.js`;
+
+    registerPluginInCache({ path, version });
+
+    const url = `${address}?_cache=${encodeURI(version)}`;
+    expect(locateWithCache({ address }, now)).toBe(url);
+  });
+
+  it('should append Date.now as cache flag if plugin is not registered in buster', () => {
+    const slug = 'bubble-chart-2';
+    const address = `http://localhost:3000/public/${resolvePath(slug)}.js`;
+
+    const url = `${address}?_cache=${encodeURI(String(now))}`;
+    expect(locateWithCache({ address }, now)).toBe(url);
+  });
+
+  it('should append Date.now as cache flag if plugin is invalidated in buster', () => {
+    const slug = 'bubble-chart-3';
+    const version = 'v1.0.0';
+    const path = resolvePath(slug);
+    const address = `http://localhost:3000/public/${path}.js`;
+
+    registerPluginInCache({ path, version });
+    invalidatePluginInCache(slug);
+
+    const url = `${address}?_cache=${encodeURI(String(now))}`;
+    expect(locateWithCache({ address }, now)).toBe(url);
+  });
+});
+
+function resolvePath(slug: string): string {
+  return `plugins/${slug}/module`;
+}

public/app/features/plugins/pluginCacheBuster.ts

@@ -0,0 +1,45 @@
+const cache: Record<string, string> = {};
+const initializedAt: number = Date.now();
+
+type CacheablePlugin = {
+  path: string;
+  version: string;
+};
+
+export function registerPluginInCache({ path, version }: CacheablePlugin): void {
+  if (!cache[path]) {
+    cache[path] = encodeURI(version);
+  }
+}
+
+export function invalidatePluginInCache(pluginId: string): void {
+  const path = `plugins/${pluginId}/module`;
+  if (cache[path]) {
+    delete cache[path];
+  }
+}
+
+export function locateWithCache(load: { address: string }, defaultBust = initializedAt): string {
+  const { address } = load;
+  const path = extractPath(address);
+
+  if (!path) {
+    return `${address}?_cache=${defaultBust}`;
+  }
+
+  const version = cache[path];
+  const bust = version || defaultBust;
+  return `${address}?_cache=${bust}`;
+}
+
+function extractPath(address: string): string | undefined {
+  const match = /\/public\/(plugins\/.+\/module)\.js/i.exec(address);
+  if (!match) {
+    return;
+  }
+  const [_, path] = match;
+  if (!path) {
+    return;
+  }
+  return path;
+}

public/app/features/plugins/plugin_loader.ts

@@ -36,6 +36,7 @@ import * as grafanaData from '@grafana/data';
 import * as grafanaUIraw from '@grafana/ui';
 import * as grafanaRuntime from '@grafana/runtime';
 import { GenericDataSourcePlugin } from '../datasources/settings/PluginSettings';
+import { locateWithCache, registerPluginInCache } from './pluginCacheBuster';
 
 // Help the 6.4 to 6.5 migration
 // The base classes were moved from @grafana/ui to @grafana/data
@@ -52,13 +53,7 @@ import * as rxjsOperators from 'rxjs/operators';
 // routing
 import * as reactRouter from 'react-router-dom';
 
-// add cache busting
-const bust = `?_cache=${Date.now()}`;
-function locate(load: { address: string }) {
-  return load.address + bust;
-}
-
-grafanaRuntime.SystemJS.registry.set('plugin-loader', grafanaRuntime.SystemJS.newModule({ locate: locate }));
+grafanaRuntime.SystemJS.registry.set('plugin-loader', grafanaRuntime.SystemJS.newModule({ locate: locateWithCache }));
 
 grafanaRuntime.SystemJS.config({
   baseURL: 'public',
@@ -178,7 +173,11 @@ for (const flotDep of flotDeps) {
   exposeToPlugin(flotDep, { fakeDep: 1 });
 }
 
-export async function importPluginModule(path: string): Promise<any> {
+export async function importPluginModule(path: string, version?: string): Promise<any> {
+  if (version) {
+    registerPluginInCache({ path, version });
+  }
+
   const builtIn = builtInPlugins[path];
   if (builtIn) {
     // for handling dynamic imports
@@ -192,7 +191,7 @@ export async function importPluginModule(path: string): Promise<any> {
 }
 
 export function importDataSourcePlugin(meta: grafanaData.DataSourcePluginMeta): Promise<GenericDataSourcePlugin> {
-  return importPluginModule(meta.module).then((pluginExports) => {
+  return importPluginModule(meta.module, meta.info?.version).then((pluginExports) => {
     if (pluginExports.plugin) {
       const dsPlugin = pluginExports.plugin as GenericDataSourcePlugin;
       dsPlugin.meta = meta;
@@ -215,7 +214,7 @@ export function importDataSourcePlugin(meta: grafanaData.DataSourcePluginMeta):
 }
 
 export function importAppPlugin(meta: grafanaData.PluginMeta): Promise<grafanaData.AppPlugin> {
-  return importPluginModule(meta.module).then((pluginExports) => {
+  return importPluginModule(meta.module, meta.info?.version).then((pluginExports) => {
     const plugin = pluginExports.plugin ? (pluginExports.plugin as grafanaData.AppPlugin) : new grafanaData.AppPlugin();
     plugin.init(meta);
     plugin.meta = meta;