RuntimeDataSource: Support in core for runtime registered data sources (#93956)

* RuntimeDataSource: Support in core for runtime registered data sources

* Added tests for runtime datasource.

* added another test to make sure runtime ds isn't included in datasource list.

* changed so we not are expecting the settings to be returned by name.

* Fixed betterer error.

* fixed type issues.

* updated comment according to feedback.

* will prevent runtime ds registration from overwriting regular ds.

---------

Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>

packages/grafana-runtime/src/services/RuntimeDataSource.ts

@@ -0,0 +1,54 @@
+import {
+  DataQuery,
+  DataSourceApi,
+  DataSourceInstanceSettings,
+  PluginType,
+  TestDataSourceResponse,
+} from '@grafana/data';
+
+export abstract class RuntimeDataSource<TQuery extends DataQuery = DataQuery> extends DataSourceApi<TQuery> {
+  public instanceSettings: DataSourceInstanceSettings;
+
+  public constructor(pluginId: string, uid: string) {
+    const instanceSettings: DataSourceInstanceSettings = {
+      name: 'RuntimeDataSource-' + pluginId,
+      uid: uid,
+      type: pluginId,
+      id: 1,
+      readOnly: true,
+      jsonData: {},
+      access: 'direct',
+      meta: {
+        id: pluginId,
+        name: 'RuntimeDataSource-' + pluginId,
+        type: PluginType.datasource,
+        info: {
+          author: {
+            name: '',
+          },
+          description: '',
+          links: [],
+          logos: {
+            large: '',
+            small: '',
+          },
+          screenshots: [],
+          updated: '',
+          version: '',
+        },
+        module: '',
+        baseUrl: '',
+      },
+    };
+
+    super(instanceSettings);
+    this.instanceSettings = instanceSettings;
+  }
+
+  public testDatasource(): Promise<TestDataSourceResponse> {
+    return Promise.resolve({
+      status: 'success',
+      message: '',
+    });
+  }
+}

packages/grafana-runtime/src/services/dataSourceSrv.ts

@@ -1,5 +1,7 @@
 import { ScopedVars, DataSourceApi, DataSourceInstanceSettings, DataSourceRef } from '@grafana/data';
 
+import { RuntimeDataSource } from './RuntimeDataSource';
+
 /**
  * This is the entry point for communicating with a datasource that is added as
  * a plugin (both external and internal). Via this service you will get access
@@ -33,6 +35,15 @@ export interface DataSourceSrv {
    * Reloads the DataSourceSrv
    */
   reload(): void;
+
+  /**
+   * Registers a runtime data source. Make sure your data source uid is unique.
+   */
+  registerRuntimeDataSource(entry: RuntimeDataSourceRegistration): void;
+}
+
+export interface RuntimeDataSourceRegistration {
+  dataSource: RuntimeDataSource;
 }
 
 /** @public */

packages/grafana-runtime/src/services/index.ts

@@ -36,5 +36,5 @@ export { setPluginLinksHook, usePluginLinks } from './pluginExtensions/usePlugin
 
 export { isPluginExtensionLink, isPluginExtensionComponent } from './pluginExtensions/utils';
 export { setCurrentUser } from './user';
-
+export { RuntimeDataSource } from './RuntimeDataSource';
 export { ScopesContext, type ScopesContextValueState, type ScopesContextValue, useScopes } from './ScopesContext';

public/app/features/alerting/unified/mocks.ts

@@ -17,7 +17,7 @@ import {
   ScopedVars,
   TestDataSourceResponse,
 } from '@grafana/data';
-import { DataSourceSrv, GetDataSourceListFilters, config } from '@grafana/runtime';
+import { DataSourceSrv, GetDataSourceListFilters, RuntimeDataSourceRegistration, config } from '@grafana/runtime';
 import { defaultDashboard } from '@grafana/schema';
 import { contextSrv } from 'app/core/services/context_srv';
 import { MOCK_GRAFANA_ALERT_RULE_TITLE } from 'app/features/alerting/unified/mocks/server/handlers/grafanaRuler';
@@ -451,6 +451,8 @@ export class MockDataSourceSrv implements DataSourceSrv {
     }
   }
 
+  registerRuntimeDataSource(entry: RuntimeDataSourceRegistration): void {}
+
   get(name?: string | null | DataSourceRef, scopedVars?: ScopedVars): Promise<DataSourceApi> {
     return DatasourceSrv.prototype.get.call(this, name, scopedVars);
     //return Promise.reject(new Error('not implemented'));

public/app/features/dashboard/containers/DashboardPage.test.tsx

@@ -204,6 +204,7 @@ describe('DashboardPage', () => {
   describe('When going into view mode', () => {
     beforeEach(() => {
       setDataSourceSrv({
+        registerRuntimeDataSource: jest.fn(),
         get: jest.fn().mockResolvedValue({ getRef: jest.fn(), query: jest.fn().mockResolvedValue([]) }),
         getInstanceSettings: jest.fn().mockReturnValue({ meta: {} }),
         getList: jest.fn(),

public/app/features/explore/hooks/useExplorePageTitle.test.tsx

@@ -18,6 +18,7 @@ describe('useExplorePageTitle', () => {
     ];
 
     setDataSourceSrv({
+      registerRuntimeDataSource: jest.fn(),
       get(datasource?: string | DataSourceRef | null) {
         let ds;
         if (!datasource) {
@@ -72,6 +73,7 @@ describe('useExplorePageTitle', () => {
     ];
 
     setDataSourceSrv({
+      registerRuntimeDataSource: jest.fn(),
       get(datasource?: string | DataSourceRef | null) {
         let ds;
         if (!datasource) {

public/app/features/explore/hooks/useStateSync/index.test.tsx

@@ -69,6 +69,7 @@ function setup({ queryParams = {}, datasourceGetter = defaultDsGetter }: SetupPa
   ];
 
   setDataSourceSrv({
+    registerRuntimeDataSource: jest.fn(),
     get: datasourceGetter(datasources),
     getInstanceSettings: jest.fn(),
     getList: jest.fn(),

public/app/features/explore/spec/helper/setup.tsx

@@ -119,6 +119,7 @@ export function setupExplore(options?: SetupOptions): {
   const previousDataSourceSrv = getDataSourceSrv();
 
   setDataSourceSrv({
+    registerRuntimeDataSource: jest.fn(),
     getList(): DataSourceInstanceSettings[] {
       return dsSettings.map((d) => d.settings);
     },

public/app/features/plugins/datasource_srv.ts

@@ -14,6 +14,8 @@ import {
   getDataSourceSrv as getDataSourceService,
   getLegacyAngularInjector,
   getTemplateSrv,
+  RuntimeDataSourceRegistration,
+  RuntimeDataSource,
   TemplateSrv,
 } from '@grafana/runtime';
 import { ExpressionDatasourceRef, isExpressionReference } from '@grafana/runtime/src/utils/DataSourceWithBackend';
@@ -32,6 +34,7 @@ export class DatasourceSrv implements DataSourceService {
   private settingsMapByName: Record<string, DataSourceInstanceSettings> = {};
   private settingsMapByUid: Record<string, DataSourceInstanceSettings> = {};
   private settingsMapById: Record<string, DataSourceInstanceSettings> = {};
+  private runtimeDataSources: Record<string, RuntimeDataSource> = {}; //
   private defaultName = ''; // actually UID
 
   constructor(private templateSrv: TemplateSrv = getTemplateSrv()) {}
@@ -51,6 +54,11 @@ export class DatasourceSrv implements DataSourceService {
       this.settingsMapById[dsSettings.id] = dsSettings;
     }
 
+    for (const ds of Object.values(this.runtimeDataSources)) {
+      this.datasources[ds.uid] = ds;
+      this.settingsMapByUid[ds.uid] = ds.instanceSettings;
+    }
+
     // Preload expressions
     this.datasources[ExpressionDatasourceRef.type] = expressionDatasource as any;
     this.datasources[ExpressionDatasourceUID] = expressionDatasource as any;
@@ -58,6 +66,19 @@ export class DatasourceSrv implements DataSourceService {
     this.settingsMapByUid[ExpressionDatasourceUID] = expressionInstanceSettings;
   }
 
+  registerRuntimeDataSource(entry: RuntimeDataSourceRegistration): void {
+    if (this.runtimeDataSources[entry.dataSource.uid]) {
+      throw new Error(`A runtime data source with uid ${entry.dataSource.uid} has already been registered`);
+    }
+    if (this.settingsMapByUid[entry.dataSource.uid]) {
+      throw new Error(`A data source with uid ${entry.dataSource.uid} has already been registered`);
+    }
+
+    this.runtimeDataSources[entry.dataSource.uid] = entry.dataSource;
+    this.datasources[entry.dataSource.uid] = entry.dataSource;
+    this.settingsMapByUid[entry.dataSource.uid] = entry.dataSource.instanceSettings;
+  }
+
   getDataSourceSettingsByUid(uid: string): DataSourceInstanceSettings | undefined {
     return this.settingsMapByUid[uid];
   }

public/app/features/plugins/tests/datasource_srv.test.ts

@@ -1,11 +1,16 @@
+import { Observable, of } from 'rxjs';
+
 import {
+  DataQuery,
+  DataQueryRequest,
+  DataQueryResponse,
   DataSourceApi,
   DataSourceInstanceSettings,
   DataSourcePlugin,
   DataSourcePluginMeta,
   ScopedVars,
 } from '@grafana/data';
-import { TemplateSrv } from '@grafana/runtime';
+import { RuntimeDataSource, TemplateSrv } from '@grafana/runtime';
 import { ExpressionDatasourceRef } from '@grafana/runtime/src/utils/DataSourceWithBackend';
 import { DatasourceSrv, getNameOrUid } from 'app/features/plugins/datasource_srv';
 
@@ -50,6 +55,12 @@ class TestDataSource {
   constructor(public instanceSettings: DataSourceInstanceSettings) {}
 }
 
+class TestRuntimeDataSource extends RuntimeDataSource {
+  query(request: DataQueryRequest<DataQuery>): Promise<DataQueryResponse> | Observable<DataQueryResponse> {
+    return of({ data: [] });
+  }
+}
+
 jest.mock('../plugin_loader', () => ({
   importDataSourcePlugin: (meta: DataSourcePluginMeta) => {
     return Promise.resolve(new DataSourcePlugin(TestDataSource as any));
@@ -138,6 +149,14 @@ describe('datasource_srv', () => {
   };
 
   describe('Given a list of data sources', () => {
+    const runtimeDataSource = new TestRuntimeDataSource('grafana-runtime-datasource', 'uuid-runtime-ds');
+
+    beforeAll(() => {
+      dataSourceSrv.registerRuntimeDataSource({
+        dataSource: runtimeDataSource,
+      });
+    });
+
     beforeEach(() => {
       dataSourceSrv.init(dataSourceInit as any, 'BBB');
     });
@@ -251,6 +270,16 @@ describe('datasource_srv', () => {
         expect(exprWithName?.uid).toBe(ExpressionDatasourceRef.uid);
         expect(exprWithName?.type).toBe(ExpressionDatasourceRef.type);
       });
+
+      it('should return settings for runtime datasource when called with uid', () => {
+        const settings = dataSourceSrv.getInstanceSettings(runtimeDataSource.uid);
+        expect(settings).toBe(runtimeDataSource.instanceSettings);
+      });
+
+      it('should not return settings for runtime datasource when called with name', () => {
+        const settings = dataSourceSrv.getInstanceSettings(runtimeDataSource.name);
+        expect(settings).toBe(undefined);
+      });
     });
 
     describe('when loading datasource', () => {
@@ -312,10 +341,17 @@ describe('datasource_srv', () => {
       expect(list[0].name).toBe('mmm');
     });
 
-    it('Can get get list and filter by pluginId', () => {
-      const list = dataSourceSrv.getList({ pluginId: 'jaeger' });
-      expect(list[0].name).toBe('Jaeger');
-      expect(list.length).toBe(1);
+    describe('get list filtered by plugin Id', () => {
+      it('should list all data sources for specified plugin id', () => {
+        const list = dataSourceSrv.getList({ pluginId: 'jaeger' });
+        expect(list.length).toBe(1);
+        expect(list[0].name).toBe('Jaeger');
+      });
+
+      it('should not include runtime datasources in list', () => {
+        const list = dataSourceSrv.getList({ pluginId: 'grafana-runtime-datasource' });
+        expect(list.length).toBe(0);
+      });
     });
 
     it('Can get get list and filter by an alias', () => {
@@ -415,20 +451,58 @@ describe('datasource_srv', () => {
       `);
     });
 
-    it('Should reload the datasource', async () => {
-      // arrange
-      getBackendSrvGetMock.mockReturnValueOnce({
-        datasources: {
-          ...dataSourceInit,
-        },
-        defaultDatasource: 'aaa',
+    describe('when calling reload', () => {
+      it('should reload the datasource list', async () => {
+        // arrange
+        getBackendSrvGetMock.mockReturnValueOnce({
+          datasources: {
+            ...dataSourceInit,
+          },
+          defaultDatasource: 'aaa',
+        });
+        const initMock = jest.spyOn(dataSourceSrv, 'init').mockImplementation(() => {});
+        // act
+        await dataSourceSrv.reload();
+        // assert
+        expect(getBackendSrvGetMock).toHaveBeenCalledWith('/api/frontend/settings');
+        expect(initMock).toHaveBeenCalledWith(dataSourceInit, 'aaa');
+      });
+
+      it('should still be possible to get registered runtime data source', async () => {
+        getBackendSrvGetMock.mockReturnValueOnce({
+          datasources: {
+            ...dataSourceInit,
+          },
+          defaultDatasource: 'aaa',
+        });
+
+        await dataSourceSrv.reload();
+        const ds = await dataSourceSrv.get(runtimeDataSource.getRef());
+        expect(ds).toBe(runtimeDataSource);
+      });
+    });
+
+    describe('when registering runtime datasources', () => {
+      it('should have registered a runtime datasource', async () => {
+        const ds = await dataSourceSrv.get(runtimeDataSource.getRef());
+        expect(ds).toBe(runtimeDataSource);
+      });
+
+      it('should throw when trying to re-register a runtime datasource', () => {
+        expect(() =>
+          dataSourceSrv.registerRuntimeDataSource({
+            dataSource: runtimeDataSource,
+          })
+        ).toThrow();
+      });
+
+      it('should throw when trying to register a runtime datasource with the same uid as an "regular" datasource', async () => {
+        expect(() =>
+          dataSourceSrv.registerRuntimeDataSource({
+            dataSource: new TestRuntimeDataSource('grafana-runtime-datasource', 'uid-code-Jaeger'),
+          })
+        ).toThrow();
       });
-      const initMock = jest.spyOn(dataSourceSrv, 'init').mockImplementation(() => {});
-      // act
-      await dataSourceSrv.reload();
-      // assert
-      expect(getBackendSrvGetMock).toHaveBeenCalledWith('/api/frontend/settings');
-      expect(initMock).toHaveBeenCalledWith(dataSourceInit, 'aaa');
     });
   });
 

public/app/features/variables/state/initVariableTransaction.test.ts

@@ -49,6 +49,7 @@ function getTestContext(variables?: VariableModel[]) {
   const templating = { list: variables ?? [constant] };
   const getInstanceSettingsMock = jest.fn().mockReturnValue(undefined);
   setDataSourceSrv({
+    registerRuntimeDataSource: jest.fn(),
     get: jest.fn().mockResolvedValue({}),
     getList: jest.fn().mockReturnValue([]),
     getInstanceSettings: getInstanceSettingsMock,

public/app/plugins/datasource/cloudwatch/__mocks__/logsTestContext.ts

@@ -51,6 +51,7 @@ export function setupForLogs() {
   queryMock.mockReturnValueOnce(of(envelope(logsFrame)));
 
   setDataSourceSrv({
+    registerRuntimeDataSource: jest.fn(),
     async get() {
       const ds: DataSourceApi = {
         name: 'Xray',

public/app/plugins/datasource/loki/configuration/DerivedField.test.tsx

@@ -13,6 +13,7 @@ const validateMock = jest.fn();
 describe('DerivedField', () => {
   beforeEach(() => {
     setDataSourceSrv({
+      registerRuntimeDataSource: jest.fn(),
       get: jest.fn(),
       reload: jest.fn(),
       getInstanceSettings: jest.fn(),

public/app/plugins/datasource/mixed/MixedDataSource.test.ts

@@ -34,6 +34,7 @@ describe('MixedDatasource', () => {
       getInstanceSettings: jest.fn().mockReturnValue({ meta: {} }),
       getList: jest.fn(),
       reload: jest.fn(),
+      registerRuntimeDataSource: jest.fn(),
     });
     setTemplateSrv(new TemplateSrv());
   });