Dashboard: SchemaV2 - arbitrary strings - dual key (#100379)

* Create element panel lookup table

* When transforming to schema v2 get the element_identifier

* Add element identifier logic to layouts

* Retrieve element id in the serialization of each layout

* Keep ElementMapping updated when adding, removing panels

* Add basic unit test

* Wip: implement element mapping in serializer

* Remove Singleton

* Apply suggestions from code review

Co-authored-by: Haris Rozajac <58232930+harisrozajac@users.noreply.github.com>

* bring back missing functions - poc works at this point

* Move getElementIdentifierForVizPanel to dashboardSceneGraph

* poc - don't keep elementMapping updated

* Move logic to layout type serializer

* Remove unused code and remove layout tests

* clean up code, remove unnecessary functions

* Fix issue with initializeMapping

* remove console errors

* Remove testing code from response transformers

* Add unit test for DashboardSceneSerializer

* reset file, change not needed

* Improve comments on getElementIDForPanel

---------

Co-authored-by: Haris Rozajac <58232930+harisrozajac@users.noreply.github.com>

public/app/features/dashboard-scene/scene/DashboardScene.tsx

@@ -687,7 +687,9 @@ export class DashboardScene extends SceneObjectBase<DashboardSceneState> impleme
     saveModel?: Dashboard | DashboardV2Spec,
     meta?: DashboardMeta | DashboardWithAccessInfo<DashboardV2Spec>['metadata']
   ): void {
-    this._serializer.initialSaveModel = sortedDeepCloneWithoutNulls(saveModel);
+    this._serializer.initializeMapping(saveModel);
+    const sortedModel = sortedDeepCloneWithoutNulls(saveModel);
+    this._serializer.initialSaveModel = sortedModel;
     this._serializer.metadata = meta;
   }
 
@@ -695,6 +697,18 @@ export class DashboardScene extends SceneObjectBase<DashboardSceneState> impleme
     return this._serializer.getTrackingInformation(this);
   }
 
+  public getPanelIdForElement(elementId: string) {
+    return this._serializer.getPanelIdForElement(elementId);
+  }
+
+  public getElementPanelMapping() {
+    return this._serializer.getElementPanelMapping();
+  }
+
+  public getElementIdentifierForPanel(panelId: number) {
+    return this._serializer.getElementIdForPanel(panelId);
+  }
+
   public async onDashboardDelete() {
     // Need to mark it non dirty to navigate away without unsaved changes warning
     this.setState({ isDirty: false });

public/app/features/dashboard-scene/serialization/DashboardSceneSerializer.test.ts

@@ -377,6 +377,82 @@ describe('DashboardSceneSerializer', () => {
 
       expect(serializer.getSnapshotUrl()).toBe('originalUrl/snapshot');
     });
+
+    describe('panel mapping methods', () => {
+      let serializer: V1DashboardSerializer;
+
+      beforeEach(() => {
+        serializer = new V1DashboardSerializer();
+      });
+
+      it('should initialize panel mapping correctly', () => {
+        const saveModel: Dashboard = {
+          title: 'hello',
+          uid: 'my-uid',
+          schemaVersion: 30,
+          panels: [
+            { id: 1, title: 'Panel 1', type: 'text' },
+            { id: 2, title: 'Panel 2', type: 'text' },
+          ],
+        };
+
+        serializer.initializeMapping(saveModel);
+        const mapping = serializer.getElementPanelMapping();
+
+        expect(mapping.size).toBe(2);
+        expect(mapping.get('panel-1')).toBe(1);
+        expect(mapping.get('panel-2')).toBe(2);
+      });
+
+      it('should handle empty or undefined panels in initializeMapping', () => {
+        serializer.initializeMapping(undefined);
+        expect(serializer.getElementPanelMapping().size).toBe(0);
+
+        serializer.initializeMapping({
+          title: 'hello',
+          uid: 'my-uid',
+          schemaVersion: 30,
+          panels: undefined,
+        });
+        expect(serializer.getElementPanelMapping().size).toBe(0);
+      });
+
+      it('should get panel id for element correctly', () => {
+        const saveModel: Dashboard = {
+          title: 'hello',
+          uid: 'my-uid',
+          schemaVersion: 30,
+          panels: [
+            { id: 1, title: 'Panel 1', type: 'text' },
+            { id: 2, title: 'Panel 2', type: 'text' },
+          ],
+        };
+
+        serializer.initializeMapping(saveModel);
+
+        expect(serializer.getPanelIdForElement('panel-1')).toBe(1);
+        expect(serializer.getPanelIdForElement('panel-2')).toBe(2);
+        expect(serializer.getPanelIdForElement('non-existent')).toBeUndefined();
+      });
+
+      it('should get element id for panel correctly', () => {
+        const saveModel: Dashboard = {
+          title: 'hello',
+          uid: 'my-uid',
+          schemaVersion: 30,
+          panels: [
+            { id: 1, title: 'Panel 1', type: 'text' },
+            { id: 2, title: 'Panel 2', type: 'text' },
+          ],
+        };
+        serializer.initializeMapping(saveModel);
+
+        expect(serializer.getElementIdForPanel(1)).toBe('panel-1');
+        expect(serializer.getElementIdForPanel(2)).toBe('panel-2');
+        // Should return default panel key for non-existent panel
+        expect(serializer.getElementIdForPanel(3)).toBe('panel-3');
+      });
+    });
   });
 
   describe('v2 schema', () => {
@@ -799,6 +875,95 @@ describe('DashboardSceneSerializer', () => {
         });
       });
     });
+
+    describe('panel mapping methods', () => {
+      let serializer: V2DashboardSerializer;
+      let saveModel: DashboardV2Spec;
+
+      beforeEach(() => {
+        serializer = new V2DashboardSerializer();
+        saveModel = {
+          ...defaultDashboardV2Spec(),
+          elements: {
+            'element-panel-a': {
+              kind: 'Panel',
+              spec: { ...defaultPanelSpec(), id: 1, title: 'Panel A' },
+            },
+            'element-panel-b': {
+              kind: 'Panel',
+              spec: { ...defaultPanelSpec(), id: 2, title: 'Panel B' },
+            },
+          },
+          layout: {
+            kind: 'GridLayout',
+            spec: {
+              items: [
+                {
+                  kind: 'GridLayoutItem',
+                  spec: {
+                    x: 0,
+                    y: 0,
+                    width: 12,
+                    height: 8,
+                    element: {
+                      kind: 'ElementReference',
+                      name: 'element-panel-a',
+                    },
+                  },
+                },
+                {
+                  kind: 'GridLayoutItem',
+                  spec: {
+                    x: 0,
+                    y: 0,
+                    width: 12,
+                    height: 8,
+                    element: {
+                      kind: 'ElementReference',
+                      name: 'element-panel-b',
+                    },
+                  },
+                },
+              ],
+            },
+          },
+        };
+      });
+
+      it('should initialize panel mapping correctly', () => {
+        serializer.initializeMapping(saveModel);
+        const mapping = serializer.getElementPanelMapping();
+
+        expect(mapping.size).toBe(2);
+        expect(mapping.get('element-panel-a')).toBe(1);
+        expect(mapping.get('element-panel-b')).toBe(2);
+      });
+
+      it('should handle empty or undefined elements in initializeMapping', () => {
+        serializer.initializeMapping({} as DashboardV2Spec);
+        expect(serializer.getElementPanelMapping().size).toBe(0);
+
+        serializer.initializeMapping({ elements: {} } as DashboardV2Spec);
+        expect(serializer.getElementPanelMapping().size).toBe(0);
+      });
+
+      it('should get panel id for element correctly', () => {
+        serializer.initializeMapping(saveModel);
+
+        expect(serializer.getPanelIdForElement('element-panel-a')).toBe(1);
+        expect(serializer.getPanelIdForElement('element-panel-b')).toBe(2);
+        expect(serializer.getPanelIdForElement('non-existent')).toBeUndefined();
+      });
+
+      it('should get element id for panel correctly', () => {
+        serializer.initializeMapping(saveModel);
+
+        expect(serializer.getElementIdForPanel(1)).toBe('element-panel-a');
+        expect(serializer.getElementIdForPanel(2)).toBe('element-panel-b');
+        // Should return default panel key for non-existent panel
+        expect(serializer.getElementIdForPanel(3)).toBe('panel-3');
+      });
+    });
   });
 
   describe('onSaveComplete', () => {

public/app/features/dashboard-scene/serialization/DashboardSceneSerializer.ts

@@ -15,6 +15,7 @@ import { DashboardMeta, SaveDashboardResponseDTO } from 'app/types';
 import { getRawDashboardChanges, getRawDashboardV2Changes } from '../saving/getDashboardChanges';
 import { DashboardChangeInfo } from '../saving/shared';
 import { DashboardScene } from '../scene/DashboardScene';
+import { getVizPanelKeyForPanelId } from '../utils/utils';
 
 import { transformSceneToSaveModel } from './transformSceneToSaveModel';
 import { transformSceneToSaveModelSchemaV2 } from './transformSceneToSaveModelSchemaV2';
@@ -25,6 +26,7 @@ export interface DashboardSceneSerializerLike<T, M> {
    */
   initialSaveModel?: T;
   metadata?: M;
+  initializeMapping(saveModel: T | undefined): void;
   getSaveModel: (s: DashboardScene) => T;
   getSaveAsModel: (s: DashboardScene, options: SaveDashboardAsOptions) => T;
   getDashboardChangesFromScene: (
@@ -38,6 +40,9 @@ export interface DashboardSceneSerializerLike<T, M> {
   onSaveComplete(saveModel: T, result: SaveDashboardResponseDTO): void;
   getTrackingInformation: (s: DashboardScene) => DashboardTrackingInfo | undefined;
   getSnapshotUrl: () => string | undefined;
+  getPanelIdForElement: (elementId: string) => number | undefined;
+  getElementIdForPanel: (panelId: number) => string | undefined;
+  getElementPanelMapping: () => Map<string, number>;
 }
 
 interface DashboardTrackingInfo {
@@ -52,6 +57,44 @@ interface DashboardTrackingInfo {
 export class V1DashboardSerializer implements DashboardSceneSerializerLike<Dashboard, DashboardMeta> {
   initialSaveModel?: Dashboard;
   metadata?: DashboardMeta;
+  protected elementPanelMap = new Map<string, number>();
+
+  initializeMapping(saveModel: Dashboard | undefined) {
+    this.elementPanelMap.clear();
+
+    if (!saveModel || !saveModel.panels) {
+      return;
+    }
+    saveModel.panels?.forEach((panel) => {
+      if (panel.id) {
+        const elementKey = getVizPanelKeyForPanelId(panel.id);
+        this.elementPanelMap.set(elementKey, panel.id);
+      }
+    });
+  }
+
+  getElementPanelMapping() {
+    return this.elementPanelMap;
+  }
+
+  getPanelIdForElement(elementId: string) {
+    return this.elementPanelMap.get(elementId);
+  }
+
+  getElementIdForPanel(panelId: number) {
+    // First try to find an existing mapping
+    for (const [elementId, id] of this.elementPanelMap.entries()) {
+      if (id === panelId) {
+        return elementId;
+      }
+    }
+
+    // For runtime-created panels, generate a new element identifier
+    const newElementId = getVizPanelKeyForPanelId(panelId);
+    // Store the new mapping for future lookups
+    this.elementPanelMap.set(newElementId, panelId);
+    return newElementId;
+  }
 
   getSaveModel(s: DashboardScene) {
     return transformSceneToSaveModel(s);
@@ -131,6 +174,46 @@ export class V2DashboardSerializer
 {
   initialSaveModel?: DashboardV2Spec;
   metadata?: DashboardWithAccessInfo<DashboardV2Spec>['metadata'];
+  protected elementPanelMap = new Map<string, number>();
+
+  getElementPanelMapping() {
+    return this.elementPanelMap;
+  }
+
+  initializeMapping(saveModel: DashboardV2Spec | undefined) {
+    this.elementPanelMap.clear();
+
+    if (!saveModel || !saveModel.elements) {
+      return;
+    }
+
+    const elementKeys = Object.keys(saveModel.elements);
+    elementKeys.forEach((key) => {
+      const elementPanel = saveModel.elements[key];
+      if (elementPanel.kind === 'Panel') {
+        this.elementPanelMap.set(key, elementPanel.spec.id);
+      }
+    });
+  }
+
+  getPanelIdForElement(elementId: string) {
+    return this.elementPanelMap.get(elementId);
+  }
+
+  getElementIdForPanel(panelId: number) {
+    // First try to find an existing mapping
+    for (const [elementId, id] of this.elementPanelMap.entries()) {
+      if (id === panelId) {
+        return elementId;
+      }
+    }
+
+    // For runtime-created panels, generate a new element identifier
+    const newElementId = getVizPanelKeyForPanelId(panelId);
+    // Store the new mapping for future lookups
+    this.elementPanelMap.set(newElementId, panelId);
+    return newElementId;
+  }
 
   getSaveModel(s: DashboardScene) {
     return transformSceneToSaveModelSchemaV2(s);

public/app/features/dashboard-scene/serialization/layoutSerializers/DefaultGridLayoutSerializer.ts

@@ -33,12 +33,8 @@ import { RowActions } from '../../scene/layout-default/row-actions/RowActions';
 import { setDashboardPanelContext } from '../../scene/setDashboardPanelContext';
 import { DashboardLayoutManager, LayoutManagerSerializer } from '../../scene/types/DashboardLayoutManager';
 import { getOriginalKey, isClonedKey } from '../../utils/clone';
-import {
-  calculateGridItemDimensions,
-  getPanelIdForVizPanel,
-  getVizPanelKeyForPanelId,
-  isLibraryPanel,
-} from '../../utils/utils';
+import { dashboardSceneGraph } from '../../utils/dashboardSceneGraph';
+import { calculateGridItemDimensions, getVizPanelKeyForPanelId, isLibraryPanel } from '../../utils/utils';
 import { GRID_ROW_HEIGHT } from '../const';
 
 import { buildVizPanel } from './utils';
@@ -147,8 +143,9 @@ function gridItemToGridLayoutItemKind(gridItem: DashboardGridItem, yOverride?: n
   width = gridItem_.state.width ?? 0;
   const repeatVar = gridItem_.state.variableName;
 
-  // FIXME: which name should we use for the element reference, key or something else ?
-  const elementName = getVizPanelKeyForPanelId(getPanelIdForVizPanel(gridItem_.state.body));
+  // For serialization we should retrieve the original element key
+  let elementKey = dashboardSceneGraph.getElementIdentifierForVizPanel(gridItem_.state.body);
+
   elementGridItem = {
     kind: 'GridLayoutItem',
     spec: {
@@ -158,7 +155,7 @@ function gridItemToGridLayoutItemKind(gridItem: DashboardGridItem, yOverride?: n
       height: height,
       element: {
         kind: 'ElementReference',
-        name: elementName,
+        name: elementKey,
       },
     },
   };

public/app/features/dashboard-scene/serialization/layoutSerializers/ResponsiveGridLayoutSerializer.ts

@@ -4,6 +4,7 @@ import { DashboardV2Spec, ResponsiveGridLayoutItemKind } from '@grafana/schema/d
 import { ResponsiveGridItem } from '../../scene/layout-responsive-grid/ResponsiveGridItem';
 import { ResponsiveGridLayoutManager } from '../../scene/layout-responsive-grid/ResponsiveGridLayoutManager';
 import { DashboardLayoutManager, LayoutManagerSerializer } from '../../scene/types/DashboardLayoutManager';
+import { dashboardSceneGraph } from '../../utils/dashboardSceneGraph';
 import { getGridItemKeyForPanelId } from '../../utils/utils';
 
 import { buildVizPanel } from './utils';
@@ -21,12 +22,15 @@ export class ResponsiveGridLayoutSerializer implements LayoutManagerSerializer {
           if (!(child instanceof ResponsiveGridItem)) {
             throw new Error('Expected ResponsiveGridItem');
           }
+          // For serialization we should retrieve the original element key
+          const elementKey = dashboardSceneGraph.getElementIdentifierForVizPanel(child.state?.body);
+
           const layoutItem: ResponsiveGridLayoutItemKind = {
             kind: 'ResponsiveGridLayoutItem',
             spec: {
               element: {
                 kind: 'ElementReference',
-                name: child.state?.body?.state.key ?? 'DefaultName',
+                name: elementKey,
               },
             },
           };

public/app/features/dashboard-scene/serialization/transformSceneToSaveModelSchemaV2.ts

@@ -45,13 +45,7 @@ import { DashboardDataLayerSet } from '../scene/DashboardDataLayerSet';
 import { DashboardScene, DashboardSceneState } from '../scene/DashboardScene';
 import { PanelTimeRange } from '../scene/PanelTimeRange';
 import { dashboardSceneGraph } from '../utils/dashboardSceneGraph';
-import {
-  getLibraryPanelBehavior,
-  getPanelIdForVizPanel,
-  getQueryRunnerFor,
-  getVizPanelKeyForPanelId,
-  isLibraryPanel,
-} from '../utils/utils';
+import { getLibraryPanelBehavior, getPanelIdForVizPanel, getQueryRunnerFor, isLibraryPanel } from '../utils/utils';
 
 import { getLayout } from './layoutSerializers/utils';
 import { sceneVariablesSetToSchemaV2Variables } from './sceneVariablesSetToVariables';
@@ -103,7 +97,7 @@ export function transformSceneToSaveModelSchemaV2(scene: DashboardScene, isSnaps
     // EOF variables
 
     // elements
-    elements: getElements(sceneDash),
+    elements: getElements(scene),
     // EOF elements
 
     // annotations
@@ -146,8 +140,8 @@ function getLiveNow(state: DashboardSceneState) {
   return Boolean(liveNow);
 }
 
-function getElements(state: DashboardSceneState) {
-  const panels = state.body.getVizPanels() ?? [];
+function getElements(scene: DashboardScene) {
+  const panels = scene.state.body.getVizPanels() ?? [];
 
   const panelsArray = panels.map((vizPanel: VizPanel) => {
     if (isLibraryPanel(vizPanel)) {
@@ -228,7 +222,7 @@ function getElements(state: DashboardSceneState) {
       return elementSpec;
     }
   });
-  return createElements(panelsArray);
+  return createElements(panelsArray, scene);
 }
 
 function getPanelLinks(panel: VizPanel): DataLink[] {
@@ -345,9 +339,10 @@ function getVizPanelQueryOptions(vizPanel: VizPanel): QueryOptionsSpec {
   return queryOptions;
 }
 
-function createElements(panels: Element[]): Record<string, Element> {
+function createElements(panels: Element[], scene: DashboardScene): Record<string, Element> {
   return panels.reduce<Record<string, Element>>((elements, panel) => {
-    elements[getVizPanelKeyForPanelId(panel.spec.id)] = panel;
+    let elementKey = scene.getElementIdentifierForPanel(panel.spec.id);
+    elements[elementKey!] = panel;
     return elements;
   }, {});
 }

public/app/features/dashboard-scene/utils/dashboardSceneGraph.ts

@@ -5,7 +5,7 @@ import { DashboardScene } from '../scene/DashboardScene';
 import { VizPanelLinks } from '../scene/PanelLinks';
 
 import { isClonedKey } from './clone';
-import { getDashboardSceneFor, getLayoutManagerFor, getPanelIdForVizPanel } from './utils';
+import { getDashboardSceneFor, getLayoutManagerFor, getPanelIdForVizPanel, getVizPanelKeyForPanelId } from './utils';
 
 function getTimePicker(scene: DashboardScene) {
   return scene.state.controls?.state.timePicker;
@@ -79,6 +79,18 @@ export function getCursorSync(scene: DashboardScene) {
 
   return;
 }
+// Functions to manage the lookup table in dashboard scene that will hold element_identifer : panel_id
+export function getElementIdentifierForVizPanel(vizPanel: VizPanel): string {
+  const scene = getDashboardSceneFor(vizPanel);
+  const panelId = getPanelIdForVizPanel(vizPanel);
+  let elementKey = scene.getElementIdentifierForPanel(panelId);
+
+  if (!elementKey) {
+    // assign a panel-id key
+    elementKey = getVizPanelKeyForPanelId(panelId);
+  }
+  return elementKey;
+}
 
 export const dashboardSceneGraph = {
   getTimePicker,
@@ -90,4 +102,5 @@ export const dashboardSceneGraph = {
   getCursorSync,
   getLayoutManagerFor,
   getNextPanelId,
+  getElementIdentifierForVizPanel,
 };

public/app/features/dashboard/api/ResponseTransformers.ts

@@ -375,6 +375,8 @@ function yOffsetInRows(p: Panel, rowY: number): number {
 }
 
 function buildElement(p: Panel): [PanelKind | LibraryPanelKind, string] {
+  const element_identifier = `panel-${p.id}`;
+
   if (p.libraryPanel) {
     // LibraryPanelKind
     const panelKind: LibraryPanelKind = {
@@ -389,7 +391,7 @@ function buildElement(p: Panel): [PanelKind | LibraryPanelKind, string] {
       },
     };
 
-    return [panelKind, `panel-${p.id}`];
+    return [panelKind, element_identifier];
   } else {
     // PanelKind
 
@@ -438,8 +440,7 @@ function buildElement(p: Panel): [PanelKind | LibraryPanelKind, string] {
         },
       },
     };
-
-    return [panelKind, `panel-${p.id}`];
+    return [panelKind, element_identifier];
   }
 }