diff --git a/public/app/features/dashboard/state/getPanelOptionsWithDefaults.test.ts b/public/app/features/dashboard/state/getPanelOptionsWithDefaults.test.ts
index df813193ca7..fdbdf90ed3c 100644
--- a/public/app/features/dashboard/state/getPanelOptionsWithDefaults.test.ts
+++ b/public/app/features/dashboard/state/getPanelOptionsWithDefaults.test.ts
@@ -8,6 +8,7 @@ import {
   standardEditorsRegistry,
   standardFieldConfigEditorRegistry,
   StandardOptionConfig,
+  ThresholdsMode,
 } from '@grafana/data';
 import { getPanelPlugin } from 'app/features/plugins/__mocks__/pluginMocks';
 import { mockStandardFieldConfigOptions } from 'test/helpers/fieldConfig';
@@ -59,9 +60,7 @@ describe('getPanelOptionsWithDefaults', () => {
       expect(result).toMatchInlineSnapshot(`
         Object {
           "fieldConfig": Object {
-            "defaults": Object {
-              "custom": Object {},
-            },
+            "defaults": Object {},
             "overrides": Array [],
           },
           "options": Object {},
@@ -232,6 +231,30 @@ describe('getPanelOptionsWithDefaults', () => {
     });
   });
 
+  describe('when changing panel type to one that does not use standard field config', () => {
+    it('should clean defaults', () => {
+      const plugin = getPanelPlugin({ id: 'graph' });
+
+      const result = getPanelOptionsWithDefaults({
+        plugin,
+        currentOptions: {},
+        currentFieldConfig: {
+          defaults: {
+            color: { mode: FieldColorModeId.Thresholds },
+            thresholds: {
+              mode: ThresholdsMode.Absolute,
+              steps: [],
+            },
+          },
+          overrides: [],
+        },
+        isAfterPluginChange: true,
+      });
+
+      expect(result.fieldConfig.defaults.thresholds).toBeUndefined();
+    });
+  });
+
   describe('when applying defaults clean properties that are no longer part of the registry', () => {
     it('should remove custom defaults that no longer exist', () => {
       const result = runScenario({
diff --git a/public/app/features/dashboard/state/getPanelOptionsWithDefaults.ts b/public/app/features/dashboard/state/getPanelOptionsWithDefaults.ts
index 8941a7a4be7..2bf513a9e9a 100644
--- a/public/app/features/dashboard/state/getPanelOptionsWithDefaults.ts
+++ b/public/app/features/dashboard/state/getPanelOptionsWithDefaults.ts
@@ -105,6 +105,8 @@ export function filterFieldConfigOverrides(
 }
 
 function cleanProperties(obj: any, parentPath: string, fieldConfigRegistry: FieldConfigOptionsRegistry) {
+  let found = false;
+
   for (const propName of Object.keys(obj)) {
     const value = obj[propName];
     const fullPath = `${parentPath}${propName}`;
@@ -112,6 +114,7 @@ function cleanProperties(obj: any, parentPath: string, fieldConfigRegistry: Fiel
 
     // need to check early here as some standard properties have nested properies
     if (existsInRegistry) {
+      found = true;
       continue;
     }
 
@@ -120,9 +123,15 @@ function cleanProperties(obj: any, parentPath: string, fieldConfigRegistry: Fiel
         unset(obj, propName);
       }
     } else {
-      cleanProperties(value, `${fullPath}.`, fieldConfigRegistry);
+      const childPropFound = cleanProperties(value, `${fullPath}.`, fieldConfigRegistry);
+      // If no child props found unset the main object
+      if (!childPropFound) {
+        unset(obj, propName);
+      }
     }
   }
+
+  return found;
 }
 
 function adaptFieldColorMode(