add logic to check if an azure VM is deallocated or not (#4908)

* add logic to check if an azure VM is deallocated or not
* update documentation  with the new azure power state label

Signed-off-by: tariqibrahim <tariq.ibrahim@microsoft.com>

discovery/azure/azure.go

@@ -47,6 +47,7 @@ const (
 	azureLabelMachinePrivateIP     = azureLabel + "machine_private_ip"
 	azureLabelMachineTag           = azureLabel + "machine_tag_"
 	azureLabelMachineScaleSet      = azureLabel + "machine_scale_set"
+	azureLabelPowerState           = azureLabel + "machine_power_state"
 )
 
 var (
@@ -228,6 +229,7 @@ type virtualMachine struct {
 	ScaleSet       string
 	Tags           map[string]*string
 	NetworkProfile compute.NetworkProfile
+	PowerStateCode string
 }
 
 // Create a new azureResource object from an ID string.
@@ -302,12 +304,21 @@ func (d *Discovery) refresh() (tg *targetgroup.Group, err error) {
 				return
 			}
 
+			// We check if the virtual machine has been deallocated.
+			// If so, we skip them in service discovery.
+			if strings.EqualFold(vm.PowerStateCode, "PowerState/deallocated") {
+				level.Debug(d.logger).Log("msg", "Skipping virtual machine", "machine", vm.Name, "power_state", vm.PowerStateCode)
+				ch <- target{}
+				return
+			}
+
 			labels := model.LabelSet{
 				azureLabelMachineID:            model.LabelValue(vm.ID),
 				azureLabelMachineName:          model.LabelValue(vm.Name),
 				azureLabelMachineOSType:        model.LabelValue(vm.OsType),
 				azureLabelMachineLocation:      model.LabelValue(vm.Location),
 				azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroup),
+				azureLabelPowerState:           model.LabelValue(vm.PowerStateCode),
 			}
 
 			if vm.ScaleSet != "" {
@@ -335,16 +346,6 @@ func (d *Discovery) refresh() (tg *targetgroup.Group, err error) {
 					continue
 				}
 
-				// Unfortunately Azure does not return information on whether a VM is deallocated.
-				// This information is available via another API call however the Go SDK does not
-				// yet support this. On deallocated machines, this value happens to be nil so it
-				// is a cheap and easy way to determine if a machine is allocated or not.
-				if networkInterface.Properties.Primary == nil {
-					level.Debug(d.logger).Log("msg", "Skipping deallocated virtual machine", "machine", vm.Name)
-					ch <- target{}
-					return
-				}
-
 				if *networkInterface.Properties.Primary {
 					for _, ip := range *networkInterface.Properties.IPConfigurations {
 						if ip.Properties.PrivateIPAddress != nil {
@@ -472,6 +473,7 @@ func mapFromVM(vm compute.VirtualMachine) virtualMachine {
 		ScaleSet:       "",
 		Tags:           tags,
 		NetworkProfile: *(vm.Properties.NetworkProfile),
+		PowerStateCode: getPowerStateFromVMInstanceView(vm.Properties.InstanceView),
 	}
 }
 
@@ -492,6 +494,7 @@ func mapFromVMScaleSetVM(vm compute.VirtualMachineScaleSetVM, scaleSetName strin
 		ScaleSet:       scaleSetName,
 		Tags:           tags,
 		NetworkProfile: *(vm.Properties.NetworkProfile),
+		PowerStateCode: getPowerStateFromVMInstanceView(vm.Properties.InstanceView),
 	}
 }
 
@@ -524,3 +527,16 @@ func (client *azureClient) getNetworkInterfaceByID(networkInterfaceID string) (n
 
 	return result, nil
 }
+
+func getPowerStateFromVMInstanceView(instanceView *compute.VirtualMachineInstanceView) (powerState string) {
+	if instanceView.Statuses == nil {
+		return
+	}
+	for _, ivs := range *instanceView.Statuses {
+		code := *(ivs.Code)
+		if strings.HasPrefix(code, "PowerState") {
+			powerState = code
+		}
+	}
+	return
+}

discovery/azure/azure_test.go

@@ -26,6 +26,10 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 	vmType := "type"
 	location := "westeurope"
 	networkProfile := compute.NetworkProfile{}
+	provisioningStatusCode := "ProvisioningState/succeeded"
+	provisionDisplayStatus := "Provisioning succeeded"
+	powerStatusCode := "PowerState/running"
+	powerDisplayStatus := "VM running"
 	properties := &compute.VirtualMachineProperties{
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
@@ -33,6 +37,20 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 			},
 		},
 		NetworkProfile: &networkProfile,
+		InstanceView: &compute.VirtualMachineInstanceView{
+			Statuses: &[]compute.InstanceViewStatus{
+				{
+					Code:          &provisioningStatusCode,
+					Level:         "Info",
+					DisplayStatus: &provisionDisplayStatus,
+				},
+				{
+					Code:          &powerStatusCode,
+					Level:         "Info",
+					DisplayStatus: &powerDisplayStatus,
+				},
+			},
+		},
 	}
 
 	testVM := compute.VirtualMachine{
@@ -52,6 +70,7 @@ func TestMapFromVMWithEmptyTags(t *testing.T) {
 		OsType:         "Linux",
 		Tags:           map[string]*string{},
 		NetworkProfile: networkProfile,
+		PowerStateCode: "PowerState/running",
 	}
 
 	actualVM := mapFromVM(testVM)
@@ -69,6 +88,10 @@ func TestMapFromVMWithTags(t *testing.T) {
 	tags := map[string]*string{
 		"prometheus": new(string),
 	}
+	provisioningStatusCode := "ProvisioningState/succeeded"
+	provisionDisplayStatus := "Provisioning succeeded"
+	powerStatusCode := "PowerState/running"
+	powerDisplayStatus := "VM running"
 	networkProfile := compute.NetworkProfile{}
 	properties := &compute.VirtualMachineProperties{
 		StorageProfile: &compute.StorageProfile{
@@ -77,6 +100,20 @@ func TestMapFromVMWithTags(t *testing.T) {
 			},
 		},
 		NetworkProfile: &networkProfile,
+		InstanceView: &compute.VirtualMachineInstanceView{
+			Statuses: &[]compute.InstanceViewStatus{
+				{
+					Code:          &provisioningStatusCode,
+					Level:         "Info",
+					DisplayStatus: &provisionDisplayStatus,
+				},
+				{
+					Code:          &powerStatusCode,
+					Level:         "Info",
+					DisplayStatus: &powerDisplayStatus,
+				},
+			},
+		},
 	}
 
 	testVM := compute.VirtualMachine{
@@ -96,6 +133,7 @@ func TestMapFromVMWithTags(t *testing.T) {
 		OsType:         "Linux",
 		Tags:           tags,
 		NetworkProfile: networkProfile,
+		PowerStateCode: "PowerState/running",
 	}
 
 	actualVM := mapFromVM(testVM)
@@ -111,6 +149,10 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 	vmType := "type"
 	location := "westeurope"
 	networkProfile := compute.NetworkProfile{}
+	provisioningStatusCode := "ProvisioningState/succeeded"
+	provisionDisplayStatus := "Provisioning succeeded"
+	powerStatusCode := "PowerState/running"
+	powerDisplayStatus := "VM running"
 	properties := &compute.VirtualMachineScaleSetVMProperties{
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
@@ -118,6 +160,20 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 			},
 		},
 		NetworkProfile: &networkProfile,
+		InstanceView: &compute.VirtualMachineInstanceView{
+			Statuses: &[]compute.InstanceViewStatus{
+				{
+					Code:          &provisioningStatusCode,
+					Level:         "Info",
+					DisplayStatus: &provisionDisplayStatus,
+				},
+				{
+					Code:          &powerStatusCode,
+					Level:         "Info",
+					DisplayStatus: &powerDisplayStatus,
+				},
+			},
+		},
 	}
 
 	testVM := compute.VirtualMachineScaleSetVM{
@@ -139,6 +195,7 @@ func TestMapFromVMScaleSetVMWithEmptyTags(t *testing.T) {
 		Tags:           map[string]*string{},
 		NetworkProfile: networkProfile,
 		ScaleSet:       scaleSet,
+		PowerStateCode: "PowerState/running",
 	}
 
 	actualVM := mapFromVMScaleSetVM(testVM, scaleSet)
@@ -157,6 +214,10 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		"prometheus": new(string),
 	}
 	networkProfile := compute.NetworkProfile{}
+	provisioningStatusCode := "ProvisioningState/succeeded"
+	provisionDisplayStatus := "Provisioning succeeded"
+	powerStatusCode := "PowerState/running"
+	powerDisplayStatus := "VM running"
 	properties := &compute.VirtualMachineScaleSetVMProperties{
 		StorageProfile: &compute.StorageProfile{
 			OsDisk: &compute.OSDisk{
@@ -164,6 +225,20 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 			},
 		},
 		NetworkProfile: &networkProfile,
+		InstanceView: &compute.VirtualMachineInstanceView{
+			Statuses: &[]compute.InstanceViewStatus{
+				{
+					Code:          &provisioningStatusCode,
+					Level:         "Info",
+					DisplayStatus: &provisionDisplayStatus,
+				},
+				{
+					Code:          &powerStatusCode,
+					Level:         "Info",
+					DisplayStatus: &powerDisplayStatus,
+				},
+			},
+		},
 	}
 
 	testVM := compute.VirtualMachineScaleSetVM{
@@ -185,6 +260,7 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		Tags:           tags,
 		NetworkProfile: networkProfile,
 		ScaleSet:       scaleSet,
+		PowerStateCode: "PowerState/running",
 	}
 
 	actualVM := mapFromVMScaleSetVM(testVM, scaleSet)
@@ -193,3 +269,52 @@ func TestMapFromVMScaleSetVMWithTags(t *testing.T) {
 		t.Errorf("Expected %v got %v", expectedVM, actualVM)
 	}
 }
+
+func TestGetPowerStatusFromVM(t *testing.T) {
+	provisioningStatusCode := "ProvisioningState/succeeded"
+	provisionDisplayStatus := "Provisioning succeeded"
+	powerStatusCode := "PowerState/running"
+	powerDisplayStatus := "VM running"
+	properties := &compute.VirtualMachineScaleSetVMProperties{
+		StorageProfile: &compute.StorageProfile{
+			OsDisk: &compute.OSDisk{
+				OsType: "Linux",
+			},
+		},
+		InstanceView: &compute.VirtualMachineInstanceView{
+			Statuses: &[]compute.InstanceViewStatus{
+				{
+					Code:          &provisioningStatusCode,
+					Level:         "Info",
+					DisplayStatus: &provisionDisplayStatus,
+				},
+				{
+					Code:          &powerStatusCode,
+					Level:         "Info",
+					DisplayStatus: &powerDisplayStatus,
+				},
+			},
+		},
+	}
+
+	testVM := compute.VirtualMachineScaleSetVM{
+		Properties: properties,
+	}
+
+	actual := getPowerStateFromVMInstanceView(testVM.Properties.InstanceView)
+
+	expected := "PowerState/running"
+
+	if actual != expected {
+		t.Errorf("expected powerStatus %s, but got %s instead", expected, actual)
+	}
+
+	// Noq we test a virtualMachine with an empty InstanceView struct.
+	testVM.Properties.InstanceView = &compute.VirtualMachineInstanceView{}
+
+	actual = getPowerStateFromVMInstanceView(testVM.Properties.InstanceView)
+
+	if actual != "" {
+		t.Errorf("expected powerStatus %s, but got %s instead", expected, actual)
+	}
+}

docs/configuration/configuration.md

@@ -263,10 +263,11 @@ The following meta labels are available on targets during relabeling:
 * `__meta_azure_machine_location`: the location the machine runs in
 * `__meta_azure_machine_name`: the machine name
 * `__meta_azure_machine_os_type`: the machine operating system
+* `__meta_azure_machine_power_state`: the current power state of the machine
 * `__meta_azure_machine_private_ip`: the machine's private IP
 * `__meta_azure_machine_resource_group`: the machine's resource group
-* `__meta_azure_machine_tag_<tagname>`: each tag value of the machine
 * `__meta_azure_machine_scale_set`: the name of the scale set which the vm is part of (this value is only set if you are using a [scale set](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/))
+* `__meta_azure_machine_tag_<tagname>`: each tag value of the machine
 
 See below for the configuration options for Azure discovery: