The open and composable observability and data visualization platform. Visualize metrics, logs, and traces from multiple sources like Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
grafana/pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go

731 lines
22 KiB

package metrics
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"path"
"regexp"
"sort"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/loganalytics"
azTime "github.com/grafana/grafana/pkg/tsdb/azuremonitor/time"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/types"
)
// AzureMonitorDatasource calls the Azure Monitor API - one of the four API's supported
type AzureMonitorDatasource struct {
Proxy types.ServiceProxy
Features featuremgmt.FeatureToggles
}
var (
// Used to convert the aggregation value to the Azure enum for deep linking
aggregationTypeMap = map[string]int{"None": 0, "Total": 1, "Minimum": 2, "Maximum": 3, "Average": 4, "Count": 7}
resourceNameLandmark = regexp.MustCompile(`(?i)(/(?P<resourceName>[\w-\.]+)/providers/Microsoft\.Insights/metrics)`)
)
const AzureMonitorAPIVersion = "2021-05-01"
func (e *AzureMonitorDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) (http.ResponseWriter, error) {
return e.Proxy.Do(rw, req, cli)
}
// executeTimeSeriesQuery does the following:
// 1. build the AzureMonitor url and querystring for each query
// 2. executes each query by calling the Azure Monitor API
// 3. parses the responses for each query into data frames
func (e *AzureMonitorDatasource) ExecuteTimeSeriesQuery(ctx context.Context, originalQueries []backend.DataQuery, dsInfo types.DatasourceInfo, client *http.Client, url string, tracer tracing.Tracer) (*backend.QueryDataResponse, error) {
result := backend.NewQueryDataResponse()
queries, err := e.buildQueries(originalQueries, dsInfo)
if err != nil {
return nil, err
}
for _, query := range queries {
res, err := e.executeQuery(ctx, query, dsInfo, client, url, tracer)
if err != nil {
result.Responses[query.RefID] = backend.DataResponse{Error: err}
continue
}
result.Responses[query.RefID] = *res
}
return result, nil
}
func (e *AzureMonitorDatasource) buildQueries(queries []backend.DataQuery, dsInfo types.DatasourceInfo) ([]*types.AzureMonitorQuery, error) {
azureMonitorQueries := []*types.AzureMonitorQuery{}
for _, query := range queries {
var target string
queryJSONModel := dataquery.AzureMonitorQuery{}
err := json.Unmarshal(query.JSON, &queryJSONModel)
if err != nil {
return nil, fmt.Errorf("failed to decode the Azure Monitor query object from JSON: %w", err)
}
azJSONModel := queryJSONModel.AzureMonitor
// Legacy: If only MetricDefinition is set, use it as namespace
if azJSONModel.MetricDefinition != nil && *azJSONModel.MetricDefinition != "" &&
azJSONModel.MetricNamespace != nil && *azJSONModel.MetricNamespace == "" {
azJSONModel.MetricNamespace = azJSONModel.MetricDefinition
}
AzureMonitor: Add support for selecting multiple options when using the equals and not equals dimension filters (#48650) * Add support for multiselect - Add filters param to Dimensions - Update existing tests - Add MultiSelect component - Add helper function to determine valid options - Update labels hook to account for custom values - Update go type - Add function to build valid filters string * Additional go tests - Ensure query targets are built correctly * Update DimensionFields frontend test - Corrently rerender components - Additional test for multiple labels selection - Better selection of options in react-select components * Fix lint issue * Reset filters when operator or dimension changes * Terminology * Update test * Add backend migration - Update types (deprecate Filter field) - Add migration logic - Update tests - Update dimension filters buliding * Add migration test code * Simplify some logic * Add frontend deprecation notice * Add frontend migration logic and migration tests * Update setting of filter values * Update DimensionFields test * Fix linting issues * PR comment updates - Remove unnecessary if/else condition - Don't set filter default value as queries should be migrated - Add comment explaining why sw operator only accepts one value - Remove unnecessary test for merging of old and new filters * Nit on terminology Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com> * Rename migrations for clarity Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>
4 years ago
azJSONModel.DimensionFilters = MigrateDimensionFilters(azJSONModel.DimensionFilters)
alias := ""
if azJSONModel.Alias != nil {
alias = *azJSONModel.Alias
}
azureURL := ""
if queryJSONModel.Subscription != nil {
azureURL = BuildSubscriptionMetricsURL(*queryJSONModel.Subscription)
}
filterInBody := true
resourceIDs := []string{}
resourceMap := map[string]dataquery.AzureMonitorResource{}
if hasOne, resourceGroup, resourceName := hasOneResource(queryJSONModel); hasOne {
ub := urlBuilder{
ResourceURI: azJSONModel.ResourceUri,
// Alternative, used to reconstruct resource URI if it's not present
DefaultSubscription: &dsInfo.Settings.SubscriptionId,
Subscription: queryJSONModel.Subscription,
ResourceGroup: resourceGroup,
MetricNamespace: azJSONModel.MetricNamespace,
ResourceName: resourceName,
}
azureURL = ub.BuildMetricsURL()
// POST requests are only supported at the subscription level
filterInBody = false
resourceUri := ub.buildResourceURI()
if resourceUri != nil {
resourceMap[*resourceUri] = dataquery.AzureMonitorResource{ResourceGroup: resourceGroup, ResourceName: resourceName}
}
} else {
for _, r := range azJSONModel.Resources {
ub := urlBuilder{
DefaultSubscription: &dsInfo.Settings.SubscriptionId,
Subscription: queryJSONModel.Subscription,
ResourceGroup: r.ResourceGroup,
MetricNamespace: azJSONModel.MetricNamespace,
ResourceName: r.ResourceName,
}
resourceUri := ub.buildResourceURI()
if resourceUri != nil {
resourceMap[*resourceUri] = r
}
resourceIDs = append(resourceIDs, fmt.Sprintf("Microsoft.ResourceId eq '%s'", *resourceUri))
}
}
// old model
dimension := ""
if azJSONModel.Dimension != nil {
dimension = strings.TrimSpace(*azJSONModel.Dimension)
}
dimensionFilter := ""
if azJSONModel.DimensionFilter != nil {
dimensionFilter = strings.TrimSpace(*azJSONModel.DimensionFilter)
}
dimSB := strings.Builder{}
if dimension != "" && dimensionFilter != "" && dimension != "None" && len(azJSONModel.DimensionFilters) == 0 {
dimSB.WriteString(fmt.Sprintf("%s eq '%s'", dimension, dimensionFilter))
} else {
for i, filter := range azJSONModel.DimensionFilters {
AzureMonitor: Add support for selecting multiple options when using the equals and not equals dimension filters (#48650) * Add support for multiselect - Add filters param to Dimensions - Update existing tests - Add MultiSelect component - Add helper function to determine valid options - Update labels hook to account for custom values - Update go type - Add function to build valid filters string * Additional go tests - Ensure query targets are built correctly * Update DimensionFields frontend test - Corrently rerender components - Additional test for multiple labels selection - Better selection of options in react-select components * Fix lint issue * Reset filters when operator or dimension changes * Terminology * Update test * Add backend migration - Update types (deprecate Filter field) - Add migration logic - Update tests - Update dimension filters buliding * Add migration test code * Simplify some logic * Add frontend deprecation notice * Add frontend migration logic and migration tests * Update setting of filter values * Update DimensionFields test * Fix linting issues * PR comment updates - Remove unnecessary if/else condition - Don't set filter default value as queries should be migrated - Add comment explaining why sw operator only accepts one value - Remove unnecessary test for merging of old and new filters * Nit on terminology Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com> * Rename migrations for clarity Co-authored-by: Andres Martinez Gotor <andres.martinez@grafana.com>
4 years ago
if len(filter.Filters) == 0 {
dimSB.WriteString(fmt.Sprintf("%s eq '*'", *filter.Dimension))
AzureMonitor: Add support for not equals and startsWith operators when creating Azure Metrics dimension filters. (#48077) * Allow dimension operator selection - Add dimension operators and function to update the operator in the query - Add logic to ensure the same dimension cannot be selected multiple times (Azure restriction) - Add selection component * Update backend logic to default operation and filter to eq '*' - This must be done as the ne and sw operators do not work with the wildcard filter * Add tests on dimension operators * Correct placement of 'and' when building query * Add comment and simplify filtering logic * Allow multiSelect for eq and ne operators - Pass PanelData to DimensionFields component - Add logic to retrieve labels from PanelData - Add MultiSelect component for relevant operators - Update frontend types to allow filter to be an array of strings - Update backend types to allow filter to be an array of strings - Update filter string building * Improve setting of labels * Update go tests * Update frontend tests - Add panelData mock (to be expanded later) - Update null check in DimensionFields * Allow custom value and set default * Add frontend test and fix lint issues * Improved handling of options for sw operator * Remove changes related to multiselect * Add check on refId to ensure dimension labels are correct for query * Extract custom hook for setting dimension labels * Add documentation around Azure Monitor metrics dimensions * Update MetricQueryEditor tests - Add missing data prop * Correctly set field values * Add additional expect for onQueryChange * Correctly set operators - Simplify onFilterInputChange * Ensure no duplicate filters appear * Ensure that filters are displayed correctly for saved queries * Update dimension filter test * Include additional test around changing dimension labels * Pass panel data through new metrics query editor
4 years ago
} else {
dimSB.WriteString(types.ConstructFiltersString(filter))
AzureMonitor: Add support for not equals and startsWith operators when creating Azure Metrics dimension filters. (#48077) * Allow dimension operator selection - Add dimension operators and function to update the operator in the query - Add logic to ensure the same dimension cannot be selected multiple times (Azure restriction) - Add selection component * Update backend logic to default operation and filter to eq '*' - This must be done as the ne and sw operators do not work with the wildcard filter * Add tests on dimension operators * Correct placement of 'and' when building query * Add comment and simplify filtering logic * Allow multiSelect for eq and ne operators - Pass PanelData to DimensionFields component - Add logic to retrieve labels from PanelData - Add MultiSelect component for relevant operators - Update frontend types to allow filter to be an array of strings - Update backend types to allow filter to be an array of strings - Update filter string building * Improve setting of labels * Update go tests * Update frontend tests - Add panelData mock (to be expanded later) - Update null check in DimensionFields * Allow custom value and set default * Add frontend test and fix lint issues * Improved handling of options for sw operator * Remove changes related to multiselect * Add check on refId to ensure dimension labels are correct for query * Extract custom hook for setting dimension labels * Add documentation around Azure Monitor metrics dimensions * Update MetricQueryEditor tests - Add missing data prop * Correctly set field values * Add additional expect for onQueryChange * Correctly set operators - Simplify onFilterInputChange * Ensure no duplicate filters appear * Ensure that filters are displayed correctly for saved queries * Update dimension filter test * Include additional test around changing dimension labels * Pass panel data through new metrics query editor
4 years ago
}
if i != len(azJSONModel.DimensionFilters)-1 {
dimSB.WriteString(" and ")
}
}
}
filterString := strings.Join(resourceIDs, " or ")
if dimSB.String() != "" {
if filterString != "" {
filterString = fmt.Sprintf("(%s) and (%s)", filterString, dimSB.String())
} else {
filterString = dimSB.String()
}
}
params, err := getParams(azJSONModel, query)
if err != nil {
return nil, err
}
target = params.Encode()
sub := ""
if queryJSONModel.Subscription != nil {
sub = *queryJSONModel.Subscription
}
query := &types.AzureMonitorQuery{
URL: azureURL,
Target: target,
Params: params,
RefID: query.RefID,
Alias: alias,
TimeRange: query.TimeRange,
Dimensions: azJSONModel.DimensionFilters,
Resources: resourceMap,
Subscription: sub,
}
if filterString != "" {
if filterInBody {
query.BodyFilter = filterString
} else {
query.Params.Add("$filter", filterString)
}
}
azureMonitorQueries = append(azureMonitorQueries, query)
}
return azureMonitorQueries, nil
}
func getParams(azJSONModel *dataquery.AzureMetricQuery, query backend.DataQuery) (url.Values, error) {
params := url.Values{}
timeGrain := azJSONModel.TimeGrain
timeGrains := azJSONModel.AllowedTimeGrainsMs
if timeGrain != nil && *timeGrain == "auto" {
var err error
timeGrain, err = azTime.SetAutoTimeGrain(query.Interval.Milliseconds(), timeGrains)
if err != nil {
return nil, err
}
}
params.Add("api-version", AzureMonitorAPIVersion)
params.Add("timespan", fmt.Sprintf("%v/%v", query.TimeRange.From.UTC().Format(time.RFC3339), query.TimeRange.To.UTC().Format(time.RFC3339)))
if timeGrain != nil {
params.Add("interval", *timeGrain)
}
if azJSONModel.Aggregation != nil {
params.Add("aggregation", *azJSONModel.Aggregation)
}
if azJSONModel.MetricName != nil {
params.Add("metricnames", *azJSONModel.MetricName)
}
if azJSONModel.CustomNamespace != nil && *azJSONModel.CustomNamespace != "" {
params.Add("metricnamespace", *azJSONModel.CustomNamespace)
} else if azJSONModel.MetricNamespace != nil {
params.Add("metricnamespace", *azJSONModel.MetricNamespace)
}
if azJSONModel.Region != nil && *azJSONModel.Region != "" {
params.Add("region", *azJSONModel.Region)
}
if azJSONModel.Top != nil && *azJSONModel.Top != "" {
params.Add("top", *azJSONModel.Top)
}
return params, nil
}
func (e *AzureMonitorDatasource) retrieveSubscriptionDetails(cli *http.Client, ctx context.Context, tracer tracing.Tracer, subscriptionId string, baseUrl string, dsId int64, orgId int64) (string, error) {
req, err := e.createRequest(ctx, fmt.Sprintf("%s/subscriptions/%s", baseUrl, subscriptionId))
if err != nil {
return "", fmt.Errorf("failed to retrieve subscription details for subscription %s: %s", subscriptionId, err)
}
values := req.URL.Query()
values.Add("api-version", "2022-12-01")
req.URL.RawQuery = values.Encode()
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("subscription", subscriptionId, attribute.Key("subscription").String(subscriptionId))
span.SetAttributes("datasource_id", dsId, attribute.Key("datasource_id").Int64(dsId))
span.SetAttributes("org_id", orgId, attribute.Key("org_id").Int64(orgId))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)
if err != nil {
return "", fmt.Errorf("failed to request subscription details: %s", err)
}
defer func() {
err := res.Body.Close()
backend.Logger.Error("Failed to close response body", "err", err)
}()
body, err := io.ReadAll(res.Body)
if err != nil {
return "", fmt.Errorf("failed to read response body: %s", err)
}
if res.StatusCode/100 != 2 {
return "", fmt.Errorf("request failed, status: %s, error: %s", res.Status, string(body))
}
var data types.SubscriptionsResponse
err = json.Unmarshal(body, &data)
if err != nil {
return "", fmt.Errorf("failed to unmarshal subscription detail response. error: %s, status: %s, body: %s", err, res.Status, string(body))
}
return data.DisplayName, nil
}
func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *types.AzureMonitorQuery, dsInfo types.DatasourceInfo, cli *http.Client,
url string, tracer tracing.Tracer) (*backend.DataResponse, error) {
req, err := e.createRequest(ctx, url)
if err != nil {
return nil, err
}
req.URL.Path = path.Join(req.URL.Path, query.URL)
req.URL.RawQuery = query.Params.Encode()
if query.BodyFilter != "" {
req.Method = http.MethodPost
req.Body = io.NopCloser(strings.NewReader(fmt.Sprintf(`{"filter": "%s"}`, query.BodyFilter)))
}
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)
if err != nil {
return nil, err
}
defer func() {
err := res.Body.Close()
backend.Logger.Error("Failed to close response body", "err", err)
}()
data, err := e.unmarshalResponse(res)
if err != nil {
return nil, err
}
azurePortalUrl, err := loganalytics.GetAzurePortalUrl(dsInfo.Cloud)
if err != nil {
return nil, err
}
subscription, err := e.retrieveSubscriptionDetails(cli, ctx, tracer, query.Subscription, dsInfo.Routes["Azure Monitor"].URL, dsInfo.DatasourceID, dsInfo.OrgID)
if err != nil {
return nil, err
}
frames, err := e.parseResponse(data, query, azurePortalUrl, subscription)
if err != nil {
return nil, err
}
dataResponse := backend.DataResponse{Frames: frames}
return &dataResponse, nil
}
func (e *AzureMonitorDatasource) createRequest(ctx context.Context, url string) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("%v: %w", "failed to create request", err)
}
req.Header.Set("Content-Type", "application/json")
return req, nil
}
func (e *AzureMonitorDatasource) unmarshalResponse(res *http.Response) (types.AzureMonitorResponse, error) {
body, err := io.ReadAll(res.Body)
if err != nil {
return types.AzureMonitorResponse{}, err
}
if res.StatusCode/100 != 2 {
return types.AzureMonitorResponse{}, fmt.Errorf("request failed, status: %s, error: %s", res.Status, string(body))
}
var data types.AzureMonitorResponse
err = json.Unmarshal(body, &data)
if err != nil {
return types.AzureMonitorResponse{}, err
}
return data, nil
}
func (e *AzureMonitorDatasource) parseResponse(amr types.AzureMonitorResponse, query *types.AzureMonitorQuery, azurePortalUrl string, subscription string) (data.Frames, error) {
if len(amr.Value) == 0 {
return nil, nil
}
frames := data.Frames{}
for _, series := range amr.Value[0].Timeseries {
labels := data.Labels{}
for _, md := range series.Metadatavalues {
labels[md.Name.LocalizedValue] = md.Value
}
frame := data.NewFrameOfFieldTypes("", len(series.Data), data.FieldTypeTime, data.FieldTypeNullableFloat64)
if e.Features.IsEnabled(featuremgmt.FlagAzureMonitorDataplane) {
frame.Meta = &data.FrameMeta{Type: data.FrameTypeTimeSeriesMulti, TypeVersion: data.FrameTypeVersion{0, 1}}
}
frame.RefID = query.RefID
timeField := frame.Fields[0]
timeField.Name = data.TimeSeriesTimeFieldName
dataField := frame.Fields[1]
dataField.Name = amr.Value[0].Name.LocalizedValue
dataField.Labels = labels
if amr.Value[0].Unit != "Unspecified" {
dataField.SetConfig(&data.FieldConfig{
Unit: toGrafanaUnit(amr.Value[0].Unit),
})
}
resourceIdLabel := "microsoft.resourceid"
resourceID, ok := labels[resourceIdLabel]
if !ok {
resourceIdLabel = "Microsoft.ResourceId"
resourceID = labels[resourceIdLabel]
}
resourceIDSlice := strings.Split(resourceID, "/")
resourceName := ""
if len(resourceIDSlice) > 1 {
resourceName = resourceIDSlice[len(resourceIDSlice)-1]
} else {
// Deprecated: This is for backward compatibility, the URL should contain
// the resource ID
resourceName = extractResourceNameFromMetricsURL(query.URL)
resourceID = extractResourceIDFromMetricsURL(query.URL)
}
if _, ok := labels[resourceIdLabel]; ok {
delete(labels, resourceIdLabel)
labels["resourceName"] = resourceName
}
if query.Alias != "" {
displayName := formatAzureMonitorLegendKey(query, resourceID, &amr, labels, subscription)
if dataField.Config != nil {
dataField.Config.DisplayName = displayName
} else {
dataField.SetConfig(&data.FieldConfig{
DisplayName: displayName,
})
}
}
requestedAgg := query.Params.Get("aggregation")
for i, point := range series.Data {
var value *float64
switch requestedAgg {
case "Average":
value = point.Average
case "Total":
value = point.Total
case "Maximum":
value = point.Maximum
case "Minimum":
value = point.Minimum
case "Count":
value = point.Count
default:
value = point.Count
}
frame.SetRow(i, point.TimeStamp, value)
}
queryUrl, err := getQueryUrl(query, azurePortalUrl, resourceID, resourceName)
if err != nil {
return nil, err
}
AzureMonitor: Application Insights Traces (#64859) * Build out barebones Traces editor - Add Traces query type and operation ID prop to query type - Add necessary header types - Update resource picker to appropriately work with traces query type - Build out TracesQueryEditor component - Include logic to retrieve operationId's for AI Workspaces - Add backend route mapping - Update macro to use timestamp as default time field for traces * AzureMonitor: Traces - Response parsing (#65442) * Update FormatAsField component - Add trace ResultFormat type - Generalise FormatAsField component - Add component to TracesQueryEditor - Remove duplicate code in setQueryValue * Add custom filter function to improve performance * Add basic conversion for logs to trace - Add serviceTags converter - Pass through required parameters (queryType and resultFormat) - Appropriately set visualisation * Update parsing to also fill trace tags - Add constant values for each table schema (include legacy mapping for now if needed) - Add constant for list of table tags - Set the foundation for dynamic query building - Update query to build tags value - Appropriately set operationName - Update tagsConverter to filter empty values * Fix lint and test issues * AzureMonitor: Traces - Data links (#65566) * Add portal link for traces - Pull out necessary values (itemId and itemType) - Appropriately construct - Fix ordering * Set default format as value - Also set default visualisation * Fix event schema * Set default formatAsField value * Include logs link on traces results - Adapt config links to allow custom title to be set * Correctly set operationId for query * Update backend types - Include OperationID in query - Pass forward datasource name and UID * Ensure setTime doesn't consistently get called if operationID is defined * Add explore link - Update util functions to allow setting custom datalinks * Fix tests * AzureMonitor: Traces - Query and Editor updates (#66076) * Add initial query - Will query the resource as soon as a resource has been selected - Updates the data links for the query without operationId - Remove initial operationId query and timeRange dependency - Update query building * Add entirely separate traces query property - Update shared types (also including future types for Azure traces) - Update backend log analytics datasource to accept both azureLogAnalytics and azureTraces queries - Update backend specific types - Update frontend datasource for new properties - Update mock query * Update FormatAsField to be entirely generic * Update query building to be done in backend - Add required mappings in backend - Update frontend querying * Fix query and explore data link * Add trace type selection * Better method for setting explore link * Fix operationId updating * Run go mod tidy * Unnecessary changes * Fix tests * AzureMonitor: Traces - Add correlation API support (#65855) Add correlation API support - Add necessary types - Add correlation API request when conditions are met - Update query * Fix property from merge * AzureMonitor: Traces - Filtering (#66303) * Add initial query - Will query the resource as soon as a resource has been selected - Updates the data links for the query without operationId - Remove initial operationId query and timeRange dependency - Update query building * Add entirely separate traces query property - Update shared types (also including future types for Azure traces) - Update backend log analytics datasource to accept both azureLogAnalytics and azureTraces queries - Update backend specific types - Update frontend datasource for new properties - Update mock query * Update FormatAsField to be entirely generic * Update query building to be done in backend - Add required mappings in backend - Update frontend querying * Fix query and explore data link * Add trace type selection * Better method for setting explore link * Fix operationId updating * Run go mod tidy * Unnecessary changes * Fix tests * Start building out Filters component - Configure component to query for Filter property values when a filter property is set - Add setFilters function - Add typing to tablesSchema - Use component in TracesQueryEditor * Update Filters - Asynchronously pull property options - Setup list of Filter components * Update filters component - Remove unused imports - Have local filters state and query filters - Correctly set filters values - Don't update query every time a filter property changes (not performant) * Update properties query - Use current timeRange - Get count to provide informative labels * Reset map when time changes * Add operation selection * Reset filters when property changes * Appropriate label name for empty values * Add filtering to query * Update filter components - Fix rendering issue - Correctly compare and update timeRange - Split out files for simplicity * Add checkbox option to multiselect - Add custom option component - Correctly call onChange - Add variableOptionGroup for template variable selection * Fix adding template vars * Improve labels and refresh labels on query prop changes * AzureMonitor: Traces - Testing (#66474) * Select ds for template variable interpolation * Update az logs ds tests - Add templateVariables test - Add filter test - Update mock - Remove anys * Update QueryEditor test - Update mocks with timeSrv for log analytics datasource - Fix query mock - Use appropriate and consistent selectors * Add TracesQueryEditor test - Update resourcePickerRows mock to include app insights resources - Remove comments and extra new line * Add FormatAsField test - Remove unneeded condition * Update resourcePicker utils test * Don't hide selected options in filters * Fix multi-selection on filters * Add TraceTypeField test - Add test file - Update selectors (remove copy/paste mistake) - Update placeholder text for select and add label * Add basic filters test * Begin filters test * Update filters test * Add final tests and simplify/generalise addFilter helper * Minor update to datasource test * Update macros test * Update selectors in tests * Add response-table-frame tests * Add datasource tests - Use sorting where JSON models are inconsistent - Update filters clause - Dedupe tags - Correct operationId conditions * Don't set a default value for blurInputOnSelect * Simplify datasource test * Update to use CheckGoldenJSON utils - Update with generated frame files - Remove redundant expected frame code - Update all usages * Fix lint * AzureMonitor: Traces feedback (#67292) * Filter traces if the visualisation is set to trace - Update build query logic - Added additional test cases - Return an error if the traces type is set by itself with the trace visualisation - Add descriptions to event types - Update tests * Fix bug for error displaying traces * Update mappings and add error field - Update tests - Remove unnecessary comments * Switch location of Operation ID field * Re-order fields * Update link title * Update label for event type selection * Update correct link title * Update logs datalink to link to Azure Logs in explore * Fix lint
3 years ago
frameWithLink := loganalytics.AddConfigLinks(*frame, queryUrl, nil)
frames = append(frames, &frameWithLink)
}
return frames, nil
}
// Gets the deep link for the given query
func getQueryUrl(query *types.AzureMonitorQuery, azurePortalUrl, resourceID, resourceName string) (string, error) {
aggregationType := aggregationTypeMap["Average"]
aggregation := query.Params.Get("aggregation")
if aggregation != "" {
if aggType, ok := aggregationTypeMap[aggregation]; ok {
aggregationType = aggType
}
}
timespan, err := json.Marshal(map[string]interface{}{
"absolute": struct {
Start string `json:"startTime"`
End string `json:"endTime"`
}{
Start: query.TimeRange.From.UTC().Format(time.RFC3339Nano),
End: query.TimeRange.To.UTC().Format(time.RFC3339Nano),
},
})
if err != nil {
return "", err
}
escapedTime := url.QueryEscape(string(timespan))
var filters []types.AzureMonitorDimensionFilterBackend
var grouping map[string]interface{}
if len(query.Dimensions) > 0 {
for _, dimension := range query.Dimensions {
var dimensionInt int
dimensionFilters := dimension.Filters
// Only the first dimension determines the splitting shown in the Azure Portal
if grouping == nil {
grouping = map[string]interface{}{
"dimension": dimension.Dimension,
"sort": 2,
"top": 10,
}
}
if len(dimension.Filters) == 0 {
continue
}
if dimension.Dimension == nil {
continue
}
if dimension.Operator == nil {
filter := types.AzureMonitorDimensionFilterBackend{
Key: *dimension.Dimension,
Operator: 0,
Values: dimensionFilters,
}
filters = append(filters, filter)
continue
}
switch *dimension.Operator {
case "eq":
dimensionInt = 0
case "ne":
dimensionInt = 1
case "sw":
dimensionInt = 3
}
filter := types.AzureMonitorDimensionFilterBackend{
Key: *dimension.Dimension,
Operator: dimensionInt,
Values: dimensionFilters,
}
filters = append(filters, filter)
}
}
chart := map[string]interface{}{
"metrics": []types.MetricChartDefinition{
{
ResourceMetadata: map[string]string{
"id": resourceID,
},
Name: query.Params.Get("metricnames"),
AggregationType: aggregationType,
Namespace: query.Params.Get("metricnamespace"),
MetricVisualization: types.MetricVisualization{
DisplayName: query.Params.Get("metricnames"),
ResourceDisplayName: resourceName,
},
},
},
}
if filters != nil {
chart["filterCollection"] = map[string]interface{}{
"filters": filters,
}
}
if grouping != nil {
chart["grouping"] = grouping
}
chartDef, err := json.Marshal(map[string]interface{}{
"v2charts": []interface{}{
chart,
},
})
if err != nil {
return "", err
}
escapedChart := url.QueryEscape(string(chartDef))
// Azure Portal will timeout if the chart definition includes a space character encoded as '+'.
// url.QueryEscape encodes spaces as '+'.
// Note: this will not encode '+' literals as those are already encoded as '%2B' by url.QueryEscape
escapedChart = strings.ReplaceAll(escapedChart, "+", "%20")
return fmt.Sprintf("%s/#blade/Microsoft_Azure_MonitoringMetrics/Metrics.ReactView/Referer/MetricsExplorer/TimeContext/%s/ChartDefinition/%s", azurePortalUrl, escapedTime, escapedChart), nil
}
// formatAzureMonitorLegendKey builds the legend key or timeseries name
// Alias patterns like {{resourcename}} are replaced with the appropriate data values.
func formatAzureMonitorLegendKey(query *types.AzureMonitorQuery, resourceId string, amr *types.AzureMonitorResponse, labels data.Labels, subscription string) string {
alias := query.Alias
subscriptionId := query.Subscription
resource := query.Resources[resourceId]
metricName := amr.Value[0].Name.LocalizedValue
namespace := amr.Namespace
// Could be a collision problem if there were two keys that varied only in case, but I don't think that would happen in azure.
lowerLabels := data.Labels{}
for k, v := range labels {
lowerLabels[strings.ToLower(k)] = v
}
keys := make([]string, 0, len(labels))
for k := range lowerLabels {
keys = append(keys, k)
}
sort.Strings(keys)
result := types.LegendKeyFormat.ReplaceAllFunc([]byte(alias), func(in []byte) []byte {
metaPartName := strings.Replace(string(in), "{{", "", 1)
metaPartName = strings.Replace(metaPartName, "}}", "", 1)
metaPartName = strings.ToLower(strings.TrimSpace(metaPartName))
if metaPartName == "subscriptionid" {
return []byte(subscriptionId)
}
if metaPartName == "subscription" {
if subscription == "" {
return []byte{}
}
return []byte(subscription)
}
if metaPartName == "resourcegroup" && resource.ResourceGroup != nil {
return []byte(*resource.ResourceGroup)
}
if metaPartName == "namespace" {
return []byte(namespace)
}
if metaPartName == "resourcename" && resource.ResourceName != nil {
return []byte(*resource.ResourceName)
}
if metaPartName == "metric" {
return []byte(metricName)
}
if metaPartName == "dimensionname" {
if len(keys) == 0 {
return []byte{}
}
return []byte(keys[0])
}
if metaPartName == "dimensionvalue" {
if len(keys) == 0 {
return []byte{}
}
return []byte(lowerLabels[keys[0]])
}
if v, ok := lowerLabels[metaPartName]; ok {
return []byte(v)
}
return in
})
return string(result)
}
// Map values from:
//
// https://docs.microsoft.com/en-us/rest/api/monitor/metrics/list#unit
//
// to
//
// https://github.com/grafana/grafana/blob/main/packages/grafana-data/src/valueFormats/categories.ts#L24
func toGrafanaUnit(unit string) string {
switch unit {
case "BitsPerSecond":
return "bps"
case "Bytes":
return "decbytes" // or ICE
case "BytesPerSecond":
return "Bps"
case "Count":
return "short" // this is used for integers
case "CountPerSecond":
return "cps"
case "Percent":
return "percent"
case "MilliSeconds":
return "ms"
case "Seconds":
return "s"
}
return unit // this will become a suffix in the display
// "ByteSeconds", "Cores", "MilliCores", and "NanoCores" all both:
// 1. Do not have a corresponding unit in Grafana's current list.
// 2. Do not have the unit listed in any of Azure Monitor's supported metrics anyways.
}
func extractResourceNameFromMetricsURL(url string) string {
matches := resourceNameLandmark.FindStringSubmatch(url)
resourceName := ""
if matches == nil {
return resourceName
}
for i, name := range resourceNameLandmark.SubexpNames() {
if name == "resourceName" {
resourceName = matches[i]
}
}
return resourceName
}
func extractResourceIDFromMetricsURL(url string) string {
return strings.Split(url, "/providers/microsoft.insights/metrics")[0]
}
func hasOneResource(query dataquery.AzureMonitorQuery) (bool, *string, *string) {
azJSONModel := query.AzureMonitor
if len(azJSONModel.Resources) > 1 {
return false, nil, nil
}
if len(azJSONModel.Resources) == 1 {
return true, azJSONModel.Resources[0].ResourceGroup, azJSONModel.Resources[0].ResourceName
}
if (azJSONModel.ResourceGroup != nil && *azJSONModel.ResourceGroup != "") || (azJSONModel.ResourceName != nil && *azJSONModel.ResourceName != "") {
// Deprecated, Resources should be used instead
return true, azJSONModel.ResourceGroup, azJSONModel.ResourceName
}
return false, nil, nil
}