apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

InfluxDB: Strong types and backend migration preparation (#69469)

Browse Source 
* Type definitions

* Move non-backend-migration code into unified place

* Update public/app/plugins/datasource/influxdb/datasource.ts
pull/69519/head
ismail simsek 3 years ago committed by GitHub
parent 6e501b9c07
commit 6c7b17b59f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 264 additions and 268 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 19
      .betterer.results
  2. 513
      public/app/plugins/datasource/influxdb/datasource.ts

19
.betterer.results
Unescape View File

@ -4153,39 +4153,34 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]
],
"public/app/plugins/datasource/influxdb/datasource.ts:5381": [
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
[0, 0, 0, "Unexpected any. Specify a different type.", "2"],
[0, 0, 0, "Unexpected any. Specify a different type.", "3"],
[0, 0, 0, "Unexpected any. Specify a different type.", "4"],
[0, 0, 0, "Do not use any type assertions.", "5"],
[0, 0, 0, "Unexpected any. Specify a different type.", "6"],
[0, 0, 0, "Unexpected any. Specify a different type.", "5"],
[0, 0, 0, "Do not use any type assertions.", "6"],
[0, 0, 0, "Unexpected any. Specify a different type.", "7"],
[0, 0, 0, "Unexpected any. Specify a different type.", "8"],
[0, 0, 0, "Unexpected any. Specify a different type.", "9"],
[0, 0, 0, "Unexpected any. Specify a different type.", "10"],
[0, 0, 0, "Unexpected any. Specify a different type.", "11"],
[0, 0, 0, "Do not use any type assertions.", "11"],
[0, 0, 0, "Unexpected any. Specify a different type.", "12"],
[0, 0, 0, "Unexpected any. Specify a different type.", "13"],
[0, 0, 0, "Unexpected any. Specify a different type.", "14"],
[0, 0, 0, "Unexpected any. Specify a different type.", "15"],
[0, 0, 0, "Do not use any type assertions.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "17"],
[0, 0, 0, "Unexpected any. Specify a different type.", "18"],
[0, 0, 0, "Unexpected any. Specify a different type.", "19"],
[0, 0, 0, "Unexpected any. Specify a different type.", "20"],
[0, 0, 0, "Do not use any type assertions.", "21"],
[0, 0, 0, "Unexpected any. Specify a different type.", "21"],
[0, 0, 0, "Unexpected any. Specify a different type.", "22"],
[0, 0, 0, "Unexpected any. Specify a different type.", "23"],
[0, 0, 0, "Unexpected any. Specify a different type.", "24"],
[0, 0, 0, "Unexpected any. Specify a different type.", "25"],
[0, 0, 0, "Unexpected any. Specify a different type.", "26"],
[0, 0, 0, "Unexpected any. Specify a different type.", "27"],
[0, 0, 0, "Unexpected any. Specify a different type.", "28"],
[0, 0, 0, "Unexpected any. Specify a different type.", "29"],
[0, 0, 0, "Unexpected any. Specify a different type.", "30"],
[0, 0, 0, "Unexpected any. Specify a different type.", "31"],
[0, 0, 0, "Unexpected any. Specify a different type.", "32"]
[0, 0, 0, "Do not use any type assertions.", "27"]
],
"public/app/plugins/datasource/influxdb/influx_query_model.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],

513
public/app/plugins/datasource/influxdb/datasource.ts
Unescape View File

@ -41,88 +41,16 @@ import { InfluxQueryBuilder } from './query_builder';
import ResponseParser from './response_parser';
import { InfluxOptions, InfluxQuery, InfluxVersion } from './types';
// we detect the field type based on the value-array
function getFieldType(values: unknown[]): FieldType {
// the values-array may contain a lot of nulls.
// we need the first not-null item
const firstNotNull = values.find((v) => v !== null);
if (firstNotNull === undefined) {
// we could not find any not-null values
return FieldType.number;
}
const valueType = typeof firstNotNull;
switch (valueType) {
case 'string':
return FieldType.string;
case 'boolean':
return FieldType.boolean;
case 'number':
return FieldType.number;
default:
// this should never happen, influxql values
// can only be numbers, strings and booleans.
throw new Error(`InfluxQL: invalid value type ${valueType}`);
}
}
// this conversion function is specialized to work with the timeseries
// data returned by InfluxDatasource.getTimeSeries()
function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame {
const times: number[] = [];
const values: unknown[] = [];
// the data we process here is not correctly typed.
// the typescript types say every data-point is number|null,
// but in fact it can be string or boolean too.
const points = timeSeries.datapoints;
for (const point of points) {
values.push(point[0]);
times.push(point[1] as number);
}
const timeField = {
name: TIME_SERIES_TIME_FIELD_NAME,
type: FieldType.time,
config: {},
values: times,
};
const valueField = {
name: TIME_SERIES_VALUE_FIELD_NAME,
type: getFieldType(values),
config: {
displayNameFromDS: timeSeries.title,
},
values: values,
labels: timeSeries.tags,
};
const fields = [timeField, valueField];
return {
name: timeSeries.target,
refId: timeSeries.refId,
meta: timeSeries.meta,
fields,
length: values.length,
};
}
export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery, InfluxOptions> {
type: string;
urls: string[];
username: string;
password: string;
name: string;
database: any;
basicAuth: any;
withCredentials: any;
database?: string;
basicAuth?: string;
withCredentials?: boolean;
access: 'direct' | 'proxy';
interval: any;
responseParser: ResponseParser;
httpMode: string;
isFlux: boolean;
@ -146,7 +74,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
this.basicAuth = instanceSettings.basicAuth;
this.withCredentials = instanceSettings.withCredentials;
this.access = instanceSettings.access;
const settingsData = instanceSettings.jsonData || ({} as InfluxOptions);
const settingsData: InfluxOptions = instanceSettings.jsonData ?? {};
this.database = settingsData.dbName ?? instanceSettings.database;
this.interval = settingsData.timeInterval;
this.httpMode = settingsData.httpMode || 'GET';
@ -297,192 +225,13 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
};
}
if (config.featureToggles.influxdbBackendMigration && this.access === 'proxy') {
if (this.isMigrationToggleOnAndIsAccessProxy()) {
query = this.applyVariables(query, scopedVars, rest);
}
return query;
}
/**
* The unchanged pre 7.1 query implementation
*/
classicQuery(options: any): Observable<DataQueryResponse> {
// migrate annotations
if (options.targets.some((target: InfluxQuery) => target.fromAnnotations)) {
const streams: Array<Observable<DataQueryResponse>> = [];
for (const target of options.targets) {
if (target.query) {
streams.push(
new Observable((subscriber) => {
this.annotationEvents(options, target)
.then((events) => subscriber.next({ data: [toDataFrame(events)] }))
.catch((ex) => subscriber.error(new Error(ex)))
.finally(() => subscriber.complete());
})
);
}
}
return merge(...streams);
}
let timeFilter = this.getTimeFilter(options);
const scopedVars = options.scopedVars;
const targets = cloneDeep(options.targets);
const queryTargets: any[] = [];
let i, y;
let allQueries = _map(targets, (target) => {
if (target.hide) {
return '';
}
queryTargets.push(target);
// backward compatibility
scopedVars.interval = scopedVars.__interval;
return new InfluxQueryModel(target, this.templateSrv, scopedVars).render(true);
}).reduce((acc, current) => {
if (current !== '') {
acc += ';' + current;
}
return acc;
});
if (allQueries === '') {
return of({ data: [] });
}
// add global adhoc filters to timeFilter
const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
const adhocFiltersFromDashboard = options.targets.flatMap((target: InfluxQuery) => target.adhocFilters ?? []);
if (adhocFilters?.length || adhocFiltersFromDashboard?.length) {
const ahFilters = adhocFilters?.length ? adhocFilters : adhocFiltersFromDashboard;
const tmpQuery = new InfluxQueryModel({ refId: 'A' }, this.templateSrv, scopedVars);
timeFilter += ' AND ' + tmpQuery.renderAdhocFilters(ahFilters);
}
// replace grafana variables
scopedVars.timeFilter = { value: timeFilter };
// replace templated variables
allQueries = this.templateSrv.replace(allQueries, scopedVars);
return this._seriesQuery(allQueries, options).pipe(
map((data: any) => {
if (!data || !data.results) {
return { data: [] };
}
const seriesList = [];
for (i = 0; i < data.results.length; i++) {
const result = data.results[i];
if (!result || !result.series) {
continue;
}
const target = queryTargets[i];
let alias = target.alias;
if (alias) {
alias = this.templateSrv.replace(target.alias, options.scopedVars);
}
const meta: QueryResultMeta = {
executedQueryString: data.executedQueryString,
};
const influxSeries = new InfluxSeries({
refId: target.refId,
series: data.results[i].series,
alias: alias,
meta,
});
switch (target.resultFormat) {
case 'logs':
meta.preferredVisualisationType = 'logs';
case 'table': {
seriesList.push(influxSeries.getTable());
break;
}
default: {
const timeSeries = influxSeries.getTimeSeries();
for (y = 0; y < timeSeries.length; y++) {
seriesList.push(timeSeriesToDataFrame(timeSeries[y]));
}
break;
}
}
}
return { data: seriesList };
})
);
}
async annotationEvents(options: DataQueryRequest, annotation: InfluxQuery): Promise<AnnotationEvent[]> {
if (this.isFlux) {
return Promise.reject({
message: 'Flux requires the standard annotation query',
});
}
// InfluxQL puts a query string on the annotation
if (!annotation.query) {
return Promise.reject({
message: 'Query missing in annotation definition',
});
}
if (config.featureToggles.influxdbBackendMigration && this.access === 'proxy') {
// We want to send our query to the backend as a raw query
const target: InfluxQuery = {
refId: 'metricFindQuery',
datasource: this.getRef(),
query: this.templateSrv.replace(annotation.query, undefined, 'regex'),
rawQuery: true,
};
return lastValueFrom(
getBackendSrv()
.fetch<BackendDataSourceResponse>({
url: '/api/ds/query',
method: 'POST',
headers: this.getRequestHeaders(),
data: {
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries: [target],
},
requestId: annotation.name,
})
.pipe(
map(
async (res: FetchResponse<BackendDataSourceResponse>) =>
await this.responseParser.transformAnnotationResponse(annotation, res, target)
)
)
);
}
const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone });
let query = annotation.query.replace('$timeFilter', timeFilter);
query = this.templateSrv.replace(query, undefined, 'regex');
return lastValueFrom(this._seriesQuery(query, options)).then((data: any) => {
if (!data || !data.results || !data.results[0]) {
throw { message: 'No results in response from InfluxDB' };
}
return new InfluxSeries({
series: data.results[0].series,
annotation: annotation,
}).getAnnotations();
});
}
targetContainsTemplate(target: any) {
// for flux-mode we just take target.query,
// for influxql-mode we use InfluxQueryModel to create the text-representation
@ -776,7 +525,259 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return date.valueOf() + 'ms';
}
// ------------------------ Legacy Code - Before Backend Migration ---------------
isMigrationToggleOnAndIsAccessProxy() {
return config.featureToggles.influxdbBackendMigration && this.access === 'proxy';
}
/**
* The unchanged pre 7.1 query implementation
*/
classicQuery(options: any): Observable<DataQueryResponse> {
// migrate annotations
if (options.targets.some((target: InfluxQuery) => target.fromAnnotations)) {
const streams: Array<Observable<DataQueryResponse>> = [];
for (const target of options.targets) {
if (target.query) {
streams.push(
new Observable((subscriber) => {
this.annotationEvents(options, target)
.then((events) => subscriber.next({ data: [toDataFrame(events)] }))
.catch((ex) => subscriber.error(new Error(ex)))
.finally(() => subscriber.complete());
})
);
}
}
return merge(...streams);
}
let timeFilter = this.getTimeFilter(options);
const scopedVars = options.scopedVars;
const targets = cloneDeep(options.targets);
const queryTargets: any[] = [];
let i, y;
let allQueries = _map(targets, (target) => {
if (target.hide) {
return '';
}
queryTargets.push(target);
// backward compatibility
scopedVars.interval = scopedVars.__interval;
return new InfluxQueryModel(target, this.templateSrv, scopedVars).render(true);
}).reduce((acc, current) => {
if (current !== '') {
acc += ';' + current;
}
return acc;
});
if (allQueries === '') {
return of({ data: [] });
}
// add global adhoc filters to timeFilter
const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
const adhocFiltersFromDashboard = options.targets.flatMap((target: InfluxQuery) => target.adhocFilters ?? []);
if (adhocFilters?.length || adhocFiltersFromDashboard?.length) {
const ahFilters = adhocFilters?.length ? adhocFilters : adhocFiltersFromDashboard;
const tmpQuery = new InfluxQueryModel({ refId: 'A' }, this.templateSrv, scopedVars);
timeFilter += ' AND ' + tmpQuery.renderAdhocFilters(ahFilters);
}
// replace grafana variables
scopedVars.timeFilter = { value: timeFilter };
// replace templated variables
allQueries = this.templateSrv.replace(allQueries, scopedVars);
return this._seriesQuery(allQueries, options).pipe(
map((data: any) => {
if (!data || !data.results) {
return { data: [] };
}
const seriesList = [];
for (i = 0; i < data.results.length; i++) {
const result = data.results[i];
if (!result || !result.series) {
continue;
}
const target = queryTargets[i];
let alias = target.alias;
if (alias) {
alias = this.templateSrv.replace(target.alias, options.scopedVars);
}
const meta: QueryResultMeta = {
executedQueryString: data.executedQueryString,
};
const influxSeries = new InfluxSeries({
refId: target.refId,
series: data.results[i].series,
alias: alias,
meta,
});
switch (target.resultFormat) {
case 'logs':
meta.preferredVisualisationType = 'logs';
case 'table': {
seriesList.push(influxSeries.getTable());
break;
}
default: {
const timeSeries = influxSeries.getTimeSeries();
for (y = 0; y < timeSeries.length; y++) {
seriesList.push(timeSeriesToDataFrame(timeSeries[y]));
}
break;
}
}
}
return { data: seriesList };
})
);
}
async annotationEvents(options: DataQueryRequest, annotation: InfluxQuery): Promise<AnnotationEvent[]> {
if (this.isFlux) {
return Promise.reject({
message: 'Flux requires the standard annotation query',
});
}
// InfluxQL puts a query string on the annotation
if (!annotation.query) {
return Promise.reject({
message: 'Query missing in annotation definition',
});
}
if (this.isMigrationToggleOnAndIsAccessProxy()) {
// We want to send our query to the backend as a raw query
const target: InfluxQuery = {
refId: 'metricFindQuery',
datasource: this.getRef(),
query: this.templateSrv.replace(annotation.query, undefined, 'regex'),
rawQuery: true,
};
return lastValueFrom(
getBackendSrv()
.fetch<BackendDataSourceResponse>({
url: '/api/ds/query',
method: 'POST',
headers: this.getRequestHeaders(),
data: {
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries: [target],
},
requestId: annotation.name,
})
.pipe(
map(
async (res: FetchResponse<BackendDataSourceResponse>) =>
await this.responseParser.transformAnnotationResponse(annotation, res, target)
)
)
);
}
const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone });
let query = annotation.query.replace('$timeFilter', timeFilter);
query = this.templateSrv.replace(query, undefined, 'regex');
return lastValueFrom(this._seriesQuery(query, options)).then((data: any) => {
if (!data || !data.results || !data.results[0]) {
throw { message: 'No results in response from InfluxDB' };
}
return new InfluxSeries({
series: data.results[0].series,
annotation: annotation,
}).getAnnotations();
});
}
}
// we detect the field type based on the value-array
function getFieldType(values: unknown[]): FieldType {
// the values-array may contain a lot of nulls.
// we need the first not-null item
const firstNotNull = values.find((v) => v !== null);
if (firstNotNull === undefined) {
// we could not find any not-null values
return FieldType.number;
}
const valueType = typeof firstNotNull;
switch (valueType) {
case 'string':
return FieldType.string;
case 'boolean':
return FieldType.boolean;
case 'number':
return FieldType.number;
default:
// this should never happen, influxql values
// can only be numbers, strings and booleans.
throw new Error(`InfluxQL: invalid value type ${valueType}`);
}
}
// this conversion function is specialized to work with the timeseries
// data returned by InfluxDatasource.getTimeSeries()
function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame {
const times: number[] = [];
const values: unknown[] = [];
// the data we process here is not correctly typed.
// the typescript types say every data-point is number|null,
// but in fact it can be string or boolean too.
const points = timeSeries.datapoints;
for (const point of points) {
values.push(point[0]);
times.push(point[1] as number);
}
const timeField = {
name: TIME_SERIES_TIME_FIELD_NAME,
type: FieldType.time,
config: {},
values: times,
};
const valueField = {
name: TIME_SERIES_VALUE_FIELD_NAME,
type: getFieldType(values),
config: {
displayNameFromDS: timeSeries.title,
},
values: values,
labels: timeSeries.tags,
};
const fields = [timeField, valueField];
return {
name: timeSeries.target,
refId: timeSeries.refId,
meta: timeSeries.meta,
fields,
length: values.length,
};
}

Write Preview
Loading…
Cancel
Save