apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Experiment: tag UI devices for anon stats (#73748)

Browse Source 
* experiment: attempt to tag only UI devices

* lint frontend

* use await

* use shorthand check

* do not assume build info exists

* do not assume build info exists

* Apply suggestions from code review

Co-authored-by: Alexander Zobnin <alexanderzobnin@gmail.com>

---------

Co-authored-by: Alexander Zobnin <alexanderzobnin@gmail.com>
pull/73839/head
Jo 2 years ago committed by GitHub
parent 5eed495cce
commit 1a281ac49d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 201 additions and 34 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      package.json
  2. 97
      pkg/services/anonymous/anonimpl/impl.go
  3. 90
      pkg/services/anonymous/anonimpl/impl_test.go
  4. 6
      pkg/services/anonymous/service.go
  5. 31
      public/app/core/services/backend_srv.ts
  6. 10
      yarn.lock

1
package.json
Unescape View File

@ -232,6 +232,7 @@
"@daybrush/utils": "1.13.0",
"@emotion/css": "11.11.2",
"@emotion/react": "11.11.1",
"@fingerprintjs/fingerprintjs": "^3.4.2",
"@glideapps/glide-data-grid": "^5.2.1",
"@grafana/aws-sdk": "0.1.2",
"@grafana/data": "workspace:*",

97
pkg/services/anonymous/anonimpl/impl.go
Unescape View File

@ -16,10 +16,12 @@ import (
"github.com/grafana/grafana/pkg/infra/remotecache"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/services/anonymous"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
)
const thirtyDays = 30 * 24 * time.Hour
const deviceIDHeader = "X-Grafana-Device-Id"
type Device struct {
Kind anonymous.DeviceKind `json:"kind"`
@ -41,6 +43,10 @@ func (a *Device) Key() (string, error) {
return strings.Join([]string{string(a.Kind), hex.EncodeToString(hash.Sum(nil))}, ":"), nil
}
func (a *Device) UIKey(deviceID string) (string, error) {
return strings.Join([]string{string(a.Kind), deviceID}, ":"), nil
}
type AnonDeviceService struct {
remoteCache remotecache.CacheStorage
log log.Logger
@ -70,9 +76,21 @@ func (a *AnonDeviceService) usageStatFn(ctx context.Context) (map[string]interfa
return nil, nil
}
anonUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AnonDeviceUI))
if err != nil {
return nil, nil
}
authedUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AuthedDeviceUI))
if err != nil {
return nil, nil
}
return map[string]interface{}{
"stats.anonymous.session.count": anonDeviceCount, // keep session for legacy data
"stats.users.device.count": authedDeviceCount,
"stats.anonymous.session.count": anonDeviceCount, // keep session for legacy data
"stats.users.device.count": authedDeviceCount,
"stats.anonymous.device.ui.count": anonUIDeviceCount,
"stats.users.device.ui.count": authedUIDeviceCount,
}, nil
}
@ -89,6 +107,72 @@ func (a *AnonDeviceService) untagDevice(ctx context.Context, device *Device) err
return nil
}
func (a *AnonDeviceService) untagUIDevice(ctx context.Context, deviceID string, device *Device) error {
key, err := device.UIKey(deviceID)
if err != nil {
return err
}
if err := a.remoteCache.Delete(ctx, key); err != nil {
return err
}
return nil
}
func (a *AnonDeviceService) tagDeviceUI(ctx context.Context, httpReq *http.Request, device Device) error {
deviceID := httpReq.Header.Get(deviceIDHeader)
if deviceID == "" {
return nil
}
if device.Kind == anonymous.AnonDevice {
device.Kind = anonymous.AnonDeviceUI
} else if device.Kind == anonymous.AuthedDevice {
device.Kind = anonymous.AuthedDeviceUI
}
key, err := device.UIKey(deviceID)
if err != nil {
return err
}
if setting.Env == setting.Dev {
a.log.Debug("tagging device for UI", "deviceID", deviceID, "device", device, "key", key)
}
if _, ok := a.localCache.Get(key); ok {
return nil
}
a.localCache.SetDefault(key, struct{}{})
deviceJSON, err := json.Marshal(device)
if err != nil {
return err
}
if err := a.remoteCache.Set(ctx, key, deviceJSON, thirtyDays); err != nil {
return err
}
// remove existing tag when device switches to another kind
untagKind := anonymous.AnonDeviceUI
if device.Kind == anonymous.AnonDeviceUI {
untagKind = anonymous.AuthedDeviceUI
}
if err := a.untagUIDevice(ctx, deviceID, &Device{
Kind: untagKind,
IP: device.IP,
UserAgent: device.UserAgent,
}); err != nil {
return err
}
return nil
}
func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request, kind anonymous.DeviceKind) error {
addr := web.RemoteAddr(httpReq)
ip, err := network.GetIPFromAddress(addr)
@ -109,11 +193,20 @@ func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request
LastSeen: time.Now().UTC(),
}
err = a.tagDeviceUI(ctx, httpReq, *taggedDevice)
if err != nil {
a.log.Debug("failed to tag device for UI", "error", err)
}
key, err := taggedDevice.Key()
if err != nil {
return err
}
if setting.Env == setting.Dev {
a.log.Debug("tagging device", "device", taggedDevice, "key", key)
}
if _, ok := a.localCache.Get(key); ok {
return nil
}

90
pkg/services/anonymous/anonimpl/impl_test.go
Unescape View File

@ -70,11 +70,13 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
kind anonymous.DeviceKind
}
testCases := []struct {
name string
req []tagReq
expectedAnonCount int64
expectedAuthedCount int64
expectedDevice *Device
name string
req []tagReq
expectedAnonCount int64
expectedAuthedCount int64
expectedAnonUICount int64
expectedAuthedUICount int64
expectedDevice *Device
}{
{
name: "no requests",
@ -112,73 +114,104 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
IP: "10.30.30.1",
UserAgent: "test"},
},
{
name: "should tag device ID once",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
},
},
kind: anonymous.AnonDevice,
},
},
expectedAnonUICount: 1,
expectedAuthedUICount: 0,
expectedAnonCount: 1,
expectedAuthedCount: 0,
expectedDevice: &Device{
Kind: anonymous.AnonDevice,
IP: "10.30.30.1",
UserAgent: "test"},
},
{
name: "repeat request should not tag",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AnonDevice,
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AnonDevice,
},
},
expectedAnonCount: 1,
expectedAnonUICount: 1,
expectedAuthedCount: 0,
}, {
name: "authed request should untag anon",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AnonDevice,
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AuthedDevice,
},
},
expectedAnonCount: 0,
expectedAuthedCount: 1,
expectedAnonCount: 0,
expectedAuthedCount: 1,
expectedAuthedUICount: 1,
}, {
name: "anon request should untag authed",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AuthedDevice,
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
kind: anonymous.AnonDevice,
},
},
expectedAnonCount: 1,
expectedAnonUICount: 1,
expectedAuthedCount: 0,
},
{
name: "tag 4 different requests",
name: "tag 4 different requests - 2 are UI",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
http.CanonicalHeaderKey("User-Agent"): []string{"test"},
http.CanonicalHeaderKey("X-Forwarded-For"): []string{"10.30.30.1"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"a"},
},
},
kind: anonymous.AnonDevice,
@ -191,8 +224,9 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
kind: anonymous.AnonDevice,
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.3"},
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.3"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"c"},
},
},
kind: anonymous.AuthedDevice,
@ -205,8 +239,10 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
kind: anonymous.AuthedDevice,
},
},
expectedAnonCount: 2,
expectedAuthedCount: 2,
expectedAnonCount: 2,
expectedAuthedCount: 2,
expectedAnonUICount: 1,
expectedAuthedUICount: 1,
},
}
@ -226,6 +262,8 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
assert.Equal(t, tc.expectedAnonCount, stats["stats.anonymous.session.count"].(int64))
assert.Equal(t, tc.expectedAuthedCount, stats["stats.users.device.count"].(int64))
assert.Equal(t, tc.expectedAnonUICount, stats["stats.anonymous.device.ui.count"].(int64))
assert.Equal(t, tc.expectedAuthedUICount, stats["stats.users.device.ui.count"].(int64))
if tc.expectedDevice != nil {
key, err := tc.expectedDevice.Key()

6
pkg/services/anonymous/service.go
Unescape View File

@ -8,8 +8,10 @@ import (
type DeviceKind string
const (
AnonDevice DeviceKind = "anon-session"
AuthedDevice DeviceKind = "authed-session"
AnonDevice DeviceKind = "anon-session"
AuthedDevice DeviceKind = "authed-session"
AnonDeviceUI DeviceKind = "ui-anon-session"
AuthedDeviceUI DeviceKind = "ui-authed-session"
)
type Service interface {

31
public/app/core/services/backend_srv.ts
Unescape View File

@ -1,3 +1,4 @@
import FingerprintJS from '@fingerprintjs/fingerprintjs';
import { from, lastValueFrom, MonoTypeOperatorFunction, Observable, Subject, Subscription, throwError } from 'rxjs';
import { fromFetch } from 'rxjs/fetch';
import {
@ -15,6 +16,7 @@ import {
import { v4 as uuidv4 } from 'uuid';
import { AppEvents, DataQueryErrorType } from '@grafana/data';
import { GrafanaEdition } from '@grafana/data/src/types/config';
import { BackendSrv as BackendService, BackendSrvRequest, config, FetchError, FetchResponse } from '@grafana/runtime';
import appEvents from 'app/core/app_events';
import { getConfig } from 'app/core/config';
@ -61,6 +63,7 @@ export class BackendSrv implements BackendService {
private readonly fetchQueue: FetchQueue;
private readonly responseQueue: ResponseQueue;
private _tokenRotationInProgress?: Observable<FetchResponse> | null = null;
private deviceID?: string | null = null;
private dependencies: BackendSrvDependencies = {
fromFetch: fromFetch,
@ -83,9 +86,26 @@ export class BackendSrv implements BackendService {
this.internalFetch = this.internalFetch.bind(this);
this.fetchQueue = new FetchQueue();
this.responseQueue = new ResponseQueue(this.fetchQueue, this.internalFetch);
this.initGrafanaDeviceID();
new FetchQueueWorker(this.fetchQueue, this.responseQueue, getConfig());
}
private async initGrafanaDeviceID() {
if (config.buildInfo?.edition === GrafanaEdition.OpenSource) {
return;
}
try {
const fp = await FingerprintJS.load();
const result = await fp.get();
this.deviceID = result.visitorId;
} catch (error) {
console.error(error);
}
}
async request<T = any>(options: BackendSrvRequest): Promise<T> {
return await lastValueFrom(this.fetch<T>(options).pipe(map((response: FetchResponse<T>) => response.data)));
}
@ -134,15 +154,18 @@ export class BackendSrv implements BackendService {
const token = loadUrlToken();
if (token !== null && token !== '') {
if (!options.headers) {
options.headers = {};
}
if (config.jwtUrlLogin && config.jwtHeaderName) {
options.headers = options.headers ?? {};
options.headers[config.jwtHeaderName] = `${token}`;
}
}
// Add device id header if not OSS build
if (config.buildInfo?.edition !== GrafanaEdition.OpenSource && this.deviceID) {
options.headers = options.headers ?? {};
options.headers['X-Grafana-Device-Id'] = `${this.deviceID}`;
}
return this.getFromFetchStream<T>(options).pipe(
this.handleStreamResponse<T>(options),
this.handleStreamError(options),

10
yarn.lock
Unescape View File

@ -3477,6 +3477,15 @@ __metadata:
languageName: node
linkType: hard
"@fingerprintjs/fingerprintjs@npm:^3.4.2":
version: 3.4.2
resolution: "@fingerprintjs/fingerprintjs@npm:3.4.2"
dependencies:
tslib: ^2.4.1
checksum: 3b9dc81e4186f1aaa39e208c17939f5747bf9a8eb1c8175264a352e46e263abd81bcf89439240bd1a4755d7e3dfb4a83164e294f940abc290e7f2076d3b603ce
languageName: node
linkType: hard
"@floating-ui/core@npm:^1.0.1":
version: 1.0.1
resolution: "@floating-ui/core@npm:1.0.1"
@ -19234,6 +19243,7 @@ __metadata:
"@emotion/css": 11.11.2
"@emotion/eslint-plugin": 11.11.0
"@emotion/react": 11.11.1
"@fingerprintjs/fingerprintjs": ^3.4.2
"@glideapps/glide-data-grid": ^5.2.1
"@grafana/aws-sdk": 0.1.2
"@grafana/data": "workspace:*"

Write Preview
Loading…
Cancel
Save