mirror of https://github.com/grafana/grafana
Crawler: use existing render service to generate dashboard thumbnails (#43515)
Co-authored-by: Artur Wierzbicki <artur@arturwierzbicki.com>pull/43871/head
Ryan McKinley
4 years ago
committed by
GitHub
parent
cc9e70be5c
commit
b404aae9c3
@ -0,0 +1,249 @@ |
||||
package thumbs |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"math/rand" |
||||
"os" |
||||
"strings" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/bus" |
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/grafana/grafana/pkg/services/live" |
||||
"github.com/grafana/grafana/pkg/services/rendering" |
||||
"github.com/grafana/grafana/pkg/services/search" |
||||
) |
||||
|
||||
type dashItem struct { |
||||
uid string |
||||
url string |
||||
} |
||||
|
||||
type simpleCrawler struct { |
||||
screenshotsFolder string |
||||
renderService rendering.Service |
||||
threadCount int |
||||
|
||||
glive *live.GrafanaLive |
||||
mode CrawlerMode |
||||
opts rendering.Opts |
||||
status crawlStatus |
||||
queue []dashItem |
||||
mu sync.Mutex |
||||
} |
||||
|
||||
func newSimpleCrawler(folder string, renderService rendering.Service, gl *live.GrafanaLive) dashRenderer { |
||||
c := &simpleCrawler{ |
||||
screenshotsFolder: folder, |
||||
renderService: renderService, |
||||
threadCount: 5, |
||||
glive: gl, |
||||
status: crawlStatus{ |
||||
State: "init", |
||||
Complete: 0, |
||||
Queue: 0, |
||||
}, |
||||
queue: make([]dashItem, 0), |
||||
} |
||||
c.broadcastStatus() |
||||
return c |
||||
} |
||||
|
||||
func (r *simpleCrawler) next() *dashItem { |
||||
if len(r.queue) < 1 { |
||||
return nil |
||||
} |
||||
r.mu.Lock() |
||||
defer r.mu.Unlock() |
||||
|
||||
v := r.queue[0] |
||||
r.queue = r.queue[1:] |
||||
return &v |
||||
} |
||||
|
||||
func (r *simpleCrawler) broadcastStatus() { |
||||
s, err := r.Status() |
||||
if err != nil { |
||||
tlog.Warn("error reading status") |
||||
return |
||||
} |
||||
msg, err := json.Marshal(s) |
||||
if err != nil { |
||||
tlog.Warn("error making message") |
||||
return |
||||
} |
||||
err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg) |
||||
if err != nil { |
||||
tlog.Warn("error Publish message") |
||||
return |
||||
} |
||||
} |
||||
|
||||
func (r *simpleCrawler) GetPreview(req *previewRequest) *previewResponse { |
||||
p := getFilePath(r.screenshotsFolder, req) |
||||
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { |
||||
return r.queueRender(p, req) |
||||
} |
||||
|
||||
return &previewResponse{ |
||||
Path: p, |
||||
Code: 200, |
||||
} |
||||
} |
||||
|
||||
func (r *simpleCrawler) queueRender(p string, req *previewRequest) *previewResponse { |
||||
go func() { |
||||
fmt.Printf("todo? queue") |
||||
}() |
||||
|
||||
return &previewResponse{ |
||||
Code: 202, |
||||
Path: p, |
||||
} |
||||
} |
||||
|
||||
func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) { |
||||
if r.status.State == "running" { |
||||
tlog.Info("already running") |
||||
return r.Status() |
||||
} |
||||
|
||||
r.mu.Lock() |
||||
defer r.mu.Unlock() |
||||
|
||||
searchQuery := search.Query{ |
||||
SignedInUser: c.SignedInUser, |
||||
OrgId: c.OrgId, |
||||
} |
||||
|
||||
err := bus.Dispatch(context.Background(), &searchQuery) |
||||
if err != nil { |
||||
return crawlStatus{}, err |
||||
} |
||||
|
||||
queue := make([]dashItem, 0, len(searchQuery.Result)) |
||||
for _, v := range searchQuery.Result { |
||||
if v.Type == search.DashHitDB { |
||||
queue = append(queue, dashItem{ |
||||
uid: v.UID, |
||||
url: v.URL, |
||||
}) |
||||
} |
||||
} |
||||
rand.Seed(time.Now().UnixNano()) |
||||
rand.Shuffle(len(queue), func(i, j int) { queue[i], queue[j] = queue[j], queue[i] }) |
||||
|
||||
r.mode = mode |
||||
r.opts = rendering.Opts{ |
||||
OrgID: c.OrgId, |
||||
UserID: c.UserId, |
||||
OrgRole: c.OrgRole, |
||||
Theme: theme, |
||||
ConcurrentLimit: 10, |
||||
} |
||||
r.queue = queue |
||||
r.status = crawlStatus{ |
||||
Started: time.Now(), |
||||
State: "running", |
||||
Complete: 0, |
||||
} |
||||
r.broadcastStatus() |
||||
|
||||
// create a pool of workers
|
||||
for i := 0; i < r.threadCount; i++ { |
||||
go r.walk() |
||||
|
||||
// wait 1/2 second before starting a new thread
|
||||
time.Sleep(500 * time.Millisecond) |
||||
} |
||||
|
||||
r.broadcastStatus() |
||||
return r.Status() |
||||
} |
||||
|
||||
func (r *simpleCrawler) Stop() (crawlStatus, error) { |
||||
// cheap hack!
|
||||
if r.status.State == "running" { |
||||
r.status.State = "stopping" |
||||
} |
||||
return r.Status() |
||||
} |
||||
|
||||
func (r *simpleCrawler) Status() (crawlStatus, error) { |
||||
status := crawlStatus{ |
||||
State: r.status.State, |
||||
Started: r.status.Started, |
||||
Complete: r.status.Complete, |
||||
Errors: r.status.Errors, |
||||
Queue: len(r.queue), |
||||
Last: r.status.Last, |
||||
} |
||||
return status, nil |
||||
} |
||||
|
||||
func (r *simpleCrawler) walk() { |
||||
for { |
||||
if r.status.State == "stopping" { |
||||
break |
||||
} |
||||
|
||||
item := r.next() |
||||
if item == nil { |
||||
break |
||||
} |
||||
|
||||
tlog.Info("GET THUMBNAIL", "url", item.url) |
||||
|
||||
// Hack (for now) pick a URL that will render
|
||||
panelURL := strings.TrimPrefix(item.url, "/") + "?kiosk" |
||||
res, err := r.renderService.Render(context.Background(), rendering.Opts{ |
||||
Width: 320, |
||||
Height: 240, |
||||
Path: panelURL, |
||||
OrgID: r.opts.OrgID, |
||||
UserID: r.opts.UserID, |
||||
ConcurrentLimit: r.opts.ConcurrentLimit, |
||||
OrgRole: r.opts.OrgRole, |
||||
Theme: r.opts.Theme, |
||||
Timeout: 10 * time.Second, |
||||
DeviceScaleFactor: -5, // negative numbers will render larger then scale down
|
||||
}) |
||||
if err != nil { |
||||
tlog.Warn("error getting image", "err", err) |
||||
r.status.Errors++ |
||||
} else if res.FilePath == "" { |
||||
tlog.Warn("error getting image... no response") |
||||
r.status.Errors++ |
||||
} else if strings.Contains(res.FilePath, "public/img") { |
||||
tlog.Warn("error getting image... internal result", "img", res.FilePath) |
||||
r.status.Errors++ |
||||
} else { |
||||
p := getFilePath(r.screenshotsFolder, &previewRequest{ |
||||
UID: item.uid, |
||||
OrgID: r.opts.OrgID, |
||||
Theme: r.opts.Theme, |
||||
Size: PreviewSizeThumb, |
||||
}) |
||||
err = os.Rename(res.FilePath, p) |
||||
if err != nil { |
||||
r.status.Errors++ |
||||
tlog.Warn("error moving image", "err", err) |
||||
} else { |
||||
r.status.Complete++ |
||||
tlog.Info("saved thumbnail", "img", item.url) |
||||
} |
||||
} |
||||
|
||||
time.Sleep(5 * time.Second) |
||||
r.status.Last = time.Now() |
||||
r.broadcastStatus() |
||||
} |
||||
|
||||
r.status.State = "stopped" |
||||
r.status.Finished = time.Now() |
||||
r.broadcastStatus() |
||||
} |
||||
@ -1,72 +0,0 @@ |
||||
package thumbs |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"os" |
||||
) |
||||
|
||||
type renderHttp struct { |
||||
crawlerURL string |
||||
config crawConfig |
||||
} |
||||
|
||||
func newRenderHttp(crawlerURL string, cfg crawConfig) dashRenderer { |
||||
return &renderHttp{ |
||||
crawlerURL: crawlerURL, |
||||
config: cfg, |
||||
} |
||||
} |
||||
|
||||
func (r *renderHttp) GetPreview(req *previewRequest) *previewResponse { |
||||
p := getFilePath(r.config.ScreenshotsFolder, req) |
||||
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { |
||||
return r.queueRender(p, req) |
||||
} |
||||
|
||||
return &previewResponse{ |
||||
Path: p, |
||||
Code: 200, |
||||
} |
||||
} |
||||
|
||||
func (r *renderHttp) CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error) { |
||||
cmd := r.config |
||||
cmd.crawlCmd = *cfg |
||||
|
||||
jsonData, err := json.Marshal(cmd) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
request, err := http.NewRequest("POST", r.crawlerURL, bytes.NewBuffer(jsonData)) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
request.Header.Set("Content-Type", "application/json; charset=UTF-8") |
||||
|
||||
client := &http.Client{} |
||||
response, error := client.Do(request) |
||||
if error != nil { |
||||
return nil, err |
||||
} |
||||
defer func() { |
||||
_ = response.Body.Close() |
||||
}() |
||||
|
||||
return ioutil.ReadAll(response.Body) |
||||
} |
||||
|
||||
func (r *renderHttp) queueRender(p string, req *previewRequest) *previewResponse { |
||||
go func() { |
||||
fmt.Printf("todo? queue") |
||||
}() |
||||
|
||||
return &previewResponse{ |
||||
Code: 202, |
||||
Path: p, |
||||
} |
||||
} |
||||
@ -0,0 +1,61 @@ |
||||
import React, { useState } from 'react'; |
||||
import { css } from '@emotion/css'; |
||||
import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui'; |
||||
import { GrafanaTheme2 } from '@grafana/data'; |
||||
import { getBackendSrv, config } from '@grafana/runtime'; |
||||
|
||||
export const CrawlerStartButton = () => { |
||||
const styles = getStyles(useTheme2()); |
||||
const [open, setOpen] = useState(false); |
||||
const [body, setBody] = useState({ |
||||
mode: 'thumbs', |
||||
theme: config.theme2.isLight ? 'light' : 'dark', |
||||
}); |
||||
const onDismiss = () => setOpen(false); |
||||
const doStart = () => { |
||||
getBackendSrv() |
||||
.post('/api/admin/crawler/start', body) |
||||
.then((v) => { |
||||
console.log('GOT', v); |
||||
onDismiss(); |
||||
}); |
||||
}; |
||||
|
||||
return ( |
||||
<> |
||||
<Modal title={'Start crawler'} isOpen={open} onDismiss={onDismiss}> |
||||
<div className={styles.wrap}> |
||||
<CodeEditor |
||||
height={200} |
||||
value={JSON.stringify(body, null, 2) ?? ''} |
||||
showLineNumbers={false} |
||||
readOnly={false} |
||||
language="json" |
||||
showMiniMap={false} |
||||
onBlur={(text: string) => { |
||||
setBody(JSON.parse(text)); // force JSON?
|
||||
}} |
||||
/> |
||||
</div> |
||||
<Modal.ButtonRow> |
||||
<Button onClick={doStart}>Start</Button> |
||||
<Button variant="secondary" onClick={onDismiss}> |
||||
Cancel |
||||
</Button> |
||||
</Modal.ButtonRow> |
||||
</Modal> |
||||
|
||||
<Button onClick={() => setOpen(true)} variant="primary"> |
||||
Start |
||||
</Button> |
||||
</> |
||||
); |
||||
}; |
||||
|
||||
const getStyles = (theme: GrafanaTheme2) => { |
||||
return { |
||||
wrap: css` |
||||
border: 2px solid #111; |
||||
`,
|
||||
}; |
||||
}; |
||||
@ -0,0 +1,79 @@ |
||||
import React, { useEffect, useState } from 'react'; |
||||
import { css } from '@emotion/css'; |
||||
import { Button, useTheme2 } from '@grafana/ui'; |
||||
import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data'; |
||||
import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime'; |
||||
import { CrawlerStartButton } from './CrawlerStartButton'; |
||||
|
||||
interface CrawlerStatusMessage { |
||||
state: string; |
||||
started: string; |
||||
finished: string; |
||||
complete: number; |
||||
queue: number; |
||||
last: string; |
||||
} |
||||
|
||||
export const CrawlerStatus = () => { |
||||
const styles = getStyles(useTheme2()); |
||||
const [status, setStatus] = useState<CrawlerStatusMessage>(); |
||||
|
||||
useEffect(() => { |
||||
const subscription = getGrafanaLiveSrv() |
||||
.getStream<CrawlerStatusMessage>({ |
||||
scope: LiveChannelScope.Grafana, |
||||
namespace: 'broadcast', |
||||
path: 'crawler', |
||||
}) |
||||
.subscribe({ |
||||
next: (evt) => { |
||||
if (isLiveChannelMessageEvent(evt)) { |
||||
setStatus(evt.message); |
||||
} else if (isLiveChannelStatusEvent(evt)) { |
||||
setStatus(evt.message); |
||||
} |
||||
}, |
||||
}); |
||||
return () => { |
||||
subscription.unsubscribe(); |
||||
}; |
||||
}, []); |
||||
|
||||
if (!status) { |
||||
return ( |
||||
<div className={styles.wrap}> |
||||
No status (never run) |
||||
<br /> |
||||
<CrawlerStartButton /> |
||||
</div> |
||||
); |
||||
} |
||||
|
||||
return ( |
||||
<div className={styles.wrap}> |
||||
<pre>{JSON.stringify(status, null, 2)}</pre> |
||||
{status.state !== 'running' && <CrawlerStartButton />} |
||||
{status.state !== 'stopped' && ( |
||||
<Button |
||||
variant="secondary" |
||||
onClick={() => { |
||||
getBackendSrv().post('/api/admin/crawler/stop'); |
||||
}} |
||||
> |
||||
Stop |
||||
</Button> |
||||
)} |
||||
</div> |
||||
); |
||||
}; |
||||
|
||||
const getStyles = (theme: GrafanaTheme2) => { |
||||
return { |
||||
wrap: css` |
||||
border: 4px solid red; |
||||
`,
|
||||
running: css` |
||||
border: 4px solid green; |
||||
`,
|
||||
}; |
||||
}; |
||||
Loading…
Reference in new issue