diff --git a/contribute/architecture/backend/communication.md b/contribute/architecture/backend/communication.md index bd1d9300fb4..d4836295d9e 100644 --- a/contribute/architecture/backend/communication.md +++ b/contribute/architecture/backend/communication.md @@ -109,7 +109,7 @@ ctx := req.Request.Context() query := &models.FindDashboardQuery{ ID: "foo", } -if err := bus.DispatchCtx(ctx, query); err != nil { +if err := bus.Dispatch(ctx, query); err != nil { return err } // The query now contains a result. diff --git a/pkg/api/api.go b/pkg/api/api.go index 6c7940b5fc3..8dc71cfa07b 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -461,6 +461,12 @@ func (hs *HTTPServer) registerRoutes() { adminRoute.Get("/stats", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionServerStatsRead)), routing.Wrap(AdminGetStats)) adminRoute.Post("/pause-all-alerts", reqGrafanaAdmin, routing.Wrap(PauseAllAlerts)) + if hs.ThumbService != nil { + adminRoute.Post("/crawler/start", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.StartCrawler)) + adminRoute.Post("/crawler/stop", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.StopCrawler)) + adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus)) + } + adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards)) adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins)) adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources)) diff --git a/pkg/services/rendering/rendering.go b/pkg/services/rendering/rendering.go index 03aada93d02..b6ec052f959 100644 --- a/pkg/services/rendering/rendering.go +++ b/pkg/services/rendering/rendering.go @@ -207,6 +207,8 @@ func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResul func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResult, error) { if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit { + rs.log.Warn("Could not render image, hit the currency limit", "concurrencyLimit", opts.ConcurrentLimit, "path", opts.Path) + theme := ThemeDark if opts.Theme != "" { theme = opts.Theme @@ -225,7 +227,7 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul } rs.log.Info("Rendering", "path", opts.Path) - if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor <= 0 { + if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor == 0 { opts.DeviceScaleFactor = 1 } renderKey, err := rs.generateAndStoreRenderKey(ctx, opts.OrgID, opts.UserID, opts.OrgRole) diff --git a/pkg/services/rendering/rendering_test.go b/pkg/services/rendering/rendering_test.go index e7776d82cc7..06ddcc7c9f3 100644 --- a/pkg/services/rendering/rendering_test.go +++ b/pkg/services/rendering/rendering_test.go @@ -110,6 +110,7 @@ func TestRenderLimitImage(t *testing.T) { HomePath: path, }, inProgressCount: 2, + log: log.New("test"), } tests := []struct { diff --git a/pkg/services/thumbs/crawler.go b/pkg/services/thumbs/crawler.go new file mode 100644 index 00000000000..884712c81c9 --- /dev/null +++ b/pkg/services/thumbs/crawler.go @@ -0,0 +1,249 @@ +package thumbs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "os" + "strings" + "sync" + "time" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/live" + "github.com/grafana/grafana/pkg/services/rendering" + "github.com/grafana/grafana/pkg/services/search" +) + +type dashItem struct { + uid string + url string +} + +type simpleCrawler struct { + screenshotsFolder string + renderService rendering.Service + threadCount int + + glive *live.GrafanaLive + mode CrawlerMode + opts rendering.Opts + status crawlStatus + queue []dashItem + mu sync.Mutex +} + +func newSimpleCrawler(folder string, renderService rendering.Service, gl *live.GrafanaLive) dashRenderer { + c := &simpleCrawler{ + screenshotsFolder: folder, + renderService: renderService, + threadCount: 5, + glive: gl, + status: crawlStatus{ + State: "init", + Complete: 0, + Queue: 0, + }, + queue: make([]dashItem, 0), + } + c.broadcastStatus() + return c +} + +func (r *simpleCrawler) next() *dashItem { + if len(r.queue) < 1 { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + + v := r.queue[0] + r.queue = r.queue[1:] + return &v +} + +func (r *simpleCrawler) broadcastStatus() { + s, err := r.Status() + if err != nil { + tlog.Warn("error reading status") + return + } + msg, err := json.Marshal(s) + if err != nil { + tlog.Warn("error making message") + return + } + err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg) + if err != nil { + tlog.Warn("error Publish message") + return + } +} + +func (r *simpleCrawler) GetPreview(req *previewRequest) *previewResponse { + p := getFilePath(r.screenshotsFolder, req) + if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { + return r.queueRender(p, req) + } + + return &previewResponse{ + Path: p, + Code: 200, + } +} + +func (r *simpleCrawler) queueRender(p string, req *previewRequest) *previewResponse { + go func() { + fmt.Printf("todo? queue") + }() + + return &previewResponse{ + Code: 202, + Path: p, + } +} + +func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) { + if r.status.State == "running" { + tlog.Info("already running") + return r.Status() + } + + r.mu.Lock() + defer r.mu.Unlock() + + searchQuery := search.Query{ + SignedInUser: c.SignedInUser, + OrgId: c.OrgId, + } + + err := bus.Dispatch(context.Background(), &searchQuery) + if err != nil { + return crawlStatus{}, err + } + + queue := make([]dashItem, 0, len(searchQuery.Result)) + for _, v := range searchQuery.Result { + if v.Type == search.DashHitDB { + queue = append(queue, dashItem{ + uid: v.UID, + url: v.URL, + }) + } + } + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(queue), func(i, j int) { queue[i], queue[j] = queue[j], queue[i] }) + + r.mode = mode + r.opts = rendering.Opts{ + OrgID: c.OrgId, + UserID: c.UserId, + OrgRole: c.OrgRole, + Theme: theme, + ConcurrentLimit: 10, + } + r.queue = queue + r.status = crawlStatus{ + Started: time.Now(), + State: "running", + Complete: 0, + } + r.broadcastStatus() + + // create a pool of workers + for i := 0; i < r.threadCount; i++ { + go r.walk() + + // wait 1/2 second before starting a new thread + time.Sleep(500 * time.Millisecond) + } + + r.broadcastStatus() + return r.Status() +} + +func (r *simpleCrawler) Stop() (crawlStatus, error) { + // cheap hack! + if r.status.State == "running" { + r.status.State = "stopping" + } + return r.Status() +} + +func (r *simpleCrawler) Status() (crawlStatus, error) { + status := crawlStatus{ + State: r.status.State, + Started: r.status.Started, + Complete: r.status.Complete, + Errors: r.status.Errors, + Queue: len(r.queue), + Last: r.status.Last, + } + return status, nil +} + +func (r *simpleCrawler) walk() { + for { + if r.status.State == "stopping" { + break + } + + item := r.next() + if item == nil { + break + } + + tlog.Info("GET THUMBNAIL", "url", item.url) + + // Hack (for now) pick a URL that will render + panelURL := strings.TrimPrefix(item.url, "/") + "?kiosk" + res, err := r.renderService.Render(context.Background(), rendering.Opts{ + Width: 320, + Height: 240, + Path: panelURL, + OrgID: r.opts.OrgID, + UserID: r.opts.UserID, + ConcurrentLimit: r.opts.ConcurrentLimit, + OrgRole: r.opts.OrgRole, + Theme: r.opts.Theme, + Timeout: 10 * time.Second, + DeviceScaleFactor: -5, // negative numbers will render larger then scale down + }) + if err != nil { + tlog.Warn("error getting image", "err", err) + r.status.Errors++ + } else if res.FilePath == "" { + tlog.Warn("error getting image... no response") + r.status.Errors++ + } else if strings.Contains(res.FilePath, "public/img") { + tlog.Warn("error getting image... internal result", "img", res.FilePath) + r.status.Errors++ + } else { + p := getFilePath(r.screenshotsFolder, &previewRequest{ + UID: item.uid, + OrgID: r.opts.OrgID, + Theme: r.opts.Theme, + Size: PreviewSizeThumb, + }) + err = os.Rename(res.FilePath, p) + if err != nil { + r.status.Errors++ + tlog.Warn("error moving image", "err", err) + } else { + r.status.Complete++ + tlog.Info("saved thumbnail", "img", item.url) + } + } + + time.Sleep(5 * time.Second) + r.status.Last = time.Now() + r.broadcastStatus() + } + + r.status.State = "stopped" + r.status.Finished = time.Now() + r.broadcastStatus() +} diff --git a/pkg/services/thumbs/crawler_http.go b/pkg/services/thumbs/crawler_http.go deleted file mode 100644 index 6f8f7525abd..00000000000 --- a/pkg/services/thumbs/crawler_http.go +++ /dev/null @@ -1,72 +0,0 @@ -package thumbs - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" -) - -type renderHttp struct { - crawlerURL string - config crawConfig -} - -func newRenderHttp(crawlerURL string, cfg crawConfig) dashRenderer { - return &renderHttp{ - crawlerURL: crawlerURL, - config: cfg, - } -} - -func (r *renderHttp) GetPreview(req *previewRequest) *previewResponse { - p := getFilePath(r.config.ScreenshotsFolder, req) - if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { - return r.queueRender(p, req) - } - - return &previewResponse{ - Path: p, - Code: 200, - } -} - -func (r *renderHttp) CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error) { - cmd := r.config - cmd.crawlCmd = *cfg - - jsonData, err := json.Marshal(cmd) - if err != nil { - return nil, err - } - request, err := http.NewRequest("POST", r.crawlerURL, bytes.NewBuffer(jsonData)) - if err != nil { - return nil, err - } - request.Header.Set("Content-Type", "application/json; charset=UTF-8") - - client := &http.Client{} - response, error := client.Do(request) - if error != nil { - return nil, err - } - defer func() { - _ = response.Body.Close() - }() - - return ioutil.ReadAll(response.Body) -} - -func (r *renderHttp) queueRender(p string, req *previewRequest) *previewResponse { - go func() { - fmt.Printf("todo? queue") - }() - - return &previewResponse{ - Code: 202, - Path: p, - } -} diff --git a/pkg/services/thumbs/dummy.go b/pkg/services/thumbs/dummy.go index 9b73951e38f..3bfd2eec692 100644 --- a/pkg/services/thumbs/dummy.go +++ b/pkg/services/thumbs/dummy.go @@ -25,8 +25,15 @@ func (ds *dummyService) StartCrawler(c *models.ReqContext) response.Response { result["error"] = "Not enabled" return response.JSON(200, result) } + func (ds *dummyService) StopCrawler(c *models.ReqContext) response.Response { result := make(map[string]string) result["error"] = "Not enabled" return response.JSON(200, result) } + +func (ds *dummyService) CrawlerStatus(c *models.ReqContext) response.Response { + result := make(map[string]string) + result["error"] = "Not enabled" + return response.JSON(200, result) +} diff --git a/pkg/services/thumbs/models.go b/pkg/services/thumbs/models.go index 17c46836e52..0e33a0b9004 100644 --- a/pkg/services/thumbs/models.go +++ b/pkg/services/thumbs/models.go @@ -1,8 +1,14 @@ package thumbs -import "encoding/json" +import ( + "time" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/rendering" +) type PreviewSize string +type CrawlerMode string const ( // PreviewSizeThumb is a small 320x240 preview @@ -13,6 +19,15 @@ const ( // PreviewSizeLarge is a large image 512x???? PreviewSizeTall PreviewSize = "tall" + + // CrawlerModeThumbs will create small thumbnails for everything + CrawlerModeThumbs CrawlerMode = "thumbs" + + // CrawlerModeAnalytics will get full page results for everythign + CrawlerModeAnalytics CrawlerMode = "analytics" + + // CrawlerModeMigrate will migrate all dashboards with old schema + CrawlerModeMigrate CrawlerMode = "migrate" ) // IsKnownSize checks if the value is a standard size @@ -39,22 +54,21 @@ func getPreviewSize(str string) (PreviewSize, bool) { return PreviewSizeThumb, false } -func getTheme(str string) (string, bool) { +func getTheme(str string) (rendering.Theme, bool) { switch str { case "light": - return str, true + return rendering.ThemeLight, true case "dark": - return str, true + return rendering.ThemeDark, true } - return "dark", false + return rendering.ThemeDark, false } type previewRequest struct { - Kind string `json:"kind"` - OrgID int64 `json:"orgId"` - UID string `json:"uid"` - Size PreviewSize `json:"size"` - Theme string `json:"theme"` + OrgID int64 `json:"orgId"` + UID string `json:"uid"` + Size PreviewSize `json:"size"` + Theme rendering.Theme `json:"theme"` } type previewResponse struct { @@ -63,35 +77,19 @@ type previewResponse struct { URL string `json:"url"` // redirect to this URL } -// export enum CrawlerMode { -// Thumbs = 'thumbs', -// Analytics = 'analytics', // Enterprise only -// Migrate = 'migrate', -// } - -// export enum CrawlerAction { -// Run = 'run', -// Stop = 'stop', -// Queue = 'queue', // TODO (later!) move some to the front -// } - type crawlCmd struct { - Mode string `json:"mode"` // thumbs | analytics | migrate - Action string `json:"action"` // run | stop | queue - Theme string `json:"theme"` // light | dark - User string `json:"user"` // :( - Password string `json:"password"` // :( - Concurrency int `json:"concurrency"` // number of pages to run in parallel - - Path string `json:"path"` // eventually for queue + Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate + Theme rendering.Theme `json:"theme"` // light | dark } -type crawConfig struct { - crawlCmd - - // Sent to the crawler with each command - URL string `json:"url"` - ScreenshotsFolder string `json:"screenshotsFolder"` +type crawlStatus struct { + State string `json:"state"` + Started time.Time `json:"started,omitempty"` + Finished time.Time `json:"finished,omitempty"` + Complete int `json:"complete"` + Errors int `json:"errors"` + Queue int `json:"queue"` + Last time.Time `json:"last,omitempty"` } type dashRenderer interface { @@ -99,5 +97,11 @@ type dashRenderer interface { GetPreview(req *previewRequest) *previewResponse // Assumes you have already authenticated as admin - CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error) + Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) + + // Assumes you have already authenticated as admin + Stop() (crawlStatus, error) + + // Assumes you have already authenticated as admin + Status() (crawlStatus, error) } diff --git a/pkg/services/thumbs/service.go b/pkg/services/thumbs/service.go index 2da855537f4..6bd41dd44d8 100644 --- a/pkg/services/thumbs/service.go +++ b/pkg/services/thumbs/service.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/guardian" + "github.com/grafana/grafana/pkg/services/live" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" @@ -28,29 +29,27 @@ var ( type Service interface { Enabled() bool GetImage(c *models.ReqContext) + + // Form post (from dashboard page) SetImage(c *models.ReqContext) // Must be admin StartCrawler(c *models.ReqContext) response.Response StopCrawler(c *models.ReqContext) response.Response + CrawlerStatus(c *models.ReqContext) response.Response } -func ProvideService(cfg *setting.Cfg, renderService rendering.Service) Service { +func ProvideService(cfg *setting.Cfg, renderService rendering.Service, gl *live.GrafanaLive) Service { if !cfg.IsDashboardPreviesEnabled() { return &dummyService{} } root := filepath.Join(cfg.DataPath, "crawler", "preview") - url := strings.TrimSuffix(cfg.RendererUrl, "/render") + "/scan" - - renderer := newRenderHttp(url, crawConfig{ - URL: strings.TrimSuffix(cfg.RendererCallbackUrl, "/"), - ScreenshotsFolder: root, - }) - tempdir := filepath.Join(cfg.DataPath, "temp") + _ = os.MkdirAll(root, 0700) _ = os.MkdirAll(tempdir, 0700) + renderer := newSimpleCrawler(root, renderService, gl) return &thumbService{ renderer: renderer, root: root, @@ -84,7 +83,6 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre } req := &previewRequest{ - Kind: "dash", OrgID: c.OrgId, UID: params[":uid"], Theme: theme, @@ -137,6 +135,7 @@ func (hs *thumbService) GetImage(c *models.ReqContext) { c.JSON(500, map[string]string{"path": rsp.Path, "error": "unknown!"}) } +// Hack for now -- lets you upload images explicitly func (hs *thumbService) SetImage(c *models.ReqContext) { req := hs.parseImageReq(c, false) if req == nil { @@ -217,29 +216,30 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response { if err != nil { return response.Error(500, "error parsing bytes", err) } - cmd.Action = "start" - - msg, err := hs.renderer.CrawlerCmd(cmd) + if cmd.Mode == "" { + cmd.Mode = CrawlerModeThumbs + } + msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme) if err != nil { return response.Error(500, "error starting", err) } - - header := make(http.Header) - header.Set("Content-Type", "application/json") - return response.CreateNormalResponse(header, msg, 200) + return response.JSON(200, msg) } func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response { - _, err := hs.renderer.CrawlerCmd(&crawlCmd{ - Action: "stop", - }) + msg, err := hs.renderer.Stop() if err != nil { - return response.Error(500, "error stopping crawler", err) + return response.Error(500, "error starting", err) } + return response.JSON(200, msg) +} - result := make(map[string]string) - result["message"] = "Stopping..." - return response.JSON(200, result) +func (hs *thumbService) CrawlerStatus(c *models.ReqContext) response.Response { + msg, err := hs.renderer.Status() + if err != nil { + return response.Error(500, "error starting", err) + } + return response.JSON(200, msg) } // Ideally this service would not require first looking up the full dashboard just to bet the id! diff --git a/public/app/features/admin/CrawlerStartButton.tsx b/public/app/features/admin/CrawlerStartButton.tsx new file mode 100644 index 00000000000..1035eac199e --- /dev/null +++ b/public/app/features/admin/CrawlerStartButton.tsx @@ -0,0 +1,61 @@ +import React, { useState } from 'react'; +import { css } from '@emotion/css'; +import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui'; +import { GrafanaTheme2 } from '@grafana/data'; +import { getBackendSrv, config } from '@grafana/runtime'; + +export const CrawlerStartButton = () => { + const styles = getStyles(useTheme2()); + const [open, setOpen] = useState(false); + const [body, setBody] = useState({ + mode: 'thumbs', + theme: config.theme2.isLight ? 'light' : 'dark', + }); + const onDismiss = () => setOpen(false); + const doStart = () => { + getBackendSrv() + .post('/api/admin/crawler/start', body) + .then((v) => { + console.log('GOT', v); + onDismiss(); + }); + }; + + return ( + <> + +
+ { + setBody(JSON.parse(text)); // force JSON? + }} + /> +
+ + + + + + + + + ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 2px solid #111; + `, + }; +}; diff --git a/public/app/features/admin/CrawlerStatus.tsx b/public/app/features/admin/CrawlerStatus.tsx new file mode 100644 index 00000000000..50dd9f97cdf --- /dev/null +++ b/public/app/features/admin/CrawlerStatus.tsx @@ -0,0 +1,79 @@ +import React, { useEffect, useState } from 'react'; +import { css } from '@emotion/css'; +import { Button, useTheme2 } from '@grafana/ui'; +import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data'; +import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime'; +import { CrawlerStartButton } from './CrawlerStartButton'; + +interface CrawlerStatusMessage { + state: string; + started: string; + finished: string; + complete: number; + queue: number; + last: string; +} + +export const CrawlerStatus = () => { + const styles = getStyles(useTheme2()); + const [status, setStatus] = useState(); + + useEffect(() => { + const subscription = getGrafanaLiveSrv() + .getStream({ + scope: LiveChannelScope.Grafana, + namespace: 'broadcast', + path: 'crawler', + }) + .subscribe({ + next: (evt) => { + if (isLiveChannelMessageEvent(evt)) { + setStatus(evt.message); + } else if (isLiveChannelStatusEvent(evt)) { + setStatus(evt.message); + } + }, + }); + return () => { + subscription.unsubscribe(); + }; + }, []); + + if (!status) { + return ( +
+ No status (never run) +
+ +
+ ); + } + + return ( +
+ 
{JSON.stringify(status, null, 2)}
+ {status.state !== 'running' && } + {status.state !== 'stopped' && ( + + )} +
+ ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 4px solid red; + `, + running: css` + border: 4px solid green; + `, + }; +}; diff --git a/public/app/features/admin/ServerStats.tsx b/public/app/features/admin/ServerStats.tsx index d34077afc7e..8a15746336d 100644 --- a/public/app/features/admin/ServerStats.tsx +++ b/public/app/features/admin/ServerStats.tsx @@ -6,6 +6,8 @@ import { AccessControlAction } from 'app/types'; import { getServerStats, ServerStat } from './state/apis'; import { contextSrv } from '../../core/services/context_srv'; import { Loader } from '../plugins/admin/components/Loader'; +import { config } from '@grafana/runtime'; +import { CrawlerStatus } from './CrawlerStatus'; export const ServerStats = () => { const [stats, setStats] = useState(null); @@ -84,6 +86,8 @@ export const ServerStats = () => { ) : (

No stats found.

)} + + {config.featureToggles.dashboardPreviews && } ); };