feat: Support rendering PNG natively

* Using capture screenshot API of chromedp, we can generate PNG of each panel using current plugin. Thus, we avoid the dependency with grafana-image-renderer

* Move all panel related JS to a separate .js file and embed that file with the app. Read the file content at runtime and inject it into chrome tab tasks.

* Add duration debug logs for better debugging experience

* Add e2e tests to test both native renderer and grafana-image-renderer integrations

Signed-off-by: Mahendra Paipuri <mahendra.paipuri@gmail.com>

.ci/docker-compose.yaml

@@ -41,6 +41,8 @@ services:
       # disable alerting because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
       # Grafana image renderer
       - GF_RENDERING_SERVER_URL=http://renderer_plain:8081/render
       - GF_RENDERING_CALLBACK_URL=http://grafana_plain:${GF_SERVER_HTTP_PORT:-3000}/
@@ -49,6 +51,7 @@ services:
       # Set CI mode to remove header in report
       - __REPORTER_APP_CI_MODE=true
       - GF_REPORTER_PLUGIN_REMOTE_CHROME_URL=${GF_REPORTER_PLUGIN_REMOTE_CHROME_URL:-}
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
 
   renderer_plain:
     image: grafana/grafana-image-renderer:latest
@@ -105,6 +108,8 @@ services:
       # disable alerting because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
        # TLS
       - GF_SERVER_PROTOCOL=https
       - GF_SERVER_CERT_KEY=/etc/grafana/tls/localhost.key
@@ -117,6 +122,7 @@ services:
       # Set CI mode to remove header in report
       - __REPORTER_APP_CI_MODE=true
       - GF_REPORTER_PLUGIN_REMOTE_CHROME_URL=${GF_REPORTER_PLUGIN_REMOTE_CHROME_URL:-}
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
       - GF_REPORTER_PLUGIN_SKIP_TLS_CHECK=true
 
   renderer_tls:

.github/workflows/step_e2e-tests.yml

@@ -18,13 +18,15 @@ jobs:
           - grafana-version: 10.3.0
             remote-chrome-url: ''
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: local-chrome
             name: local-chrome-10.3.0-with-features
 
           # Grafana v10 without user cookie and with feature flags
           - grafana-version: 10.4.5
             remote-chrome-url: ''
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: true
             # snapshots-folder: local-chrome
             name: local-chrome-10.4.5-with-features
 
@@ -33,22 +35,33 @@ jobs:
           - grafana-version: 10.4.7
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
             name: remote-chrome-10.4.7-without-features
 
           # Grafana v11 with remote chrome
           - grafana-version: 11.1.0
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
             name: remote-chrome-11.1.0-with-features
 
-          # Latest Grafana with local chrome
-          - grafana-version: 11.3.0
+          # Latest Grafana with local chrome and grafana-image-renderer
+          - grafana-version: 11.4.0
             remote-chrome-url: ws://localhost:9222
             feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: false
             # snapshots-folder: remote-chrome
-            name: local-chrome-11.3.0-with-features
+            name: local-chrome-11.4.0-with-features
+
+          # Latest Grafana with local chrome and native-renderer
+          - grafana-version: 11.4.0
+            remote-chrome-url: ws://localhost:9222
+            feature-flags: 'accessControlOnCall,idForwarding,externalServiceAccounts'
+            native-rendering: true
+            # snapshots-folder: remote-chrome
+            name: local-chrome-11.4.0-with-features-native-renderer
 
     steps:
       - uses: actions/checkout@v4
@@ -76,6 +89,7 @@ jobs:
         env:
           GRAFANA_VERSION: ${{ matrix.grafana-version }}
           GF_REPORTER_PLUGIN_REMOTE_CHROME_URL: ${{ matrix.remote-chrome-url }}
+          GF_REPORTER_PLUGIN_NATIVE_RENDERER: ${{ matrix.native-rendering }}
           GF_FEATURE_TOGGLES_ENABLE: ${{ matrix.feature-flags }}
         run: |
           # Upload/Download artifacts wont preserve permissions

docker-compose.yaml

@@ -40,20 +40,24 @@ services:
       # the token from a service account to read dashboards
       - GF_FEATURE_TOGGLES_ENABLE=${GF_FEATURE_TOGGLES_ENABLE:-accessControlOnCall,idForwarding,externalServiceAccounts}
       - GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED=${GF_AUTH_MANAGED_SERVICE_ACCOUNTS_ENABLED:-true}
-      # disable alerting because it vomits logs
+      # disable alerting and Grafana live because it vomits logs
       - GF_ALERTING_ENABLED=false
       - GF_UNIFIED_ALERTING_ENABLED=false
+      - GF_LIVE_MAX_CONNECTIONS=0
+      - GF_PLUGINS_DISABLE_PLUGINS=grafana-lokiexplore-app
       # Grafana image renderer
       - GF_RENDERING_SERVER_URL=http://renderer:8081/render
       - GF_RENDERING_CALLBACK_URL=http://grafana:${GF_SERVER_HTTP_PORT:-3000}/
       - "GF_LOG_FILTERS=rendering:debug plugin.mahendrapaipuri-dashboardreporter-app:debug"
+      # Current plugin config
+      - GF_REPORTER_PLUGIN_NATIVE_RENDERER=${GF_REPORTER_PLUGIN_NATIVE_RENDERER:-false}
   renderer:
     image: grafana/grafana-image-renderer:latest
     environment:
       # Recommendation of grafana-image-renderer for optimal performance
       # https://grafana.com/docs/grafana/latest/setup-grafana/image-rendering/#configuration
       - RENDERING_MODE=clustered
-      - RENDERING_CLUSTERING_MODE=browser
+      - RENDERING_CLUSTERING_MODE=context
       - RENDERING_CLUSTERING_MAX_CONCURRENCY=5
       - RENDERING_CLUSTERING_TIMEOUT=60
       - IGNORE_HTTPS_ERRORS=true

pkg/plugin/chrome/local.go

@@ -141,10 +141,4 @@ func (i *LocalInstance) Close(logger log.Logger) {
 			logger.Error("got error from cancel browser context", "error", err)
 		}
 	}
-
-	if i.allocCtx != nil {
-		if err := chromedp.Cancel(i.allocCtx); err != nil {
-			logger.Error("got error from cancel browser allocator context", "error", err)
-		}
-	}
 }

pkg/plugin/chrome/tab.go

@@ -102,6 +102,11 @@ func (t *Tab) Context() context.Context {
 	return t.ctx
 }
 
+// Target returns tab's target ID.
+func (t *Tab) Target() *chromedp.Target {
+	return chromedp.FromContext(t.Context()).Target
+}
+
 // PrintToPDF returns chroms tasks that print the requested HTML into a PDF and returns the PDF stream handle.
 func (t *Tab) PrintToPDF(options PDFOptions, writer io.Writer) error {
 	err := chromedp.Run(t.ctx, chromedp.Tasks{

pkg/plugin/config/settings.go

@@ -41,6 +41,7 @@ type Config struct {
 	MaxBrowserWorkers   int    `env:"GF_REPORTER_PLUGIN_MAX_BROWSER_WORKERS, overwrite"    json:"maxBrowserWorkers"`
 	MaxRenderWorkers    int    `env:"GF_REPORTER_PLUGIN_MAX_RENDER_WORKERS, overwrite"     json:"maxRenderWorkers"`
 	RemoteChromeURL     string `env:"GF_REPORTER_PLUGIN_REMOTE_CHROME_URL, overwrite"      json:"remoteChromeUrl"`
+	NativeRendering     bool   `env:"GF_REPORTER_PLUGIN_NATIVE_RENDERER, overwrite"        json:"nativeRenderer"`
 	IncludePanelIDs     []string
 	ExcludePanelIDs     []string
 	IncludePanelDataIDs []string
@@ -141,10 +142,12 @@ func (c *Config) String() string {
 		"Theme: %s; Orientation: %s; Layout: %s; Dashboard Mode: %s; "+
 			"Time Zone: %s; Time Format: %s; Encoded Logo: %s; "+
 			"Max Renderer Workers: %d; Max Browser Workers: %d; Remote Chrome Addr: %s; App URL: %s; "+
-			"TLS Skip verify: %v; Included Panel IDs: %s; Excluded Panel IDs: %s Included Data for Panel IDs: %s",
+			"TLS Skip verify: %v; Included Panel IDs: %s; Excluded Panel IDs: %s Included Data for Panel IDs: %s; "+
+			"Native Renderer: %v; Client Timeout: %d",
 		c.Theme, c.Orientation, c.Layout, c.DashboardMode, c.TimeZone, c.TimeFormat,
 		encodedLogo, c.MaxRenderWorkers, c.MaxBrowserWorkers, c.RemoteChromeURL, appURL,
-		c.SkipTLSCheck, includedPanelIDs, excludedPanelIDs, includeDataPanelIDs,
+		c.SkipTLSCheck, includedPanelIDs, excludedPanelIDs, includeDataPanelIDs, c.NativeRendering,
+		int(c.HTTPClientOptions.Timeouts.Timeout.Seconds()),
 	)
 }
 

pkg/plugin/dashboard/dashboard.go

@@ -2,50 +2,58 @@ package dashboard
 
 import (
 	"context"
+	"embed"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/chrome"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/config"
-	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/worker"
+	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/helpers"
 )
 
-// Regex for parsing X and Y co-ordinates from CSS
-// Scales for converting width and height to Grafana units.
+// Embed the entire directory.
 //
-// This is based on viewportWidth that we used in client.go which
-// is 1952px. Stripping margin 32px we get 1920px / 24 = 80px
-// height scale should be fine with 36px as width and aspect ratio
-// should choose a height appropriately.
-var (
-	scales = map[string]float64{
-		"width":  80,
-		"height": 36,
-	}
-)
+//go:embed js
+var jsFS embed.FS
 
 // New creates a new instance of the Dashboard struct.
 func New(logger log.Logger, conf *config.Config, httpClient *http.Client, chromeInstance chrome.Instance,
-	pools worker.Pools, appURL, appVersion string, model *Model, authHeader http.Header,
-) *Dashboard {
+	appURL, appVersion string, model *Model, authHeader http.Header,
+) (*Dashboard, error) {
+	// Parse app URL
+	u, err := url.Parse(appURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse app URL: %w", errors.Unwrap(err))
+	}
+
+	// Read JS from embedded file
+	js, err := jsFS.ReadFile("js/panels.js")
+	if err != nil {
+		return nil, fmt.Errorf("failed to load JS: %w", err)
+	}
+
 	return &Dashboard{
 		logger,
 		conf,
 		httpClient,
 		chromeInstance,
-		pools,
-		appURL,
+		u,
 		appVersion,
+		string(js),
 		model,
 		authHeader,
-	}
+	}, nil
 }
 
 // GetData fetches dashboard related data.
 func (d *Dashboard) GetData(ctx context.Context) (*Data, error) {
+	defer helpers.TimeTrack(time.Now(), "dashboard data", d.logger)
+
 	// Make panels from loading the dashboard in a browser instance
 	panels, err := d.panels(ctx)
 	if err != nil {

pkg/plugin/dashboard/data.go

@@ -3,28 +3,32 @@ package dashboard
 import (
 	"context"
 	"encoding/csv"
-	"errors"
 	"fmt"
 	"maps"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/chromedp/cdproto/browser"
+	"github.com/chromedp/cdproto/runtime"
 	"github.com/chromedp/chromedp"
 	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/chrome"
+	"github.com/mahendrapaipuri/grafana-dashboard-reporter-app/pkg/plugin/helpers"
 )
 
 // PanelCSV returns CSV data of a given panel.
 func (d *Dashboard) PanelCSV(_ context.Context, p Panel) (CSVData, error) {
 	// Get panel CSV data URL
 	panelURL := d.panelCSVURL(p)
 
+	defer helpers.TimeTrack(time.Now(), "fetch panel CSV data", d.logger, "fetcher", "native", "panel_id", p.ID, "url", panelURL.String())
+
 	// Create a new tab
 	tab := d.chromeInstance.NewTab(d.logger, d.conf)
 	// Set a timeout for the tab
 	// Fail-safe for newer Grafana versions, if css has been changed.
-	tab.WithTimeout(d.conf.HTTPClientOptions.Timeouts.Timeout)
+	tab.WithTimeout(2 * d.conf.HTTPClientOptions.Timeouts.Timeout)
 	defer tab.Close(d.logger)
 
 	headers := make(map[string]any)
@@ -35,9 +39,7 @@ func (d *Dashboard) PanelCSV(_ context.Context, p Panel) (CSVData, error) {
 		}
 	}
 
-	d.logger.Debug("fetch table data via browser", "url", panelURL)
-
-	err := tab.NavigateAndWaitFor(panelURL, headers, "networkIdle")
+	err := tab.NavigateAndWaitFor(panelURL.String(), headers, "networkIdle")
 	if err != nil {
 		return nil, fmt.Errorf("NavigateAndWaitFor: %w", err)
 	}
@@ -51,46 +53,34 @@ func (d *Dashboard) PanelCSV(_ context.Context, p Panel) (CSVData, error) {
 	// Listen for download events. Downloading from JavaScript won't emit any network events.
 	chromedp.ListenTarget(tab.Context(), func(event interface{}) {
 		if eventDownloadWillBegin, ok := event.(*browser.EventDownloadWillBegin); ok {
-			d.logger.Debug("got CSV download URL", "url", eventDownloadWillBegin.URL)
+			d.logger.Debug("got CSV download URL", "panel_id", p.ID, "url", eventDownloadWillBegin.URL)
 			// once we have the download URL, we can fetch the CSV data via JavaScript.
 			blobURLCh <- eventDownloadWillBegin.URL
 		}
 	})
 
+	js := fmt.Sprintf(
+		`waitForCSVData(version = '%s', timeout = %d);`,
+		d.appVersion, d.conf.HTTPClientOptions.Timeouts.Timeout.Milliseconds(),
+	)
+
 	downTasks := chromedp.Tasks{
 		// Downloads needs to be allowed, otherwise the CSV request will be denied.
 		// Allow download events to emit so we can get the download URL.
 		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).
 			WithDownloadPath("/dev/null").
 			WithEventsEnabled(true),
-	}
-
-	if err = tab.RunWithTimeout(2*time.Second, downTasks); err != nil {
-		return nil, fmt.Errorf("error setting download behavior: %w", err)
-	}
-
-	if err = tab.RunWithTimeout(2*time.Second, chromedp.WaitVisible(selDownloadCSVButton, chromedp.ByQuery)); err != nil {
-		return nil, fmt.Errorf("error waiting for download CSV button: %w", err)
-	}
-
-	if err = tab.RunWithTimeout(2*time.Second, chromedp.Click(selInspectPanelDataTabExpandDataOptions, chromedp.ByQuery)); err != nil {
-		return nil, fmt.Errorf("error clicking on expand data options: %w", err)
-	}
-
-	if err = tab.RunWithTimeout(1*time.Second, chromedp.Click(selInspectPanelDataTabApplyTransformationsToggle, chromedp.ByQuery)); err != nil && !errors.Is(err, context.DeadlineExceeded) {
-		return nil, fmt.Errorf("error clicking on apply transformations toggle: %w", err)
-	}
-
-	if err = tab.RunWithTimeout(1*time.Second, chromedp.Click(selInspectPanelDataTabApplyTransformationsToggle, chromedp.ByQuery)); err != nil && !errors.Is(err, context.DeadlineExceeded) {
-		return nil, fmt.Errorf("error clicking on apply transformations toggle: %w", err)
+		chromedp.Evaluate(d.jsContent, nil),
+		chromedp.Evaluate(js, nil, func(p *runtime.EvaluateParams) *runtime.EvaluateParams {
+			return p.WithAwaitPromise(true)
+		}),
 	}
 
 	// Run all tasks in a goroutine.
 	// If an error occurs, it will be sent to the errCh channel.
 	// If a element can't be found, a timeout will occur and the context will be canceled.
 	go func() {
-		task := chromedp.Evaluate(clickDownloadCSVButton, nil)
-		if err := tab.Run(task); err != nil {
+		if err := tab.Run(downTasks); err != nil {
 			errCh <- fmt.Errorf("error fetching CSV URL from browser %s: %w", panelURL, err)
 		}
 	}()
@@ -120,7 +110,7 @@ func (d *Dashboard) PanelCSV(_ context.Context, p Panel) (CSVData, error) {
 		chrome.WithAwaitPromise,
 	)
 
-	if err := tab.RunWithTimeout(45*time.Second, task); err != nil {
+	if err := tab.RunWithTimeout(d.conf.HTTPClientOptions.Timeouts.Timeout, task); err != nil {
 		return nil, fmt.Errorf("error fetching CSV data from URL from browser %s: %w", panelURL, err)
 	}
 
@@ -144,13 +134,18 @@ func (d *Dashboard) PanelCSV(_ context.Context, p Panel) (CSVData, error) {
 }
 
 // panelCSVURL returns URL to fetch panel's CSV data.
-func (d *Dashboard) panelCSVURL(p Panel) string {
+func (d *Dashboard) panelCSVURL(p Panel) *url.URL {
 	values := maps.Clone(d.model.Dashboard.Variables)
 	values.Add("theme", d.conf.Theme)
 	values.Add("viewPanel", p.ID)
 	values.Add("inspect", p.ID)
 	values.Add("inspectTab", "data")
 
+	// Make a copy of appURL
+	panelURL := *d.appURL
+	panelURL.Path = fmt.Sprintf("/d/%s/_", d.model.Dashboard.UID)
+	panelURL.RawQuery = values.Encode()
+
 	// Get Panel API endpoint
-	return fmt.Sprintf("%s/d/%s/_?%s", d.appURL, d.model.Dashboard.UID, values.Encode())
+	return &panelURL
 }