Adds Cloud Logging Service Endpoint option to configuration (#6)

* Adds Cloud Logging Service Endpoint option to configuration

* Adds screenshots to plugin.json and README

Author: StefanKurek

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 1.1.0 (Unreleased)
+
+* Adds Cloud Logging Service Endpoint to configuration
+* Fixes hide not working for Cloud Logging query
+* Fixes inability to retrieve projects displaying an error
+
 ## 1.0.0 (2023-01-17)
 
 Initial release.

README.md

@@ -1,7 +1,7 @@
 # Google Cloud Logging Data Source
 
 ## Overview
-
+![image info](./src/img/cloud_logging_explore_view.png)
 The Google Cloud Logging Data Source is a backend data source plugin for Grafana,
 which allows users to query and visualize their Google Cloud logs in Grafana.
 
@@ -32,12 +32,13 @@ You can following the those steps to enable it:
 8.  Choose key type `JSON` and click `Create`. A JSON key file will be created and downloaded to your computer
 
 ### Grafana Configuration
-
+![image info](./src/img/cloud_logging_config.png)
 1. With Grafana restarted, navigate to `Configuration -> Data sources` (or the route `/datasources`)
 2. Click "Add data source"
 3. Select "Google Cloud Logging"
 4. Provide credentials in a JWT file, either by using the file selector or pasting the contents of the file.
-5. Click "Save & test" to test that logs can be queried from Cloud Logging.
+5. If desired, provide a regional [Cloud Logging service endpoint](https://cloud.google.com/vpc/docs/regional-service-endpoints#cloud-logging) in order to only collect logs from a specific log bucket region
+6. Click "Save & test" to test that logs can be queried from Cloud Logging.
 
 ## Licenses
 

pkg/plugin/cloudlogging/client.go

@@ -30,6 +30,8 @@ import (
 	loggingpb "google.golang.org/genproto/googleapis/logging/v2"
 )
 
+const testConnectionTimeout = time.Minute * 1
+
 // API implements the methods we need to query logs and list projects from GCP
 type API interface {
 	// ListLogs retrieves all logs matching some query filter up to the given limit
@@ -50,9 +52,9 @@ type Client struct {
 }
 
 // NewClient creates a new Client using jsonCreds for authentication
-func NewClient(ctx context.Context, jsonCreds []byte) (*Client, error) {
+func NewClient(ctx context.Context, jsonCreds []byte, endpoint string) (*Client, error) {
 	client, err := logging.NewClient(ctx, option.WithCredentialsJSON(jsonCreds),
-		option.WithUserAgent("googlecloud-logging-datasource"))
+		option.WithUserAgent("googlecloud-logging-datasource"), option.WithEndpoint(endpoint))
 	if err != nil {
 		return nil, err
 	}
@@ -112,19 +114,30 @@ func (c *Client) ListProjects(ctx context.Context) ([]string, error) {
 // TestConnection queries for any log from the given project
 func (c *Client) TestConnection(ctx context.Context, projectID string) error {
 	start := time.Now()
+
+	listCtx, cancel := context.WithTimeout(ctx, time.Duration(testConnectionTimeout))
+
 	defer func() {
+		cancel()
 		log.DefaultLogger.Debug("Finished testConnection", "duration", time.Since(start).String())
 	}()
 
-	it := c.lClient.ListLogEntries(ctx, &loggingpb.ListLogEntriesRequest{
+	it := c.lClient.ListLogEntries(listCtx, &loggingpb.ListLogEntriesRequest{
 		ResourceNames: []string{projectResourceName(projectID)},
 		PageSize:      1,
 	})
 
+	if listCtx.Err() != nil {
+		return errors.New("list entries: timeout")
+	}
+
 	entry, err := it.Next()
 	if err == iterator.Done {
 		return errors.New("no entries")
 	}
+	if err == context.DeadlineExceeded {
+		return errors.New("list entries: timeout")
+	}
 	if err != nil {
 		return fmt.Errorf("list entries: %w", err)
 	}

pkg/plugin/plugin.go

@@ -42,17 +42,17 @@ const (
 	privateKeyKey = "privateKey"
 )
 
-// jwtToken is the fields parsed from a JWT token and sent from the front end
-type jwtToken struct {
+// config is the fields parsed from the front end
+type config struct {
 	AuthType       string `json:"authenticationType"`
 	ClientEmail    string `json:"clientEmail"`
 	DefaultProject string `json:"defaultProject"`
 	TokenURI       string `json:"tokenUri"`
+	Endpoint       string `json:"endpoint"`
 }
 
-// toServiceAccountJSON creates the serviceAccountJSON bytes from the JWT token fields
-// in gcpCreds
-func (c jwtToken) toServiceAccountJSON(privateKey string) ([]byte, error) {
+// toServiceAccountJSON creates the serviceAccountJSON bytes from the config fields
+func (c config) toServiceAccountJSON(privateKey string) ([]byte, error) {
 	return json.Marshal(serviceAccountJSON{
 		Type:        "service_account",
 		ProjectID:   c.DefaultProject,
@@ -74,8 +74,8 @@ type serviceAccountJSON struct {
 
 // NewCloudLoggingDatasource creates a new datasource instance.
 func NewCloudLoggingDatasource(settings backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-	var creds jwtToken
-	if err := json.Unmarshal(settings.JSONData, &creds); err != nil {
+	var conf config
+	if err := json.Unmarshal(settings.JSONData, &conf); err != nil {
 		return nil, fmt.Errorf("unmarshal: %w", err)
 	}
 
@@ -84,12 +84,12 @@ func NewCloudLoggingDatasource(settings backend.DataSourceInstanceSettings) (ins
 		return nil, errMissingCredentials
 	}
 
-	serviceAccount, err := creds.toServiceAccountJSON(privateKey)
+	serviceAccount, err := conf.toServiceAccountJSON(privateKey)
 	if err != nil {
 		return nil, fmt.Errorf("create credentials: %w", err)
 	}
 
-	client, err := cloudlogging.NewClient(context.TODO(), serviceAccount)
+	client, err := cloudlogging.NewClient(context.TODO(), serviceAccount, conf.Endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -136,11 +136,7 @@ func (d *CloudLoggingDatasource) CallResource(ctx context.Context, req *backend.
 
 	projects, err := d.client.ListProjects(ctx)
 	if err != nil {
-		log.DefaultLogger.Error("error listing", "error", err)
-		return sender.Send(&backend.CallResourceResponse{
-			Status: http.StatusInternalServerError,
-			Body:   []byte(`Unable to list projects`),
-		})
+		log.DefaultLogger.Warn("problem listing projects", "error", err)
 	}
 
 	body, err := json.Marshal(&ListProjectsResponse{Projects: projects})
@@ -182,9 +178,9 @@ func (d *CloudLoggingDatasource) QueryData(ctx context.Context, req *backend.Que
 
 // queryModel is the fields needed to query from Grafana
 type queryModel struct {
-	QueryText     string `json:"queryText"`
-	ProjectID     string `json:"projectId"`
-	MaxDataPoints int    `json:"MaxDataPoints"`
+	QueryText string `json:"queryText"`
+	ProjectID string `json:"projectId"`
+	Hide      bool   `json:"hide"`
 }
 
 func (d *CloudLoggingDatasource) query(ctx context.Context, pCtx backend.PluginContext, query backend.DataQuery) backend.DataResponse {
@@ -196,6 +192,11 @@ func (d *CloudLoggingDatasource) query(ctx context.Context, pCtx backend.PluginC
 		return response
 	}
 
+	// Don't query if query should be hidden
+	if q.Hide {
+		return response
+	}
+
 	clientRequest := cloudlogging.Query{
 		ProjectID: q.ProjectID,
 		Filter:    q.QueryText,
@@ -254,8 +255,8 @@ func (d *CloudLoggingDatasource) CheckHealth(ctx context.Context, req *backend.C
 	var status = backend.HealthStatusOk
 	settings := req.PluginContext.DataSourceInstanceSettings
 
-	var creds jwtToken
-	if err := json.Unmarshal(settings.JSONData, &creds); err != nil {
+	var conf config
+	if err := json.Unmarshal(settings.JSONData, &conf); err != nil {
 		return nil, fmt.Errorf("unmarshal: %w", err)
 	}
 
@@ -264,12 +265,12 @@ func (d *CloudLoggingDatasource) CheckHealth(ctx context.Context, req *backend.C
 		return nil, errMissingCredentials
 	}
 
-	serviceAccount, err := creds.toServiceAccountJSON(privateKey)
+	serviceAccount, err := conf.toServiceAccountJSON(privateKey)
 	if err != nil {
 		return nil, fmt.Errorf("create credentials: %w", err)
 	}
 
-	client, err := cloudlogging.NewClient(ctx, serviceAccount)
+	client, err := cloudlogging.NewClient(ctx, serviceAccount, conf.Endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -280,7 +281,7 @@ func (d *CloudLoggingDatasource) CheckHealth(ctx context.Context, req *backend.C
 		}
 	}()
 
-	if err := client.TestConnection(ctx, creds.DefaultProject); err != nil {
+	if err := client.TestConnection(ctx, conf.DefaultProject); err != nil {
 		return &backend.CheckHealthResult{
 			Status:  backend.HealthStatusError,
 			Message: fmt.Sprintf("failed to run test query: %s", err),
@@ -289,6 +290,6 @@ func (d *CloudLoggingDatasource) CheckHealth(ctx context.Context, req *backend.C
 
 	return &backend.CheckHealthResult{
 		Status:  status,
-		Message: fmt.Sprintf("Successfully queried logs from GCP project %s", creds.DefaultProject),
+		Message: fmt.Sprintf("Successfully queried logs from GCP project %s", conf.DefaultProject),
 	}, nil
 }

src/ConfigEditor.tsx

@@ -16,12 +16,13 @@
 
 import React from 'react';
 import { DataSourcePluginOptionsEditorProps, onUpdateDatasourceJsonDataOption } from '@grafana/data';
-import { DataSourceOptions, DataSourceSecureJsonData, GoogleAuthType } from '@grafana/google-sdk';
-import { FieldSet } from '@grafana/ui';
+import { DataSourceSecureJsonData, GoogleAuthType } from '@grafana/google-sdk';
+import { Field, FieldSet, Input } from '@grafana/ui';
 import { JWTConfigEditor } from './components/JWTConfigEditor';
 import { JWTForm } from './components/JWTForm';
+import { CloudLoggingOptions } from 'types';
 
-type Props = DataSourcePluginOptionsEditorProps<DataSourceOptions, DataSourceSecureJsonData>;
+type Props = DataSourcePluginOptionsEditorProps<CloudLoggingOptions, DataSourceSecureJsonData>;
 
 /**
  * Config page that accepts a JWT token either through upload or pasting
@@ -47,7 +48,7 @@ export const ConfigEditor: React.FC<Props> = (props: Props) => {
    *
    * @param jsonData Remaining data source options
    */
-  const onResetJWTToken = (jsonData?: Partial<DataSourceOptions> | null) => {
+  const onResetJWTToken = (jsonData?: Partial<CloudLoggingOptions> | null) => {
     const nextSecureJsonData = { ...secureJsonData };
     const nextJsonData = !jsonData ? { ...options.jsonData } : { ...options.jsonData, ...jsonData };
 
@@ -63,7 +64,7 @@ export const ConfigEditor: React.FC<Props> = (props: Props) => {
     });
   };
 
-  const onJWTFormChange = (key: keyof DataSourceOptions) => onUpdateDatasourceJsonDataOption(props, key);
+  const onJWTFormChange = (key: keyof CloudLoggingOptions) => onUpdateDatasourceJsonDataOption(props, key);
 
   return (
     <>
@@ -91,6 +92,13 @@ export const ConfigEditor: React.FC<Props> = (props: Props) => {
           />
         )}{' '}
       </FieldSet>
+      <Field label="Cloud Logging Service Endpoint" description="Optional">
+        <Input
+          id="endpoint" 
+          value={options.jsonData.endpoint || ''}
+          onChange={onJWTFormChange('endpoint')}
+        />
+      </Field>
     </>
   );
 };

src/QueryEditor.tsx

@@ -16,12 +16,11 @@
 
 import React, { KeyboardEvent, useEffect, useMemo, useState } from 'react';
 import { QueryEditorProps, SelectableValue } from '@grafana/data';
-import { DataSourceOptions } from '@grafana/google-sdk';
 import { InlineField, InlineFieldRow, LinkButton, Select, TextArea, Tooltip } from '@grafana/ui';
 import { DataSource } from './datasource';
-import { defaultQuery, Query } from './types';
+import { CloudLoggingOptions, defaultQuery, Query } from './types';
 
-type Props = QueryEditorProps<DataSource, Query, DataSourceOptions>;
+type Props = QueryEditorProps<DataSource, Query, CloudLoggingOptions>;
 
 /**
  * This is basically copied from {MQLQueryEditor} from the cloud-monitoring data source

src/img/cloud_logging_config.png

@@ -31,7 +31,11 @@
         "url": "https://github.com/GoogleCloudPlatform/cloud-logging-data-source-plugin/blob/master/LICENSE"
       }
     ],
-    "screenshots": [],
+    "screenshots": [
+      {"name": "cloud_logging_config", "path": "img/cloud_logging_config.png"},
+      {"name": "cloud_logging_explore_view", "path": "img/cloud_logging_explore_view.png"},
+      {"name": "cloud_logging_dashboard_view", "path": "img/cloud_logging_dashboard_view.png"}
+    ],
     "version": "%VERSION%",
     "updated": "%TODAY%"
   },