Merge pull request #15 from netsage-project/feature/distinct

Adds filtering for dashboard and datasource

.drone.yml

@@ -2,11 +2,11 @@ kind: pipeline
 name: default
 
 steps:
-- name: build
-  image: golang:1.16
-  commands:
-  - make linux
-  - cp conf/importer-example.yml conf/importer.yml
-  - make test
-  - ./bin/grafana-dashboard-manager_linux version
+  - name: build
+    image: golang:1.16
+    commands:
+      - make linux
+      - cp conf/importer-example.yml conf/importer.yml
+      - make test
+      - ./bin/gdg_linux version
 

.goreleaser.yml

@@ -9,7 +9,8 @@ before:
 builds:
   - env:
       - CGO_ENABLED=0
-    ldflags: -X github.com/netsage-project/grafana-dashboard-manager/version.GitCommit={{ .Commit }} -X github.com/netsage-project/grafana-dashboard-manager/version.BuildDate={{ .Date }}  -X github.com/netsage-project/grafana-dashboard-manager/version.Version={{ .Tag }} 
+    binary: gdg
+    ldflags: -X github.com/netsage-project/grafana-dashboard-manager/version.GitCommit={{ .Commit }} -X github.com/netsage-project/grafana-dashboard-manager/version.BuildDate={{ .Date }}  -X github.com/netsage-project/grafana-dashboard-manager/version.Version={{ .Tag }}
     goos:
       - linux
       - windows
@@ -23,16 +24,16 @@ archives:
       386: i386
       amd64: x86_64
     files:
-     - README*
-     - conf/importer-example.yml
+      - README*
+      - conf/importer-example.yml
 
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "{{ .Tag }}-next"
 changelog:
   sort: asc
   filters:
     exclude:
-      - '^docs:'
-      - '^test:'
+      - "^docs:"
+      - "^test:"

.vscode/launch.json

@@ -13,8 +13,6 @@
             "args": [
                 "users",
                 "list",
-                // "--context",
-                // "prod"
                 //"promote",
                 //"-u",
                 //"[email protected]"

Makefile

@@ -3,15 +3,15 @@
 
 
 
-BIN_NAME=grafana-dashboard-manager
+BIN_NAME=gdg
 
 VERSION := $(shell grep "const Version " version/version.go | sed -E 's/.*"(.+)"$$/\1/')
 GIT_COMMIT=$(shell git rev-parse HEAD)
 GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true)
 BUILD_DATE=$(shell date '+%Y-%m-%d-%H:%M:%S')
 IMAGE_NAME := "netsage-project/grafana-dashboard-manager"
 
-default: test
+default: build
 
 linux: clean 
 	env GOOS='linux' GOARCH='amd64' go build -ldflags "-X github.com/netsage-project/grafana-dashboard-manager/version.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X github.com/netsage-project/grafana-dashboard-manager/version.BuildDate=${BUILD_DATE}" -o bin/${BIN_NAME}_linux

README.md

@@ -1,6 +1,6 @@
-# grafana-dashboard-manager
+# Grafana dash-n-grab 
 
-Grafana Dashboard Manager (GDM)
+Grafana Dash-n-Grab (GDG) -- Dashboard/DataSource Manager.  The purpose of this project is to provide an easy to use CLI to interact with the grafana API allowing you to backup and restore dashboard and datasources.
 
 ## Getting started
 

api/common.go

@@ -4,14 +4,29 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/gosimple/slug"
 	"github.com/netsage-project/grafana-dashboard-manager/config"
+	"github.com/netsage-project/sdk"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
 var DefaultFolderName = "General"
 
+func GetSlug(title string) string {
+	return strings.ToLower(slug.Make(title))
+}
+
+//Update the slug in the board returned
+func updateSlug(board *sdk.FoundBoard) {
+	elements := strings.Split(board.URI, "/")
+	if len(elements) > 1 {
+		board.Slug = elements[len(elements)-1]
+	}
+}
+
 //buildDashboardPath returns the dashboard path for a given folder
 func buildDashboardPath(conf *viper.Viper, folderName string) string {
 	if folderName == "" {

api/dashboards.go

@@ -6,14 +6,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"regexp"
 	"strings"
 
-	"github.com/gosimple/slug"
 	"github.com/spf13/viper"
 	"github.com/tidwall/pretty"
 
-	"github.com/netsage-project/grafana-dashboard-manager/config"
-
 	"github.com/netsage-project/sdk"
 	log "github.com/sirupsen/logrus"
 	"github.com/thoas/go-funk"
@@ -22,32 +21,56 @@ import (
 
 //ListDashboards: List all dashboards optionally filtered by folder name. If folderFilters
 // is blank, defaults to the configured Monitored folders
-func ListDashboards(client *sdk.Client, folderFilters []string, query string) []sdk.FoundBoard {
+func ListDashboards(client *sdk.Client, filters *DashboardFilter) []sdk.FoundBoard {
 	ctx := context.Background()
 	var boardsList []sdk.FoundBoard = make([]sdk.FoundBoard, 0)
-	boardLinks, err := client.SearchDashboards(ctx, query, false)
+	boardLinks, err := client.SearchDashboards(ctx, "", false)
 	if err != nil {
 		panic(err)
 	}
-	if len(folderFilters) == 0 {
-		folderFilters = config.GetDefaultGrafanaConfig().GetMonitoredFolders()
+	//Fallback on defaults
+	if filters == nil {
+		filters = &DashboardFilter{}
 	}
+
+	folderFilters := filters.GetFolders()
+	var validFolder bool = false
+	var validUid bool = false
 	for _, link := range boardLinks {
 		if funk.Contains(folderFilters, link.FolderTitle) {
-			boardsList = append(boardsList, link)
+			validFolder = true
 		} else if funk.Contains(folderFilters, DefaultFolderName) && link.FolderID == 0 {
 			link.FolderTitle = DefaultFolderName
+			validFolder = true
+		}
+		if !validFolder {
+			continue
+		}
+		updateSlug(&link)
+		if filters.DashFilter != "" {
+			if link.Slug == filters.DashFilter {
+				validUid = true
+			} else {
+				validUid = false
+			}
+		} else {
+			validUid = true
+		}
+
+		if validFolder && validUid {
 			boardsList = append(boardsList, link)
 		}
 
+		validFolder, validUid = false, false
+
 	}
 
 	return boardsList
 
 }
 
 //ImportDashboards saves all dashboards matching query to configured location
-func ImportDashboards(client *sdk.Client, query string, conf *viper.Viper) []string {
+func ImportDashboards(client *sdk.Client, filter DashboardFilter, conf *viper.Viper) []string {
 	var (
 		boardLinks []sdk.FoundBoard
 		rawBoard   []byte
@@ -56,7 +79,7 @@ func ImportDashboards(client *sdk.Client, query string, conf *viper.Viper) []str
 	)
 	ctx := context.Background()
 
-	boardLinks = ListDashboards(client, config.GetDefaultGrafanaConfig().GetMonitoredFolders(), query)
+	boardLinks = ListDashboards(client, &filter)
 	var boards []string = make([]string, 0)
 	for _, link := range boardLinks {
 		if rawBoard, meta, err = client.GetRawDashboardByUID(ctx, link.UID); err != nil {
@@ -87,7 +110,7 @@ func getFolderNameIDMap(client *sdk.Client, ctx context.Context) map[string]int
 
 //ExportDashboards finds all the dashboards in the configured location and exports them to grafana.
 // if the folde doesn't exist, it'll be created.
-func ExportDashboards(client *sdk.Client, folderFilters []string, query string, conf *viper.Viper) {
+func ExportDashboards(client *sdk.Client, filters DashboardFilter, conf *viper.Viper) {
 	filesInDir := findAllFiles(getResourcePath(conf, "dashboard"))
 	ctx := context.Background()
 	var rawBoard []byte
@@ -97,41 +120,59 @@ func ExportDashboards(client *sdk.Client, folderFilters []string, query string,
 	var folderId int
 
 	for _, file := range filesInDir {
+		baseFile := filepath.Base(file)
+		baseFile = strings.ReplaceAll(baseFile, ".json", "")
 		if strings.HasSuffix(file, ".json") {
 			if rawBoard, err = ioutil.ReadFile(file); err != nil {
 				log.Println(err)
 				continue
 			}
+			var board = make(map[string]interface{})
+			if err = json.Unmarshal(rawBoard, &board); err != nil {
+				log.Println(err)
+				log.Printf("Failed to unmarshall file: %s", file)
+				continue
+			}
+
 			elements := strings.Split(file, "/")
 			if len(elements) >= 2 {
 				folderName = elements[len(elements)-2]
 			}
 			if folderName == "" || folderName == DefaultFolderName {
 				folderId = sdk.DefaultFolderId
+				folderName = DefaultFolderName
+			}
+
+			if folderName == DefaultFolderName {
+				folderId = sdk.DefaultFolderId
 			} else {
 				if val, ok := folderMap[folderName]; ok {
 					folderId = val
 				} else {
-					folder := sdk.Folder{Title: folderName}
-					folder, err = client.CreateFolder(ctx, folder)
-					if err != nil {
-						panic(err)
+					createFolder := filters.ValidateFolder(folderName)
+					validUid := filters.ValidateDashboard(baseFile)
+
+					if createFolder && validUid {
+						folder := sdk.Folder{Title: folderName}
+						folder, err = client.CreateFolder(ctx, folder)
+						if err != nil {
+							panic(err)
+						}
+						folderMap[folderName] = folder.ID
+						folderId = folder.ID
 					}
-					folderMap[folderName] = folder.ID
-					folderId = folder.ID
 				}
 			}
 
-			var board = make(map[string]interface{})
-			if err = json.Unmarshal(rawBoard, &board); err != nil {
-				log.Println(err)
-				log.Printf("Failed to unmarshall file: %s", file)
+			//If folder OR slug is filtered, then skip if it doesn't match
+			if !filters.Validate(folderName, baseFile) {
 				continue
 			}
+
 			title, err := jsonpath.Read(board, "$.title")
 
 			rawTitle := fmt.Sprintf("%v", title)
-			slugName := strings.ToLower(slug.Make(rawTitle))
+			slugName := GetSlug(rawTitle)
 			if _, err = client.DeleteDashboard(ctx, slugName); err != nil {
 				log.Println(err)
 				continue
@@ -156,16 +197,25 @@ func ExportDashboards(client *sdk.Client, folderFilters []string, query string,
 
 //DeleteAllDashboards clears all current dashboards being monitored.  Any folder not white listed
 // will not be affected
-func DeleteAllDashboards(client *sdk.Client, folderFilters []string) []string {
+func DeleteAllDashboards(client *sdk.Client, filter DashboardFilter) []string {
 	ctx := context.Background()
 	var dashboards []string = make([]string, 0)
-	items := ListDashboards(client, config.GetDefaultGrafanaConfig().GetMonitoredFolders(), "")
+
+	items := ListDashboards(client, &filter)
 	for _, item := range items {
-		_, err := client.DeleteDashboardByUID(ctx, item.UID)
-		if err == nil {
-			dashboards = append(dashboards, item.Title)
+		if filter.Validate(item.FolderTitle, item.Slug) {
+			_, err := client.DeleteDashboardByUID(ctx, item.UID)
+			if err == nil {
+				dashboards = append(dashboards, item.Title)
+			}
 		}
 	}
 	return dashboards
 
 }
+
+var quoteRegex *regexp.Regexp
+
+func init() {
+	quoteRegex, _ = regexp.Compile("['\"]+")
+}

api/datasources.go

@@ -15,28 +15,34 @@ import (
 )
 
 //ListDataSources: list all the currently configured datasources
-func ListDataSources(client *sdk.Client, folderFilters []string) []sdk.Datasource {
+func ListDataSources(client *sdk.Client, filter DatasourceFilter) []sdk.Datasource {
 
 	ctx := context.Background()
 	ds, err := client.GetAllDatasources(ctx)
 	if err != nil {
 		panic(err)
 	}
+	result := make([]sdk.Datasource, 0)
+	for _, item := range ds {
+		if filter.ValidateDatasource(GetSlug(item.Name)) {
+			result = append(result, item)
+		}
+	}
 
-	return ds
+	return result
 }
 
 //ImportDataSources: will read in all the configured datasources.
 //NOTE: credentials cannot be retrieved and need to be set via configuration
-func ImportDataSources(client *sdk.Client, conf *viper.Viper) []string {
+func ImportDataSources(client *sdk.Client, filter DatasourceFilter, conf *viper.Viper) []string {
 	var (
 		datasources []sdk.Datasource
 		dsPacked    []byte
 		meta        sdk.BoardProperties
 		err         error
 		dataFiles   []string
 	)
-	datasources = ListDataSources(client, nil)
+	datasources = ListDataSources(client, filter)
 	for _, ds := range datasources {
 		if dsPacked, err = json.MarshalIndent(ds, "", "	"); err != nil {
 			fmt.Fprintf(os.Stderr, "%s for %s\n", err, ds.Name)
@@ -53,10 +59,10 @@ func ImportDataSources(client *sdk.Client, conf *viper.Viper) []string {
 }
 
 //Removes all current datasources
-func DeleteAllDataSources(client *sdk.Client) []string {
+func DeleteAllDataSources(client *sdk.Client, filter DatasourceFilter) []string {
 	ctx := context.Background()
 	var ds []string = make([]string, 0)
-	items := ListDataSources(client, nil)
+	items := ListDataSources(client, filter)
 	for _, item := range items {
 		client.DeleteDatasource(ctx, item.ID)
 		ds = append(ds, item.Name)
@@ -65,14 +71,14 @@ func DeleteAllDataSources(client *sdk.Client) []string {
 }
 
 //ExportDataSources: exports all datasources to grafana using the credentials configured in config file.
-func ExportDataSources(client *sdk.Client, folderFilters []string, query string, conf *viper.Viper) []string {
+func ExportDataSources(client *sdk.Client, filter DatasourceFilter, conf *viper.Viper) []string {
 	var datasources []sdk.Datasource
 	var status sdk.StatusMessage
 	var exported []string = make([]string, 0)
 
 	ctx := context.Background()
 	filesInDir, err := ioutil.ReadDir(getResourcePath(conf, "ds"))
-	datasources = ListDataSources(client, nil)
+	datasources = ListDataSources(client, filter)
 
 	var rawDS []byte
 	if err != nil {
@@ -91,6 +97,9 @@ func ExportDataSources(client *sdk.Client, folderFilters []string, query string,
 				fmt.Fprint(os.Stderr, err)
 				continue
 			}
+			if !filter.ValidateDatasource(GetSlug(newDS.Name)) {
+				continue
+			}
 			dsConfig := config.GetDefaultGrafanaConfig()
 			var creds *config.GrafanaDataSource