add gitlab datasource

cmd/dads/dads.go

@@ -24,6 +24,7 @@ import (
 	"github.com/LF-Engineering/da-ds/pipermail"
 
 	"github.com/LF-Engineering/da-ds/bugzilla"
+	"github.com/LF-Engineering/da-ds/gitlab"
 
 	jsoniter "github.com/json-iterator/go"
 
@@ -90,6 +91,13 @@ func runDS(ctx *lib.Ctx) (err error) {
 			return err
 		}
 		return manager.Sync()
+	case gitlab.Gitlab:
+		manager, err := buildGitlabManager(ctx)
+		if err != nil {
+			fmt.Println(err)
+			return err
+		}
+		return manager.Sync()
 	default:
 		err = fmt.Errorf("unknown data source type: " + ctx.DS)
 		return
@@ -445,6 +453,51 @@ func buildGoogleGroupsManager(ctx *lib.Ctx) (*googlegroups.Manager, error) {
 	return mgr, err
 }
 
+func buildGitlabManager(ctx *lib.Ctx) (*gitlab.Manager, error) {
+
+	params := &gitlab.MgrParams{}
+	authData, err := getAuthData()
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	params.Fetch = true
+	params.Enrich = ctx.BoolEnv("ENRICH")
+	params.ESBulkSize, _ = strconv.Atoi(ctx.Env("ES_BULK_SIZE"))
+	params.ESIndex = ctx.Env("RICH_INDEX")
+	params.ESUrl = ctx.ESURL
+	params.ESPassword = ""
+	params.ESUsername = ""
+	params.ProjectSlug = ctx.Env("PROJECT_SLUG")
+	params.Project = ctx.Env("PROJECT")
+	params.ESCacheURL = authData["es_url"]
+	params.ESCachePassword = authData["es_pass"]
+	params.ESCacheUsername = authData["es_user"]
+	params.AuthGrantType = authData["grant_type"]
+	params.AuthClientID = authData["client_id"]
+	params.AuthClientSecret = authData["client_secret"]
+	params.AuthAudience = authData["audience"]
+	params.Auth0URL = authData["url"]
+	params.Environment = authData["env"]
+	params.AffBaseURL = ctx.Env("AFFILIATION_API_URL") + "/v1"
+	params.Repo = ctx.Env("URL")
+	params.Token = ctx.Env("TOKEN")
+
+	timeout, err := time.ParseDuration("60s")
+	if err != nil {
+		fmt.Println(err)
+	}
+	params.HTTPTimeout = timeout
+
+	mgr, err := gitlab.NewManager(params)
+	if err != nil {
+		fmt.Println("manager error:", mgr)
+	}
+
+	return mgr, err
+
+}
+
 func getAuthData() (map[string]string, error) {
 	var data map[string]string
 

gitlab/const.go

@@ -0,0 +1,52 @@
+package gitlab
+
+import "time"
+
+const (
+	// GitlabAPIVersion ...
+	GitlabAPIVersion = "v4"
+	//GitlabAPIBase ...
+	GitlabAPIBase = "https://gitlab.com/api"
+	//Gitlab datasource name
+	Gitlab = "gitlab"
+	//Unknown ...
+	Unknown = "Unknown"
+)
+
+var (
+	// DefaultDateTime ...
+	DefaultDateTime = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
+	// GitlabRawMapping ...
+	GitlabRawMapping = []byte(`{"mappings":{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"body":{"dynamic":false,"properties":{}}}}}}}`)
+	// GitlabRichMapping ...
+	GitlabRichMapping = []byte(`{
+		"mappings": {
+			"dynamic_templates": [
+			  {
+				"notanalyzed": {
+				  "match": "*",
+				  "match_mapping_type": "string",
+				  "mapping": {
+					"type": "keyword"
+				  }
+				}
+			  },
+			  {
+				"formatdate": {
+				  "match": "*",
+				  "match_mapping_type": "date",
+				  "mapping": {
+					"format": "strict_date_optional_time||epoch_millis",
+					"type": "date"
+				  }
+				}
+			  }
+			],
+			"properties": { 
+			  "metadata__updated_on": {
+				"type": "date"
+			  }
+			}
+		}
+	  }`)
+)