Add support for missing services

.github/workflows/test.yaml

@@ -23,7 +23,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.2
+          version: v1.55.2
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build

README.md

@@ -1,11 +1,11 @@
 # CloudQuery OpenStack Source Plugin
 
-[![test](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml/badge.svg)](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml)
-[![Go Report Card](https://goreportcard.com/badge/github.com/github.com/dihedron/cq-source-openstack)](https://goreportcard.com/report/github.com/github.com/dihedron/cq-source-openstack)
+[![test](https://github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml/badge.svg)](https://github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml)
+[![Go Report Card](https://goreportcard.com/badge/github.com/dihedron/cq-source-openstack)](https://goreportcard.com/report/github.com/dihedron/cq-source-openstack)
 
 A source plugin for CloudQuery that loads data from OpenStack to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
 
-OpenStack is the most widely used open source cloud platform; it powers hundreds of private clouds and some public clouds too;. 
+OpenStack is the most widely used open source cloud platform; it powers hundreds of private clouds and some public clouds too. 
 
 ## Links
 

_test/docker-compose.yaml

@@ -1,19 +1,19 @@
 version: '3.7'
 services:
-    cloudquery:
-        image: postgres:14
-        restart: unless-stopped
-        environment:
-          - POSTGRES_USER=cloudquery
-          - POSTGRES_PASSWORD=cloudquery
-        logging:
-          options:
-            max-size: 10m
-            max-file: "3"
-        ports:
-          - '5433:5432'
-        volumes: 
-          - cloudquery_data:/var/lib/postgresql/data
+  cloudquery:
+    image: postgres:14
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=cloudquery
+      - POSTGRES_PASSWORD=cloudquery
+    logging:
+      options:
+        max-size: 10m
+        max-file: "3"
+    ports:
+      - '5433:5432'
+    volumes:
+      - cloudquery_data:/var/lib/postgresql/data
 volumes:
   cloudquery_data:
     external: false

_test/openstack.yaml

@@ -2,13 +2,12 @@ kind: source
 spec:
   name: openstack_lab
   registry: local
-  path: "/data/workspaces/gomods/cq-source-openstack/dist/cq-source-openstack_linux_amd64_v1/cq-source-openstack"
+  path: "../dist/cq-source-openstack_linux_amd64_v1/cq-source-openstack"
   version: v1.0.0
-  tables: 
-    ["openstack*"]
+  tables: ["openstack*"]
   destinations:
-    - sqlite
-#    - postgresql
+    # - sqlite
+    - postgresql
   spec:
     endpoint_url: ${OPENSTACK_ENDPOINT_URL}
     username: ${OPENSTACK_USERNAME}
@@ -18,14 +17,24 @@ spec:
     domain_name: ${OPENSTACK_DOMAIN_NAME}
     app_credential_id: ${OPENSTACK_APP_CREDENTIAL_ID}
     app_credential_secret: ${OPENSTACK_APP_CREDENTIAL_SECRET}
+    included_tables: ["openstack*"]
+    excluded_tables:
+      [
+        "openstack_ironic*",
+        "openstack_nova_images",
+        "openstack_nova_networks",
+        "openstack_nova_secgroups",
+        "openstack_nova_tenant*",
+        "openstack_keystone_tenants",
+      ]
 # ---
 # kind: source
 # spec:
 #   name: gitlab_cloud
 #   path: cloudquery/gitlab
 #   version: v3.0.0
 #   tables: ["*"]
-#   destinations: 
+#   destinations:
 #     - sqlite
 #     - postgresql
 #   spec:
@@ -45,8 +54,8 @@ spec:
   name: postgresql
   registry: github
   path: cloudquery/postgresql
-  version: v7.1.6
+  version: v7.3.4
   spec:
-    connection_string: postgresql://cloudquery:cloudquery@localhost:15433/cloudquery?sslmode=disable    
-#    connection_string: postgresql://cloudquery:cloudquery@localhost:5432/cloudquery?sslmode=disable  
+    connection_string: postgresql://cloudquery:cloudquery@${DB_POSTGRES_URL}:5433/cloudquery?sslmode=disable
+#    connection_string: postgresql://cloudquery:cloudquery@localhost:5432/cloudquery?sslmode=disable
 #    connection_string: postgresql://superset:superset@localhost:15432/superset?sslmode=disable

client/client.go

@@ -33,6 +33,12 @@ func New(ctx context.Context, logger zerolog.Logger, spec *Spec) (*Client, error
 
 	logger.Debug().Str("spec", format.ToJSON(spec)).Msg("plugin configuration")
 
+	err := spec.Validate()
+	if err != nil {
+		logger.Error().Err(err).Msg("invalid spec configuration")
+		return nil, fmt.Errorf("error spec not valid: %w", err)
+	}
+
 	auth := gophercloud.AuthOptions{
 		AllowReauth: true,
 	}
@@ -142,25 +148,28 @@ func (c *Client) initServiceClient(key ServiceType) (*gophercloud.ServiceClient,
 
 const (
 	// defaults currently referring to Train
-	DefaultComputeV2Microversion      = "2.79"
-	DefaultIdentityV3Microversion     = "3.13"
-	DefaultBlockStorageV3Microversion = "3.59"
-	DefaultImageServiceV2Microversion = "2.9"
+	DefaultIronicV1Microversion   = "1.58"
+	DefaultNovaV2Microversion     = "2.79"
+	DefaultKeystoneV3Microversion = "3.13"
+	DefaultCinderV3Microversion   = "3.59"
+	DefaultGlanceV2Microversion   = "2.9"
 )
 
 type ServiceType string
 
 const (
-	// IdentityV3 identifies the OpenStack Identity V3 service (Keystone).
-	IdentityV3 ServiceType = "openstack_identity_v3"
+	// IronicV1 identifies the OpenStack Baremetal V1 service (Ironic).
+	IronicV1 = "openstack_ironic_v1"
+	// KeystoneV3 identifies the OpenStack Identity V3 service (Keystone).
+	KeystoneV3 ServiceType = "openstack_keystone_v3"
 	// Compute identifies the penStack Compute V2 service (Nova).
-	ComputeV2 = "openstack_compute_v2"
-	// NetworkV2 identifies the OpenStack Network V2 service (Neutron).
-	NetworkV2 = "openstack_network_v2"
-	// BlockStorageV3 identifies the OpenStack Block Storage V3 service (Cinder).
-	BlockStorageV3 = "openstack_blockstorage_v3"
-	// ImageServiceV2 identifies the OpenStack Image Service V2 service (Glance).
-	ImageServiceV2 = "openstack_imageservice_v2"
+	NovaV2 = "openstack_nova_v2"
+	// NeutronV2 identifies the OpenStack Network V2 service (Neutron).
+	NeutronV2 = "openstack_neutron_v2"
+	// CinderV3 identifies the OpenStack Block Storage V3 service (Cinder).
+	CinderV3 = "openstack_cinder_v3"
+	// GlanceV2 identifies the OpenStack Image Service V2 service (Glance).
+	GlanceV2 = "openstack_glance_v2"
 )
 
 type serviceConfig struct {
@@ -169,49 +178,59 @@ type serviceConfig struct {
 }
 
 var serviceConfigMap = map[ServiceType]serviceConfig{
-	IdentityV3: {
+	IronicV1: {
+		newClient: openstack.NewBareMetalV1,
+		getMicroversion: func(spec *Spec) string {
+			microversion := DefaultIronicV1Microversion
+			if spec.IronicV1Microversion != nil {
+				microversion = *spec.IronicV1Microversion
+			}
+			return microversion
+		},
+	},
+	KeystoneV3: {
 		newClient: openstack.NewIdentityV3,
 		getMicroversion: func(spec *Spec) string {
-			microversion := DefaultIdentityV3Microversion
-			if spec.IdentityV3Microversion != nil {
-				microversion = *spec.IdentityV3Microversion
+			microversion := DefaultKeystoneV3Microversion
+			if spec.KeystoneV3Microversion != nil {
+				microversion = *spec.KeystoneV3Microversion
 			}
 			return microversion
 		},
 	},
-	ComputeV2: {
+	NovaV2: {
 		newClient: openstack.NewComputeV2,
 		getMicroversion: func(spec *Spec) string {
-			microversion := DefaultComputeV2Microversion
-			if spec.ComputeV2Microversion != nil {
-				microversion = *spec.ComputeV2Microversion
+			microversion := DefaultNovaV2Microversion
+			if spec.NovaV2Microversion != nil {
+				microversion = *spec.NovaV2Microversion
 			}
 			return microversion
 		},
 	},
-	NetworkV2: {
+	NeutronV2: {
 		newClient: openstack.NewNetworkV2,
 		getMicroversion: func(spec *Spec) string {
 			// TODO: check if we need to leverage/support micro-versions
 			return ""
 		},
 	},
-	BlockStorageV3: {
+	CinderV3: {
 		newClient: openstack.NewBlockStorageV3,
 		getMicroversion: func(spec *Spec) string {
-			microversion := DefaultBlockStorageV3Microversion
-			if spec.BlockStorageV3Microversion != nil {
-				microversion = *spec.BlockStorageV3Microversion
+			microversion := DefaultCinderV3Microversion
+			if spec.CinderV3Microversion != nil {
+				microversion = *spec.CinderV3Microversion
 			}
 			return microversion
 		},
 	},
-	ImageServiceV2: {
+	GlanceV2: {
 		newClient: openstack.NewImageServiceV2,
 		getMicroversion: func(spec *Spec) string {
-			microversion := DefaultImageServiceV2Microversion
-			if spec.ImageServiceV2Microversion != nil {
-				microversion = *spec.ImageServiceV2Microversion
+			microversion := DefaultGlanceV2Microversion
+			if spec.GlanceV2Microversion != nil {
+				microversion = *spec.GlanceV2Microversion
 			}
 			return microversion
 		},

client/spec.go

@@ -1,28 +1,49 @@
 package client
 
+import (
+	"net/http"
+	"net/url"
+
+	"github.com/rs/zerolog/log"
+)
+
 type Spec struct {
-	EndpointUrl                *string `json:"endpoint_url,omitempty" yaml:"endpoint_url,omitempty"`
-	UserID                     *string `json:"userid,omitempty" yaml:"userid,omitempty"`
-	Username                   *string `json:"username,omitempty" yaml:"username,omitempty"`
-	Password                   *string `json:"password,omitempty" yaml:"password,omitempty"`
-	Region                     *string `json:"region,omitempty" yaml:"region,omitempty"`
-	ProjectID                  *string `json:"project_id,omitempty" yaml:"project_id,omitempty"`
-	ProjectName                *string `json:"project_name,omitempty" yaml:"project_name,omitempty"`
-	DomainID                   *string `json:"domain_id,omitempty" yaml:"domain_id,omitempty"`
-	DomainName                 *string `json:"domain_name,omitempty" yaml:"domain_name,omitempty"`
-	AccessToken                *string `json:"access_token,omitempty" yaml:"access_token,omitempty"`
-	AppCredentialID            *string `json:"app_credential_id,omitempty" yaml:"app_credential_id,omitempty"`
-	AppCredentialSecret        *string `json:"app_credential_secret,omitempty" yaml:"app_credential_secret,omitempty"`
-	AllowReauth                *bool   `json:"allow_reauth,omitempty" yaml:"allow_reauth,omitempty"`
-	IdentityV3Microversion     *string `json:"identity_v3_microversion,omitempty" yaml:"identity_v3_microversion,omitempty"`
-	ComputeV2Microversion      *string `json:"compute_v2_microversion,omitempty" yaml:"compute_v2_microversion,omitempty"`
-	NetworkV2Microversion      *string `json:"network_v2_microversion,omitempty" yaml:"network_v2_microversion,omitempty"`
-	BlockStorageV3Microversion *string `json:"blockstorage_v3_microversion,omitempty" yaml:"blockstorage_v3_microversion,omitempty"`
-	ImageServiceV2Microversion *string `json:"imageservice_v2_microversion,omitempty" yaml:"imageservice_v2_microversion,omitempty"`
+	EndpointUrl            *string  `json:"endpoint_url,omitempty" yaml:"endpoint_url,omitempty"`
+	UserID                 *string  `json:"userid,omitempty" yaml:"userid,omitempty"`
+	Username               *string  `json:"username,omitempty" yaml:"username,omitempty"`
+	Password               *string  `json:"password,omitempty" yaml:"password,omitempty"`
+	Region                 *string  `json:"region,omitempty" yaml:"region,omitempty"`
+	ProjectID              *string  `json:"project_id,omitempty" yaml:"project_id,omitempty"`
+	ProjectName            *string  `json:"project_name,omitempty" yaml:"project_name,omitempty"`
+	DomainID               *string  `json:"domain_id,omitempty" yaml:"domain_id,omitempty"`
+	DomainName             *string  `json:"domain_name,omitempty" yaml:"domain_name,omitempty"`
+	AccessToken            *string  `json:"access_token,omitempty" yaml:"access_token,omitempty"`
+	AppCredentialID        *string  `json:"app_credential_id,omitempty" yaml:"app_credential_id,omitempty"`
+	AppCredentialSecret    *string  `json:"app_credential_secret,omitempty" yaml:"app_credential_secret,omitempty"`
+	AllowReauth            *bool    `json:"allow_reauth,omitempty" yaml:"allow_reauth,omitempty"`
+	IronicV1Microversion   *string  `json:"ironic_v1_microversion,omitempty" yaml:"ironic_v1_microversion,omitempty"`
+	KeystoneV3Microversion *string  `json:"keyston_v3_microversion,omitempty" yaml:"keyston_v3_microversion,omitempty"`
+	NovaV2Microversion     *string  `json:"nova_v2_microversion,omitempty" yaml:"nova_v2_microversion,omitempty"`
+	NeutronV2Microversion  *string  `json:"neutron_v2_microversion,omitempty" yaml:"neutron_v2_microversion,omitempty"`
+	CinderV3Microversion   *string  `json:"cinder_v3_microversion,omitempty" yaml:"cinder_v3_microversion,omitempty"`
+	GlanceV2Microversion   *string  `json:"glance_v2_microversion,omitempty" yaml:"glance_v2_microversion,omitempty"`
+	IncludedTables         []string `json:"included_tables,omitempty" yaml:"included_tables,omitempty"`
+	ExcludedTables         []string `json:"excluded_tables,omitempty" yaml:"excluded_tables,omitempty"`
 }
 
 func (s *Spec) Validate() error {
-	// TODO: implement
+	// Check that the endpoint URL is a valid URL
+	_, err := url.ParseRequestURI(*s.EndpointUrl)
+	if err != nil {
+		log.Error().Err(err).Msg("invalid endpoint URL")
+		return err
+	}
+	// Check that the endpoint URL is reachable
+	_, err = http.Get(*s.EndpointUrl)
+	if err != nil {
+		log.Error().Err(err).Msg("endpoint URL is unreachable")
+		return err
+	}
 	return nil
 }