Add support for missing services

README.md

@@ -1,11 +1,11 @@
 # CloudQuery OpenStack Source Plugin
 
-[![test](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml/badge.svg)](https://github.com/github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml)
-[![Go Report Card](https://goreportcard.com/badge/github.com/github.com/dihedron/cq-source-openstack)](https://goreportcard.com/report/github.com/github.com/dihedron/cq-source-openstack)
+[![test](https://github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml/badge.svg)](https://github.com/dihedron/cq-source-openstack/actions/workflows/test.yaml)
+[![Go Report Card](https://goreportcard.com/badge/github.com/dihedron/cq-source-openstack)](https://goreportcard.com/report/github.com/dihedron/cq-source-openstack)
 
 A source plugin for CloudQuery that loads data from OpenStack to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
 
-OpenStack is the most widely used open source cloud platform; it powers hundreds of private clouds and some public clouds too;. 
+OpenStack is the most widely used open source cloud platform; it powers hundreds of private clouds and some public clouds too. 
 
 ## Links
 

_test/openstack.yaml

@@ -2,7 +2,7 @@ kind: source
 spec:
   name: openstack_lab
   registry: local
-  path: "/data/workspaces/gomods/cq-source-openstack/dist/cq-source-openstack_linux_amd64_v1/cq-source-openstack"
+  path: "../dist/cq-source-openstack_linux_amd64_v1/cq-source-openstack"
   version: v1.0.0
   tables: 
     ["openstack*"]

plugin/plugin.go

@@ -0,0 +1,7 @@
+package plugin
+
+var (
+	Name	= "dihedron-openstack"
+	Kind   	= "source"
+	Version = "development"
+)

resources/data_structs.go → resources/internal/utils/data_structs.go

@@ -1,4 +1,4 @@
-package resources
+package utils
 
 type Single[V any] struct {
 	Name V `cq-name:"name"`
@@ -8,3 +8,7 @@ type Pair[K comparable, V any] struct {
 	Key   K `cq-name:"key"`
 	Value V `cq-name:"value"`
 }
+
+type Tag struct {
+	Value string `cq-name:"value"`
+}

resources/time.go → resources/internal/utils/time.go

@@ -1,4 +1,4 @@
-package resources
+package utils
 
 import (
 	"fmt"

resources/time_test.go → resources/internal/utils/time_test.go

@@ -1,4 +1,4 @@
-package resources
+package utils
 
 import (
 	"encoding/json"

resources/plugin/client.go

@@ -11,7 +11,11 @@ import (
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-source-openstack/client"
-	"github.com/dihedron/cq-source-openstack/resources"
+	"github.com/dihedron/cq-source-openstack/resources/services/blockstorage"
+	"github.com/dihedron/cq-source-openstack/resources/services/compute"
+	"github.com/dihedron/cq-source-openstack/resources/services/identity"
+	"github.com/dihedron/cq-source-openstack/resources/services/image_service"
+	"github.com/dihedron/cq-source-openstack/resources/services/networking"
 	"github.com/rs/zerolog"
 )
 
@@ -76,20 +80,19 @@ func (*Client) Close(_ context.Context) error {
 
 func getTables() schema.Tables {
 	tables := schema.Tables{
-		resources.Aggregates(),
-		resources.Attachments(),
-		resources.Flavors(),
-		resources.Hypervisors(),
-		resources.Instances(),
-		resources.Images(),
-		//resources.KeyPairs(),
-		resources.Networks(),
-		resources.Ports(),
-		resources.Projects(),
-		resources.SecurityGroups(),
-		resources.SecurityGroupRules(),
-		resources.Users(),
-		resources.Volumes(),
+		blockstorage.Attachments(),
+		blockstorage.Volumes(),
+		compute.Aggregates(),
+		compute.Flavors(),
+		compute.Hypervisors(),
+		compute.Instances(),
+		identity.Projects(),
+		identity.Users(),
+		image_service.Images(),
+		networking.Networks(),
+		networking.Ports(),
+		networking.SecurityGroups(),
+		networking.SecurityGroupRules(),
 	}
 	if err := transformers.TransformTables(tables); err != nil {
 		panic(err)

resources/plugin/plugin.go

@@ -2,12 +2,13 @@ package plugin
 
 import (
 	"github.com/cloudquery/plugin-sdk/v4/plugin"
-)
-
-var (
-	Version = "development"
+	internalPlugin "github.com/dihedron/cq-source-openstack/plugin"
 )
 
 func Plugin() *plugin.Plugin {
-	return plugin.NewPlugin("dihedron-openstack", Version, Configure)
-}
+	return plugin.NewPlugin(
+		internalPlugin.Name,
+		internalPlugin.Version,
+		Configure,
+	)
+}

resources/attachment_hosts.go → resources/services/blockstorage/attachment_hosts.go

@@ -1,4 +1,4 @@
-package resources
+package blockstorage
 
 import (
 	"context"

resources/attachments.go → resources/services/blockstorage/attachments.go

@@ -1,8 +1,10 @@
-package resources
+package blockstorage
 
 import (
 	"context"
 
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
+
 	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
@@ -264,8 +266,8 @@ func fetchAttachments(ctx context.Context, meta schema.ClientMeta, parent *schem
 
 type Attachment struct {
 	ID             string `json:"id"`
-	AttachedAt     Time   `json:"attached_at"`
-	DetachedAt     Time   `json:"detached_at"`
+	AttachedAt     utils.Time   `json:"attached_at"`
+	DetachedAt     utils.Time   `json:"detached_at"`
 	AttachmentID   string `json:"attachment_id"`
 	VolumeID       string `json:"volume_id"`
 	Instance       string `json:"instance" cq-name:"instance_id"`

resources/volumes.go → resources/services/blockstorage/volumes.go

@@ -1,8 +1,10 @@
-package resources
+package blockstorage
 
 import (
 	"context"
 
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
+
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/format"
@@ -86,9 +88,9 @@ type Volume struct {
 	// AvailabilityZone is which availability zone the volume is in.
 	AvailabilityZone string `json:"availability_zone"`
 	// The date when this volume was created.
-	CreatedAt Time `json:"created_at"`
+	CreatedAt utils.Time `json:"created_at"`
 	// The date when this volume was last updated
-	UpdatedAt Time `json:"updated_at"`
+	UpdatedAt utils.Time `json:"updated_at"`
 	// Instances onto which the volume is attached.
 	Attachments []volumes.Attachment `json:"attachments"`
 	// Human-readable display name for the volume.

resources/aggregate_hosts.go → resources/services/compute/aggregate_hosts.go

@@ -1,8 +1,10 @@
-package resources
+package compute
 
 import (
 	"context"
 
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
+
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/transform"
@@ -15,7 +17,7 @@ func AggregateHosts() *schema.Table {
 		Name:     "openstack_aggregate_hosts",
 		Resolver: fetchAggregateHosts,
 		Transform: transformers.TransformWithStruct(
-			&Single[string]{},
+			&utils.Single[string]{},
 			transformers.WithNameTransformer(transform.TagNameTransformer), // use cq-name tags to translate name
 			transformers.WithTypeTransformer(transform.TagTypeTransformer), // use cq-type tags to translate type
 		),
@@ -26,7 +28,7 @@ func fetchAggregateHosts(ctx context.Context, meta schema.ClientMeta, parent *sc
 	api := meta.(*client.Client)
 	aggregate := parent.Item.(aggregates.Aggregate)
 	for _, v := range aggregate.Hosts {
-		host := &Single[string]{Name: v}
+		host := &utils.Single[string]{Name: v}
 		api.Logger().Debug().Int("aggregate id", aggregate.ID).Str("host", v).Msg("streaming aggregate host")
 		res <- host
 	}

resources/aggregates.go → resources/services/compute/aggregates.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/flavor_accesses.go → resources/services/compute/flavor_accesses.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/flavor_extra_specs.go → resources/services/compute/flavor_extra_specs.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"
@@ -7,14 +7,15 @@ import (
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/transform"
 	"github.com/dihedron/cq-source-openstack/client"
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
 )
 
 func FlavorExtraSpecs() *schema.Table {
 	return &schema.Table{
 		Name:     "openstack_flavor_extra_specs",
 		Resolver: fetchFlavorExtraSpecs,
 		Transform: transformers.TransformWithStruct(
-			&Pair[string, string]{},
+			&utils.Pair[string, string]{},
 			transformers.WithNameTransformer(transform.TagNameTransformer), // use cq-name tags to translate name
 			transformers.WithTypeTransformer(transform.TagTypeTransformer), // use cq-type tags to translate type
 		),
@@ -29,7 +30,7 @@ func fetchFlavorExtraSpecs(ctx context.Context, meta schema.ClientMeta, parent *
 
 	if flavor.ExtraSpecsMap != nil {
 		for k, v := range *flavor.ExtraSpecsMap {
-			pair := &Pair[string, string]{
+			pair := &utils.Pair[string, string]{
 				Key:   k,
 				Value: v,
 			}

resources/flavors.go → resources/services/compute/flavors.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/hypervisors.go → resources/services/compute/hypervisors.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/instance_addresses.go → resources/services/compute/instance_addresses.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/instance_attached_volumes.go → resources/services/compute/instance_attached_volumes.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/instance_flavor_extra_specs.go → resources/services/compute/instance_flavor_extra_specs.go

@@ -1,8 +1,10 @@
-package resources
+package compute
 
 import (
 	"context"
 
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
+
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/transform"
@@ -14,7 +16,7 @@ func InstanceFlavorExtraSpecs() *schema.Table {
 		Name:     "openstack_instance_flavor_extra_specs",
 		Resolver: fetchInstanceFlavorExtraSpecs,
 		Transform: transformers.TransformWithStruct(
-			&Pair[string, string]{},
+			&utils.Pair[string, string]{},
 			transformers.WithNameTransformer(transform.TagNameTransformer), // use cq-name tags to translate name
 			transformers.WithTypeTransformer(transform.TagTypeTransformer), // use cq-type tags to translate type
 		),
@@ -32,7 +34,7 @@ func fetchInstanceFlavorExtraSpecs(ctx context.Context, meta schema.ClientMeta,
 
 	if instance.Flavor.ExtraSpecsMap != nil {
 		for k, v := range *instance.Flavor.ExtraSpecsMap {
-			pair := &Pair[string, string]{
+			pair := &utils.Pair[string, string]{
 				Key:   k,
 				Value: v,
 			}

resources/instance_flavors.go → resources/services/compute/instance_flavors.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/instance_metadata.go → resources/services/compute/instance_metadata.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"
@@ -7,14 +7,15 @@ import (
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/transform"
 	"github.com/dihedron/cq-source-openstack/client"
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
 )
 
 func InstanceMetadata() *schema.Table {
 	return &schema.Table{
 		Name:     "openstack_instance_metadata",
 		Resolver: fetchInstanceMetadata,
 		Transform: transformers.TransformWithStruct(
-			&Pair[string, string]{},
+			&utils.Pair[string, string]{},
 			transformers.WithNameTransformer(transform.TagNameTransformer), // use cq-name tags to translate name
 			transformers.WithTypeTransformer(transform.TagTypeTransformer), // use cq-type tags to translate type
 			//transformers.WithSkipFields("OriginalName", "ExtraSpecs"),
@@ -29,7 +30,7 @@ func fetchInstanceMetadata(ctx context.Context, meta schema.ClientMeta, parent *
 	instance := parent.Item.(*Instance)
 
 	for k, v := range instance.Metadata {
-		pair := &Pair[string, string]{
+		pair := &utils.Pair[string, string]{
 			Key:   k,
 			Value: v,
 		}

resources/instance_security_groups.go → resources/services/compute/instance_security_groups.go

@@ -1,4 +1,4 @@
-package resources
+package compute
 
 import (
 	"context"

resources/instance_tags.go → resources/services/compute/instance_tags.go

@@ -1,8 +1,10 @@
-package resources
+package compute
 
 import (
 	"context"
 
+	"github.com/dihedron/cq-source-openstack/resources/internal/utils"
+
 	"github.com/cloudquery/plugin-sdk/v4/schema"
 	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/dihedron/cq-plugin-utils/transform"
@@ -14,7 +16,7 @@ func InstanceTags() *schema.Table {
 		Name:     "openstack_instance_tags",
 		Resolver: fetchInstanceTags,
 		Transform: transformers.TransformWithStruct(
-			&Tag{},
+			&utils.Tag{},
 			transformers.WithNameTransformer(transform.TagNameTransformer), // use cq-name tags to translate name
 			transformers.WithTypeTransformer(transform.TagTypeTransformer), // use cq-type tags to translate type
 		),
@@ -29,14 +31,10 @@ func fetchInstanceTags(ctx context.Context, meta schema.ClientMeta, parent *sche
 
 	if instance.Tags != nil {
 		for _, v := range *instance.Tags {
-			tag := &Tag{Value: v}
+			tag := &utils.Tag{Value: v}
 			api.Logger().Debug().Str("instance id", instance.ID).Msg("streaming instance tag")
 			res <- tag
 		}
 	}
 	return nil
 }
-
-type Tag struct {
-	Value string `cq-name:"value"`
-}