add crd for pulsar package & function & connector

.ci/clusters/values.yaml

@@ -0,0 +1,133 @@
+# Copyright 2022 StreamNative
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## deployed withh emptyDir
+volumes:
+  persistence: false
+
+# disabled AntiAffinity
+affinity:
+  anti_affinity: false
+
+# disable auto recovery
+components:
+  autorecovery: false
+  pulsar_manager: false
+  sql_worker: false
+  proxy: false
+  toolset: false
+
+## disable monitoring stack
+monitoring:
+  # monitoring - prometheus
+  prometheus: false
+  # monitoring - grafana
+  grafana: false
+  # monitoring - node_exporter
+  node_exporter: false
+  # alerting - alert-manager
+  alert_manager: false
+  # monitoring - loki
+  loki: false
+  # monitoring - datadog
+  datadog: false
+
+images:
+  zookeeper:
+    repository: streamnative/sn-platform
+    tag: 2.10.4.3
+  bookie:
+    repository: streamnative/sn-platform
+    tag: 2.10.4.3
+  broker:
+    repository: streamnative/sn-platform
+    tag: 2.10.4.3
+  functions:
+    repository: streamnative/sn-platform
+    tag: 2.10.4.3
+
+zookeeper:
+  replicaCount: 1
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 10m
+
+bookkeeper:
+  replicaCount: 1
+  metadata:
+    image:
+      repository: streamnative/sn-platform
+      tag: 2.10.4.3
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 10m
+  configData:
+    PULSAR_PREFIX_autoRecoveryDaemonEnabled: "false"
+    PULSAR_PREFIX_dlog.bkcEnsembleSize: "1"
+    PULSAR_PREFIX_dlog.bkcWriteQuorumSize: "1"
+    PULSAR_PREFIX_dlog.bkcAckQuorumSize: "1"
+    # `BOOKIE_MEM` is used for `bookie shell`
+    BOOKIE_MEM: >
+      -Xms128m
+      -Xmx256m
+      -XX:MaxDirectMemorySize=256m
+    # we use `bin/pulsar` for starting bookie daemons
+    PULSAR_MEM: >
+      -Xms128m
+      -Xmx256m
+      -XX:MaxDirectMemorySize=256m
+    PULSAR_GC: >
+      -XX:+UseG1GC
+      -XX:MaxGCPauseMillis=10
+      -XX:+ParallelRefProcEnabled
+      -XX:+UnlockExperimentalVMOptions
+      -XX:+AggressiveOpts
+      -XX:+DoEscapeAnalysis
+      -XX:ParallelGCThreads=4
+      -XX:ConcGCThreads=4
+      -XX:G1NewSizePercent=50
+      -XX:+DisableExplicitGC
+      -XX:-ResizePLAB
+      -XX:+ExitOnOutOfMemoryError
+      -XX:+PerfDisableSharedMem
+
+pulsar_metadata:
+  image:
+    repository: streamnative/sn-platform
+    tag: 2.10.4.3
+
+broker:
+  replicaCount: 1
+  configData:
+    ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
+    ## without persistence
+    autoSkipNonRecoverableData: "true"
+    # storage settings
+    managedLedgerDefaultEnsembleSize: "1"
+    managedLedgerDefaultWriteQuorum: "1"
+    managedLedgerDefaultAckQuorum: "1"
+    enablePackagesManagement: "true"
+    PULSAR_PREFIX_enablePackagesManagement: "true"
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 10m
+
+functions:
+  functionState: false
+  useDedicatedRunner: false
+  configData:
+    narExtractionDirectory: /pulsar/data

.github/workflows/chart-test.yml

@@ -42,6 +42,10 @@ jobs:
             diff config/crd/bases/resource.streamnative.io_pulsarnamespaces.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarnamespaces.yaml  && \
             diff config/crd/bases/resource.streamnative.io_pulsarpermissions.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarpermissions.yaml  && \
             diff config/crd/bases/resource.streamnative.io_pulsartenants.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsartenants.yaml  && \
+            diff config/crd/bases/resource.streamnative.io_pulsarfunctions.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarfunctions.yaml  && \
+            diff config/crd/bases/resource.streamnative.io_pulsarsinks.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarsinks.yaml  && \
+            diff config/crd/bases/resource.streamnative.io_pulsarsources.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarsources.yaml  && \
+            diff config/crd/bases/resource.streamnative.io_pulsarpackages.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsarpackages.yaml  && \
             diff config/crd/bases/resource.streamnative.io_pulsartopics.yaml charts/pulsar-resources-operator/crds/resource.streamnative.io_pulsartopics.yaml
 
       - name: Set up Helm

.github/workflows/e2e_test.yml

@@ -32,7 +32,7 @@ jobs:
       ALWAYS_UPDATE_PULSAR_RESOURCE: ${{ matrix.alwaysUpdatePulsar }}
       GOPRIVATE: github.com/streamnative
       ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
-      IMAGE: apachepulsar/pulsar:latest
+      IMAGE: streamnative/sn-platform:2.10.4.3
       WATCH_CERT_MANAGER_CRDS: "false"
     steps:
       - name: clean disk
@@ -54,6 +54,13 @@ jobs:
         run: |
           git config --global url."https://${ACCESS_TOKEN}:@github.com/".insteadOf "https://github.com/"
 
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKER_USER }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Set up Ginkgo
         run: |
           go install github.com/onsi/ginkgo/ginkgo@latest
@@ -64,7 +71,7 @@ jobs:
       # TODO the k8s version should be configurable
       - name: Setup K8s cluster
         run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
           chmod +x ./kind
           export PATH="$PWD:$PATH"
           kind version
@@ -79,15 +86,21 @@ jobs:
           df -h
 
 
-      - name: Install Pulsar Operators and sn-platform 
+      - name: Install Pulsar Operators and sn-platform
         run: |
           helm repo add streamnative https://charts.streamnative.io
           helm repo add jetstack https://charts.jetstack.io
           helm repo update
-          kubectl create namespace sn-system
-          helm install pulsar-operators --namespace sn-system streamnative/pulsar-operator
-          helm install cert-manager --namespace sn-system jetstack/cert-manager --set installCRDs=true
-          helm install test streamnative/sn-platform --version 1.2.38 --set initialize=true,auth.authentication.enabled=false,auth.authorization.enabled=false,components.toolset=false,components.streamnative_console=false,components.vault=false,components.pulsar_detector=false,monitoring.prometheus=false,monitoring.grafana=false,monitoring.node_exporter=false,monitoring.alert_manager=false,zookeeper.resources.requests.cpu=10m,bookkeeper.resources.requests.cpu=10m,broker.resources.requests.cpu=10m,proxy.resources.requests.cpu=10m
+          helm install cert-manager jetstack/cert-manager --set installCRDs=true --version v1.8.2
+
+          rm -rf pulsar-charts/
+          git clone --branch pulsar-operator-0.17.10 https://github.com/streamnative/charts.git pulsar-charts
+          cd pulsar-charts/
+          ./scripts/pulsar/prepare_helm_release.sh -n default -k test -c
+          helm repo add grafana https://grafana.github.io/helm-charts
+          helm repo update
+          helm dependency update charts/pulsar
+          helm install test --set initialize=true --values ../.ci/clusters/values.yaml charts/pulsar
           df -h
 
       - name: Build and Setup Pulsar Resources Operator

Makefile

@@ -54,6 +54,9 @@ IMG ?= controller:latest
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.23
 
+# Architecture to use envtest with
+ENVTEST_ARCH ?= amd64
+
 KUBE_RBAC_PROXY_IMG ?= gcr.io/kubebuilder/kube-rbac-proxy:v0.14.4
 
 REDHAT_SCAN_REGITRY ?= "quay.io"
@@ -112,7 +115,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --arch=${ENVTEST_ARCH} -p path)" go test ./... -coverprofile cover.out
 
 ##@ Build
 
@@ -181,7 +184,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0)
 
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 .PHONY: kustomize
@@ -289,7 +292,7 @@ husky:
 LICENSE_EYE = $(shell pwd)/bin/license-eye
 .PHONY: license-eye
 license-eye: ## Download license-eye locally if necessary. https://github.com/apache/skywalking-eyes
-	$(call go-get-tool,$(LICENSE_EYE),github.com/apache/skywalking-eyes/cmd/license-eye@0.4.0)
+	$(call go-get-tool,$(LICENSE_EYE),github.com/apache/skywalking-eyes/cmd/license-eye@v0.4.0)
 
 ## Check if the specified files have the license header in the config file.
 .PHONY: license-check

README.md

@@ -8,6 +8,10 @@ Currently, the Pulsar Resources Operator provides full lifecycle management for
 - Namespaces
 - Topics
 - Permissions
+- Packages
+- Functions
+- Sinks
+- Sources
 
 
 # Installation
@@ -98,6 +102,10 @@ In this tutorial, a Kubernetes namespace called `test` is used for examples, whi
 - [PulsarNamespace](docs/pulsar_namespace.md)
 - [PulsarTopic](docs/pulsar_topic.md)
 - [PulsarPermission](docs/pulsar_permission.md)
+- [PulsarPackage](docs/pulsar_package.md)
+- [PulsarFunction](docs/pulsar_function.md)
+- [PulsarSink](docs/pulsar_sink.md)
+- [PulsarSource](docs/pulsar_source.md)
 
 ## Contributing
 

api/v1alpha1/common.go

@@ -86,3 +86,52 @@ func IsPulsarResourceReady(instance reconciler.Object) bool {
 		instance.GetGeneration() == observedGeneration &&
 		meta.IsStatusConditionTrue(conditions, ConditionReady)
 }
+
+// PackageContentRef indicates the package content reference
+type PackageContentRef struct {
+	// +optional
+	// persistentVolumeTemplate *corev1.PersistentVolumeClaim `json:"persistentVolumeTemplate,omitempty"`
+
+	// +optional
+	URL string `json:"url,omitempty"`
+}
+
+// Resources indicates the resources for the pulsar functions and connectors
+type Resources struct {
+	CPU  string `json:"cpu"`
+	Disk int64  `json:"disk"`
+	RAM  int64  `json:"ram"`
+}
+
+// ProducerConfig represents the configuration for the producer of the pulsar functions and connectors
+type ProducerConfig struct {
+	MaxPendingMessages                 int `json:"maxPendingMessages" yaml:"maxPendingMessages"`
+	MaxPendingMessagesAcrossPartitions int `json:"maxPendingMessagesAcrossPartitions" yaml:"maxPendingMessagesAcrossPartitions"`
+
+	UseThreadLocalProducers bool          `json:"useThreadLocalProducers" yaml:"useThreadLocalProducers"`
+	CryptoConfig            *CryptoConfig `json:"cryptoConfig" yaml:"cryptoConfig"`
+	BatchBuilder            string        `json:"batchBuilder" yaml:"batchBuilder"`
+	CompressionType         string        `json:"compressionType" yaml:"compressionType"`
+}
+
+// ConsumerConfig represents the configuration for the consumer of the pulsar functions and connectors
+type ConsumerConfig struct {
+	SchemaType         string            `json:"schemaType,omitempty" yaml:"schemaType"`
+	SerdeClassName     string            `json:"serdeClassName,omitempty" yaml:"serdeClassName"`
+	RegexPattern       bool              `json:"regexPattern,omitempty" yaml:"regexPattern"`
+	ReceiverQueueSize  int               `json:"receiverQueueSize,omitempty" yaml:"receiverQueueSize"`
+	SchemaProperties   map[string]string `json:"schemaProperties,omitempty" yaml:"schemaProperties"`
+	ConsumerProperties map[string]string `json:"consumerProperties,omitempty" yaml:"consumerProperties"`
+	CryptoConfig       *CryptoConfig     `json:"cryptoConfig,omitempty" yaml:"cryptoConfig"`
+	PoolMessages       bool              `json:"poolMessages,omitempty" yaml:"poolMessages"`
+}
+
+// CryptoConfig represents the configuration for the crypto of the pulsar functions and connectors
+type CryptoConfig struct {
+	CryptoKeyReaderClassName string            `json:"cryptoKeyReaderClassName" yaml:"cryptoKeyReaderClassName"`
+	CryptoKeyReaderConfig    map[string]string `json:"cryptoKeyReaderConfig" yaml:"cryptoKeyReaderConfig"`
+
+	EncryptionKeys              []string `json:"encryptionKeys" yaml:"encryptionKeys"`
+	ProducerCryptoFailureAction string   `json:"producerCryptoFailureAction" yaml:"producerCryptoFailureAction"`
+	ConsumerCryptoFailureAction string   `json:"consumerCryptoFailureAction" yaml:"consumerCryptoFailureAction"`
+}