Skip to content

Commit

Permalink
feat: support manage sn cloud secrets resources (#289)
Browse files Browse the repository at this point in the history
* stash

* support manage sn cloud secrets

* fix lint

* address the latest changes

* fix lint

* upgrade golangci

* fix lint

* add docs

* using larger runner for e2e tests

* fix ci
  • Loading branch information
freeznet authored Feb 27, 2025
1 parent 51b9363 commit 88fdc02
Show file tree
Hide file tree
Showing 62 changed files with 4,563 additions and 1,329 deletions.
31 changes: 20 additions & 11 deletions .github/workflows/e2e_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,27 +29,36 @@ jobs:
strategy:
matrix:
alwaysUpdatePulsar: [ "true", "false" ]
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
env:
ALWAYS_UPDATE_PULSAR_RESOURCE: ${{ matrix.alwaysUpdatePulsar }}
GOPRIVATE: github.com/streamnative
ACCESS_TOKEN: ${{ secrets.SNBOT_GITHUB_TOKEN }}
IMAGE: streamnative/sn-platform:2.10.4.3
WATCH_CERT_MANAGER_CRDS: "false"
steps:
- name: clean disk
run: |
df -h
sudo swapoff -a
sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo apt clean
docker rmi $(docker images -q) -f
df -h
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true

- name: Cleanup disk
run: sudo rm -rf /usr/local/.ghcup

- name: Set up Go 1.22.7
- name: Set up Go 1.22.11
uses: actions/setup-go@v3
with:
go-version: '1.22.7'
go-version: '1.22.11'
id: go

- name: Set up Git token
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/golangci-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ jobs:

- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
uses: golangci/golangci-lint-action@v6
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
version: v1.55.2
version: v1.64
args: --timeout=5m

# Optional: working directory, useful for monorepos
Expand Down
19 changes: 8 additions & 11 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ linters:
# - depguard
- dogsled
- errcheck
- exportloopref
- copyloopvar
# - gci
- gocritic
# - gocyclo
Expand Down Expand Up @@ -61,7 +61,8 @@ linters-settings:
- '^ \+.*'
- '^ ANCHOR.*'
gci:
local-prefixes: "github.com/streamnative"
sections:
- prefix(github.com/streamnative)

importas:
no-unaliased: true
Expand All @@ -82,12 +83,7 @@ linters-settings:
alias: ctrl
nolintlint:
allow-unused: false
allow-leading-space: false
require-specific: true
staticcheck:
go: "1.17"
stylecheck:
go: "1.17"
gosec:
excludes:
- G307 # Deferring unsafe method "Close" on type "\*os.File"
Expand Down Expand Up @@ -115,6 +111,10 @@ issues:
# We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant
# changes in PRs and avoid nitpicking.
exclude-use-default: false
exclude-files:
- "zz_generated.*\\.go$"
exclude-dirs:
- third_party
exclude-rules:
- linters:
- revive
Expand Down Expand Up @@ -204,8 +204,5 @@ run:
build-tags:
- tools
- e2e
skip-files:
- "zz_generated.*\\.go$"
skip-dirs:
- third_party
allow-parallel-runners: true
go: "1.22"
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

# Build the manager binary
FROM golang:1.22.7-alpine3.20 as builder
FROM golang:1.22.11-alpine3.20 as builder

ARG ACCESS_TOKEN="none"

Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ In this tutorial, a Kubernetes namespace called `test` is used for examples, whi
- [StreamNativeCloudConnection](docs/streamnative_cloud_connection.md)
- [ComputeWorkspace](docs/compute_workspace.md)
- [ComputeFlinkDeployment](docs/compute_flink_deployment.md)
- [StreamNative Cloud Secret](docs/secret.md)

# Contributing

Expand Down
40 changes: 40 additions & 0 deletions api/v1alpha1/computeflinkdeployment_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ type ComputeFlinkDeploymentSpec struct {
// DefaultPulsarCluster is the default pulsar cluster to use
// +optional
DefaultPulsarCluster *string `json:"defaultPulsarCluster,omitempty"`

// Configuration is the list of configurations to deploy with the Flink deployment.
// +optional
Configuration *Configuration `json:"configuration,omitempty"`

// ImagePullSecrets is the list of image pull secrets to use for the deployment.
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}

// ComputeFlinkDeploymentStatus defines the observed state of ComputeFlinkDeployment
Expand Down Expand Up @@ -267,6 +275,9 @@ type Artifact struct {

// +optional
URI string `json:"uri,omitempty"`

// +optional
ArtifactImage string `json:"artifactImage,omitempty"`
}

// VvpDeploymentStatus defines the deployment status
Expand Down Expand Up @@ -409,6 +420,35 @@ type VvpDeploymentDetailsTemplateSpecKubernetesSpec struct {
Labels map[string]string `json:"labels,omitempty"`
}

// SecretReference references a StreamNative Cloud secret.
type SecretReference struct {
// Name of the ENV variable.
Name string `json:"name" protobuf:"bytes,1,opt,name=name"`

// ValueFrom references a secret in the same namespace.
ValueFrom *corev1.SecretKeySelector `json:"valueFrom,omitempty" protobuf:"bytes,2,opt,name=valueFrom"`
}

// EnvVar defines an environment variable.
type EnvVar struct {
// Name of the environment variable.
Name string `json:"name" protobuf:"bytes,1,opt,name=name"`

// Value of the environment variable.
Value string `json:"value" protobuf:"bytes,2,opt,name=value"`
}

// Configuration defines the additional configuration for the Flink deployment
type Configuration struct {
// Envs is the list of environment variables to set in the Flink deployment.
// +optional
Envs []EnvVar `json:"envs,omitempty" protobuf:"bytes,1,opt,name=envs"`

// Secrets is the list of secrets referenced to deploy with the Flink deployment.
// +optional
Secrets []SecretReference `json:"secrets,omitempty" protobuf:"bytes,2,opt,name=secrets"`
}

func init() {
SchemeBuilder.Register(&ComputeFlinkDeployment{}, &ComputeFlinkDeploymentList{})
}
151 changes: 151 additions & 0 deletions api/v1alpha1/secret_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
// Copyright 2025 StreamNative
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import (
"github.com/streamnative/cloud-api-server/pkg/apis/cloud/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)

// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

// SecretSpec defines the desired state of StreamNative Cloud Secret
type SecretSpec struct {
// APIServerRef is the reference to the StreamNativeCloudConnection
// +required
APIServerRef corev1.LocalObjectReference `json:"apiServerRef"`

// InstanceName is the name of the instance this secret is for (e.g. pulsar-instance)
// +optional
InstanceName string `json:"instanceName"`

// Location is the location of the secret.
// +optional
Location string `json:"location"`

// the value should be base64 encoded
// +optional
Data map[string]string `json:"data,omitempty"`

// SecretRef is the reference to the kubernetes secret
// When SecretRef is set, it will be used to fetch the secret data.
// Data will be ignored.
// +optional
SecretRef *KubernetesSecretReference `json:"secretRef,omitempty"`

// PoolMemberName is the pool member to deploy the secret.
// +optional
PoolMemberName *string `json:"poolMemberName,omitempty"`

// Toleration is the toleration for the secret.
// +optional
// +listType=atomic
Tolerations []Toleration `json:"tolerations,omitempty"`

// Type Used to facilitate programmatic handling of secret data.
// +optional
Type *corev1.SecretType `json:"type,omitempty"`
}

// SecretStatus defines the observed state of StreamNative Cloud Secret
type SecretStatus struct {
// Conditions represent the latest available observations of an object's state
// +optional
Conditions []metav1.Condition `json:"conditions,omitempty"`

// ObservedGeneration is the last observed generation.
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:resource:scope=Namespaced,categories={streamnative,all}
//+kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp"
//+kubebuilder:printcolumn:name="READY",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status"

// Secret is the Schema for the StreamNative Cloud Secret API
type Secret struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec SecretSpec `json:"spec,omitempty"`
Status SecretStatus `json:"status,omitempty"`
}

// PoolMemberReference is a reference to a pool member with a given name.
type PoolMemberReference struct {
Namespace string `json:"namespace" protobuf:"bytes,1,opt,name=namespace"`
Name string `json:"name" protobuf:"bytes,2,opt,name=name"`
}

func (r PoolMemberReference) ToNamespacedName() types.NamespacedName {
return types.NamespacedName{
Namespace: r.Namespace,
Name: r.Name,
}
}

// KubernetesSecretReference is a reference to a Kubernetes Secret with a given name.
type KubernetesSecretReference struct {
Namespace string `json:"namespace" protobuf:"bytes,1,opt,name=namespace"`
Name string `json:"name" protobuf:"bytes,2,opt,name=name"`
}

func (r KubernetesSecretReference) ToNamespacedName() types.NamespacedName {
return types.NamespacedName{
Namespace: r.Namespace,
Name: r.Name,
}
}

// Toleration The workload this Toleration is attached to tolerates any taint that matches
// the triple <key,value,effect> using the matching operator <operator>.
type Toleration struct {
// Key is the taint key that the toleration applies to. Empty means match all taint keys.
// If the key is empty, operator must be Exists; this combination means to match all values and all keys.
// +optional
Key string `json:"key,omitempty" protobuf:"bytes,1,opt,name=key"`
// Operator represents a key's relationship to the value.
// Valid operators are Exists and Equal. Defaults to Equal.
// Exists is equivalent to wildcard for value, so that a workload can
// tolerate all taints of a particular category.
// +optional
Operator v1alpha1.TolerationOperator `json:"operator,omitempty" protobuf:"bytes,2,opt,name=operator,casttype=TolerationOperator"`
// Value is the taint value the toleration matches to.
// If the operator is Exists, the value should be empty, otherwise just a regular string.
// +optional
Value string `json:"value,omitempty" protobuf:"bytes,3,opt,name=value"`
// Effect indicates the taint effect to match. Empty means match all taint effects.
// When specified, allowed values are NoSchedule and PreferNoSchedule.
// +optional
Effect v1alpha1.TaintEffect `json:"effect,omitempty" protobuf:"bytes,4,opt,name=effect,casttype=TaintEffect"`
}

//+kubebuilder:object:root=true

// SecretList contains a list of Secret
type SecretList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Secret `json:"items"`
}

func init() {
SchemeBuilder.Register(&Secret{}, &SecretList{})
}
Loading

0 comments on commit 88fdc02

Please sign in to comment.