Skip to content

Commit 010080f

Browse files
pcfreak30pcfreak30
committed
feat(cluster): expose NodePorts via environment file
This commit adds a configuration system that discovers and exposes NodePorts: - Add init container that detects service NodePorts and writes to config file - Replace static AKASH_EXTERNAL_PORT env var with dynamic config file - Mount config volume in containers to access port information - Add RBAC permissions for pods to query their own service details - Support fallback for different service types (NodePort, LoadBalancer, ClusterIP)
1 parent a23c32e commit 010080f

File tree

7 files changed

+368
-22
lines changed

7 files changed

+368
-22
lines changed

‎cluster/kube/builder/builder.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ const (
5959
envVarAkashClusterPublicHostname = "AKASH_CLUSTER_PUBLIC_HOSTNAME"
6060
envVarAkashIngressHostname = "AKASH_INGRESS_HOST"
6161
envVarAkashIngressCustomHostname = "AKASH_INGRESS_CUSTOM_HOST"
62-
envVarAkashExternalPort = "AKASH_EXTERNAL_PORT"
6362
)
6463

6564
var (

‎cluster/kube/builder/config.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package builder
2+
3+
const (
4+
// Config system constants
5+
AkashConfigVolume = "akash-cfg"
6+
AkashConfigMount = "/akash-cfg"
7+
AkashConfigInitName = "akash-init"
8+
AkashConfigEnvFile = "config.env"
9+
10+
// RBAC constants
11+
AkashRoleName = "akash-role"
12+
AkashRoleBinding = "akash-binding"
13+
14+
// Init container script
15+
akashInitScript = `
16+
# Install jq
17+
apk add --no-cache jq curl &>/dev/null
18+
19+
# Define default paths if not set
20+
AKASH_CONFIG_PATH="${AKASH_CONFIG_PATH:-/akash/config}"
21+
AKASH_CONFIG_FILE="${AKASH_CONFIG_FILE:-env.sh}"
22+
23+
# Validate paths
24+
[ "$AKASH_CONFIG_PATH" = "/" ] && AKASH_CONFIG_PATH="/tmp/akash"
25+
AKASH_CONFIG_PATH="${AKASH_CONFIG_PATH%/}"
26+
27+
# Create config directory if it doesn't exist
28+
mkdir -p "${AKASH_CONFIG_PATH}"
29+
30+
if [ "$AKASH_REQUIRES_NODEPORT" != "true" ]; then
31+
touch "${AKASH_CONFIG_PATH}/${AKASH_CONFIG_FILE}"
32+
echo "# No NodePorts required" >> "${AKASH_CONFIG_PATH}/${AKASH_CONFIG_FILE}"
33+
exit 0
34+
fi
35+
36+
# Get service information using the Kubernetes API
37+
NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
38+
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
39+
CACERT=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
40+
API_SERVER="https://kubernetes.default.svc:443"
41+
BASE_API_URL="${API_SERVER}/api/v1/namespaces/${NAMESPACE}"
42+
43+
# Function to get first valid service using jq
44+
get_valid_service() {
45+
local service_name="$1"
46+
local services_json
47+
48+
services_json=$(curl -s --cacert "${CACERT}" -H "Authorization: Bearer ${TOKEN}" "${BASE_API_URL}/services/")
49+
50+
echo "$services_json" | jq -r "
51+
(.items[] | select(.metadata.name == \"${service_name}\") | .metadata.name) //
52+
(.items[] | select(.metadata.name == \"${service_name}-np\") | .metadata.name) //
53+
(.items[] | select(.metadata.name | contains(\"${service_name}\")) | .metadata.name) //
54+
empty
55+
" | head -n 1
56+
}
57+
58+
# Get the valid service name
59+
ACTUAL_SERVICE_NAME=$(get_valid_service "${SERVICE_NAME}")
60+
61+
[ -z "$ACTUAL_SERVICE_NAME" ] && ACTUAL_SERVICE_NAME="${SERVICE_NAME}"
62+
63+
API_URL="${BASE_API_URL}/services/${ACTUAL_SERVICE_NAME}"
64+
TEMP_FILE="${AKASH_CONFIG_PATH}/.tmp.${AKASH_CONFIG_FILE}"
65+
CONFIG_FILE="${AKASH_CONFIG_PATH}/${AKASH_CONFIG_FILE}"
66+
67+
# Create initial config file header
68+
echo "# Akash config generated on $(date)" > "$TEMP_FILE"
69+
echo "# Service: ${ACTUAL_SERVICE_NAME}" >> "$TEMP_FILE"
70+
71+
# Add retries with exponential backoff
72+
MAX_ATTEMPTS=30
73+
for i in $(seq 1 $MAX_ATTEMPTS); do
74+
# Query the service to get NodePort mappings
75+
RESPONSE=$(curl -s --max-time 5 --retry 3 --retry-delay 1 --cacert "${CACERT}" \
76+
-H "Authorization: Bearer ${TOKEN}" \
77+
"${API_URL}")
78+
79+
# Check service type first
80+
SERVICE_TYPE=$(echo "$RESPONSE" | jq -r '.spec.type // "unknown"')
81+
82+
if [ "$SERVICE_TYPE" = "NodePort" ]; then
83+
# Service is NodePort, extract nodePort values
84+
NODE_PORTS=$(echo "$RESPONSE" | jq -r '.spec.ports[] | select(.nodePort != null) | "export AKASH_EXTERNAL_PORT_\(.targetPort)+=\(.nodePort)"' 2>/dev/null || echo "")
85+
86+
if [ -n "$NODE_PORTS" ]; then
87+
echo "$NODE_PORTS" >> "$TEMP_FILE"
88+
mv "$TEMP_FILE" "$CONFIG_FILE"
89+
exit 0
90+
fi
91+
elif [ "$SERVICE_TYPE" = "LoadBalancer" ]; then
92+
# Service is LoadBalancer, check for external IPs
93+
EXTERNAL_IPS=$(echo "$RESPONSE" | jq -r '.status.loadBalancer.ingress[]?.ip // empty' 2>/dev/null || echo "")
94+
if [ -n "$EXTERNAL_IPS" ]; then
95+
echo "export AKASH_EXTERNAL_IP+=${EXTERNAL_IPS}" >> "$TEMP_FILE"
96+
mv "$TEMP_FILE" "$CONFIG_FILE"
97+
exit 0
98+
fi
99+
elif [ "$SERVICE_TYPE" = "ClusterIP" ]; then
100+
# Service is ClusterIP with dedicated IP
101+
echo "# Service type is ClusterIP with dedicated IP" >> "$TEMP_FILE"
102+
103+
# Get service ports for reference
104+
PORTS=$(echo "$RESPONSE" | jq -r '.spec.ports[] | "# Port \(.port) -> \(.targetPort)"' 2>/dev/null || echo "# No port mappings found")
105+
echo "$PORTS" >> "$TEMP_FILE"
106+
107+
# Move to final file after waiting for some time, in case it's still being configured
108+
if [ $i -gt 5 ]; then
109+
mv "$TEMP_FILE" "$CONFIG_FILE"
110+
exit 0
111+
fi
112+
fi
113+
114+
# Exponential backoff with max of 10 seconds
115+
SLEEP_TIME=$((2 ** ((i-1) > 3 ? 3 : (i-1))))
116+
sleep $SLEEP_TIME
117+
done
118+
119+
# Create empty config file to prevent container from failing
120+
echo "# Warning: Service configuration timeout after $MAX_ATTEMPTS attempts" >> "$TEMP_FILE"
121+
echo "# Service type: ${SERVICE_TYPE}" >> "$TEMP_FILE"
122+
mv "$TEMP_FILE" "$CONFIG_FILE"
123+
exit 0
124+
`
125+
)

‎cluster/kube/builder/deployment.go‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package builder
22

33
import (
4+
"strconv"
5+
46
appsv1 "k8s.io/api/apps/v1"
57
corev1 "k8s.io/api/core/v1"
68
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,12 +33,66 @@ func NewDeployment(workload Workload) Deployment {
3133

3234
func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:golint,unparam
3335
falseValue := false
36+
trueValue := true
3437

3538
revisionHistoryLimit := int32(10)
3639

3740
maxSurge := intstr.FromInt32(0)
3841
maxUnavailable := intstr.FromInt32(1)
3942

43+
// Add config volume
44+
configVolume := corev1.Volume{
45+
Name: AkashConfigVolume,
46+
VolumeSource: corev1.VolumeSource{
47+
EmptyDir: &corev1.EmptyDirVolumeSource{},
48+
},
49+
}
50+
51+
// Calculate if NodePort is required
52+
requiresNodePort := false
53+
service := &b.deployment.ManifestGroup().Services[b.serviceIdx]
54+
for _, expose := range service.Expose {
55+
if expose.Global {
56+
requiresNodePort = true
57+
break
58+
}
59+
}
60+
61+
// Add init container
62+
initContainer := corev1.Container{
63+
Name: AkashConfigInitName,
64+
Image: "alpine/curl:3.14",
65+
Command: []string{
66+
"/bin/sh",
67+
"-c",
68+
akashInitScript,
69+
},
70+
Env: []corev1.EnvVar{
71+
{
72+
Name: "SERVICE_NAME",
73+
Value: b.Name(),
74+
},
75+
{
76+
Name: "AKASH_CONFIG_PATH",
77+
Value: AkashConfigMount,
78+
},
79+
{
80+
Name: "AKASH_CONFIG_FILE",
81+
Value: AkashConfigEnvFile,
82+
},
83+
{
84+
Name: "AKASH_REQUIRES_NODEPORT",
85+
Value: strconv.FormatBool(requiresNodePort),
86+
},
87+
},
88+
VolumeMounts: []corev1.VolumeMount{
89+
{
90+
Name: AkashConfigVolume,
91+
MountPath: AkashConfigMount,
92+
},
93+
},
94+
}
95+
4096
kdeployment := &appsv1.Deployment{
4197
ObjectMeta: metav1.ObjectMeta{
4298
Name: b.Name(),
@@ -65,10 +121,11 @@ func (b *deployment) Create() (*appsv1.Deployment, error) { // nolint:golint,unp
65121
SecurityContext: &corev1.PodSecurityContext{
66122
RunAsNonRoot: &falseValue,
67123
},
68-
AutomountServiceAccountToken: &falseValue,
69-
Containers: []corev1.Container{b.container()},
70-
ImagePullSecrets: b.secretsRefs,
71-
Volumes: b.volumesObjs,
124+
AutomountServiceAccountToken: &trueValue,
125+
InitContainers: []corev1.Container{initContainer},
126+
Containers: []corev1.Container{b.container()},
127+
ImagePullSecrets: b.secretsRefs,
128+
Volumes: append(b.volumesObjs, configVolume),
72129
},
73130
},
74131
},

‎cluster/kube/builder/rbac.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package builder
2+
3+
import (
4+
rbacv1 "k8s.io/api/rbac/v1"
5+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
6+
)
7+
8+
// Create role that only grants access to specific service
9+
func CreateRole(namespace string, serviceName string) *rbacv1.Role {
10+
return &rbacv1.Role{
11+
ObjectMeta: metav1.ObjectMeta{
12+
Name: AkashRoleName,
13+
Namespace: namespace,
14+
},
15+
Rules: []rbacv1.PolicyRule{
16+
{
17+
APIGroups: []string{""},
18+
Resources: []string{"services", "services/status"},
19+
Verbs: []string{"get", "list", "watch"},
20+
},
21+
},
22+
}
23+
}
24+
25+
// Create role binding between service account and role
26+
func CreateRoleBinding(namespace string) *rbacv1.RoleBinding {
27+
return &rbacv1.RoleBinding{
28+
ObjectMeta: metav1.ObjectMeta{
29+
Name: AkashRoleBinding,
30+
Namespace: namespace,
31+
},
32+
Subjects: []rbacv1.Subject{
33+
{
34+
Kind: "ServiceAccount",
35+
Name: "default",
36+
Namespace: namespace,
37+
},
38+
},
39+
RoleRef: rbacv1.RoleRef{
40+
APIGroup: "rbac.authorization.k8s.io",
41+
Kind: "Role",
42+
Name: AkashRoleName,
43+
},
44+
}
45+
}

‎cluster/kube/builder/statefulset.go‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
corev1 "k8s.io/api/core/v1"
66
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
77
"k8s.io/apimachinery/pkg/util/intstr"
8+
"strconv"
89
)
910

1011
type StatefulSet interface {
@@ -31,12 +32,66 @@ func BuildStatefulSet(workload Workload) StatefulSet {
3132

3233
func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:golint,unparam
3334
falseValue := false
35+
trueValue := true
3436

3537
revisionHistoryLimit := int32(1)
3638

3739
partition := int32(0)
3840
maxUnavailable := intstr.FromInt32(1)
3941

42+
// Add config volume
43+
configVolume := corev1.Volume{
44+
Name: AkashConfigVolume,
45+
VolumeSource: corev1.VolumeSource{
46+
EmptyDir: &corev1.EmptyDirVolumeSource{},
47+
},
48+
}
49+
50+
// Calculate if NodePort is required
51+
requiresNodePort := false
52+
service := &b.deployment.ManifestGroup().Services[b.serviceIdx]
53+
for _, expose := range service.Expose {
54+
if expose.Global {
55+
requiresNodePort = true
56+
break
57+
}
58+
}
59+
60+
// Add init container
61+
initContainer := corev1.Container{
62+
Name: AkashConfigInitName,
63+
Image: "alpine/curl:3.14",
64+
Command: []string{
65+
"/bin/sh",
66+
"-c",
67+
akashInitScript,
68+
},
69+
Env: []corev1.EnvVar{
70+
{
71+
Name: "SERVICE_NAME",
72+
Value: b.Name(),
73+
},
74+
{
75+
Name: "AKASH_CONFIG_PATH",
76+
Value: AkashConfigMount,
77+
},
78+
{
79+
Name: "AKASH_CONFIG_FILE",
80+
Value: AkashConfigEnvFile,
81+
},
82+
{
83+
Name: "AKASH_REQUIRES_NODEPORT",
84+
Value: strconv.FormatBool(requiresNodePort),
85+
},
86+
},
87+
VolumeMounts: []corev1.VolumeMount{
88+
{
89+
Name: AkashConfigVolume,
90+
MountPath: AkashConfigMount,
91+
},
92+
},
93+
}
94+
4095
kdeployment := &appsv1.StatefulSet{
4196
ObjectMeta: metav1.ObjectMeta{
4297
Name: b.Name(),
@@ -65,10 +120,11 @@ func (b *statefulSet) Create() (*appsv1.StatefulSet, error) { // nolint:golint,u
65120
SecurityContext: &corev1.PodSecurityContext{
66121
RunAsNonRoot: &falseValue,
67122
},
68-
AutomountServiceAccountToken: &falseValue,
123+
AutomountServiceAccountToken: &trueValue,
124+
InitContainers: []corev1.Container{initContainer},
69125
Containers: []corev1.Container{b.container()},
70126
ImagePullSecrets: b.secretsRefs,
71-
Volumes: b.volumesObjs,
127+
Volumes: append(b.volumesObjs, configVolume),
72128
},
73129
},
74130
VolumeClaimTemplates: b.pvcsObjs,

0 commit comments

Comments
 (0)