diff --git a/.gitignore b/.gitignore index de25c7922..8b0e95f71 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ tsconfig.json !/.backportrc.json !/.github/workflows/backport.yml .vscode/ +.idea/ docs/typescript.md docs/python.md docs/java.md diff --git a/.projenrc.ts b/.projenrc.ts index 471994428..e9f245f88 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -76,7 +76,7 @@ const project = new Cdk8sTeamJsiiProject({ // identical to npm defaults. project.package.addField('publishConfig', { access: 'public' }); -project.gitignore.exclude('.vscode/'); +project.gitignore.exclude('.vscode/', '.idea/'); const importdir = path.join('src', 'imports'); diff --git a/docs/plus/volume.md b/docs/plus/volume.md index a71378033..de210ecca 100644 --- a/docs/plus/volume.md +++ b/docs/plus/volume.md @@ -36,3 +36,49 @@ const redis = pod.addContainer({ // mount to the redis container. redis.mount('/var/lib/redis', data); ``` + +## Using PersistentVolumeClaim Templates with StatefulSets + +When working with StatefulSets, you can use PersistentVolumeClaim templates to create stable storage for each pod in the StatefulSet. This allows each pod to have its own storage that persists even if the pod is rescheduled to a different node. + +```typescript +import * as kplus from 'cdk8s-plus-32'; +import { Size } from 'cdk8s'; + +const dataVolume = Volume.fromName(chart, "pvc-template-data", "data-volume") + +// Create a StatefulSet with a PVC template +const statefulSet = new kplus.StatefulSet(this, 'StatefulSet', { + containers: [{ + image: 'nginx', + volumeMounts: [{ + volume: dataVolume, + path: '/data', + }, { + volume: Volume.fromName(chart, "pvc-template-temp", "temp-volume"), + path: '/data', + }], + }], + // Define PVC templates during initialization + volumeClaimTemplates: [{ + name: dataVolume.name, + storage: Size.gibibytes(10), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE], + storageClassName: 'standard', // Optional: Specify the storage class + }, { + name: 'temp-volume', // Must match a volume mount name in a container + storage: Size.gibibytes(10), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE], + storageClassName: 'standard', // Optional: Specify the storage class + }], +}); + +// Or add PVC templates after creation +statefulSet.addVolumeClaimTemplate({ + name: 'logs-volume', + storage: Size.gibibytes(5), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE], +}); +``` + +Each pod in the StatefulSet will get its own PVC instance based on these templates, with names like `data-volume-my-statefulset-0`, `data-volume-my-statefulset-1`, etc. diff --git a/src/pod.ts b/src/pod.ts index e4afe2468..fd1e9bede 100644 --- a/src/pod.ts +++ b/src/pod.ts @@ -486,7 +486,8 @@ export interface AbstractPodProps extends base.ResourceProps { /** * Properties for `Pod`. */ -export interface PodProps extends AbstractPodProps {} +export interface PodProps extends AbstractPodProps { +} /** * Options for `LabelSelector.of`. @@ -515,7 +516,8 @@ export class LabelSelector { private constructor( private readonly expressions: LabelExpression[], - private readonly labels: { [key: string]: string }) {} + private readonly labels: { [key: string]: string }) { + } public isEmpty() { return this.expressions.length === 0 && Object.keys(this.labels).length === 0; @@ -529,7 +531,11 @@ export class LabelSelector { return {}; } return { - matchExpressions: undefinedIfEmpty(this.expressions.map(q => ({ key: q.key, operator: q.operator, values: q.values }))), + matchExpressions: undefinedIfEmpty(this.expressions.map(q => ({ + key: q.key, + operator: q.operator, + values: q.values, + }))), matchLabels: undefinedIfEmpty(this.labels), }; } @@ -1199,10 +1205,10 @@ export interface PodsSelectOptions { readonly labels?: { [key: string]: string }; /** - * Expressions the pods must satisify. - * - * @default - no expressions requirements. - */ + * Expressions the pods must satisify. + * + * @default - no expressions requirements. + */ readonly expressions?: LabelExpression[]; /** @@ -1234,7 +1240,8 @@ export class Pods extends Construct implements IPodSelector { return Pods.select(scope, id, { namespaces: options.namespaces }); } - constructor(scope: Construct, id: string, + constructor( + scope: Construct, id: string, private readonly expressions?: LabelExpression[], private readonly labels?: { [key: string]: string }, private readonly namespaces?: namespace.INamespaceSelector) { @@ -1271,21 +1278,24 @@ export class Pods extends Construct implements IPodSelector { * A node that is matched by label selectors. */ export class LabeledNode { - public constructor(public readonly labelSelector: NodeLabelQuery[]) {}; + public constructor(public readonly labelSelector: NodeLabelQuery[]) { + }; } /** * A node that is matched by taint selectors. */ export class TaintedNode { - public constructor(public readonly taintSelector: NodeTaintQuery[]) {}; + public constructor(public readonly taintSelector: NodeTaintQuery[]) { + }; } /** * A node that is matched by its name. */ export class NamedNode { - public constructor(public readonly name: string) {}; + public constructor(public readonly name: string) { + }; } /** @@ -1361,7 +1371,8 @@ export class Topology { return new Topology(key); } - private constructor(public readonly key: string) {}; + private constructor(public readonly key: string) { + }; } /** @@ -1428,7 +1439,8 @@ export class PodScheduling { private _tolerations: k8s.Toleration[] = []; private _nodeName?: string; - constructor(protected readonly instance: AbstractPod) {} + constructor(protected readonly instance: AbstractPod) { + } /** * Assign this pod a specific node by name. @@ -1690,7 +1702,8 @@ export interface PodConnectionsAllowFromOptions { */ export class PodConnections { - constructor(protected readonly instance: AbstractPod) {} + constructor(protected readonly instance: AbstractPod) { + } /** * Allow network traffic from this pod to the peer. diff --git a/src/stateful-set.ts b/src/stateful-set.ts index 79b8527b1..c4ef0ac70 100644 --- a/src/stateful-set.ts +++ b/src/stateful-set.ts @@ -3,6 +3,8 @@ import { Construct } from 'constructs'; import * as container from './container'; import { IScalable, ScalingTarget } from './horizontal-pod-autoscaler'; import * as k8s from './imports/k8s'; +import { KubePersistentVolumeClaimProps, PodSpec, Quantity, VolumeResourceRequirements } from './imports/k8s'; +import { PersistentVolumeClaimProps } from './pvc'; import * as service from './service'; import * as workload from './workload'; @@ -34,17 +36,17 @@ export interface StatefulSetProps extends workload.WorkloadProps { readonly service?: service.Service; /** - * Number of desired pods. - * - * @default 1 - */ + * Number of desired pods. + * + * @default 1 + */ readonly replicas?: number; /** - * Pod management policy to use for this statefulset. - * - * @default PodManagementPolicy.ORDERED_READY - */ + * Pod management policy to use for this statefulset. + * + * @default PodManagementPolicy.ORDERED_READY + */ readonly podManagementPolicy?: PodManagementPolicy; /** @@ -65,6 +67,16 @@ export interface StatefulSetProps extends workload.WorkloadProps { */ readonly minReady?: Duration; + /** + * A list of PersistentVolumeClaim templates that will be created for each pod in the StatefulSet. + * The StatefulSet controller creates a PVC and a PV for each template based on the pod's ordinal index, + * ensuring stable storage across pod restarts and rescheduling. + * + * Each claim in this list must have at least one matching (by name) volumeMount in one of the containers. + * + * @default - No volume claim templates will be created. + */ + readonly volumeClaimTemplates?: PersistentVolumeClaimTemplateProps[]; } /** @@ -95,13 +107,13 @@ export interface StatefulSetProps extends workload.WorkloadProps { */ export class StatefulSet extends workload.Workload implements IScalable { /** - * Number of desired pods. - */ + * Number of desired pods. + */ public readonly replicas?: number; /** - * Management policy to use for the set. - */ + * Management policy to use for the set. + */ public readonly podManagementPolicy: PodManagementPolicy; /** @@ -116,8 +128,8 @@ export class StatefulSet extends workload.Workload implements IScalable { public readonly minReady: Duration; /** - * @see base.Resource.apiObject - */ + * @see base.Resource.apiObject + */ protected readonly apiObject: ApiObject; public readonly resourceType = 'statefulsets'; @@ -126,6 +138,8 @@ export class StatefulSet extends workload.Workload implements IScalable { public readonly service: service.Service; + public volumeClaimTemplates?: PersistentVolumeClaimTemplateProps[]; + constructor(scope: Construct, id: string, props: StatefulSetProps) { super(scope, id, props); @@ -138,23 +152,35 @@ export class StatefulSet extends workload.Workload implements IScalable { this.apiObject.addDependency(this.service); this.replicas = props.replicas; - this.strategy = props.strategy ?? StatefulSetUpdateStrategy.rollingUpdate(), + this.strategy = props.strategy ?? StatefulSetUpdateStrategy.rollingUpdate(); this.podManagementPolicy = props.podManagementPolicy ?? PodManagementPolicy.ORDERED_READY; this.minReady = props.minReady ?? Duration.seconds(0); - + props?.volumeClaimTemplates?.forEach(template => this.addVolumeClaimTemplate(template)); this.service.select(this); if (this.isolate) { this.connections.isolate(); } + } + public addVolumeClaimTemplate(template: PersistentVolumeClaimTemplateProps) { + if (this.volumeClaimTemplates?.some(t => t.name === template.name)) { + throw new Error(`A volume claim template with name "${template.name}" already exists`); + } + this.volumeClaimTemplates = this.volumeClaimTemplates + ? [...this.volumeClaimTemplates, template] + : [template]; } private _createHeadlessService() { - const myPorts = container.extractContainerPorts(this); const myPortNumbers = myPorts.map(p => p.number); - const ports: service.ServicePort[] = myPorts.map(p => ({ port: p.number, targetPort: p.number, protocol: p.protocol, name: p.name })); + const ports: service.ServicePort[] = myPorts.map(p => ({ + port: p.number, + targetPort: p.number, + protocol: p.protocol, + name: p.name, + })); if (ports.length === 0) { throw new Error(`Unable to create a service for the stateful set ${this.name}: StatefulSet ports cannot be determined.`); } @@ -179,8 +205,8 @@ export class StatefulSet extends workload.Workload implements IScalable { } /** - * @internal - */ + * @internal + */ public _toKube(): k8s.StatefulSetSpec { return { replicas: this.hasAutoscaler ? undefined : (this.replicas ?? 1), @@ -192,10 +218,41 @@ export class StatefulSet extends workload.Workload implements IScalable { }, selector: this._toLabelSelector(), podManagementPolicy: this.podManagementPolicy, + volumeClaimTemplates: this._toPersistentVolumeClaims(), updateStrategy: this.strategy._toKube(), }; } + /** + * @internal + */ + public _toPodSpec(): k8s.PodSpec { + const podSpec = super._toPodSpec(); + return { + ...podSpec, + volumes: this._filterPodSpecVolumes(podSpec), + }; + } + + /** + * @internal + */ + private _filterPodSpecVolumes(podSpec: PodSpec) { + // When using volumeClaimTemplates, the volumes with matching names should not be included in the pod spec + if (this.volumeClaimTemplates && podSpec.volumes) { + const volumeClaimNames = this.volumeClaimTemplates.map(vct => vct.name); + const volumesWithNoTemplates = podSpec.volumes.filter(vol => !volumeClaimNames.includes(vol.name)); + + // If there are no volumes after filtering, don't include volumes property + if (volumesWithNoTemplates.length === 0) { + return undefined; + } else { + return volumesWithNoTemplates; + } + } + return undefined; + } + /** * @see IScalable.markHasAutoscaler() */ @@ -215,6 +272,33 @@ export class StatefulSet extends workload.Workload implements IScalable { replicas: this.replicas, }; } + + /** + * @internal + */ + private _toPersistentVolumeClaims(): KubePersistentVolumeClaimProps[] | undefined { + const volumeNames = this.containers.flatMap(it => it.mounts).map(mount => mount.volume.name); + this.volumeClaimTemplates?.forEach(t => { + if (!volumeNames.includes(t.name)) { + throw new Error(`Volume claim template with name "${t.name}" is not used by any container mount`); + } + }); + return this.volumeClaimTemplates?.map(template => { + const resources: VolumeResourceRequirements = template.storage + ? { requests: { storage: Quantity.fromString(template.storage.asString()) } } + : {}; + return { + metadata: { + name: template.name, + }, + spec: { + accessModes: template.accessModes, + storageClassName: template.storageClassName, + resources, + }, + }; + }); + } } /** @@ -269,7 +353,8 @@ export class StatefulSetUpdateStrategy { }); } - private constructor(private readonly strategy: k8s.StatefulSetUpdateStrategy) {} + private constructor(private readonly strategy: k8s.StatefulSetUpdateStrategy) { + } /** * @internal @@ -279,3 +364,16 @@ export class StatefulSetUpdateStrategy { } } + +/** + * A PersistentVolumeClaim template for StatefulSets + */ +export interface PersistentVolumeClaimTemplateProps extends PersistentVolumeClaimProps { + /** + * The name of the claim that the StatefulSet controller will create for each pod. + * This will be used to name the created PVC in the format - + * + * This name should match the name of a volume mount in one of the containers. + */ + readonly name: string; +} diff --git a/src/volume.ts b/src/volume.ts index bafd4921d..a74ec7e64 100644 --- a/src/volume.ts +++ b/src/volume.ts @@ -1,5 +1,5 @@ import { Names, Size } from 'cdk8s'; -import { IConstruct, Construct } from 'constructs'; +import { Construct, IConstruct } from 'constructs'; import * as configmap from './config-map'; import * as k8s from './imports/k8s'; import * as pvc from './pvc'; @@ -127,7 +127,7 @@ export class Volume extends Construct implements IStorage { * @param configMap The config map to use to populate the volume. * @param options Options */ - public static fromConfigMap(scope: Construct, id: string, configMap: configmap.IConfigMap, options: ConfigMapVolumeOptions = { }): Volume { + public static fromConfigMap(scope: Construct, id: string, configMap: configmap.IConfigMap, options: ConfigMapVolumeOptions = {}): Volume { return new Volume(scope, id, options.name ?? `configmap-${configMap.name}`, { configMap: { name: configMap.name, @@ -150,7 +150,7 @@ export class Volume extends Construct implements IStorage { * * @param options - Additional options. */ - public static fromEmptyDir(scope: Construct, id: string, name: string, options: EmptyDirVolumeOptions = { }): Volume { + public static fromEmptyDir(scope: Construct, id: string, name: string, options: EmptyDirVolumeOptions = {}): Volume { return new Volume(scope, id, name, { emptyDir: { medium: options.medium, @@ -176,7 +176,7 @@ export class Volume extends Construct implements IStorage { * @param secr The secret to use to populate the volume. * @param options Options */ - public static fromSecret(scope: Construct, id: string, secr: secret.ISecret, options: SecretVolumeOptions = { }): Volume { + public static fromSecret(scope: Construct, id: string, secr: secret.ISecret, options: SecretVolumeOptions = {}): Volume { return new Volume(scope, id, options.name ?? `secret-${secr.name}`, { secret: { secretName: secr.name, @@ -194,7 +194,8 @@ export class Volume extends Construct implements IStorage { * * @see https://kubernetes.io/docs/concepts/storage/persistent-volumes/ */ - public static fromPersistentVolumeClaim(scope: Construct, id: string, + public static fromPersistentVolumeClaim( + scope: Construct, id: string, claim: pvc.IPersistentVolumeClaim, options: PersistentVolumeClaimVolumeOptions = {}): Volume { return new Volume(scope, id, options.name ?? `pvc-${claim.name}`, { @@ -246,7 +247,7 @@ export class Volume extends Construct implements IStorage { * @param driver The name of the CSI driver to use to populate the volume. * @param options Options for the CSI volume, including driver-specific ones. */ - public static fromCsi(scope: Construct, id: string, driver: string, options: CsiVolumeOptions = { }): Volume { + public static fromCsi(scope: Construct, id: string, driver: string, options: CsiVolumeOptions = {}): Volume { return new Volume(scope, id, options.name ?? Names.toDnsLabel(scope, { extra: [id] }), { csi: { driver: driver, @@ -258,10 +259,19 @@ export class Volume extends Construct implements IStorage { } /** - * @internal + * Create a volume with an arbitrary name and no configuration. + */ + public static fromName(scope: Construct, id: string, name: string): Volume { + return new Volume(scope, id, name, {}); + } + + /** + * @internal */ private static renderItems = (items?: { [key: string]: PathMapping }): undefined | Array => { - if (!items) { return undefined; } + if (!items) { + return undefined; + } const result = new Array(); for (const key of Object.keys(items).sort()) { result.push({ @@ -274,7 +284,8 @@ export class Volume extends Construct implements IStorage { }; - private constructor(scope: Construct, id: string, + private constructor( + scope: Construct, id: string, public readonly name: string, private readonly config: Omit) { super(scope, id); @@ -751,4 +762,4 @@ export interface CsiVolumeOptions { * @default - undefined */ readonly attributes?: { [key: string]: string }; -} \ No newline at end of file +} diff --git a/test/__snapshots__/statefulset.test.ts.snap b/test/__snapshots__/statefulset.test.ts.snap index 92e3117c7..4d07c492c 100644 --- a/test/__snapshots__/statefulset.test.ts.snap +++ b/test/__snapshots__/statefulset.test.ts.snap @@ -212,3 +212,172 @@ Array [ }, ] `; + +exports[`volumeClaimTemplates 1`] = ` +Array [ + Object { + "apiVersion": "v1", + "immutable": false, + "kind": "Secret", + "metadata": Object { + "name": "test-awssecret-c8d3e80d", + }, + "stringData": Object {}, + }, + Object { + "apiVersion": "v1", + "kind": "Service", + "metadata": Object { + "name": "test-statefulset-service-c8d576f5", + }, + "spec": Object { + "clusterIP": "None", + "externalIPs": Array [], + "ports": Array [ + Object { + "port": 80, + "targetPort": 80, + }, + ], + "selector": Object { + "cdk8s.io/metadata.addr": "test-StatefulSet-c809b559", + }, + "type": "ClusterIP", + }, + }, + Object { + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": Object { + "name": "test-statefulset-c8a6ec86", + }, + "spec": Object { + "minReadySeconds": 0, + "podManagementPolicy": "OrderedReady", + "replicas": 1, + "selector": Object { + "matchLabels": Object { + "cdk8s.io/metadata.addr": "test-StatefulSet-c809b559", + }, + }, + "serviceName": "test-statefulset-service-c8d576f5", + "template": Object { + "metadata": Object { + "labels": Object { + "cdk8s.io/metadata.addr": "test-StatefulSet-c809b559", + }, + }, + "spec": Object { + "automountServiceAccountToken": false, + "containers": Array [ + Object { + "image": "foobar", + "imagePullPolicy": "Always", + "name": "main", + "ports": Array [ + Object { + "containerPort": 80, + }, + ], + "resources": Object { + "limits": Object { + "cpu": "1500m", + "memory": "2048Mi", + }, + "requests": Object { + "cpu": "1000m", + "memory": "512Mi", + }, + }, + "securityContext": Object { + "allowPrivilegeEscalation": false, + "privileged": false, + "readOnlyRootFilesystem": true, + "runAsNonRoot": true, + }, + "startupProbe": Object { + "failureThreshold": 3, + "tcpSocket": Object { + "port": 80, + }, + }, + "volumeMounts": Array [ + Object { + "mountPath": "/mnt/data", + "name": "data", + }, + Object { + "mountPath": "/mnt/temp", + "name": "temp", + }, + Object { + "mountPath": "/mnt/secret", + "name": "secret-test-awssecret-c8d3e80d", + }, + ], + }, + ], + "dnsPolicy": "ClusterFirst", + "hostNetwork": false, + "restartPolicy": "Always", + "securityContext": Object { + "fsGroupChangePolicy": "Always", + "runAsNonRoot": true, + }, + "setHostnameAsFQDN": false, + "shareProcessNamespace": false, + "terminationGracePeriodSeconds": 30, + "volumes": Array [ + Object { + "name": "secret-test-awssecret-c8d3e80d", + "secret": Object { + "secretName": "test-awssecret-c8d3e80d", + }, + }, + ], + }, + }, + "updateStrategy": Object { + "rollingUpdate": Object { + "partition": 0, + }, + "type": "RollingUpdate", + }, + "volumeClaimTemplates": Array [ + Object { + "metadata": Object { + "name": "data", + }, + "spec": Object { + "accessModes": Array [ + "ReadWriteOncePod", + ], + "resources": Object { + "requests": Object { + "storage": "20Gi", + }, + }, + "storageClassName": "standard", + }, + }, + Object { + "metadata": Object { + "name": "temp", + }, + "spec": Object { + "accessModes": Array [ + "ReadWriteOncePod", + ], + "resources": Object { + "requests": Object { + "storage": "20Gi", + }, + }, + "storageClassName": "standard", + }, + }, + ], + }, + }, +] +`; diff --git a/test/statefulset.test.ts b/test/statefulset.test.ts index b80d1b604..3cd414272 100644 --- a/test/statefulset.test.ts +++ b/test/statefulset.test.ts @@ -1,7 +1,7 @@ -import { Testing, ApiObject, Duration } from 'cdk8s'; +import { Testing, ApiObject, Duration, Size } from 'cdk8s'; import { Node } from 'constructs'; import * as kplus from '../src'; -import { StatefulSetUpdateStrategy, k8s } from '../src'; +import { StatefulSetUpdateStrategy, k8s, Volume } from '../src'; test('defaultChild', () => { @@ -205,4 +205,89 @@ test('Can be isolated', () => { const networkPolicy = manifest[2].spec; expect(networkPolicy.podSelector.matchLabels).toBeDefined; expect(networkPolicy.policyTypes).toEqual(['Egress', 'Ingress']); -}); \ No newline at end of file +}); + +test('volumeClaimTemplates', () => { + const chart = Testing.chart(); + const secret = new kplus.Secret(chart, 'AwsSecret'); + new kplus.StatefulSet(chart, 'StatefulSet', { + containers: [ + { + image: 'foobar', + portNumber: 80, + volumeMounts: [ + { + volume: Volume.fromName(chart, 'data', 'data'), + path: '/mnt/data', + }, { + volume: Volume.fromName(chart, 'temp', 'temp'), + path: '/mnt/temp', + }, { + volume: Volume.fromSecret(chart, 'secret', secret), + path: '/mnt/secret', + }, + ], + }, + ], + volumeClaimTemplates: [{ + name: 'data', + storage: Size.gibibytes(20), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE_POD], + storageClassName: 'standard', + }], + }).addVolumeClaimTemplate({ + name: 'temp', + storage: Size.gibibytes(20), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE_POD], + storageClassName: 'standard', + }); + + const synthesized = Testing.synth(chart); + const spec: k8s.StatefulSetSpec = synthesized[2].spec; + expect(spec.template.spec?.volumes).toEqual([{ + name: 'secret-test-awssecret-c8d3e80d', + secret: { secretName: 'test-awssecret-c8d3e80d' }, + }]); + expect(spec.volumeClaimTemplates).toEqual([ + { + metadata: { name: 'data' }, + spec: { + accessModes: ['ReadWriteOncePod'], + resources: { requests: { storage: '20Gi' } }, + storageClassName: 'standard', + }, + }, + { + metadata: { name: 'temp' }, + spec: { + accessModes: ['ReadWriteOncePod'], + resources: { requests: { storage: '20Gi' } }, + storageClassName: 'standard', + }, + }, + ]); + + expect(Testing.synth(chart)).toMatchSnapshot(); +}); + +test('missing volumeMount for volumeClaimTemplate', () => { + const chart = Testing.chart(); + new kplus.StatefulSet(chart, 'StatefulSet', { + containers: [ + { + image: 'foobar', + portNumber: 80, + }, + ], + volumeClaimTemplates: [{ + name: 'data', + storage: Size.gibibytes(20), + accessModes: [kplus.PersistentVolumeAccessMode.READ_WRITE_ONCE_POD], + storageClassName: 'standard', + }], + }); + + expect(() => { + Testing.synth(chart); + }).toThrow('Volume claim template with name "data" is not used by any container mount'); +});