Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
SHELL = bash

PKG_VERSION ?= v1.4.2
PKG_VERSION ?= v1.4.3
OCI_DRIVER_VERSION ?= v1.32.0

PRE_COMMIT := $(shell command -v pre-commit 2> /dev/null)
Expand Down Expand Up @@ -66,6 +66,13 @@ ifdef PODMAN
@echo '---' >> custom_manifests/manifests/05-oci-eval-user-data.yml
@podman run -i --rm quay.io/coreos/butane:release --pretty --strict < custom_manifests/butane/oci-eval-user-data-worker.bu >> custom_manifests/manifests/05-oci-eval-user-data.yml
@echo '---' >> custom_manifests/manifests/05-oci-eval-user-data.yml

@podman run -i --rm quay.io/coreos/butane:release --pretty --strict < custom_manifests/butane/vlan-bm-mtu-configure-master.bu > custom_manifests/manifests/07-configure-bm-vlan-mtu.yml
@echo '---' >> custom_manifests/manifests/07-configure-bm-vlan-mtu.yml
@podman run -i --rm quay.io/coreos/butane:release --pretty --strict < custom_manifests/butane/vlan-bm-mtu-configure-worker.bu >> custom_manifests/manifests/07-configure-bm-vlan-mtu.yml
@echo '---' >> custom_manifests/manifests/07-configure-bm-vlan-mtu.yml


else
$(warning podman not installed. Skipping...)
endif
Expand Down
113 changes: 113 additions & 0 deletions custom_manifests/butane/vlan-bm-mtu-configure-master.bu
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
variant: openshift
version: 4.16.0
metadata:
name: 00-master-bm-vlan-mtu-9000
labels:
machineconfiguration.openshift.io/role: master
storage:
files:
- mode: 0755
path: "/usr/local/sbin/bm-mtu-configure.sh"
contents:
inline: |
#!/bin/bash
# Configure MTU=9000 on VLAN-backed bare metal interfaces
# This script runs at boot to ensure OVN-Kubernetes has sufficient MTU
# for its Geneve overlay (8900+ 58 bytes encapsulation = 8958 minimum)
#
# Problem: Bare metal shapes with VLAN-backed networking (like BM.Standard.E5.192,
# BM.Standard.E6.256) default to MTU 1500 on RHCOS, causing OVN to fail.
#
# Solution: Dynamically detect VLAN interfaces and set MTU=9000 on parent+VLAN.
# Uses ip commands directly (no NetworkManager dependency).
# Includes guards to skip VM workers and already-configured nodes.

set -euo pipefail

# Guard 1: No VLAN interfaces => no-op (virtual workers)
vlans=$(ip -o link show type vlan | awk '{print $2}' | sed 's/:$//' | sed 's/@.*//' || true)
if [ -z "$vlans" ]; then
echo "[INFO] No VLAN interfaces found, exiting (no-op for VM workers)"
exit 0
fi

echo "[INFO] Found VLAN interfaces: $vlans"

# Process each VLAN interface
for vlan in $vlans; do
# Extract parent interface name
# VLAN interfaces are named: parent.vlanid or parent.vlanid@parent
# Strip everything from first dot onwards to get parent
parent=$(echo "$vlan" | sed 's/\..*//')

if [ -z "$parent" ] || [ "$parent" = "$vlan" ]; then
echo "[WARN] Could not determine parent for $vlan, skipping"
continue
fi

# Get current MTU from actual devices (not NetworkManager)
parent_mtu=$(ip link show "$parent" 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)
vlan_mtu=$(ip link show "$vlan" 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)

echo "[INFO] $parent: MTU=$parent_mtu, $vlan: MTU=$vlan_mtu"

# Guard 2: Already jumbo => no-op (idempotent)
if [ "$parent_mtu" -ge 9000 ] && [ "$vlan_mtu" -ge 9000 ]; then
echo "[INFO] $parent and $vlan already have MTU >= 9000, no changes needed"
continue
fi

echo "[ACTION] Setting MTU=9000 on $parent and $vlan"

# Set MTU on parent device (if needed)
if [ "$parent_mtu" -lt 9000 ]; then
ip link set dev "$parent" mtu 9000
echo "[SUCCESS] Set MTU=9000 on $parent"
fi

# Set MTU on VLAN device (if needed)
if [ "$vlan_mtu" -lt 9000 ]; then
ip link set dev "$vlan" mtu 9000
echo "[SUCCESS] Set MTU=9000 on $vlan"
fi

echo "[SUCCESS] MTU configuration complete for $parent and $vlan"
done

# Also set br-ex MTU if it exists (OVS bridge created by ovs-configuration.service)
# This handles the case where the script runs after OVS has started
if ip link show br-ex &>/dev/null; then
br_ex_mtu=$(ip link show br-ex 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)
if [ "$br_ex_mtu" -lt 9000 ]; then
echo "[ACTION] Setting MTU=9000 on br-ex"
if command -v ovs-vsctl &>/dev/null; then
ovs-vsctl set Interface br-ex mtu_request=9000 && echo "[SUCCESS] Set MTU=9000 on br-ex"
fi
else
echo "[INFO] br-ex already has MTU >= 9000"
fi
fi

echo "[INFO] bm-mtu-configure.sh completed successfully"
exit 0
systemd:
units:
- name: bm-mtu-configure.service
enabled: true
contents: |
[Unit]
Description=Configure MTU=9000 on VLAN-backed bare metal interfaces
Documentation=https://github.com/oracle-quickstart/oci-openshift/issues/78
After=NetworkManager.service oci-eval-user-data.service
Before=ovs-configuration.service kubelet.service
ConditionPathExists=/usr/local/sbin/bm-mtu-configure.sh

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/bm-mtu-configure.sh
RemainAfterExit=yes
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
113 changes: 113 additions & 0 deletions custom_manifests/butane/vlan-bm-mtu-configure-worker.bu
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
variant: openshift
version: 4.16.0
metadata:
name: 00-worker-bm-vlan-mtu-9000
labels:
machineconfiguration.openshift.io/role: worker
storage:
files:
- mode: 0755
path: "/usr/local/sbin/bm-mtu-configure.sh"
contents:
inline: |
#!/bin/bash
# Configure MTU=9000 on VLAN-backed bare metal interfaces
# This script runs at boot to ensure OVN-Kubernetes has sufficient MTU
# for its Geneve overlay (8900+ 58 bytes encapsulation = 8958 minimum)
#
# Problem: Bare metal shapes with VLAN-backed networking (like BM.Standard.E5.192,
# BM.Standard.E6.256) default to MTU 1500 on RHCOS, causing OVN to fail.
#
# Solution: Dynamically detect VLAN interfaces and set MTU=9000 on parent+VLAN.
# Uses ip commands directly (no NetworkManager dependency).
# Includes guards to skip VM workers and already-configured nodes.

set -euo pipefail

# Guard 1: No VLAN interfaces => no-op (virtual workers)
vlans=$(ip -o link show type vlan | awk '{print $2}' | sed 's/:$//' | sed 's/@.*//' || true)
if [ -z "$vlans" ]; then
echo "[INFO] No VLAN interfaces found, exiting (no-op for VM workers)"
exit 0
fi

echo "[INFO] Found VLAN interfaces: $vlans"

# Process each VLAN interface
for vlan in $vlans; do
# Extract parent interface name
# VLAN interfaces are named: parent.vlanid or parent.vlanid@parent
# Strip everything from first dot onwards to get parent
parent=$(echo "$vlan" | sed 's/\..*//')

if [ -z "$parent" ] || [ "$parent" = "$vlan" ]; then
echo "[WARN] Could not determine parent for $vlan, skipping"
continue
fi

# Get current MTU from actual devices (not NetworkManager)
parent_mtu=$(ip link show "$parent" 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)
vlan_mtu=$(ip link show "$vlan" 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)

echo "[INFO] $parent: MTU=$parent_mtu, $vlan: MTU=$vlan_mtu"

# Guard 2: Already jumbo => no-op (idempotent)
if [ "$parent_mtu" -ge 9000 ] && [ "$vlan_mtu" -ge 9000 ]; then
echo "[INFO] $parent and $vlan already have MTU >= 9000, no changes needed"
continue
fi

echo "[ACTION] Setting MTU=9000 on $parent and $vlan"

# Set MTU on parent device (if needed)
if [ "$parent_mtu" -lt 9000 ]; then
ip link set dev "$parent" mtu 9000
echo "[SUCCESS] Set MTU=9000 on $parent"
fi

# Set MTU on VLAN device (if needed)
if [ "$vlan_mtu" -lt 9000 ]; then
ip link set dev "$vlan" mtu 9000
echo "[SUCCESS] Set MTU=9000 on $vlan"
fi

echo "[SUCCESS] MTU configuration complete for $parent and $vlan"
done

# Also set br-ex MTU if it exists (OVS bridge created by ovs-configuration.service)
# This handles the case where the script runs after OVS has started
if ip link show br-ex &>/dev/null; then
br_ex_mtu=$(ip link show br-ex 2>/dev/null | grep -oP 'mtu \K[0-9]+' || echo 0)
if [ "$br_ex_mtu" -lt 9000 ]; then
echo "[ACTION] Setting MTU=9000 on br-ex"
if command -v ovs-vsctl &>/dev/null; then
ovs-vsctl set Interface br-ex mtu_request=9000 && echo "[SUCCESS] Set MTU=9000 on br-ex"
fi
else
echo "[INFO] br-ex already has MTU >= 9000"
fi
fi

echo "[INFO] bm-mtu-configure.sh completed successfully"
exit 0
systemd:
units:
- name: bm-mtu-configure.service
enabled: true
contents: |
[Unit]
Description=Configure MTU=9000 on VLAN-backed bare metal interfaces
Documentation=https://github.com/oracle-quickstart/oci-openshift/issues/78
After=NetworkManager.service oci-eval-user-data.service
Before=ovs-configuration.service kubelet.service
ConditionPathExists=/usr/local/sbin/bm-mtu-configure.sh

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/bm-mtu-configure.sh
RemainAfterExit=yes
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
80 changes: 80 additions & 0 deletions custom_manifests/manifests/07-configure-bm-vlan-mtu.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# Generated by Butane; do not edit
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
labels:
machineconfiguration.openshift.io/role: master
name: 00-master-bm-vlan-mtu-9000
spec:
config:
ignition:
version: 3.4.0
storage:
files:
- contents:
compression: gzip
source: data:;base64,H4sIAAAAAAAC/5xW71PbuBb97r/ivISB5JWElBk6kE46pby2j+kjdBrgfaCZjmJdx1psyasfgeyy//uOZCd2CN1tN59iWzr33HPPvVL7XwczIQ9mzKRRG2dKJmLuNOHi6np0MhgMoCRu/nc67s1YfEccM6YJOVmWQUhLOmExmaiNq1QYmFiLwkI7acAsZkpZWAWSxkNe3ox7n9yMtCRLBikzMC5JRCxIWh8waiNRGsIafCRJC4JakM7YEp3jk8HgBY6OMVv6vSRjVhiXMSuUxAjHJ0fHyIUUucu7UTtq47NWs4zyId7VhE3KCjK4FzbdyEmSvVf6Tsg5Opm4I7y76E8sk5xp3n9/1H95crgftbH+bXx+1T88etUFp4S5LKR7cXWNl0eldF/+e3Y52UfMnPHwlzdjvyJhIusHmhOVOZ/DEP9ZSpaLmGXZEpwsxTZwbKgMJjkM2Y3aFEyTtC/80n6T47UhA1EgVnnOJDfgQlNssyU6UmFcZnzBJJuTBqeCJCcZL7sbIOcyzhwng7ljmhtP3dyJAjcX8NtJl5xYponxZS9euYdDKk6mH0Webo+cQiEK8mlHURsfPRpeDjFWWzmO3kCqnirQWQhtHctWkbrRImPSjHY6okBPIRPyDiZV97DLguA/4hHs/g57vxdaSIudwz/28AhDHHvmYLhzcNB4fNv/d3h+hNWOupFIcIveb2jthDAtTF/DpiQjgOJUoXV7Pv5wOX2OcqKc5PugB2GDhUr+3sm1Tt2WB3oQFoMoEVG0gfnBAzyFHaJiEpVujskYEIvTJwsjHyhkL2S15TW4ioA23j9YzWJbmaTeA8lyCiu2LKbLj3xYbep7RMGh9OaLt+VTAJlYLQrQgvTSpl6CRKscidDGgisLJe9X9pnTik2E6s9op1OqEci36hJ97YcadaMIqMtTbmph6mt323gxWiPUpVsX7/+nX8ZTnCmXeW/a0GE6F5JW2ngVw/b9YPFCyHkrAMRKWiGdl8sXzuf7kSxip3U1tsp0WRzcymkhvJAdH2WzzbrrlL/l1pVGrl1cJ3L45oDT4kC6LMMj5pq83z9jL7cOXz/dDnon0xfBuSG3gYf1xJ8HLRX5acintq/IDcPs2amz2C9Fq96vaLQqnUKfHw5xWg4I/OLymWq0uOCUF8qStN1VjVsN8BZ6c0KYdFPs7oav6xCNb9vl3iAdZlTYuJpUSNkiHHF4MwoY+5AKccrknAwkESf+nepXAU7Prs4vx1NMyIamb47krbCVHJNydNdTuzILOiKpgn5Phsxup7ouNAWghoF8Vf3ypiKT67Oz95PJdMXiKdtWw98NomE+/BXNRj1+jGRpyJ+jWIr4pATr5Z7q6ugpLwSxyouMLJVdvV0OriT5sXqaGRWozXSPHgKQSCCsn9TGGnQubyaYacHnhFgTs/4CtIRamN5GwL4h7SXqri5CKZM8I+N1QMwM4T4lTeFx446UWNLwQcJlyDJtifujaKOJS3K7dQuv9Z3pb/TwXN+XW/7BIKmqugb+Tll/oAsChbKJRLK6hqC3COotTGyzZ1Pyv3qFr835+tQqs8qt+6bpV0fGlsF2d//WQQ0yifAWygxtz4sSv54RpjkiKv9tHd6zvJdbV999+iZd+4/DuNgf3InLsmUrqi4AfwYAAP//4DC51nQLAAA=
mode: 493
path: /usr/local/sbin/bm-mtu-configure.sh
systemd:
units:
- contents: |
[Unit]
Description=Configure MTU=9000 on VLAN-backed bare metal interfaces
Documentation=https://github.com/oracle-quickstart/oci-openshift/issues/78
After=NetworkManager.service oci-eval-user-data.service
Before=ovs-configuration.service kubelet.service
ConditionPathExists=/usr/local/sbin/bm-mtu-configure.sh

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/bm-mtu-configure.sh
RemainAfterExit=yes
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
enabled: true
name: bm-mtu-configure.service
---
# Generated by Butane; do not edit
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
labels:
machineconfiguration.openshift.io/role: worker
name: 00-worker-bm-vlan-mtu-9000
spec:
config:
ignition:
version: 3.4.0
storage:
files:
- contents:
compression: gzip
source: data:;base64,H4sIAAAAAAAC/5xW71PbuBb97r/ivISB5JWElBk6kE46pby2j+kjdBrgfaCZjmJdx1psyasfgeyy//uOZCd2CN1tN59iWzr33HPPvVL7XwczIQ9mzKRRG2dKJmLuNOHi6np0MhgMoCRu/nc67s1YfEccM6YJOVmWQUhLOmExmaiNq1QYmFiLwkI7acAsZkpZWAWSxkNe3ox7n9yMtCRLBikzMC5JRCxIWh8waiNRGsIafCRJC4JakM7YEp3jk8HgBY6OMVv6vSRjVhiXMSuUxAjHJ0fHyIUUucu7UTtq47NWs4zyId7VhE3KCjK4FzbdyEmSvVf6Tsg5Opm4I7y76E8sk5xp3n9/1H95crgftbH+bXx+1T88etUFp4S5LKR7cXWNl0eldF/+e3Y52UfMnPHwlzdjvyJhIusHmhOVOZ/DEP9ZSpaLmGXZEpwsxTZwbKgMJjkM2Y3aFEyTtC/80n6T47UhA1EgVnnOJDfgQlNssyU6UmFcZnzBJJuTBqeCJCcZL7sbIOcyzhwng7ljmhtP3dyJAjcX8NtJl5xYponxZS9euYdDKk6mH0Webo+cQiEK8mlHURsfPRpeDjFWWzmO3kCqnirQWQhtHctWkbrRImPSjHY6okBPIRPyDiZV97DLguA/4hHs/g57vxdaSIudwz/28AhDHHvmYLhzcNB4fNv/d3h+hNWOupFIcIveb2jthDAtTF/DpiQjgOJUoXV7Pv5wOX2OcqKc5PugB2GDhUr+3sm1Tt2WB3oQFoMoEVG0gfnBAzyFHaJiEpVujskYEIvTJwsjHyhkL2S15TW4ioA23j9YzWJbmaTeA8lyCiu2LKbLj3xYbep7RMGh9OaLt+VTAJlYLQrQgvTSpl6CRKscidDGgisLJe9X9pnTik2E6s9op1OqEci36hJ97YcadaMIqMtTbmph6mt323gxWiPUpVsX7/+nX8ZTnCmXeW/a0GE6F5JW2ngVw/b9YPFCyHkrAMRKWiGdl8sXzuf7kSxip3U1tsp0WRzcymkhvJAdH2WzzbrrlL/l1pVGrl1cJ3L45oDT4kC6LMMj5pq83z9jL7cOXz/dDnon0xfBuSG3gYf1xJ8HLRX5acintq/IDcPs2amz2C9Fq96vaLQqnUKfHw5xWg4I/OLymWq0uOCUF8qStN1VjVsN8BZ6c0KYdFPs7oav6xCNb9vl3iAdZlTYuJpUSNkiHHF4MwoY+5AKccrknAwkESf+nepXAU7Prs4vx1NMyIamb47krbCVHJNydNdTuzILOiKpgn5Phsxup7ouNAWghoF8Vf3ypiKT67Oz95PJdMXiKdtWw98NomE+/BXNRj1+jGRpyJ+jWIr4pATr5Z7q6ugpLwSxyouMLJVdvV0OriT5sXqaGRWozXSPHgKQSCCsn9TGGnQubyaYacHnhFgTs/4CtIRamN5GwL4h7SXqri5CKZM8I+N1QMwM4T4lTeFx446UWNLwQcJlyDJtifujaKOJS3K7dQuv9Z3pb/TwXN+XW/7BIKmqugb+Tll/oAsChbKJRLK6hqC3COotTGyzZ1Pyv3qFr835+tQqs8qt+6bpV0fGlsF2d//WQQ0yifAWygxtz4sSv54RpjkiKv9tHd6zvJdbV999+iZd+4/DuNgf3InLsmUrqi4AfwYAAP//4DC51nQLAAA=
mode: 493
path: /usr/local/sbin/bm-mtu-configure.sh
systemd:
units:
- contents: |
[Unit]
Description=Configure MTU=9000 on VLAN-backed bare metal interfaces
Documentation=https://github.com/oracle-quickstart/oci-openshift/issues/78
After=NetworkManager.service oci-eval-user-data.service
Before=ovs-configuration.service kubelet.service
ConditionPathExists=/usr/local/sbin/bm-mtu-configure.sh

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/bm-mtu-configure.sh
RemainAfterExit=yes
StandardOutput=journal+console
StandardError=journal+console

[Install]
WantedBy=multi-user.target
enabled: true
name: bm-mtu-configure.service
---
4 changes: 2 additions & 2 deletions terraform-stacks/add-nodes/schema.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
title: Add Nodes to OpenShift Cluster on OCI - v1.4.2
title: Add Nodes to OpenShift Cluster on OCI - v1.4.3
description: A Terraform Stack for creating resources required for adding nodes to OpenShift Cluster on OCI
schemaVersion: 1.4.2
schemaVersion: 1.4.3
version: "20250916"
locale: "en"

Expand Down
2 changes: 1 addition & 1 deletion terraform-stacks/add-nodes/version.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
locals {
stack_version = "v1.4.2"
stack_version = "v1.4.3"
}
4 changes: 2 additions & 2 deletions terraform-stacks/create-cluster/schema.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
title: OpenShift Cluster on OCI - v1.4.2
title: OpenShift Cluster on OCI - v1.4.3
description: A Terraform Stack for creating resources required for installing OpenShift on OCI
schemaVersion: 1.4.2
schemaVersion: 1.4.3
version: "20250916"
locale: "en"

Expand Down
2 changes: 1 addition & 1 deletion terraform-stacks/create-cluster/version.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
locals {
stack_version = "v1.4.2"
stack_version = "v1.4.3"
}
4 changes: 2 additions & 2 deletions terraform-stacks/create-instance-role-tags/schema.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
title: OpenShift on OCI Instance Role Tags - v1.4.2
title: OpenShift on OCI Instance Role Tags - v1.4.3
description: A Terraform Stack for creating OpenShift Instance Role Tags
schemaVersion: 1.4.2
schemaVersion: 1.4.3
version: "20250916"
locale: "en"

Expand Down
2 changes: 1 addition & 1 deletion terraform-stacks/create-instance-role-tags/version.tf
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
locals {
stack_version = "v1.4.2"
stack_version = "v1.4.3"
}
4 changes: 2 additions & 2 deletions terraform-stacks/create-resource-attribution-tags/schema.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
title: OpenShift on OCI Resource Attribution Tags - v1.4.2
title: OpenShift on OCI Resource Attribution Tags - v1.4.3
description: A Terraform Stack for creating OpenShift Resource Attribution Tags.
schemaVersion: 1.4.2
schemaVersion: 1.4.3
version: "20250916"
locale: "en"

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
locals {
stack_version = "v1.4.2"
stack_version = "v1.4.3"
}
1 change: 1 addition & 0 deletions terraform-stacks/shared_modules/manifest/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ output "dynamic_custom_manifest" {
${file("${path.module}/manifests/03-machineconfig-consistent-device-path.yml")}
${file("${path.module}/manifests/04-cluster-network.yml")}
${file("${path.module}/manifests/05-oci-eval-user-data.yml")}
${file("${path.module}/manifests/07-configure-bm-vlan-mtu.yml")}
EOT
}

Expand Down