Squash Commit of changes for v2.11.1

Author: arnaudfroidmont

README.md

@@ -27,7 +27,9 @@ Allow dynamic-group instance_principal to manage dns in compartment compartmentN
 ```
 or:
 
-`Allow dynamic-group instance_principal to manage all-resources in compartment compartmentName`
+```
+Allow dynamic-group instance_principal to manage all-resources in compartment compartmentName
+```
 
 
 ## Supported OS: 
@@ -67,21 +69,24 @@ The resize.sh is deployed on the controller node as part of the HPC cluster Stac
 usage: resize.sh [-h] [--compartment_ocid COMPARTMENT_OCID]
                  [--cluster_name CLUSTER_NAME] [--nodes NODES [NODES ...]]
                  [--no_reconfigure] [--user_logging] [--force] [--remove_unreachable]
-                 [{add,remove,list,reconfigure}] [number]
-
+                 [{add,remove,remove_unreachable,list,reconfigure}] [number] [--quiet]
 Script to resize the CN
 
 positional arguments:
   {add,remove,remove_unreachable,list,reconfigure}
-                        Mode type. add/remove node options, implicitly
-                        configures newly added nodes. Also implicitly
-                        reconfigure/restart services like Slurm to recognize
-                        new nodes. Similarly for remove option, terminates
-                        nodes and implicitly reconfigure/restart services like
-                        Slurm on rest of the cluster nodes to remove reference
-                        to deleted nodes.
-  number                Number of nodes to add or delete if a list of
-                        hostnames is not defined
+                              Mode type. add/remove node options, implicitly
+                              configures newly added nodes. Also implicitly
+                              reconfigure/restart services like Slurm to recognize
+                              new nodes. Similarly for remove option, terminates
+                              nodes and implicitly reconfigure/restart services like
+                              Slurm on rest of the cluster nodes to remove reference
+                              to deleted nodes. IMPORTANT: remove or remove_unreachable 
+                              means delete the node from the cluster which means terminate 
+                              the node. remove_unreachable should be used to remove specific 
+                              nodes which are no longer reachable via ssh. It gives you control 
+                              on which nodes will be terminated by passing the --nodes parameter.
+number                        Number of nodes to add or delete if a list of
+                              hostnames is not defined.
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -102,9 +107,11 @@ optional arguments:
   --ansible_crucial     If present during reconfiguration, only crucial
                         ansible playbooks will be executed on the live nodes.
                         Non live nodes will be removed
-  --remove_unreachable  If present, nodes that are not sshable will be terminated 
-                        before running the action that was requested
-                        (Example Adding a node)
+  --remove_unreachable  If present, ALL nodes that are not sshable will be terminated 
+                        before running the action that was requested (Example Adding a node). 
+                        CAUTION: Use this only if you want to remove ALL nodes that 
+                        are unreachable. Instead, remove specific nodes that are 
+                        unreachable by using positional argument remove_unreachable. 
   --quiet               If present, the script will not prompt for a response when 
                         removing nodes and will not give a reminder to save data 
                         from nodes that are being removed

autoscaling/crontab/autoscale_slurm.sh

@@ -214,7 +214,12 @@ def getstatus_slurm():
 
                 nodes=int(new_line[3])
                 jobID=int(new_line[1])
-                cluster_to_build.append([nodes,instanceType,queue,jobID,user])
+                if isPermanent(config,queue,instanceType) is None :
+                    continue
+                elif isPermanent(config,queue,instanceType):
+                    continue
+                else:
+                    cluster_to_build.append([nodes,instanceType,queue,jobID,user])
 
     cluster_to_destroy=[]
     current_nodes={}

autoscaling/tf_init/cluster-network-configuration.tf

@@ -11,7 +11,10 @@ resource "oci_core_instance_configuration" "cluster-network-instance_configurati
       compartment_id      = var.targetCompartment
       create_vnic_details {
       }
-      display_name = local.cluster_name
+      freeform_tags = {
+        "cluster_name"   = local.cluster_name
+        "parent_cluster" = local.cluster_name
+      }
       metadata = {
 # TODO: add user key to the authorized_keys 
         ssh_authorized_keys = file("/home/${var.controller_username}/.ssh/id_rsa.pub")

autoscaling/tf_init/controller_update.tf

@@ -33,6 +33,7 @@ resource "local_file" "inventory" {
     rdma_netmask = cidrnetmask(var.rdma_subnet),
     zone_name = var.zone_name,
     dns_entries = var.dns_entries,
+    vcn_compartment = var.vcn_compartment,
     nfs = var.use_scratch_nfs ? local.cluster_instances_names[0] : "",
     scratch_nfs = var.use_scratch_nfs,
     cluster_nfs = var.use_cluster_nfs,
@@ -82,7 +83,8 @@ resource "local_file" "inventory" {
     use_compute_agent=var.use_compute_agent,
     healthchecks=var.healthchecks,
     change_hostname=var.change_hostname,
-    hostname_convention=var.hostname_convention
+    hostname_convention=var.hostname_convention,
+    ons_topic_ocid=var.ons_topic_ocid
     })
   filename   = "${local.controller_path}/inventory"
 }

autoscaling/tf_init/data.tf

@@ -55,13 +55,13 @@ data "oci_core_vcn" "vcn" {
 } 
 
 data "oci_dns_views" "dns_views" {
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   scope = "PRIVATE"
   display_name = data.oci_core_vcn.vcn.display_name
 }
 
 data "oci_dns_zones" "dns_zones" {
-    compartment_id = var.targetCompartment
+    compartment_id = var.vcn_compartment
     name = "${var.zone_name}"
     zone_type = "PRIMARY"
     scope = "PRIVATE"

autoscaling/tf_init/instance-pool-configuration.tf

@@ -11,7 +11,10 @@ resource "oci_core_instance_configuration" "instance_pool_configuration" {
       compartment_id      = var.targetCompartment
       create_vnic_details {
       }
-      display_name = local.cluster_name
+      freeform_tags = {
+        "cluster_name"   = local.cluster_name
+        "parent_cluster" = local.cluster_name
+      }
       metadata = {
 # TODO: add user key to the authorized_keys 
         ssh_authorized_keys = file("/home/${var.controller_username}/.ssh/id_rsa.pub")

autoscaling/tf_init/inventory.tpl

@@ -76,6 +76,8 @@ sacct_limits=${sacct_limits}
 use_compute_agent=${use_compute_agent}
 zone_name=${zone_name}
 dns_entries=${dns_entries}
+vcn_compartment=${vcn_compartment}
 healthchecks=${healthchecks}
 change_hostname=${change_hostname}
-hostname_convention=${hostname_convention}
+hostname_convention=${hostname_convention}
+ons_topic_ocid=${ons_topic_ocid}

autoscaling/tf_init/network.tf

@@ -1,15 +1,15 @@
 resource "oci_core_vcn" "vcn" {
   count          = var.use_existing_vcn ? 0 : 1
   cidr_block     = var.vcn_subnet
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   display_name   = "${local.cluster_name}_VCN"
   dns_label      = "cluster"
 }
 
 resource "oci_core_security_list" "internal-security-list" {
   count          = var.use_existing_vcn ? 0 : 1
   vcn_id         = oci_core_vcn.vcn[0].id
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
 
   ingress_security_rules {
     protocol = "all"
@@ -41,7 +41,7 @@ resource "oci_core_security_list" "internal-security-list" {
 resource "oci_core_security_list" "public-security-list" {
   count          = var.use_existing_vcn ? 0 : 1
   vcn_id         = oci_core_vcn.vcn[0].id
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
 
   ingress_security_rules {
     protocol = "all"
@@ -83,22 +83,22 @@ resource "oci_core_security_list" "public-security-list" {
 resource "oci_core_internet_gateway" "ig1" {
   count          = var.use_existing_vcn ? 0 : 1
   vcn_id         = oci_core_vcn.vcn[0].id
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   display_name   = "${local.cluster_name}_internet-gateway"
 }
 
 resource "oci_core_nat_gateway" "ng1" {
   count          = var.use_existing_vcn ? 0 : 1
   vcn_id         = oci_core_vcn.vcn[0].id
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   display_name   = "${local.cluster_name}_nat-gateway"
 }
 
 
 resource "oci_core_service_gateway" "sg1" {
   count          = var.use_existing_vcn ? 0 : 1
   vcn_id         = oci_core_vcn.vcn[0].id
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   display_name   = "${local.cluster_name}_service-gateway"
 
   services {
@@ -108,7 +108,7 @@ resource "oci_core_service_gateway" "sg1" {
 
 resource "oci_core_route_table" "public_route_table" {
   count          = var.use_existing_vcn ? 0 : 1
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   vcn_id         = oci_core_vcn.vcn[0].id
   display_name   = "${local.cluster_name}_public_route_table"
 
@@ -122,7 +122,7 @@ resource "oci_core_route_table" "public_route_table" {
 resource "oci_core_route_table" "private_route_table" {
   count          = var.use_existing_vcn ? 0 : 1
   display_name   = "${local.cluster_name}_private_route_table"
-  compartment_id = var.targetCompartment
+  compartment_id = var.vcn_compartment
   vcn_id         = oci_core_vcn.vcn[0].id
 
   route_rules {
@@ -142,7 +142,7 @@ resource "oci_core_subnet" "public-subnet" {
   count               = var.use_existing_vcn ? 0 : 1
   # availability_domain = var.ad
   vcn_id              = oci_core_vcn.vcn[0].id
-  compartment_id      = var.targetCompartment
+  compartment_id      = var.vcn_compartment
   cidr_block          = trimspace(var.public_subnet)
   security_list_ids   = [oci_core_security_list.public-security-list[0].id]
   dns_label           = "public"
@@ -154,7 +154,7 @@ resource "oci_core_subnet" "private-subnet" {
   count                      = var.use_existing_vcn ? 0 : 1
   # availability_domain        = var.ad
   vcn_id                     = oci_core_vcn.vcn[0].id
-  compartment_id             = var.targetCompartment
+  compartment_id             = var.vcn_compartment
   cidr_block                 = trimspace(var.private_subnet)
   security_list_ids          = [oci_core_security_list.internal-security-list[0].id]
   dns_label                  = "private"

bin/controller.sh

@@ -36,17 +36,22 @@ if [ $ID == "ol" ] || [ $ID == "centos" ] ; then
     sudo yum-config-manager --save --setopt=ol7_oci_included.skip_if_unavailable=true
     sudo yum makecache --enablerepo=$repo
     sudo yum install --enablerepo=$repo -y ansible python-netaddr
+    sudo yum install -y https://archive.releases.hashicorp.com/RHEL/7.9/x86_64/stable/terraform-1.6.6-1.x86_64.rpm
   elif [ $vid == 8 ] ; then
     sudo yum makecache --enablerepo=$repo
     sudo yum install --enablerepo=$repo -y python38.x86_64
     sudo python3.8 -m pip install ansible cryptography netaddr > /dev/null
     sudo mkdir /etc/ansible
     sudo ln -s /usr/local/bin/ansible-playbook /bin/ansible-playbook
     sudo ln -s /usr/local/bin/ansible /bin/ansible
+    sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+    sudo sed -i 's/$releasever/'"${vid}"'/g' /etc/yum.repos.d/hashicorp.repo
+    sudo yum install -y terraform
   fi
-  sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
-  sudo sed -i 's/$releasever/'"${vid}"'/g' /etc/yum.repos.d/hashicorp.repo
-  sudo yum install -y terraform
+#  sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/RHEL/hashicorp.repo
+#  sudo sed -i 's/$releasever/'"${vid}"'/g' /etc/yum.repos.d/hashicorp.repo
+#  wget https://archive.releases.hashicorp.com/RHEL/7.9/x86_64/stable/terraform-1.6.6-1.x86_64.rpm
+#  sudo yum install -y terraform
   sudo python3 -m pip install -U pip > /dev/null
   sudo python3 -m pip install netaddr --upgrade > /dev/null
   sudo python3 -m pip install setuptools_rust --upgrade > /dev/null

bin/create_cluster.sh

@@ -100,6 +100,7 @@ do
   if [ $status -eq 0 ]
   then
     echo "Successfully created $2 in $runtime seconds"
+    sleep 60 # Give the jobs some time to start
     rm currently_building
     if [ -f $monitoring_folder/activated ]
     then
@@ -161,6 +162,7 @@ do
       mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; INSERT INTO cluster_log.errors_timeserie (cluster_id,state,error_log,error_type,nodes,created_on_m,class_name) VALUES ('$2_${date}','creation','$logs_folder/create_$2_${date}.log','$ERROR_MSG $inst_pool_work_request_error_messages $cn_work_request_error_messages','$1','$end_timestamp','$4');" >> $logs_folder/create_$2_${date}.log 2>&1
       mysql -u $ENV_MYSQL_USER -p$ENV_MYSQL_PASS -e "use $ENV_MYSQL_DATABASE_NAME; UPDATE cluster_log.clusters SET state='deleting',creation_error='`tail $logs_folder/create_$2_${date}.log | grep Error`' WHERE id='$2_${date}';" >> $logs_folder/create_$2_${date}.log 2>&1
     fi
+    sleep 60 # Give the jobs some time to start
     rm currently_building
   fi
 done