Skip to content

Commit

Permalink
Merge pull request #159 from Barts-Life-Science/deploy-custom-vms
Browse files Browse the repository at this point in the history 
Deploy custom vms
  • Loading branch information
@BIOKU-BH
BIOKU-BH authored Sep 18, 2024
2 parents 9304426 + e4d5f03 commit 71e11f0
Show file tree
Hide file tree
Showing 3 changed files with 104 additions and 109 deletions.
22 changes: 13 additions & 9 deletions templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,24 @@ custom:
"36 CPU | 440GB | 1 x A10 GPU": Standard_NV36ads_A10_v5 # 2375.82
"36 CPU | 880GB | 1 x A10 GPU": Standard_NV36adms_A10_v5 # 3323.17
image_options:
"Ubuntu 22.04 LTS":
source_image_reference:
publisher: canonical
offer: 0001-com-ubuntu-server-jammy
sku: 22_04-lts-gen2
version: latest
apt_sku: 22.04
install_ui: true
conda_config: false
# "Ubuntu 22.04 LTS":
# source_image_reference:
# publisher: canonical
# offer: 0001-com-ubuntu-server-jammy
# sku: 22_04-lts-gen2
# version: latest
# apt_sku: 22.04
# install_ui: true
# conda_config: false
# For information on using custom images, see README.me in the guacamole/user-resources folder
"Custom Ubuntu 22.04 LTS":
source_image_name: imgdef-linux-dsvm-rpython
install_ui: true
conda_config: true
"Custom Ubuntu 24.04 LTS":
source_image_name: imgdef-linux24-dsvm-rpython
install_ui: true
conda_config: true

credentials:
- name: azure_tenant_id
Expand Down
4 changes: 2 additions & 2 deletions .../workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/template_schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
"title": "Linux image",
"description": "Select Linux image to use for VM",
"enum": [
"Custom Ubuntu 22.04 LTS",
"Ubuntu 22.04 LTS"
"Custom Ubuntu 24.04 LTS",
"Custom Ubuntu 22.04 LTS"
]
},
"vm_size": {
Expand Down
187 changes: 89 additions & 98 deletions ...orkspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/vm_config.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,48 +4,69 @@ set -o errexit
set -o pipefail
set -o nounset
# Uncomment this line to see each command for debugging (careful: this will show secrets!)
# set -o xtrace
set -o xtrace

echo "init_vm.sh: START"

echo "init_vm.sh: Acquire lock"
timeout 900 bash -c -- 'while fuser /var/lib/dpkg/lock-frontend > /dev/null 2>&1
do
echo "Waiting to get lock /var/lib/dpkg/lock-frontend..."
sleep 5
done'

echo "init_vm.sh: Currently installed packages:"
apt list --installed

# Remove apt sources not included in sources.list file
echo "init_vm.sh: APT sources"
sudo rm -f /etc/apt/sources.list.d/*
rm -f /etc/apt/sources.list.d/*

# shellcheck disable=SC1091
. /etc/os-release
sed -i "s%__VERSION_ID__%$VERSION_ID%" /etc/apt/sources.list
if [ "$VERSION_ID" == "24.04" ]; then
echo "init_vm.sh: Fix APT for Ubuntu 24.04"
# azuredatastudio seems to be broken, at least, that's what it reports when we run this...
apt --fix-broken install -y

# While we're here, disable bombing out, so we can debug this thing easier
set +o errexit
set +o pipefail
fi

# Update apt packages from configured Nexus sources
echo "init_vm.sh: START"
sudo apt update || true
sudo apt upgrade -y
sudo apt install -y software-properties-common apt-transport-https wget dirmngr gdebi-core
sudo apt-get update || true
echo "init_vm.sh: Update OS"
export DEBIAN_FRONTEND=noninteractive
export DEBIAN_PRIORITY=critical
apt upgrade -y
apt-get update -y
rm -f /etc/apt/sources.list.d/* # Again, because of VS Code
apt install -y software-properties-common apt-transport-https wget dirmngr gdebi-core
# apt-get update || true

## Desktop
echo "init_vm.sh: Desktop"
sudo systemctl start gdm3 || true
DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true dpkg-reconfigure gdm3 || true
sudo apt install -y xfce4 xfce4-goodies xorg dbus-x11 x11-xserver-utils
echo /usr/sbin/gdm3 > /etc/X11/default-display-manager

## Install xrdp so Guacamole can connect via RDP
echo "init_vm.sh: xrdp"
sudo apt install -y xrdp xorgxrdp xfce4-session
sudo adduser xrdp ssl-cert
if [ "$VERSION_ID" == "24.04" ]; then
echo "init_vm.sh: Desktop"

# This next line causes problems. Force it to succeed, and hope for the best
apt install -y xfce4 xfce4-goodies xorg dbus-x11 --fix-missing || true # --fix-missing for Ubuntu 24.04

## Install xrdp so Guacamole can connect via RDP
echo "init_vm.sh: xrdp"
apt install -y xrdp xorgxrdp xfce4-session
adduser xrdp ssl-cert
fi
sudo -u "${VM_USER}" -i bash -c 'echo xfce4-session > ~/.xsession'
sudo -u "${VM_USER}" -i bash -c 'echo xset s off >> ~/.xsession'
sudo -u "${VM_USER}" -i bash -c 'echo xset -dpms >> ~/.xsession'

# Make sure xrdp service starts up with the system
sudo systemctl enable xrdp
sudo service xrdp restart

# Prevent screen timeout
echo "init_vm.sh: Preventing Timeout"
sudo mkdir -p /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml
sudo touch /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml
sudo chmod 664 /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml
sudo tee /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml << END
mkdir -p /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml
touch /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml
chmod 664 /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml
tee /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml << END
<?xml version="1.0" encoding="UTF-8"?>
<channel name="xfce4-screensaver" version="1.0">
<property name="saver" type="empty">
Expand All @@ -57,15 +78,12 @@ sudo tee /home/"${VM_USER}"/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-scree
</property>
</ channel>
END
sudo chown -Rf "${VM_USER}":"${VM_USER}" /home/"${VM_USER}"/.config

# Fix for blank screen on DSVM (/sh -> /bash due to conflict with profile.d scripts)
sudo sed -i 's|!/bin/sh|!/bin/bash|g' /etc/xrdp/startwm.sh
chown -Rf "${VM_USER}":"${VM_USER}" /home/"${VM_USER}"/.config

if [ "${SHARED_STORAGE_ACCESS}" -eq 1 ]; then
# Install required packages
echo "init_vm.sh: Shared storage"
sudo apt-get install autofs -y
apt-get install -y autofs

# Pass in required variables
storageAccountName="${STORAGE_ACCOUNT_NAME}"
Expand All @@ -76,68 +94,35 @@ if [ "${SHARED_STORAGE_ACCESS}" -eq 1 ]; then
mntRoot="/shared-storage"
credentialRoot="/etc/smbcredentials"

# mntPath="$mntRoot/$fileShareName"
# shellcheck disable=SC2308
smbPath=$(echo "$httpEndpoint" | cut -c7-"$(expr length "$httpEndpoint")")$fileShareName
smbCredentialFile="$credentialRoot/$storageAccountName.cred"

# Create required file paths
# sudo mkdir -p "$mntPath"
sudo mkdir -p $credentialRoot
sudo mkdir -p $mntRoot
mkdir -p $credentialRoot
mkdir -p $mntRoot

### Auto FS to persist storage
# Create credential file
if [ ! -f "$smbCredentialFile" ]; then
echo "username=$storageAccountName" | sudo tee "$smbCredentialFile" > /dev/null
echo "password=$storageAccountKey" | sudo tee -a "$smbCredentialFile" > /dev/null
echo "username=$storageAccountName" | tee "$smbCredentialFile" > /dev/null
echo "password=$storageAccountKey" | tee -a "$smbCredentialFile" > /dev/null
else
echo "The credential file $smbCredentialFile already exists, and was not modified."
fi

# Change permissions on the credential file so only root can read or modify the password file.
sudo chmod 600 "$smbCredentialFile"
chmod 600 "$smbCredentialFile"

echo "$smbPath $mntRoot cifs rw,vers=default,dir_mode=0777,file_mode=0777,uid=1000,gid=1000,credentials=$smbCredentialFile 0 0" | sudo tee -a /etc/fstab >/dev/null
sudo mount $mntRoot
echo "$smbPath $mntRoot cifs rw,vers=default,dir_mode=0777,file_mode=0777,uid=1000,gid=1000,credentials=$smbCredentialFile 0 0" | tee -a /etc/fstab >/dev/null
mount $mntRoot
fi

# set +o errexit
# set +o pipefail
# set +o nounset
set -o xtrace

## Python 3.8 and Jupyter
echo "init_vm.sh: Jupyter, Edge"
sudo apt install -y jupyter-notebook microsoft-edge-dev

## VS Code
echo "init_vm.sh: VS Code"
sudo apt install -y code
sudo apt install -y gvfs-bin || true

echo "init_vm.sh: Folders"
sudo mkdir -p /opt/vscode/user-data
sudo mkdir -p /opt/vscode/extensions

echo "init_vm.sh: azure-cli"
sudo apt install azure-cli -y

# TODO: need to look at proxy extentions
## VSCode Extensions
# echo "init_vm.sh: VSCode extensions"
# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension ms-python.python
# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension REditorSupport.r
# code --extensions-dir="/opt/vscode/extensions" --user-data-dir="/opt/vscode/user-data" --install-extension RDebugger.r-debugger

# Azure Storage Explorer
sudo apt install gnome-keyring dotnet-sdk-8.0 -y
wget -q "${NEXUS_PROXY_URL}"/repository/microsoft-download/A/E/3/AE32C485-B62B-4437-92F7-8B6B2C48CB40/StorageExplorer-linux-x64.tar.gz -P /tmp
sudo mkdir -p /opt/storage-explorer
sudo tar xvf /tmp/StorageExplorer-linux-x64.tar.gz -C /opt/storage-explorer
sudo chmod +x /opt/storage-explorer/*

sudo tee /usr/share/applications/storage-explorer.desktop << END
apt install -y jupyter-notebook microsoft-edge-dev

tee /usr/share/applications/storage-explorer.desktop << END
[Desktop Entry]
Name=Storage Explorer
Comment=Azure Storage Explorer
Expand All @@ -150,15 +135,21 @@ StartupWMClass=Code
Categories=Development;
END

## R
echo "init_vm.sh: R Setup"
sudo apt install -y r-base

# RStudio Desktop
echo "init_vm.sh: RStudio"
wget "${NEXUS_PROXY_URL}"/repository/r-studio-download/electron/jammy/amd64/rstudio-2023.12.1-402-amd64.deb -P /tmp/
# wget "${NEXUS_PROXY_URL}"/repository/r-studio-download/electron/focal/amd64/rstudio-2023.12.1-402-amd64.deb -P /tmp/
sudo gdebi --non-interactive /tmp/rstudio-2023.12.1-402-amd64.deb
if [ "$VERSION_ID" == "24.04" ]; then
echo "init_vm.sh: RStudio"
echo "Sadly, this won't work, there's a problem with the proxy configuration for RStudio"
# # wget "${NEXUS_PROXY_URL}"/repository/r-studio-download/electron/jammy/amd64/rstudio-2023.12.1-402-amd64.deb -P /tmp/
# # wget "${NEXUS_PROXY_URL}"/repository/r-studio-download/electron/focal/amd64/rstudio-2023.12.1-402-amd64.deb -P /tmp/
# # gdebi --non-interactive /tmp/rstudio-2023.12.1-402-amd64.deb

# # https://download1.rstudio.org/electron/focal/amd64/rstudio-2024.04.2-764-amd64.deb
# wget "${NEXUS_PROXY_URL}"/repository/r-studio-download/electron/focal/amd64/rstudio-2024.04.2-764-amd64.deb -P /tmp/
# gdebi --non-interactive /tmp/rstudio-2024.04.2-764-amd64.deb
fi

# R config
echo -e "local({\n r <- getOption(\"repos\")\n r[\"Nexus\"] <- \"""${NEXUS_PROXY_URL}\"/repository/r-proxy/\"\n options(repos = r)\n})" | tee /etc/R/Rprofile.site

### Anaconda Config
if [ "${CONDA_CONFIG}" -eq 1 ]; then
Expand All @@ -169,35 +160,35 @@ if [ "${CONDA_CONFIG}" -eq 1 ]; then
if [ -d "/opt/anaconda" ]; then
export PATH="/opt/anaconda/condabin:/opt/anaconda/bin":$PATH
fi
conda config --add channels "${NEXUS_PROXY_URL}"/repository/conda-mirror/main/ --system
conda config --add channels "${NEXUS_PROXY_URL}"/repository/conda-repo/main/ --system
conda config --add channels "${NEXUS_PROXY_URL}"/repository/conda-mirror/main/ --system
conda config --add channels "${NEXUS_PROXY_URL}"/repository/conda-repo/main/ --system
conda config --remove channels defaults --system
conda config --set channel_alias "${NEXUS_PROXY_URL}"/repository/conda-mirror/ --system
conda config --set channel_alias "${NEXUS_PROXY_URL}"/repository/conda-mirror/ --system
fi

# Docker install and config
sudo apt-get remove -y moby-tini || true
sudo apt-get install -y r-base-core
sudo apt-get install -y ca-certificates curl gnupg lsb-release
sudo apt-get install -y docker-compose-plugin docker-ce-cli containerd.io jq
sudo apt-get install -y docker-ce
echo "init_vm.sh: Docker"
apt-get remove -y moby-tini || true
apt-get install -y ca-certificates curl gnupg lsb-release
apt-get install -y docker-compose-plugin docker-ce-cli containerd.io jq
apt-get install -y docker-ce
jq -n --arg proxy "${NEXUS_PROXY_URL}:8083" '{"registry-mirrors": [$proxy]}' > /etc/docker/daemon.json
sudo systemctl daemon-reload
sudo systemctl restart docker
systemctl daemon-reload
systemctl restart docker

# R config
sudo echo -e "local({\n r <- getOption(\"repos\")\n r[\"Nexus\"] <- \"""${NEXUS_PROXY_URL}\"/repository/r-proxy/\"\n options(repos = r)\n})" | sudo tee /etc/R/Rprofile.site
echo "init_vm.sh: odds and ends"

# Jupiter Notebook Config
sudo sed -i -e 's/Terminal=true/Terminal=false/g' /usr/share/applications/jupyter-notebook.desktop
sed -i -e 's/Terminal=true/Terminal=false/g' /usr/share/applications/jupyter-notebook.desktop

# Default Browser
sudo update-alternatives --config x-www-browser
update-alternatives --config x-www-browser

echo "init_vm.sh: environment"
echo "export NEXUS_PROXY_URL=${NEXUS_PROXY_URL}" > /etc/profile.d/99-sde-environment.sh
env | sort

## Cleanup
echo "init_vm.sh: Cleanup"
sudo shutdown -r now
echo "init_vm.sh: Cleanup & restart"
rm -f /etc/apt/sources.list.d/* # Again, because of VS Code
set +o xtrace # Avoid Python stack dump from myself
shutdown -r now

0 comments on commit 71e11f0

Please sign in to comment.