git-mrt

#!/bin/bash

VERSION="1.5"

# Exit with return code
# $1 - message to output in stderr
# $2 - return code
function die() {
    echo "$1" >&2
    exit ${2:-1}
}

#declare -A LOG_LEVELS
LOG_LEVELS=([0]="ERROR" [1]="WARN" [2]="INFO" [3]="DEBUG")

VERBOSITY=2

function log() {
    if [ "$#" -eq 0 ]; then
        die "ASSERT: log() called with no arguments"
    elif [ "$#" -eq 1 ]; then
        # default to INFO level if not set
        local level=2
    else
        local level=${1}
        shift
    fi
    if [ "$VERBOSITY" -ge "$level" ]; then
        printf "[${LOG_LEVELS[$level]}] $@\n"
    fi
}

function usage() {
    printf "Usage: %s [-v|--version] [-h|--help]
               [-m|--monolocation <dir>] [-d|--deploykeyfile <file>]
               [-a|--allowdirs] [-p | --pushbranch]

               command [<args>]\n" `basename $0`
    echo "
Available options:
  -v or --version                      - show version
  -h or --help                         - show this help
  -m or --monolocation                 - local directory where the monorepo will be cloned
  -r or --remote                       - monorepo remote URL (e.g. https://...)
  -d or --deploykeyfile                - private deploy key to use for monorepo clone
  -a or --allowdirs                    - allow creation of new directories in monorepo on push
  -p or --pushbranch                   - override the name of the monorepo branch used by push

Available commands:
  clone <subdir_in_monorepo>           - extract a monorepo subdirectory as a local repo
  push [subdir_in_monorepo]            - push local repo changes as a monorepo branch
  pull [subdir_in_monorepo]            - update local repo from a monorepo subdirectory
  status [subdir_in_monorepo]          - show the local repo status compared to the remote"

    exit 1
}

function set_script_vars() {
    # Primary monorepo branch to use, e.g. main or master
    if [ -z "${MONOBRANCH+x}" ]; then
        MONOBRANCH="main"
    fi

    # Clone link - https default
    if [ ! -z "${MONOHTTPSLINK}" ]; then
        # Use existing definition if present
        MONOHTTPSLINK=${MONOHTTPSLINK}
    elif [ -d ".git" ] && [ "$(git config --get mrt.git_mrt_repo_url &> /dev/null)" ]; then
        # Get from subrepo metadata if available
        MONOHTTPSLINK="$(git config --get mrt.git_mrt_repo_url)"
    elif [ ! -z "${GIT_MRT_REPO_URL+x}" ]; then
        # Use environment variable
        MONOHTTPSLINK=${GIT_MRT_REPO_URL}
    else
        # Use default location
        MONOHTTPSLINK=https://github.com/NSLS2/app-deploy-epics.git
    fi
    # support URL without the suffix ".git"
    MONOHTTPSLINK=${MONOHTTPSLINK%.git}.git
    log "Using monorepo link ${MONOHTTPSLINK}"

    # Location to which monorepo is cloned
    if [ -z "${MONOHOME+x}" ]; then
        MONOHOME="${HOME}/.monorepo"
    fi
    log "Using monorepo home ${MONOHOME}"

    # Name of the cloned repository
    MONODIR="$(echo ${MONOHTTPSLINK} | sed -nE 's|^https://.*/.*/(.*).git$|\1|p')"
    [ -z ${MONODIR} ] && die "[FATAL] Failed to determine the monorepo dir name. Wrong URL?"
    log "Using monorepo dir name ${MONODIR}"

    # Actual path to the local monorepo
    MONOPATH=$MONOHOME/$MONODIR
    [ -z ${MONOPATH} ] && die "[FATAL] Failed to determine the local monorepo path. Wrong URL?"
    log "Using local monorepo path ${MONOPATH}"

    # Clone link - when using deploy key
    MONOGITLINK="$(echo ${MONOHTTPSLINK} | sed -nE 's|^https://([^/]*)/(.*)$|git@\1:\2|p')"
    [ -z ${MONOGITLINK} ] && die "[FATAL] Failed to construct monorepo git link. Wrong URL?"
    log "Using monorepo git link ${MONOGITLINK}"
}

function get_valid_subdirectory() {
    local subdir="$1"
    local _subdirectory="$1"
    [ -d ".git" ] && basepath="$(git config --get mrt.basepath)"

    if [ $# -ge 2 ]; then
        _subdirectory="$2"
        log "Using the specified location as the monorepo subdirectory"
    elif [ ! -z "$basepath" ]; then
        subdir=${basepath}/${PWD##*/}
        log "The monorepo subdirectory is constructed from the file .git/config"
    elif [ ! -z "${GIT_MRT_BASE_PATH+x}" ]; then
        subdir=$GIT_MRT_BASE_PATH/${PWD##*/}
        log "The monorepo subdirectory is constructed from the variable GIT_MRT_BASE_PATH"
    else
        die "[FATAL] Cannot determine the monorepo subdirectory.
        Must specify a subdirectory, Or type 'git config mrt.basepath base_path_in_monorepo',
        Or setup the environment variable GIT_MRT_BASE_PATH"
    fi

    # validate subdirectory by removing extra slashes; /acc/diag//diag-mc/ --> acc/diag/diag-mc
    subdir="$(echo "$subdir" | sed -E 's://*:/:g; s:^/?::g; s:/*$::g')"
    log "The monorepo subdirectory is validated to $subdir"

    # the eval statement below returns/sets the caller's variable subdirectory
    eval $_subdirectory="'$subdir'"
}

function check_uncommited_changes () {
    log "Checking for uncommited changes in local subrepo"
    num_uncommited_changes=$(git status -s --untracked-files=no | wc -l)
    if [ "$num_uncommited_changes" != "0" ]; then
        git status --untracked-files=no
        die "[FATAL] $PWD is a git repository but has uncommited changes!
        Commit them first before pushing or pulling"
    fi
}

function sparse_pull_mono() {
    # $1 - subdirectory to sparse-checkout
    log "Creating and validating local monorepo"

    # Remember original dir
    pushd ${PWD} &> /dev/null

    # have to install the python script 'git-filter-repo' for the command 'git filter-repo'
    git filter-repo --version &> /dev/null
    [ $? -ne 0 ] && die "[FATAL] Please install the script 'git-filter-repo' in your \$PATH
        (i.e. /usr/local/bin/git-filter-repo)"

    # Make sure that monorepo home dir exists
    if [ ! -d $MONOHOME ]; then
        mkdir -p $MONOHOME
        [ ! -d $MONOHOME ] && die "[FATAL] Failed to create $MONOHOME"
    fi
    cd $MONOHOME

    if [ -d $MONODIR ]; then
        # Sanity check in case if local monorepo dir already exists
        if [ ! -d $MONODIR/.git ]; then
            die "[FATAL] $MONOPATH exists but is not a git repository, determine the cause manually"
        fi
    fi

    # Clone the monorepo if it is absent
    if [ ! -d $MONODIR ]; then
        #local gceargs="--filter=blob:none --sparse"
        local gceargs="--sparse"
        if [ ! "$DEPLOYKEYFILE" ]; then
            log "Cloning monorepo from $MONOHTTPSLINK"
            git clone $gceargs $MONOHTTPSLINK
        else
            log "Cloning monorepo from $MONOGITLINK"
            git clone -c core.sshCommand="/usr/bin/ssh -i $DEPLOYKEYFILE" $gceargs $MONOGITLINK
        fi
        if [ $? -ne 0 ]; then
            die "[FATAL] Monorepo clone failed"
        fi
    fi

    # since the first step of all operations (clone, pull, push) is calling 'sparse_pull_mono', it 
    # makes sense clean-up only needs to be done here. 
    cleanup_monorepo

    log "Performing monorepo sparse checkout"
    git sparse-checkout set $1
    [ $? -ne 0 ] && die "[FATAL] Monorepo sparse-checkout failed"

    log "Checking for monorepo updates"
    git pull -q origin $MONOBRANCH
    [ $? -ne 0 ] && die "[FATAL] Monorepo pull failed"

    # Restore original dir
    popd &> /dev/null
}

function cleanup_monorepo() {
    log "Cleaning up the local monorepo"
    cd $MONOPATH
    git checkout -q $MONOBRANCH
    git branch | grep -v "* $MONOBRANCH" | xargs git branch -D &> /dev/null
    git remote | grep -v origin | xargs git remote rm &> /dev/null
    git remote prune origin
}

function extract_subdirectory() {
    # $1: $subdirectory
    [ -z $1 ] && die "[FATAL] Subdirectory argument is not specified"
    log "Extracting the subdirectory into the temporary monorepo branch 'filter-repo'"
    cd $MONOPATH
    git checkout -b filter-repo || die "[FATAL] Failed to create the branch 'filter-repo'"
    # Rewrite the history of $subdirectory on the temporary branch 'filter-repo' created above
    # '--prune*': remove merge commits "Merge pull request ..." to get a clean history for the subrepo
    # '--refs filter-repo': keep the branch 'filter-repo' (otherwise it is deleted) after rewriting
    extra_args="--prune-empty=always --prune-degenerate=always"
    git filter-repo --force $extra_args --subdirectory-filter "$1" --refs filter-repo
    [ $? -ne 0 ] && die "[FATAL] Failed to perform filter-repo on $1"

    #gitk
    log "Removing 'Merge remote-tracking branch ...' to get a clean history for the subrepo"   
    git log -1 --oneline | grep "Merge remote-tracking branch 'temp/main' into"
    if [ $? -eq 0 ]; then
        #'git reset --hard HEAD^' does not work because the HEAD has two parents.
        # So get the latest new commit as the parent, then do 'git reset ...'
        local new_parent=( $(git log --oneline --skip 1 -1) )
        git reset --merge ${new_parent[0]}
    fi
}

function update_subrepo() {
    [ -z $1 ] && die "[FATAL] Subrepopath argument is not specified"
    log "Pulling changes from the branch 'filter-repo' into the subrepo"
    cd $1
    # '--rebase' solves the fast-forward problem when users forget 'git-mrt pull' before 'git-mrt push'
    git pull $MONOPATH filter-repo --rebase
    [ $? -ne 0 ] && die "[FATAL] Pull in the subrepo failed"
}

function update_basepath() {
    [ $# -ne 2 ] && die "ASSERT: missing arguments in update_basepath()"
    log "Updating subrepo basepath metadata"
    #cd $subrepopath
    cd $1
    local subdir="$(dirname $2)"
    local basepath="$(git config --get mrt.basepath)"
    if [ -z "$basepath" ]; then
        git config mrt.basepath $subdir
    else
        if [ "$basepath" != "$subdir" ]; then
            die "[FATAL] The subrepo basepath is ${basepath} but expecting ${subdir}.
            To update, run: git config mrt.basepath ${subdir}"
        fi
    fi
}

function mrt__clone() {
    if [ "$1" ]; then
        local subdirectory=$1
    else
        die 'ASSERT: mrt__clone() is called without subdirectory argument being specified'
    fi

    log "Preparing to extract monorepo subdir as local subrepo"
    get_valid_subdirectory $1 subdirectory
    local subrepopath="$(pwd)/`basename $subdirectory`"
    [ -d $subrepopath ] && die "[FATAL] `basename $subdirectory` directory already exists"

    sparse_pull_mono $subdirectory
    [ ! -d $MONOPATH/$subdirectory ] && die "[FATAL] Monorepo $MONODIR does not contain $subdirectory"

    log "Initializing a subrepo"
    mkdir $subrepopath || die "[FATAL] Failed to create the subrepo dir"
    cd $subrepopath 
    git init -q -b main

    extract_subdirectory $subdirectory
    update_subrepo $subrepopath

    log "Writing subrepo metadata"
    cd $subrepopath
    git config mrt.basepath "$(dirname $subdirectory)"

    log "Local subrepo clone complete"
}

function mrt__pull() {
    log "Preparing to pull local subrepo updates from the monorepo"
    local subrepopath=$(pwd)
    [ ! -d ".git" ] && die "[FATAL] $subrepopath is not a git repository"
    check_uncommited_changes

    get_valid_subdirectory $1 subdirectory
    sparse_pull_mono $subdirectory
    [ ! -d $MONOPATH/$subdirectory ] && die "[FATAL] Monorepo $MONODIR does not contain $subdirectory"

    extract_subdirectory $subdirectory
    update_subrepo $subrepopath
    #gitk

    update_basepath $subrepopath $subdirectory
    log "Local subrepo pull complete"
}

function mrt__push() {
    log "Preparing to push local subrepo updates to the upstream monorepo"
    local subrepopath=$(pwd)
    [ ! -d ".git" ] && die "[FATAL] $subrepopath is not a git repository"

    log "Determining the relevant local subrepo branch and the monorepo subdirectory"
    check_uncommited_changes
    local subrepobranch=$(git rev-parse --abbrev-ref HEAD)
    get_valid_subdirectory $1 subdirectory
    sparse_pull_mono $subdirectory

    # Check if the containing subdir is already present in the index
    log "Ensuring that no intermediate subdirs will be created unintentionally by the push"
    cd $MONOPATH
    git ls-files --error-unmatch $(dirname $subdirectory) &> /dev/null
    if [ $? -eq 1 ] && [ ! -n "$ALLOWDIRS" ]; then
        die "[FATAL] Performing the push would create a new intermediate monorepo subdir.
        Really wanna push? Repeat your command but with '-a', i.e. '$0 push $subdirectory -a'
        Or create '$(dirname $subdirectory)' subdir in the upstream monorepo directly"
    fi

    log "Rewriting local subrepo to a temporary repo with git filter-repo" 
    local tempdir="$(mktemp -d -t "${subdirectory##*/}.XXXX")"
    git clone $subrepopath $tempdir
    cd $tempdir
    local extra_args="--replace-refs delete-no-add --preserve-commit-hashes"
    git filter-repo $extra_args --force --to-subdirectory-filter $subdirectory 
    [ $? -ne 0 ] && die "[FATAL] git filter-repo failed to re-write history"
    #gitk

    log "Switching the local monorepo to a branch to be pushed"
    cd $MONOPATH
    if [ -z "${PUSHBRANCH+x}" ]; then
        local monorepobranch=$subdirectory/$subrepobranch
    else
        local monorepobranch="${PUSHBRANCH}"
    fi
    git checkout -b "$monorepobranch" || die "[FATAL] branch switch failed, check subdirectory name"
    #gitk

    # If the branch exists on remote, rebase on top of it to absorb changes
    # This allows multiple different pushes to the same branch by using --pushbranch
    git ls-remote --exit-code --heads origin "${monorepobranch}" &> /dev/null
    if [ $? -eq 0 ]; then
        git pull --rebase -q origin "${monorepobranch}"
        [ $? -ne 0 ] && die "[FATAL] Failed to rebase on origin branch ${monorepobranch}"
    fi

    log "Merging the temporary repo to the local monorepo branch"
    git remote add temp $tempdir
    git fetch temp
    local merge_args="--allow-unrelated-histories --no-edit"
    # DO NOT push if 'Already up to date.'
    git merge $merge_args temp/$subrepobranch | xargs | grep "Already up to date"
    [ $? -eq 0 ] && die "[INFO] No pushing because of already up to date"
    # Have to merge again?
    git merge $merge_args temp/$subrepobranch
    if [ $? -ne 0 ]; then
        log "Aborting merge"
        git merge --abort
        die "[FATAL] Merge failed - your local subrepo might be out of sync, try pull"
    fi

    log "Pushing local monorepo changes to upstream"
    git push --set-upstream origin $monorepobranch
    [ $? -ne 0 ] && die "[FATAL] Push to upstream failed"

    update_basepath $subrepopath $subdirectory
    log "Local subrepo push complete"
}

function mrt__status() {
    local subrepopath=${PWD}
    [ ! -d ".git" ] && die "[FATAL] ${subrepopath} is not a git repository"

    get_valid_subdirectory $1 subdirectory
    sparse_pull_mono $subdirectory
    if [ ! -d $MONOPATH/$subdirectory ]; then
        printf "\nThe remote monorepo is: ${MONOHTTPSLINK}\n"
        die "The remote monorepo does not contain the subdirectory $subdirectory"
    fi

    local branch=$(git rev-parse --abbrev-ref HEAD)
    local localcount=$(git rev-list --count ${branch})

    local tempdir="$(mktemp -d -t "${subdirectory##*/}.XXXX")"
    cd $tempdir
    mrt__clone $subdirectory &> /dev/null

    cd $subrepopath
    git remote add monorepo $tempdir/${subdirectory##*/}
    git fetch monorepo
    git branch -u monorepo/main
    git status | sed 's/git push/git-mrt push/g' | sed 's/git pull/git-mrt pull/g'
    git remote remove monorepo
}

function mrt_main() {
    POSITIONAL_ARGS=()
    while [[ $# -gt 0 ]]; do
        case $1 in
            -h|--help)
                usage
                exit 0
                ;;
            -v|--version)
                echo "git-mrt version $VERSION"
                exit 0
                ;;
            -m|--monolocation)
                if [ "$2" ]; then
                    MONOHOME=$2
                    shift 2
                else
                    die 'ERROR: -m|--monolocation requires a non-empty option argument'
                fi
                ;;
            -r|--remote)
                if [ "$2" ]; then
                    MONOHTTPSLINK=$2
                    shift 2
                else
                    die 'ERROR: -r|--remote requires a non-empty option argument'
                fi
                ;;
            -d|--deploykeyfile)
                if [ "$2" ]; then
                    DEPLOYKEYFILE=$2
                    shift 2
                else
                    die 'ERROR: -d|--deploykeyfile requires a non-empty option argument'
                fi
                ;;
            -a|--allowdirs)
                ALLOWDIRS=1
                shift
                ;;
            -p|--pushbranch)
                if [ "$2" ]; then
                    PUSHBRANCH=$2
                    shift 2
                else
                    die 'ERROR: -p|--pushbranch requires a non-empty option argument'
                fi
                ;;
            clone|pull|push|status)
                if [ -z ${cmdname+x} ]; then
                    cmdname=$1
                    shift
                else
                    die "ERROR: '$1' command requested but '$cmdname' was already specified"
                fi
                ;;
            -*|--*)
                log 0 "Unknown option '$1'"
                usage
                ;;
            *)
                POSITIONAL_ARGS+=("$1")
                shift
                ;;
        esac
    done

    set -- "${POSITIONAL_ARGS[@]}"

    if [ -z ${cmdname+x} ]; then
        log 0 "No command is specified"
        usage
    fi

    set_script_vars

    "mrt__$cmdname" "$@"
}

mrt_main $@