#!/bin/bash
#
#  File: install.sh
#
#  Copyright 2018-2023 Penguin Computing Inc.  All Rights Reserved.
#
DIR="$( cd "$( dirname $( realpath "${BASH_SOURCE[0]}" ) )" && pwd )"

DB_RPM="${DB_RPM:-clusterware-etcd}"

INFLUXDB2="clusterware-telegraf influxdb2 influxdb2-cli grafana grafana-pcp"
INSTALL_RPMS="${DB_RPM} clusterware-docs clusterware-dnsmasq clusterware-grafana clusterware-chrony clusterware-ipxe ${INFLUXDB2} scyld-nss"
# Old TICK packages that need to be erased when updating to CW12
TICK_OLD="influxdb chronograf kapacitor"

MAX_USER_LOGS=10
CWLOGS="${HOME}/.scyldcw/logs"
YUMDIR_CWREPO="/etc/yum.repos.d/clusterware.repo"
RESTORE_CWREPO=
MISMATCH_ERR="99"               # must be a unique cleanup() arg value
DEFAULT_NO="no"                 # default answer for ask_continue
BASE_INI="/opt/scyld/clusterware/conf/base.ini"
MANAGEDB="/opt/scyld/clusterware/bin/managedb"
USER="$(whoami)"

# Remove any crufty old lingering temp files.
rm -f /tmp/scyld-base.ini*

# The following line is parsed by src/cw_backend/views/cluster.py
pack_files="clusterware.repo.remote.template clusterware.repo.local.template"


spin[0]="-"
spin[1]="\\"
spin[2]="|"
spin[3]="/"
display_spinner()
{
    local PID=$1
    local MSG=$2

    log "${MSG}..."
    echo -n "${MSG}... ${spin[0]}"
    while kill -0 ${PID} 2>/dev/null; do
        for i in "${spin[@]}"
        do
            echo -ne "\b$i"
            sleep 0.1
        done
    done

    wait ${PID}
    RES=$?
    if [ "${RES}" == "0" ]; then
        msg_success "\bdone."
    else
        msg_error "\bFAILED."
    fi
    return $RES
}


prepare_colors()
{
    RED=
    YELLOW=
    GREEN=
    BLUE=
    NC=
    STATUS=
    if [ -t 1 ]; then
        RED='\033[0;31m'
        YELLOW='\033[0;33m'
        GREEN='\033[0;32m'
        BLUE='\033[0;34m'
        NC='\033[0m'
#        printf "${RED}Terminal ${YELLOW}output ${GREEN}coloring ${BLUE}enabled.${NC}\n"
    fi
}
prepare_colors


log()
{
    local MSG="$@"
    if [ -n "${LOG_FILE}" ]; then
        echo -e "$MSG" >> "${LOG_FILE}"
    fi
}


__msg()
{
    local COLOR="${!1}"
    local MSG="$2"

    log "${MSG}"
    echo -e "${COLOR}${MSG}${NC}"
}


msg_error()
{
    __msg RED "$@"
}


msg_warn()
{
    __msg YELLOW "$@"
}


msg_info()
{
    __msg BLUE "$@"
}


msg_success()
{
    __msg GREEN "$@"
}


msg_status()
{
    __msg STATUS "$@"
}


ctrl_c()
{
    echo
    stty sane
    cleanup 1 "Exiting on ctrl-c"
}
trap ctrl_c INT


print_help()
{
    local EXITVAL="$1"
    if [ "$2" != "" ]; then
        msg_error "$2"
        echo
    fi

    echo "Usage: $0 [OPTION]..."
    echo ""
    echo "Tool for installing ClusterWare head nodes, joining them to existing"
    echo "clusters, or saving and loading database backups."
    echo ""
    echo "optional arguments:"
    echo "  -h, --help        Print this help message."
    echo "  --config          Specify a cluster configuration file to load."
    echo "  --token           Specify a cluster serial number or other authentication"
    echo "                       token to use in the yum repository file."
    echo "  --yum-repo        Provide a complete yum repository file."
    echo ""
    echo "database load/save options:"
    echo "  -l, --load FILE   Load the ClusterWare database."
    echo "  -s, --save FILE   Save the ClusterWare database and exit."
    echo "  --without-files   Do NOT include the contents of images and boot files"
    echo "                      when loading or saving."
    echo ""
    echo "advanced options:"
    echo "  --non-interactive Execute installer in non-interactive mode that supplies"
    echo "                    default answers to otherwise interactive questions."
    echo "  --iso             Provide a URL or local path to the Clusterware ISO the"
    echo "                    system should use to install or upgrade."
    echo "  --os-iso          Provide a URL or local path to the ISO the system should"
    echo "                    use to create DefaultImage and DefaultBoot."
    echo "  --clear           DEPRECATED. Use --clear-all."
    echo "  --clear-all       Remove clusterware* packages except clusterware-installer,"
    echo "                    and clear the ClusterWare database, which removes all"
    echo "                    images and configurations."
    echo "                    This allows for a subsequent fresh install."
    echo "  --no-tools        Do not install the ClusterWare tools on the server."
    echo "  --join EXISTING_HEAD  Join this head node to an existing head node cluster."
    echo "  --skip-version-check  Use this installer without checking for a newer"
    echo "                        version online."
    echo "  --database-passwd Specify the root password for the database."
    echo "  --reconfigure     During an update most steps that alter the head node OS"
    echo "                    will be skipped, but this option will execute those."
    echo "  -u, --update      If ClusterWare is already installed, then skip asking user"
    echo "                      for confirmation that a software update is requested."

    cleanup "${EXITVAL}"
}


ask_continue()
{
    local EXITVAL=$1
    local ANSWER=$2
    local QUESTION=$3

    if [ -n "${NON_INTERACTIVE}" ]; then
        if [ -n "${ANSWER}" -a "${ANSWER}" == "no" ]; then
            msg_info "Presume default intention is to quit."
            cleanup "${EXITVAL}"
        else
            if [ -z "${ANSWER}" ]; then
                msg_info "Presume default intention is to continue."
            fi
            true; return
        fi
    else
        if [ -z "${QUESTION}" ]; then
            QUESTION="Press c to continue, any other key to quit: "
        fi
        read -r -n 1 -s -p "${QUESTION}" continue
        echo
        if [ "$continue" == "c" ]; then
            true; return
        else
            if [ -z "${EXITVAL}" ]; then
                cleanup 1 "Quitting"
            else
                if [ ${EXITVAL} -ge 0 ]; then
                    cleanup "${EXITVAL}" "Quitting"
                fi
                false; return
                # else EXITVAL < 0 means silently continue
            fi
        fi
    fi
}

yum_clean_all_if_needed()
{
    # If the clusterware.repo file has changed since the last 'yum clean all'
    #  then do another "clean all".
    if [ -e "${YUMDIR_CWREPO}" ]; then
        if ! diff "${LAST_YUM_CLEAN_REPO}" "${YUMDIR_CWREPO}" >& /dev/null; then
            run_with_sudo "yum --assumeyes --enablerepo=scyld* clean all >& /dev/null"
            cp "${YUMDIR_CWREPO}" "${LAST_YUM_CLEAN_REPO}"
        fi
    else
        # ${YUMDIR_CWREPO} doesn't exist yet, but 'clean all' to be safe.
        run_with_sudo "yum --assumeyes --enablerepo=scyld* clean all >& /dev/null"
    fi
}

cleanup()
{
    local EXITVAL=$1
    if [ -z "${EXITVAL}" ]; then
        EXITVAL=0
    fi
    local MSG=$2

    log "Cleaning up: $(date)"

    if [ "${EXITVAL}" == "${MISMATCH_ERR}" -a -n "${RESTORE_CWREPO}" ]; then
        # Caller used --yum-repo which might be bad - restore original file.
        if [ -s "${RESTORE_CWREPO}" ]; then
            # Backup file is non-empty: copy contents into ${YUMDIR_CWREPO}.
            run_with_sudo "cp -a ${RESTORE_CWREPO} ${YUMDIR_CWREPO}"
        else
            # Backup file is empty: delete ${YUMDIR_CWREPO}.
            run_with_sudo "rm -f ${YUMDIR_CWREPO}"
        fi
    fi
    yum_clean_all_if_needed
    rm -f "${LAST_YUM_CLEAN_REPO}"
    run_with_sudo "rm -f ${RESTORE_CWREPO}"

    # disable the repos
    if grep --silent '^\s*enabled\s*=\s*1\s*$' ${YUMDIR_CWREPO} 2>/dev/null; then
        run_with_sudo "sed --in-place 's/^\s*enabled\s*=\s*[01].*$/enabled=0/' ${YUMDIR_CWREPO}"
    fi

    # remove the temporary folder
    if [ -n "${TMPDIR}" ]; then
        log "Deleting temporary folder: ${TMPDIR}"
        rm -rf ${TMPDIR}
    fi

    if [ -n "${TMP_ISO}" -a -d "${TMP_ISO}" ]; then
        if [ -d "${TMP_ISO}"/mount ]; then
            # umount the ISO if we mounted it, but do it in background because
            #  we're likely executing the 'scyld-install' in that directory.
            log "Umounting and deleting temporary iso"
            run_with_sudo LIVE bash -c "sleep 1; umount -l ${TMP_ISO}/mount; sleep 1; rm -rf ${TMP_ISO}" &
        else
            # No mountpoint exists, so no need to umount.  Just delete TMP_ISO dir.
            log "Deleting temporary iso directory"
            run_with_sudo "rm -fr ${TMP_ISO}"
        fi
    fi

    # print the message and exit
    if [ -n "${MSG}" ]; then
        if [ "${EXITVAL}" == "0" ]; then
            msg_success "${MSG}"
        else
            msg_error "${MSG}"
        fi
    fi
    exit "${EXITVAL}"
}

check_file()
{
    local VAR=$1
    local FILE=$2
    local PARAMNAME=$3
    local MISSINGOK=$4
    if [ -n "${!VAR}" ]; then
        if [ -z "${!FILE}" ]; then
            print_help 1 "A file name must follow ${PARAMNAME}"
        elif [ -z "${MISSINGOK}" -a ! -e "${!FILE}" ]; then
            print_help 2 "The file '${!FILE}' does not exist."
        fi
    fi
}

using_mounted_ISO()
{
    # are we running from a mounted ISO?
    path=${DIR}
    read -r DEV MOUNT < <(df "${path}" | tail -n+2 | awk '{ print $1 " " $NF }')
    mount | grep "${path} type iso9660" &> /dev/null
    if [ "$?" == "0" ]; then
        true
    else
        false
    fi
    return
}

pending_clusterware_pkg()
{
    local list_type
    local list
    if rpm -q clusterware >& /dev/null; then
        list_type="updates"
    else
        list_type="available"
    fi
    yum_clean_all_if_needed
    list=$(run_with_sudo LIVE bash -c "yum --assumeyes --disablerepo=\* --enablerepo=scyld\* list ${list_type} clusterware" 2>/dev/null | grep ^clusterware | head -1)
    if [ -n "${list}" ]; then
        CW_PKG_VER=$( echo ${list} | awk '{print $2}')
        CW_PKG_ARCH=$(echo ${list} | awk '{print $1}' | sed -e 's/clusterware.//')
    fi
}

check_clusterware_major_for_cw12()
{
    pending_clusterware_pkg

    if [ -z "${CW_PKG_VER}" ]; then
        # No pending install or update, so what version is this now?
        CW_PKG_VER=$(rpm -q clusterware >& /dev/null)
    fi
    if [ -n "${CW_PKG_VER}" ] && [[ ! "${CW_PKG_VER}" =~ ^"12" ]]; then
        msg_error "\nThis ClusterWare 12 scyld-install only supports installing ClusterWare 12,"
        msg_error "updating to ClusterWare 12, or joining a ClusterWare 12 cluster."
        msg_error "Otherwise revert to a ClusterWare 11 scyld-install and retry."
        cleanup 1
    fi
}

check_clusterware_version_against_base()
{
    local cw_os_base=$1 # optional
    local base_os_ver
    local base_os_arch

    # If we'e already done this, then just quietly leave.
    if [ -n "${CHECKED_OS_COMPATIBLE}" ]; then
        return
    fi

    if [ -n "${cw_os_base}" ]; then
        CW_BASE_MAJOR=$(echo ${cw_os_base} | sed 's/^.*el//' | sed 's/\..*$//')
        CW_PKG_ARCH=$(echo ${cw_os_base} | sed 's/^.*\.//')
    else
        pending_clusterware_pkg
        CW_BASE_MAJOR=$(echo "${CW_PKG_VER}" | sed 's/^.*el//' | sed 's/\..*$//')
    fi

    if [ -n "${CW_PKG_VER}" ]; then
        base_os_ver=${os_release_major}
        base_os_arch=$(uname -p)
        if [ "${CW_BASE_MAJOR}" != "${base_os_ver}"   -o \
             "${CW_PKG_ARCH}"   != "${base_os_arch}" ]; then
            if [ -z "${CW_BASE_MAJOR}" ]; then
                CW_BASE_MAJOR="?"
            fi
            if [ -z "${base_os_arch}" ]; then
                base_os_arch="?"
            fi
            msg_warn "Mismatched ${JOIN_CLUSTER} ClusterWare version ${CW_BASE_MAJOR} for ${CW_PKG_ARCH} on base distro version ${base_os_ver} for ${base_os_arch}."
            ask_continue "${MISMATCH_ERR}" "${DEFAULT_NO}"
        fi
        CHECKED_OS_COMPATIBLE=yes
    fi

}

num_joined_head_nodes()
{
    # Is this node already joined to a multi-head cluster?
    local num_heads
    if $(which "${MANAGEDB}" >& /dev/null); then
        local err=$(mktemp -t stderr.XXXXXXXX)
        num_heads=$(sudo "${MANAGEDB}" --heads 2>${err} | wc -l)
        # That counts the header line in the output, so subtract that.
        num_heads=$(( ${num_heads} - 1))
        if $(grep -q grpc ${err}); then
            log "'managedb --heads' detects grpc error, so presume head node has been ejected:"
            cat ${err} >> "${LOG_FILE}"
            log "Ignore error and continue."
            num_heads=1
        fi
        rm -f ${err}
    else
        num_heads=1
    fi

    echo ${num_heads}
}

check_jointo_cred_database()
{
    local db_this_head
    local ret
    if [ -z "${DB_PASSWD}" ]; then
        log "Requesting the database password from the user."
        echo -n "Database password for ${JOIN_CLUSTER}: "
        read -s DB_PASSWD
        echo
    fi
    if rpm -q clusterware-etcd >& /dev/null; then
        db_this_head=etcd
    elif rpm -q clusterware-couchbase >& /dev/null; then
        db_this_head=couchbase
    fi

    # Ask join-to head node to validate db password and return db type.
    ret=$(curl -s --data "{\"admin_pass\": \"${DB_PASSWD}\"}" "${JOIN_CLUSTER}"/api/v1/install/password)

    PASSWD_OK=
    if $(echo "${ret}" | grep -q "\"success\": true"); then
        log "Database password for ${JOIN_CLUSTER} is correct."
        if $(echo "${ret}" | grep -q "\"database\"":); then
            DB_JOIN_TO=$(echo "${ret}" | sed 's/^.*"database": "//' | sed 's/".*}//')
            DB_JOIN_TO=$(echo ${DB_JOIN_TO,,})
        fi
        if [ -n "${db_this_head}" ]; then
            if [ "${DB_JOIN_TO}" == "${db_this_head}" ]; then
                log "Database type '${DB_JOIN_TO}' for ${JOIN_CLUSTER} matches this head node."
            else
                msg_error "Database type '${DB_JOIN_TO}' for ${JOIN_CLUSTER} mismatches '${db_this_head}' on this node, so cannot join it."
                cleanup 1
            fi
        fi
        PASSWD_OK=yes
    elif $(echo "${ret}" | grep -q "\"success\": false.*password is incorrect"); then
        msg_error "Database password for ${JOIN_CLUSTER} is incorrect, so cannot join it."
        cleanup 1
    fi

    # We no longer support Couchbase post-CW11, so such references to Couchbase
    #  should eventually go away.  Let's retain this through the first half of 2023.
    if [ -z "${PASSWD_OK}" -a "${db_this_head}" == "couchbase" ]; then
        # We didn't get an explicit verification from the join-to node and we
        #  use Couchbase.  Perhaps the join-to node is an older node that
        #  understands an older question that authenticates the password?
        ret=$(curl -s -o /dev/null -w "%{http_code}" -u "root:${DB_PASSWD}" ${JOIN_CLUSTER}:8091/settings/stats)
        if [ "$ret" == "200" ]; then
            msg_error "Couchbase cluster at $JOIN_CLUSTER authenticates the database password."
            PASSWD_OK=yes
        elif [ "$ret" == "401" ]; then
            msg_error "Couchbase cluster at $JOIN_CLUSTER rejects authentication of database password."
        fi
    fi

    if [ -z "${PASSWD_OK}" ]; then
        msg_error "Unable to authenticate ${JOIN_CLUSTER} password and database type."
        ask_continue 1 "${DEFAULT_NO}"
    fi

    # Now if ${PASSWD_OK} then we have the correct ${DB_PASSWD} password,
    #  and if ${DB_JOIN_TO} we have the join-to head node's database type.
}

report_cw_rpmnew_rpmsave()
{
    local files
    local rpmnew
    local rpmsave
    local newer
    local saved
    local existing
    local f

    # Deal with a special case of cw11 updating to cw12 and leaving a radically
    #  different /etc/telegraf/telegraf.conf.rpmnew.
    if [ -n "${UPDATE_CW11_TO_CW12}" ]; then
        if [ -e /etc/telegraf/telegraf.conf.rpmnew ]; then
            # Rename the old file to telegraf.conf.rpmsave and rename the
            #  new file to telegraf.conf.  The different files will still
            #  be reported, below, but telegraf.conf will be the cw12 default.
            run_with_sudo "mv -f /etc/telegraf/telegraf.conf /etc/telegraf/telegraf.conf.rpmsave"
            run_with_sudo "mv /etc/telegraf/telegraf.conf.rpmnew /etc/telegraf/telegraf.conf"
        fi
    fi

    # Collect the config file names into a local file for non-sudo access
    files=$(mktemp -t config_files.XXXXXXXX)
    run_with_sudo "cat /opt/scyld/clusterware/conf/config.files >${files}"

    # Search for *.rpmnew and *.rpmsave files
    rpmnew=$(mktemp -t rpmnew.XXXXXXXX)
    rpmsave=$(mktemp -t rpmsave.XXXXXXXX)
    for f in $(cat ${files}); do
        run_with_sudo "find ${f}.rpmnew  2>/dev/null >>${rpmnew}"
        run_with_sudo "find ${f}.rpmsave 2>/dev/null >>${rpmsave}"
    done

    # Any 'rpmnew' files?
    for newer in $(sort ${rpmnew} | uniq); do
        existing=$(echo ${newer} | sed -e 's/.rpmnew$//')
        if $(run_with_sudo diff -s ${existing} ${newer} >& /dev/null); then
            msg_info "Newer ${newer}"
            msg_info "  and ${existing} are the same, so deleting .rpmnew"
            run_with_sudo rm -f ${newer}
        else
            msg_warn "Newer ${newer} may need merging"
            msg_warn " into ${existing}"
        fi
    done

    # Any 'rpmsave' files?
    for saved in $(sort ${rpmsave} | uniq); do
        existing=$(echo ${saved} | sed -e 's/.rpmsave$//')
        if $(run_with_sudo diff -s ${existing} ${saved} >& /dev/null); then
            msg_info "Saved ${saved}"
            msg_info "  and ${existing} are the same, so deleting .rpmsave"
            run_with_sudo rm -f ${saved}
        else
            msg_warn "Saved ${saved} may need merging"
            msg_warn " into ${existing}"
        fi
    done

    rm -f ${files} ${rpmnew} ${rpmsave}
}


generate_excludes()
{
    local only="$1"

    run_with_sudo LIVE bash -c "yum --assumeyes --disablerepo=\* --enablerepo=scyld\* clean metadata >/dev/null"
    while read -r pkg; do
        match=$(echo "${pkg}" | grep "${only}")
        if [ -n "${match}" ]; then
            found=yes
        else
            if [ -n "${args}" ]; then
                args="${args} "
            fi
            args="${args} --exclude $(echo "${pkg}" | sed 's/ware-/ware*-/' | sed 's/-[^-]*$/-*/')"
        fi
    done < <(run_with_sudo LIVE bash -c "yum --assumeyes --disablerepo=\* --enablerepo=scyld\* list clusterware --showduplicates" | \
             awk "/^clusterware/ { print \"clusterware-\" \$2 \".$(uname --machine)\" }")

    echo "${args}"
}


excludes=
install_latest_package()
{
    # see if we should exclude some packages
    if [ -n "${matchver}" -a -z "${excludes}" ]; then
        excludes=$(generate_excludes "${matchver}")
    fi

    check_clusterware_version_against_base

    # Check if package is already installed
    local NAME="$1"
    RPMVER=$(rpm -q "${NAME}" 2>/dev/null)
    if [ "$?" == "0" ]; then

        # Already installed so try to update.
        RPMVER=$(echo "${RPMVER}" | sed 's/\.scyld\..*//')
        run_with_sudo SPINNER "Updating ${RPMVER}" "yum -y update ${excludes} ${NAME} > >(tee --append '${LOG_FILE}') 2>&1"
        RPMNEWVER=$(rpm -q "${NAME}" 2>/dev/null | sed 's/\.scyld\..*//')
        if [ "${RPMVER}" == "${RPMNEWVER}" ]; then
            if $(tail -8 "${LOG_FILE}" | grep -q "HTTPS.*401.*Unauthorized"); then
                msg_error "    Failed to update ${NAME} package: Unauthorized access to repo."
                msg_error "    Please confirm your credentials in the clusterware.repo file."
                false; return
            fi
            msg_status "         no newer version found"
        else
            msg_status "         updated to ${RPMNEWVER}"
        fi

        true; return 
    fi

    # Package wasn't installed so try to install.
    run_with_sudo SPINNER "Installing ${NAME}" "yum -y install ${excludes} ${NAME} > >(tee --append '${LOG_FILE}') 2>&1"
    RPMVER=$(rpm -q "${NAME}" 2>/dev/null)
    if [ "$?" == "0" ]; then
        msg_status "           installed $(echo "${RPMVER}" | sed 's/\.scyld\..*//')" 
        true; return
    fi

    # Could not install.
    if $(tail -8 "${LOG_FILE}" | grep -q "HTTPS.*401.*Unauthorized"); then
        msg_error "    Unauthorized access to repo.\n    Please confirm your credentials in the clusterware.repo file."
    else
        msg_error "           failed installing ${NAME}"
    fi
    false; return
}

enable_start_service()
{
    local service="$1"
    local action="$2"
    msg_status "Enabling and starting service ${service}"

    if [ "$(systemctl is-enabled "${service}" 2>/dev/null)" != "enabled" ]; then
        run_with_sudo "systemctl enable ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    fi
    if [ "$(systemctl is-active "${service}" 2>/dev/null)" != "active" ]; then
        run_with_sudo "systemctl start ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    elif [ -n "${action}" ]; then
        run_with_sudo "systemctl ${action} ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    fi
}

stop_disable_service()
{
    local service="$1"
    local action="$2"
    msg_status "Stopping and disabling service ${service}"
    if systemctl is-active "${service}" &>/dev/null; then
        run_with_sudo "systemctl stop ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    fi
    if systemctl is-enabled "${service}" &>/dev/null; then
        run_with_sudo "systemctl disable ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    elif [ -n "${action}" ]; then
        run_with_sudo "systemctl ${action} ${service} > >(tee --append ${LOG_FILE}) 2>&1"
    fi
}

open_port_or_service()
{
    local NAME=$1
    echo ${NAME} | grep '^[0-9]*/' 2>/dev/null 1>/dev/null
    if [ "$?" == "0" ]; then
        msg_status "Opening firewall port ${NAME}"
        run_with_sudo "firewall-cmd --add-port ${NAME} > >(tee --append ${LOG_FILE}) 2>&1" || \
            cleanup 1 "  opening firewall to ${NAME} failed: $?"
        run_with_sudo "firewall-cmd --permanent --add-port ${NAME} > >(tee --append ${LOG_FILE}) 2>&1" || \
            cleanup 1 "  opening firewall to ${NAME} with --permanent failed: $?"
    else
        msg_status "Opening firewall for service ${NAME}"
        run_with_sudo "firewall-cmd --add-service ${NAME} > >(tee --append ${LOG_FILE}) 2>&1" || \
            cleanup 1 "  opening firewall for ${NAME} failed: $?"
        run_with_sudo "firewall-cmd --permanent --add-service ${NAME} > >(tee --append ${LOG_FILE}) 2>&1" || \
            cleanup 1 "  opening firewall for ${NAME} with --permanent failed: $?"
    fi
}


open_firewall_for_tftp()
{
    ARG=$1
    run_with_sudo "firewall-cmd ${ARG} --add-service tftp > >(tee --append ${LOG_FILE}) 2>&1"
    RES=$?
    if [ "${RES}" != "0" ]; then
        # ignore the module loading error if it crops up inside docker
        if [ -e "/.dockerenv" ]; then
            msg_warn "  opening firewall for tftp failed (${RES}), but this is expected in a container."
        else
            cleanup 1 "  opening firewall for tftp failed: ${RES}"
        fi
    fi
}


proxy_vars_seen=
__run_sudo()
{
    local proxy_vars
    if [ -n "${http_proxy}${https_proxy}${no_proxy}" ]; then
        if [ -z "${proxy_vars_seen}" ]; then
            msg_warn "One or more proxy variables are set.\n If installation fails, please inspect your proxy settings."
            proxy_vars_seen=yes
        fi
        proxy_vars="export http_proxy=${http_proxy}; export https_proxy=${https_proxy}; export no_proxy=${no_proxy};"
    fi

    # force the use of the bash builtin echo to avoid SUDO_PASS hitting
    # a command line and showing up in /proc
    SUDO_OUTPUT=$(eval "PATH= command echo '${SUDO_PASS}' | sudo --prompt= --stdin bash -c '${proxy_vars} $@'")
}


__run_sudo_live()
{
    # force the use of the bash builtin echo to avoid SUDO_PASS hitting
    # a command line and showing up in /proc
    PATH= command echo "${SUDO_PASS}" | sudo --prompt= --stdin "$@"
}


run_with_sudo()
{
    local SPINMSG=
    if [ "$1" == "SPINNER" ]; then
        shift
        SPINMSG=$1
        shift
    fi

    local LIVE=
    if [ "$1" == "LIVE" ]; then
        shift
        LIVE=yes
    fi

    if [ -z "${ASKED_PASS}" -a "$(whoami)" != "root" ]; then
        # check if sudo is asking for a password
        first=yes
        while true; do
            if sudo --non-interactive true 2>/dev/null; then
                break
            elif [ -n "${first}" ]; then
                echo "This script uses sudo to configure the system. Please provide the"
                first=
            fi

            log "Requesting user's sudo password."
            echo -n "[sudo] password for ${USER}: "
            read -s SUDO_PASS
            echo
            ASKED_PASS=yes
            __run_sudo true
        done
    fi

    log "Running: $@"
    if [ -n "${SPINMSG}" ]; then
        #echo "====$@===="
        __run_sudo "$@" & CPID=$!
        display_spinner ${CPID} "${SPINMSG}"
    else
        if [ -n "${LIVE}" ]; then
            __run_sudo_live "$@"
        else
            __run_sudo "$@"
        fi
    fi
    return $?
}


# This function will find INFLUXDB2 and scyld-nss packages so we can upgrade
#   clusterware\*, INFLUXDB2, and scyld-nss but not mess with middleware or
#   schedulers or the like.  We also need to do this for --clear-all.
scyld_influxdb_nss()
{
    local INFLUXDB_NSS
    local RPM_Q
    for pkg in ${INFLUXDB2} scyld-nss
    do
        RPM_Q=$(rpm -q ${pkg})
        if [ $? -eq 0 ]; then
            # ${pkg} is installed...
            if $(echo "${RPM_Q}" | grep -q scyld); then
                # and ${pkg} is a "scyld" package.
                INFLUXDB_NSS="${pkg} ${INFLUXDB_NSS}"
            fi
        fi
    done
    echo "${INFLUXDB_NSS}"
}


clear_all()
{
    local SCYLDCW_DIR=$(realpath ~/.scyldcw)
    local ROOT_SCYLDCW_DIR="/root/.scyldcw"
    local next_line
    local extra_text

    msg_warn "\nThis operation will remove all ClusterWare (except clusterware-installer)"
    msg_warn "and Scyld status monitoring packages (e.g., influxdb, telegraf),"
    if rpm -q couchbase-server >& /dev/null; then
        msg_warn "plus the couchbase-server, libcouchbase and libcouchbase-devel RPMs,"
    fi

    next_line="and delete directories /var/log/clusterware/,"
    if [ -d /opt/couchbase/ ]  &&  ! rpm -q couchbase-server >& /dev/null ]; then
        next_line+=" /opt/couchbase/,"
    fi
    msg_warn "${next_line}"

    msg_warn "/opt/scyld/clusterware*/ (except /opt/scyld/clusterware-installer/),"

    next_line=${SCYLDCW_DIR}
    if [ "${SCYLDCW_DIR}" != "${ROOT_SCYLDCW_DIR}" ]; then
        next_line+=" and ${ROOT_SCYLDCW_DIR}"
    fi
    next_line+=" (except scyldiso.iso and logs/),"
    msg_warn "${next_line}"

    if $(grep -q ^baseurl.*localhost.*scyldiso "${YUMDIR_CWREPO}"); then
        extra_text="and unmount the ClusterWare ISO, "
    fi
    msg_warn "${extra_text}before reinstalling ClusterWare."
    echo ""

    # We delete ~/.scyldcw/ for the current admin and root, but not for other admins!
    if $(which scyld-adminctl >& /dev/null); then
        local other_admins=$(scyld-adminctl ls | grep -v ^"Administrators" | grep -v $(whoami) | grep -v "root")
        if [ -n "${other_admins}" ]; then
            local list=$(echo ${other_admins} | tr '\n' ' ' | sed 's/ $//')
            msg_warn "Other administrators (${list}) should also --clear-all!"
            echo ""
        fi
    fi

    msg_warn "Are you sure you want to proceed?"
    ask_continue 1

    if $(which scyld-clusterctl >& /dev/null); then
        if $(scyld-clusterctl repos -i scyldiso ls >& /dev/null); then
            echo ""
            msg_warn "Do you want to capture the scyldiso repo into an ISO file?"
            if $(ask_continue -1 "yes" "Press c to save ISO file, any other key to not save: "); then
                echo ""
                if $(scyld-clusterctl repos -i scyldiso download --dest $HOME/.scyldcw iso 1>/dev/null); then
                    msg_warn "Captured scyldiso repo saved as '$HOME/.scyldcw/scyldiso.iso'"
                else
                    msg_warn "Cannot save repo.  Do you want to proceed?"
                    ask_continue 1 "${DEFAULT_NO}"
                fi
            else
                echo ""
            fi
        fi
    fi

    # Is this a head node in a multi-head node cluster?
    # If so, then issue some warnings and be cautious about proceeding.
    if $(which "${MANAGEDB}" >& /dev/null); then
        local num_heads=$(num_joined_head_nodes)
        if [ ${num_heads} -ge 2 ]; then
            echo ""
            msg_warn "This head node is part of a ${num_heads} head node configuration."
            if [ ${num_heads} -eq 2 ] && rpm -q clusterware-etcd >& /dev/null; then
                msg_warn "You must execute 'managedb recover' on the other head node,"
                msg_warn "either before doing this --clear-all or immediately afterwards."
            else
                msg_warn "Before doing the --clear-all you should execute either 'managedb leave' on"
                msg_warn "this head node or 'managedb eject <IP-this-headnode>' on another head node."
            fi
            echo ""
            msg_warn "Do you want to quit and perform that action, or do you want to continue?"
            ask_continue 1 "${DEFAULT_NO}"
        fi
    fi

    log "Starting --clear-all"

    # The current directory might be in a directory that will soon be deleted.
    # Switch to a safe dir (reverting back at the end if the CWD still exists).
    CWD=$(pwd)
    cd /tmp

    # Copy the current clusterware.repo file to a backup.
    if [ -e ${YUMDIR_CWREPO} ]; then
        BACKUP=$(echo -n "${YUMDIR_CWREPO}_"; date --iso-8601=seconds | sed 's/:/./g' | tr -d '\n')
        run_with_sudo "cp -f ${YUMDIR_CWREPO} ${BACKUP}"
    fi

    # Do we need to prune the number of clusterware.repo backup files?
    local files=$(ls "${YUMDIR_CWREPO}"_20[0-9][0-9]-[0-1][0-9]-* 2>/dev/null | sort)
    local files_count=$(echo "${files}" | wc -l)
    local rm_count=$(( files_count - MAX_USER_LOGS ))
    if [ ${rm_count} -gt 0 ]; then
        log "Remove excess clusterware.repo backups"
    fi
    local f
    for f in $files ; do
        if [ $rm_count -le 0 ]; then break ; fi
        run_with_sudo "rm -f ${f}"
        rm_count=$(( rm_count - 1 ))
    done

    local service
    for service in httpd influxdb grafana-server telegraf; do
        stop_disable_service $service
    done

    if [ -x /opt/scyld/scyld-nss/bin/scyld-nssctl ]; then
        sudo /opt/scyld/scyld-nss/bin/scyld-nssctl stop
    fi

    if [ "${os_release_major}" == "7" ]; then
        stop_disable_service xinetd
    else
        stop_disable_service tftp
    fi

    mountpoint /opt/scyld/clusterware/storage > /dev/null
    if [ "$?" == "0" ]; then
        run_with_sudo SPINNER "Unmounting default storage directory" "umount /opt/scyld/clusterware/storage"
    else
       echo "Unmounting default storage directory... not mounted."
    fi
    local installed_pkgs=$(rpm -qa --qf "%{NAME}\n" | grep clusterware | sort; echo couchbase-server libcouchbase libcouchbase-devel $(scyld_influxdb_nss) ${TICK_OLD} python39-scyld)
    local pkg
    for pkg in ${installed_pkgs}; do
        if [ "${pkg}" != "clusterware-installer" ]; then
            run_with_sudo SPINNER "Removing ${pkg}" "yum -y remove ${pkg} >& /dev/null"
            if rpm -q ${pkg} >& /dev/null; then
                args=
                for arg in --nodeps --noscripts; do
                    args="${args} ${arg}"
                    run_with_sudo SPINNER "Trying rpm -e ${args} ${pkg}" "rpm -e ${args} ${pkg} >& /dev/null"
                    if ! rpm -q ${pkg} >& /dev/null; then
                        break
                    fi
                done
            fi

            if [ "${pkg}" == "python39-scyld" ]; then
                # Special case!
                path="/opt/scyld/python39"
            else
                path="/opt/scyld/${pkg}"
            fi
            if [ -e "${path}" ]; then
                run_with_sudo SPINNER "Erasing ${path}" "rm -rf ${path}"
            fi
        fi
    done

    if [ -e "/opt/couchbase" ]; then
        run_with_sudo SPINNER "Erasing /opt/couchbase" "rm -rf /opt/couchbase"
    fi
    run_with_sudo SPINNER "Erasing /var/log/clusterware" "rm -rf /var/log/clusterware"
    log "Erasing database services from firewalld"
    # Remove both databases, just to be safe.
    run_with_sudo "firewall-cmd --remove-service couchbase >& /dev/null"
    run_with_sudo "firewall-cmd --permanent --remove-service couchbase >& /dev/null"
    run_with_sudo "firewall-cmd --remove-service cw-etcd >& /dev/null"
    run_with_sudo "firewall-cmd --permanent --remove-service cw-etcd >& /dev/null"

    # TODO: unclear why directory gets replaced by telegraf.conf file but delete it.
    run_with_sudo "rm -fr /etc/telegraf"
    run_with_sudo "systemctl daemon-reload"
    CWINSTALLED=

    local scyldcw_dir
    for scyldcw_dir in ${SCYLDCW_DIR} ${ROOT_SCYLDCW_DIR}; do
        run_with_sudo "test -d ${scyldcw_dir}"
        if [ "$?" == "0" ]; then
            # This user has a .scyldcw/ subdirectory.

            run_with_sudo "test -d ${scyldcw_dir}/workspace"
            if [ "$?" == "0" ]; then
                # This user has a .scyldcw/workspace/ subdirectory.
                # Find any bind mounts.
                #  We can't simply read /proc/mounts, umount, and keep reading because
                #  the unmount perturbs the contents of /proc/mounts and confuses bash.
                local workspace=${scyldcw_dir}/workspace
                local mounts_list=$(mktemp -t mounts.XXXXXXXX)
                grep ${workspace} /proc/mounts >${mounts_list}
                if [ -s ${mounts_list} ]; then
                    msg_info "Unmount mounts in ${workspace}..."
                    while read -r ; do
                        local mnt=$(echo ${REPLY} | awk '{print $2}')
                        run_with_sudo "umount '${mnt}'"
                    done < "${mounts_list}"
                fi
                rm -f ${mounts_list}
            fi

            msg_info "Removing ${scyldcw_dir}/* (retaining scyldiso.iso and logs/)..."
            run_with_sudo "ls ${scyldcw_dir}"
            scyldcw_dir_files="${SUDO_OUTPUT}"
            for f in ${scyldcw_dir_files}; do
                if [ "${f}" != "logs" -a "${f}" != "scyldiso.iso" ]; then
                    run_with_sudo "rm -fr ${scyldcw_dir}/${f}"
                fi
            done
        fi
    done

    # Lobotomize yum's memory of rpms
    run_with_sudo "yum --assumeyes --enablerepo=scyld* clean all"
    # And because "clean all" doesn't fully "clean all"...
    PATHS=$(find /var/cache/yum -name scyld\*)
    for P in ${PATHS}; do
        run_with_sudo "rm -fr ${P}"
    done

    echo
    if [ ! -d "${CWD}" ]; then
        msg_warn "Erasing complete."
        msg_warn "Current working directory ${CWD} has been deleted."
        msg_warn "You should change to a valid directory before re-installing ClusterWare."
        cleanup 1 "Quitting"
    else
        # Revert to the original current working directory.
        cd ${CWD}
    fi

    msg_info "Erasing complete.  Ready to start new installation?"
    if [ -z "${LOAD_CONFIG}" -o -n "${LOAD_CONFIG_SKIP}" ]; then
        ask_continue 1 "${DEFAULT_NO}"
    else
        ask_continue 1
    fi

    run_with_sudo "rm -f ${RESTORE_CWREPO}"

    # Start a new LOG_FILE to get a timestamp after the erasing
    LOG_FILE=$(echo -n "${CWLOGS}/install_"; date --iso-8601=seconds | sed 's/:/./g' | tr -d '\n'; echo -n '.log')
}


unpack()
{
    # either copy the local install files or unpack the compressed payload
    if [ -f "${DIR}/clusterware.repo.remote.template" ]; then
        for name in ${pack_files}; do
            cp "${DIR}"/${name}  "${TMPDIR}"
        done
        msg_status "Copied installation files."
    else
        match=$(grep --text --line-number '^PAYLOAD:$' $0 | cut -d ':' -f 1)
        if [ -z "${match}" ]; then
            cleanup 6 "No compressed payload found, exiting."
        fi

        payload_start=$((match + 1))
        tail -n +$payload_start $0 | base64 --decode | tar --warning=no-timestamp -C "${TMPDIR}" -xzf -
        msg_status "Unpacked installation files."
    fi
}


# parse the command line
parse_args()
{
    #################################################################
    # The following lines will be located and replaced whenever this
    # is repacked by the cw_backend code.
    SERIAL_NUM=
    SERIAL_NUM_TARGET=
    SKIP_VER_CHECK=
    JOIN_CLUSTER=
    #################################################################

    ARCHIVE_ARGS=
    INSTALL_TOOLS=yes
    DO_CLEAR_ALL=
    LOAD_CONFIG=
    LOAD_CONFIG_SKIP=
    OSISO_SKIP=
    RECONF=
    DEVEL=
    while test $# -gt 0
    do
        case "$1" in
            --help|-h)
                print_help 0
                ;;

            --devel)
                DEVEL=$(pwd)
                ;;

            --loop-404)
                # A "hidden" option to support in-house testing.
                # Allows '--iso' and waiting for ISO to be available.
                LOOP_404=yes
                ;;

            --skip-version-check)
                SKIP_VER_CHECK=yes
                ;;

            --config)
                LOAD_CONFIG=$2
                if [ "${LOAD_CONFIG}" == "skip" ]; then
                    # Special case: don't load config file
                    LOAD_CONFIG_SKIP=yes
                elif [ ! -e "${LOAD_CONFIG}" ]; then
                    msg_error "--config file \"${LOAD_CONFIG}\" does not exist!"
                    cleanup 1
                fi
                shift
                ;;

            --match)
                matchver="$2"
                shift
                ;;

            --token)
                SERIAL_NUM=yes
                SERIAL_NUM_TARGET=$2
                shift
                ;;

            --yum-repo)
                YUM_REPO=yes
                YUM_REPO_TARGET=$2
                shift
                ;;

            --no-tools)
                INSTALL_TOOLS=
                ;;

            --clear-all)
                if [ -n "${ERROR_ON_ARGS}" ]; then
                    DO_CLEAR_ALL=$1
                fi
                ;;

            --load|-l)
                LOAD_ARCHIVE=yes
                TARGET=$2
                shift
                ;;

            --save|-s)
                SAVE_ARCHIVE=yes
                TARGET=$2
                shift
                ;;

            --update|-u)
                DO_UPDATE=$1
                ;;

            --without-files)
                ARCHIVE_ARGS="--without-all"
                ;;

            --clear)
                REINIT=yes
                ;;

            --non-interactive)
                NON_INTERACTIVE=yes
                ;;

            --iso)
                ISO=$2
                shift
                ;;

            --os-iso)
                OSISO=$2
                if [ "${OSISO}" == "skip" ]; then
                    # Special case: don't build images or load cluster config
                    OSISO_SKIP=yes
                fi
                shift
                ;;

            --join)
                JOIN_CLUSTER=$2
                shift
                ;;

            --database-passwd)
                DB_PASSWD=$2
                LEN=$(echo "${DB_PASSWD}" | wc -c)
                if [ ${LEN} -le 8 ]; then
                    msg_warn "Warning: influx2 requires database passwd to be minimum of 8 chars"
                fi
                shift
                ;;

            --reconfigure)
                RECONF=yes
                ;;

            --pack)
                echo "Packing a stand-alone ./scyld-install"
                # confirm we have what we need
                unpack
                for name in ${pack_files}; do
                    if [ ! -e "${TMPDIR}/${name}" ]; then
                        echo "  required file missing: ${TMPDIR}/${name}"
                        cleanup 1
                    fi
                done

                # package the installer
                outfile=./scyld-install
                cp "$0" "${TMPDIR}"/install.sh
                match=$(grep --text --line-number '^PAYLOAD:$' "${TMPDIR}"/install.sh | cut -d ':' -f 1)
                if [ -n "${match}" ]; then
                    head -n$((${match} - 1)) "${TMPDIR}"/install.sh > "${outfile}"
                else
                    cat "${TMPDIR}"/install.sh > "${outfile}"
                fi
                echo "PAYLOAD:" >> "${outfile}"
                tar -C "${TMPDIR}" -chzf- ${pack_files} | base64 >> "${outfile}"
                chmod 755 "${outfile}"

                cleanup
                ;;

            *)
                if [ -n "${ERROR_ON_ARGS}" ]; then
                    print_help 3 "Unknown argument: $1"
                fi
                ;;
        esac
        shift
    done

    if [ -n "${DO_CLEAR_ALL}" ]; then
        clear_all
    fi
}

#
#  BEGIN INSTALLER
#

# open the log file
if [ -z "${LOG_FILE}" ]; then
    LOG_FILE=$(echo -n "${CWLOGS}/install_"; date --iso-8601=seconds | sed 's/:/./g' | tr -d '\n'; echo -n '.log')
    mkdir -p "${CWLOGS}"
fi

# If not a restart of the installer, then use a new LAST_YUM_CLEAN_REPO file.
if [ -z "${LAST_YUM_CLEAN_REPO}" ]; then
    LAST_YUM_CLEAN_REPO=$(mktemp -t yum_clean_repo.XXXXXXXX)
fi

# If not a restart of the installer, then use a new TMPDIR directory.
if [ -z "${TMPDIR}" ]; then
    # Create a temporary directory to hold temp installer files
    TMPDIR=$(mktemp --directory --tmpdir clusterware-inst.XXXXXX)
    log "Created temporary folder: ${TMPDIR}"
fi

# Do we need to prune the number of log files?
touch "${LOG_FILE}"                  # count new file as one of the retained
FILES=$(ls "${CWLOGS}"/install_* 2>/dev/null | sort)
FILES_COUNT=$(echo "${FILES}" | wc -l)
RM_COUNT=$(( FILES_COUNT - MAX_USER_LOGS ))
for F in $FILES ; do
    if [ $RM_COUNT -le 0 ]; then break ; fi
    rm -f "$F"
    RM_COUNT=$(( RM_COUNT - 1 ))
done

ERROR_ON_ARGS= parse_args "$@"

echo "ClusterWare installer starts."
PKG=$(rpm -qf $0)
if [ $? -eq 0 ] && [ $(echo ${PKG} | grep -qv "is not owned") ]; then
    log "ClusterWare installer $0 (${PKG}) starts."
else
    log "ClusterWare installer $0 starts."
fi
if [ "$#" == "0" ]; then
    log "  no arguments passed"
else
    log "  arguments: $@"
fi
log "  $(echo -n "when: "; date --iso-8601=ns)"
log "----------------------------------------"

# pull in os-release fields for multiple uses
. /etc/os-release

# simplify down to just the major number
os_release_major=$(echo "${VERSION_ID}" | sed 's/^\(.*\)\..*$/\1/')

# check if we appear to be running from an ISO with a local repository
UPDATED=
LOCAL_REPO=
if [ -d "${DIR}/ScyldPackages" -a -d "${DIR}/repodata" ]; then
    if ! using_mounted_ISO ; then
        msg_error "${DIR} must be a mounted ISO, not a copy."
        cleanup 1
    fi
    LOCAL_REPO="${DIR}"
    echo "Using local repository at ${LOCAL_REPO}"
    yum_clean_all_if_needed

    CW_OS_RPM=$(ls "${DIR}"/ScyldPackages/clusterware-[1-9]*\.scyld\.* 2>/dev/null)
    if [ -z "${CW_OS_RPM}" ]; then
        msg_error "${DIR}/ScyldPackages/ does not contain a clusterware 'scyld' RPM"
        cleanup 1
    fi
    CW_OS_BASE=$(echo ${CW_OS_RPM} | sed 's/^.*ScyldPackages.clusterware-.*\.scyld.//' | sed 's/.rpm//')
    if [ -n "${CW_OS_BASE}" ]; then
        check_clusterware_version_against_base "${CW_OS_BASE}"
    fi

    # If 'clusterware-installer' is present, then try to update it using
    #  the clusterware-installer in the ISO.
    log "Checking for newer installer in ISO."
    run_with_sudo "rpm -Uvh '${DIR}/ScyldPackages/clusterware-installer*' 2>&1 | cat"
    echo ${SUDO_OUTPUT} | grep 'Updating / installing...' &>/dev/null
    if [ "$?" == "0" ]; then
         # Assume ${DIR}/scyld-install is same version
         #     as ${DIR}/ScyldPackages/clusterware-installer RPM
         echo "Updated to a new installer RPM and currently running that installer."
         SCYLD_INSTALL="${DIR}/scyld-install"
    fi

    SKIP_VER_CHECK=silent
else
    if [ -e ${YUMDIR_CWREPO} ] && \
       grep -q ^baseurl.*http:\/\/localhost\/api\/v.\/isomount\/scyldiso ${YUMDIR_CWREPO}; then
           SKIP_VER_CHECK=yes
    fi
fi

# Prior to checking the command line arguments check for a newer script via RPM.
if [ -n "${SKIP_VER_CHECK}" ]; then
    if [ "${SKIP_VER_CHECK}" != "silent" ]; then
        echo "Skipping check for a newer installer rpm."
    fi
else
    # default the RPM path to the generally correct value
    if [ -z "${RPM_URL}" ]; then
        # NO trailing slash on this one
        RPM_BASE_URL=https://updates.penguincomputing.com/clusterware/11/installer
        RPM_URL_DETECT=yes
    fi

    if [ -z "${RPM_URL}" ]; then
        if [ -n "${RPM_URL_DETECT}" ]; then
            REPO_FILE=
            if [ -n "${YUM_REPO}" ]; then
                REPO_FILE=${YUM_REPO_TARGET}
                # We will replace the current clusterware.repo with a new file,
                #  so first make a special backup in case we need to restore.
                RESTORE_CWREPO=$(mktemp -t repofile.orig.XXXXXX)
                if [ -e "${YUMDIR_CWREPO}" ]; then
                    run_with_sudo "cp -a ${YUMDIR_CWREPO} ${RESTORE_CWREPO}"
                fi
            elif [ -e "${YUMDIR_CWREPO}" ]; then
                REPO_FILE=${YUMDIR_CWREPO}
            fi
            if [ -n "${REPO_FILE}" ]; then
                URL=$(grep '^\s*baseurl=' ${REPO_FILE} | \
                      head -n1 | sed 's/^baseurl=//' | sed 's/\/*$//')
                REPO_COUNT=$(grep '^\s*baseurl=' ${REPO_FILE} | wc -l)
                if [ ${REPO_COUNT} -gt 1 ]; then
                    msg_warn "Checking only 1st of ${REPO_COUNT} repos in ${REPO_FILE}."
                fi
            fi

            if [ -n "${URL}" ]; then
                path=$(echo "${URL}" | sed 's|FILE://||i')
                if $(echo ${path} | grep -q repo\/scyldiso\/); then
                    # This is a locally mounted repo.
                    RPM_BASE_URL="${path}/ScyldPackages"
                    RPM_URL=${RPM_BASE_URL}/$(curl -s ${RPM_BASE_URL}/ | grep clusterware-installer | sed 's/^.*="//' | sed 's/".*$//')
                elif [ "${path::1}" == "/" ]; then
                    RPM_URL=$(ls "${path}/clusterware-installer-"*)
                else
                    INST_DIR=$(echo "${URL}" | awk --field-separator / '{ print $(NF-1) }' | sed 's/[^-]*/installer/')
                    RPM_BASE_URL="$(echo "${URL}" | sed 's/\/[^\/]*\/[^\/]*$//')/${INST_DIR}"
                fi
            fi
        fi

        if [ -z "${RPM_URL}" ]; then
            RPM_URL="${RPM_BASE_URL}/clusterware-installer.rpm"
        fi
    fi

    log "Checking for newer at ${RPM_URL}."
    echo "Checking for a newer installer."
    run_with_sudo "rpm -Uvh '${RPM_URL}' 2>&1 | cat"

    # see what RPM did
    REASON=""
    echo ${SUDO_OUTPUT} | grep 'Could not resolve host' &>/dev/null
    if [ "$?" == "0" ]; then
        REASON="Could not resolve host: $(echo ${RPM_URL} | sed 's/.*:\/\///' | sed 's/\/.*//')"
    fi
    if [ -z "${REASON}" ]; then
        echo ${SUDO_OUTPUT} | grep 'Failed connect to' &>/dev/null
        if [ "$?" == "0" ]; then
            REASON="Failed connect to $(echo ${RPM_URL} | sed 's/.*:\/\///' | sed 's/\/.*//'); No route to host"
        fi
    fi
    if [ -z "${REASON}" ]; then
        echo ${SUDO_OUTPUT} | grep 'URL returned error' &>/dev/null
        if [ "$?" == "0" ]; then
            if $(echo ${RPM_BASE_URL} | grep -q "localhost.*\/isomount\/scyldiso\/ScyldPackages"); then
                REASON="Is the 'scyldiso' repo a ClusterWare ISO?"
            else
                REASON="Bad URL: ${RPM_URL}"
            fi
        else
            echo ${SUDO_OUTPUT} | grep 'is already installed$' &>/dev/null
            if [ "$?" == "0" ]; then
                REASON=up-to-date
            else
                echo ${SUDO_OUTPUT} | grep 'Updating / installing...' &>/dev/null
                if [ "$?" == "0" ]; then
                    REASON=updated
                fi
            fi
        fi
    fi
    if [ -z "${REASON}" ]; then
        echo ${SUDO_OUTPUT} | grep 'No such file or directory' &>/dev/null
        if [ "$?" == "0" ]; then
            REASON="No such file or directory"
        fi
    fi
    if [ -z "${REASON}" ]; then
        echo ${SUDO_OUTPUT} | grep 'transfer failed' &>/dev/null
        if [ "$?" == "0" ]; then
            REASON="Transfer of $(echo ${RPM_URL})"
        fi
    fi
    if [ -z "${REASON}" ]; then
        echo ${SUDO_OUTPUT} | grep 'conflicts with file from' >&/dev/null
        if [ "$?" == "0" ]; then
            OUT1=$(echo ${SUDO_OUTPUT} | sed 's/^.* installed file/File/')
            OUT2=$(echo ${OUT1} | sed 's/ install of//' | sed 's/package/installed package/')
            OUT3=$(echo ${OUT2} | sed 's/from/from\n /' | sed 's/file from/file from\n /')
            REASON="${OUT3}"
        fi
    fi
    if [ -z "${REASON}" ]; then
        REASON="Unknown error"
    fi

    if [ "${REASON}" == "updated" ]; then
        echo "Installed a new installer RPM, running the newly installed script."
        UPDATED=yes
        SCYLD_INSTALL="/usr/bin/scyld-install"
    else
        if [ "${REASON}" != "up-to-date" ]; then
            # TODO: May be a bad URL in ${CWDIR_YUMREPO}, in which case better to ignore,
            #       or even ignore quietly.
            msg_error "An error occurred when fetching the installer RPM:\n  ${REASON}"
            ask_continue 2
        fi

        rpm -q clusterware-installer &>/dev/null
        if [ "$?" == "0" -a \
             "$( realpath "${BASH_SOURCE[0]}" )" != "$( realpath "/usr/bin/scyld-install" )" ]; then
            echo "Running the script provided by the clusterware-installer rpm."
            UPDATED=yes
            SCYLD_INSTALL="/usr/bin/scyld-install"
        fi
    fi
fi

if [ -n "${UPDATED}" ]; then
    msg_status
    export LOG_FILE
    export ASKED_PASS
    export SUDO_PASS
    export LAST_YUM_CLEAN_REPO
    export TMPDIR
    exec ${SCYLD_INSTALL} "$@"
    cleanup 2 "Unreachable after exec"
fi
echo "Proceeding with current script."

# reparse the arguments before proceeding
ERROR_ON_ARGS=yes parse_args "$@"

if $(rpm -q clusterware-couchbase >& /dev/null); then
    msg_error "\nThis version of ClusterWare does not support the Couchbase database."
    msg_error "Either convert to the etcd database and retry, or revert to a"
    msg_error "ClusterWare 11 scyld-install and continue using ClusterWare 11."
    cleanup 1
fi
if [ "${DB_RPM}" != "clusterware-etcd" ]; then
    msg_error "\nOnly \"DB_RPM=clusterware-etcd\" is valid for this version of ClusterWare."
    cleanup 1
fi

# If doing a join, then query the join-to head node with its database password
#  to validate the password and receive the head node's database type.
#  Then ask for the join-to repo file, check for incompatibilities with the
#  joining node, and if okay then make it our own clusterware.repo.
if [ -n "${JOIN_CLUSTER}" ]; then
    # Don't do a join if already joined to a cluster.
    NUM_HEADS=$(num_joined_head_nodes)
    if [ ${NUM_HEADS} -ge 2 ]; then
        msg_error "This head node is already part of a ${NUM_HEADS} head node configuration."
        cleanup 1
    fi

    # Query the join-to database password and database type.
    check_jointo_cred_database
    if [ -z "${PASSWD_OK}" -o -z "${DB_JOIN_TO}" ]; then
        msg_error "Invalid or undetermined join-to password and/or database."
        cleanup 1
    fi

    #  Download the join-to clusterware.repo and examine it.
    TMP_CWREPO=$(mktemp -t jointo.repofile.XXXXXX)
    USE_TMP_CWREPO=
    RET=$(curl -s "${JOIN_CLUSTER}"/api/v1/install/repo > "${TMP_CWREPO}")
    if [ -n "${RET}" ] || [ ! -s "${TMP_CWREPO}" ]        \
                       || ! grep -q scyld "${TMP_CWREPO}" \
                       || ! grep -q baseurl "${TMP_CWREPO}" ; then
        # curl returned an error, or an empty file, or file without "scyld" and
        #  "baseurl", so it doesn't appear to be a clusterware.repo file.
        msg_error "Cannot download ${JOIN_CLUSTER} yum repo file."
        msg_error "   $(cat ${TMP_CWREPO})"
    else
        RESTORE_CWREPO=$(mktemp -t repofile.orig.XXXXXX)
        touch "${RESTORE_CWREPO}"
        if [ -e "${YUMDIR_CWREPO}" ]; then
            run_with_sudo "cp -a ${YUMDIR_CWREPO} ${RESTORE_CWREPO}"
        fi
        run_with_sudo "cp ${TMP_CWREPO} ${YUMDIR_CWREPO}"
        run_with_sudo "chmod 644 ${YUMDIR_CWREPO}"
    fi
    rm -f "${TMP_CWREPO}"

    # Check the join-to clusterware.repo now.
    yum_clean_all_if_needed
    clusterware_list=$(run_with_sudo "yum --assumeyes --enablerepo=scyld* list available clusterware 2>/dev/null" | grep ^clusterware | head -1)
    if [ -n "${clusterware_list}" ]; then
        clusterware_pkg=$(echo "${clusterware_list}" | awk '{print $2}')
        if [ -n "${clusterware_pkg}" ]; then
            if [[ ! "${clusterware_pkg}" =~ ^"12" ]]; then
                msg_error "\nThis version of ClusterWare only supports installing ClusterWare 12,"
                msg_error "updating to ClusterWare 12, or joining a ClusterWare 12 cluster."
                msg_error "Otherwise revert to a ClusterWare 11 scyld-install and retry."
                cleanup 1
            fi
        fi
    fi

    # managedb needs this for the join
    DBARG="--purge"
fi

if [ ! -e "${YUMDIR_CWREPO}" ] && [ -n "${YUM_REPO}" -o -n "${YUM_REPO_TARGET}" ]; then
    # No current repo file and arg specifies it, so use that file if ok.
    check_file YUM_REPO YUM_REPO_TARGET "--yum-repo"
    msg_status "Using yum repo file: ${YUM_REPO_TARGET}"
    run_with_sudo "cp ${YUM_REPO_TARGET} ${YUMDIR_CWREPO}"
    # New repo file is now installed, so forget the "--yum-repo" arg.
    YUM_REPO=
    YUM_REPO_TARGET=
fi

# check if iso specified
if [ -n "${ISO}" ]; then
    TMP_ISO=$(mktemp --directory --tmpdir scyld-iso.XXXXX)

    # Download iso if not local
    echo ${ISO} |  grep "://" > /dev/null
    if [ $? -eq 0 ]; then
        echo -n "Waiting for installation iso   "
        download_ready=
        iso_size=0
        pos=0
        while true; do
            if [ -z "${download_ready}" ]; then
                # do HEAD requests until the size stabilizes
                curl_output=$(curl --silent --head "${ISO}")
                res=$?
                curl_status=$(echo "${curl_output}" | head -n1 | awk '{ print $2 }')
            else
                # download the file once the size is stable
                curl_status=$(curl --silent --write-out %{http_code} -o "${TMP_ISO}/scyld.iso" "${ISO}")
                res=$?
            fi

            if [ "${res}" != "0" ]; then
                # fail out on curl errors
                msg_error "\nCurl failed with exit code ${res}"
                cleanup 1
            elif [ ${curl_status} -eq 200 ]; then
                new_size=$(echo "${curl_output}" | awk '/^Content-Length:/ { print $2 }')
                if [ "${iso_size}" == "${new_size}" ]; then
                    # trigger the actual download
                    if [ -z "${download_ready}" ]; then
                        download_ready=yes
                        echo -en "\nDownloading installation iso..."
                    else
                        echo
                        break
                    fi
                else
                    # loop on success until size stabilizes
                    iso_size=${new_size}
                    echo -ne "\b${spin[pos]}"
                    sleep 0.1
                    pos=$(( (${pos} + 1) % 4 ))
                fi
            elif [ ${curl_status} = 404 -a -n "${LOOP_404}" ]; then
                echo -ne "\b${spin[pos]}"
                sleep 0.1
                pos=$(( (${pos} + 1) % 4 ))
            else
                # Fail if we saw any failure, unless it's 404 and --loop-404,
                #  which means the user wants curl to wait for the ISO to appear.
                msg_error "\nDownloading ISO failed with http code ${curl_status}."
                cleanup 1
            fi
        done
        ISO=${TMP_ISO}/scyld.iso
    fi

    # If we don't have an ${ISO} file by now, then fail.
    if [ ! -e "${ISO}" ]; then
        msg_error "${ISO} does not exist"
        cleanup 1 "Quitting"
    fi

    # mount $ISO
    mkdir -p "${TMP_ISO}/mount"
    run_with_sudo "mount -o loop,ro $ISO $TMP_ISO/mount"
    if [ $? -ne 0 ]; then
        echo "Mount ${ISO} failed"
        # Delete $TMP_ISO mountpoint here to avoid confusing cleanup's umount.
        rm -rf ${TMP_ISO}/mount
        cleanup 1
    fi

    # Check that we've mounted a ClusterWare ISO with the right version.
    # Otherwise, it's a waste of time to re-exec the installer in the mount.
    DIR="${TMP_ISO}/mount"
    if [ ! -e "${DIR}/scyld-install" -o ! -d "${DIR}/ScyldPackages" ]; then
        msg_error "${ISO} is not a valid ClusterWare 11 ISO"
        cleanup 1
    fi
    CW_OS_RPM=$(ls "${DIR}"/ScyldPackages/clusterware-[1-9]*\.scyld\.* 2>/dev/null)
    if [ -z "${CW_OS_RPM}" ]; then
        msg_error "${ISO} does not contain a 'clusterware' RPM"
        cleanup 1
    fi
    CW_OS_BASE=$(echo ${CW_OS_RPM} | sed 's/^.*ScyldPackages.clusterware-.*\.scyld.//' | sed 's/.rpm//')
    if [ -n "${CW_OS_BASE}" ]; then
        check_clusterware_version_against_base "${CW_OS_BASE}"
    fi

    # remove --iso from args and recurse
    args=("$@")
    for ((i=0; i<"${#args[@]}"; ++i)); do
        case ${args[i]} in
            --iso) unset args[i]; unset args[i+1]; break;;
        esac
    done
    export LOG_FILE
    export TMP_ISO
    export DB_RPM="${DB_RPM}"
    export LAST_YUM_CLEAN_REPO
    export TMPDIR
    exec ${TMP_ISO}/mount/scyld-install ${args[@]}
    cleanup 2 "Unreachable after exec from iso"
fi

# check if the clusterware rpm is installed
CWINSTALLED=
if rpm -q clusterware &>/dev/null; then
    CWINSTALLED=yes
    if [ -n "${JOIN_CLUSTER}" ]; then
        msg_warn "\nThis node has previously been installed, but a join was requested."
        msg_warn "This join will purge the existing database contents, including all images"
        msg_warn "and boot configs, and will just have access to the joined head node(s)'s"
        msg_warn "images and boot configs."
        msg_warn "If you want to save any images or configs, then do not continue the join"
        msg_warn "and instead use 'scyld-bootctl export' or 'managedb save' before retrying."
        ask_continue
    fi
    if [ -n "${DO_UPDATE}" ]; then
        msg_status "User wants to continue and to update existing software."
    else
        if [ -n "${JOIN_CLUSTER}" ]; then
            msg_warn "ClusterWare is already installed.  Update ClusterWare using ${JOIN_CLUSTER} clusterware.repo?"
        else
            msg_warn "ClusterWare is already installed.  Update ClusterWare?"
        fi
        ask_continue "${MISMATCH_ERR}"
        DO_UPDATE=yes
    fi
else
    msg_status "The 'clusterware' package is not currently installed."
    if [ -n "${DO_UPDATE}" ]; then
        msg_warn "The ${DO_UPDATE} argument presumes an existing installation.  Install ClusterWare now? "
        ask_continue
    fi
fi

check_clusterware_version_against_base

# check if system has enough memory
SYSTEM_MEMORY=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
if [ 3750000 -gt $SYSTEM_MEMORY ]; then
    cleanup 1 "A minimum of 4 GB of memory is required, exiting"
fi

# check if the minor release is high enough
read -r MAJOR_RELEASE MINOR_RELEASE < <(grep -oE '[0-9]+(\.[0-9]+)?' /etc/redhat-release | awk -F. '{ print $1 " " $2 }')
if [ -z "${MINOR_RELEASE}" ]; then
    # No MINOR_RELEASE field, so set to zero to keep bash happy.
    MINOR_RELEASE=0
fi
if [ "${MAJOR_RELEASE}" -le "6" ] ||  \
   [ "${MAJOR_RELEASE}" == "7" -a "${MINOR_RELEASE}" -lt "6" ]; then
    cleanup 1 "RHEL-based 7.6+ required, ${MAJOR_RELEASE}.${MINOR_RELEASE} installed"
else
    if [ "${MAJOR_RELEASE}" == "7" ]; then
        msg_info "RHEL-based 7.6+ detected."
    else
        if grep --silent ' Stream ' /etc/redhat-release; then
            msg_info "CentOS Stream ${MAJOR_RELEASE} detected."
        else
            msg_info "RHEL-based ${MAJOR_RELEASE} detected."
        fi
        install_latest_package tar || cleanup 1
    fi
fi

# check if conflicting services are active or enabled
if [ "$(systemctl is-active libvirtd)" == "active" ] || [ "$(systemctl is-enabled libvirtd 2> /dev/null)" == "enabled" ] ; then
    cleanup 1 "This software is incompatible with libvirt. Please stop and disable libvirtd, and kill the associated dnsmasq processes or reboot."
fi
if [ "$(systemctl is-active dnsmasq)" == "active" ] || [ "$(systemctl is-enabled dnsmasq 2> /dev/null)" == "enabled" ] ; then
    msg_warn "Active dnsmasq service will conflict with clusterware-dnsmasq. Stop and disable dnsmasq, or remove clusteware-dnsmasq after installation."
fi

# check the arguments in more depth
check_file SAVE_ARCHIVE TARGET "--save" missingok
check_file LOAD_ARCHIVE TARGET "--load"

if [ -n "${ARCHIVE_ARGS}" -a -z "${LOAD_ARCHIVE}" ]; then
    print_help 4 "The --without-files options requires --load."
fi
if [ -n "${JOIN_CLUSTER}" ] && \
   [ -n "${LOAD_ARCHIVE}" -o -n "${SAVE_ARCHIVE}" ]; then
    print_help 5 "Cannot load or save while joining an existing cluster."
fi

if [ -z "${SAVE_ARCHIVE}" ]; then
    # cannot install without configuring but passing an empty file avoids this exit
    if [ -z "${LOAD_CONFIG}" -a -z "${LOAD_ARCHIVE}" -a -z "${CWINSTALLED}" -a -z "${JOIN_CLUSTER}" ]; then
        print_help 7 "A cluster configuration file is required for new installations.\nSee '--config' below:"
    fi

    if [ -z "${LOCAL_REPO}" ]; then
        if [ -n "${SERIAL_NUM}" -a -z "${SERIAL_NUM_TARGET}" ]; then
            print_help 8 "A string must follow --token."
        fi
        check_file YUM_REPO YUM_REPO_TARGET "--yum-repo"

        # if a /etc/yum.repos.d/clusterware.repo file exists assume it's correct and use it
        if [ -z "${SERIAL_NUM}" -a -z "${YUM_REPO}" -a ! -e "${YUMDIR_CWREPO}" ]; then
            msg_info "Neither a clusterware.repo file nor an authentication token has been provided."
            read -p 'Please provide an authentication token: ' SERIAL_NUM_TARGET
            if [ -z "${SERIAL_NUM_TARGET}" ]; then
                # TODO: other option is to install an evaluation version here
                print_help 9 "Either an authentication token or a yum repo file must be provided."
            else
                SERIAL_NUM=yes
            fi
        fi
    fi
fi

# save the objects and exit
if [ -n "${SAVE_ARCHIVE}" ]; then
    if [ -z "${CWINSTALLED}" ]; then
        cleanup 5 "ClusterWare does not appear to be installed, cannot save '${TARGET}'"
    else
        if [ -z "${ARCHIVE_ARGS}" ]; then
            # default to saving db plus all files
            ARCHIVE_ARGS="--with-all"
        fi
        msg_info "Saving ClusterWare database to ${TARGET}"
        run_with_sudo "${MANAGEDB} save ${ARCHIVE_ARGS} '${TARGET}'"
        cleanup 0 "  saved"
    fi
fi

# unpack the payload
unpack

# Put some repo file into ${TMPDIR} based on the local path or serial
# number or passed or installed repo.
if [ -n "${LOCAL_REPO}" ]; then
    msg_status "Using local yum repo: ${LOCAL_REPO}"
    sed "s\\<REPO_ROOT>\\${LOCAL_REPO}\\" "${TMPDIR}"/clusterware.repo.local.template > "${TMPDIR}"/clusterware.repo
    sed --in-place "s/<VERSION_ID>/${os_release_major}/" "${TMPDIR}"/clusterware.repo
elif [ -n "${SERIAL_NUM}" ]; then
    if [ "${SERIAL_NUM_TARGET}" == "--serial-from-head--" ]; then
        msg_status "Using authentication token already embedded in clusterware.repo"
    else
        msg_status "Using authentication token: ${SERIAL_NUM_TARGET}"
        # append a colon if none is found in the string
        if [[ "${SERIAL_NUM_TARGET}" != *:* ]]; then
            SERIAL_NUM_TARGET="${SERIAL_NUM_TARGET}:"
        fi
    fi
    sed "s/<AUTHENTICATION_TOKEN>/${SERIAL_NUM_TARGET}/" \
        "${TMPDIR}"/clusterware.repo.remote.template > "${TMPDIR}"/clusterware.repo
    sed --in-place "s/<VERSION_ID>/${os_release_major}/" "${TMPDIR}"/clusterware.repo
elif [ -n "${YUM_REPO}" -o -n "${YUM_REPO_TARGET}" ]; then
    check_file YUM_REPO YUM_REPO_TARGET "--yum-repo"
    msg_status "Using yum repo file: ${YUM_REPO_TARGET}"
    cp "${YUM_REPO_TARGET}" "${TMPDIR}"/clusterware.repo
    # If we haven't already created a special backup, then do it now.
    if [ -z "${RESTORE_CWREPO}" ]; then
        RESTORE_CWREPO=$(mktemp -t repofile.orig.XXXXXX)
        if [ -e "${YUMDIR_CWREPO}" ]; then
            run_with_sudo "cp -a ${YUMDIR_CWREPO} ${RESTORE_CWREPO}"
        fi
    fi
elif [ -e "${YUMDIR_CWREPO}" ]; then
    msg_status "Using existing /etc/yum.repos.d/clusterware.repo file."
    cp -a ${YUMDIR_CWREPO} "${TMPDIR}"/clusterware.repo
else
    cleanup 7 "Could not find or construct a clusterware.repo file."
fi

# Copy possibly new repo file in ${TMPDIR} to the system location if necessary
if [ -e ${YUMDIR_CWREPO} ] && \
    diff -s "${TMPDIR}"/clusterware.repo ${YUMDIR_CWREPO} >&/dev/null; then
    msg_status "Repo file is unchanged."
else
    msg_status "Copying the repo file to ${YUMDIR_CWREPO}"
    run_with_sudo "cp ${TMPDIR}/clusterware.repo ${YUMDIR_CWREPO}"
    run_with_sudo "chmod 644 ${YUMDIR_CWREPO}"
    yum_clean_all_if_needed
fi

# restore SELinux context for the repo
if [ -e /usr/sbin/restorecon ]; then
    run_with_sudo "/usr/sbin/restorecon -F ${YUMDIR_CWREPO}"
fi

# enable the repos
run_with_sudo "sed --in-place 's/^\s*enabled\s*=\s*[01].*$/enabled=1/' ${YUMDIR_CWREPO}"
check_clusterware_major_for_cw12

# ensure yum-plugin-priorities is installed
if [ "${os_release_major}" == "7" ]; then
    if [ "$ID" == "centos" -o "$ID" == "ol" -o "$ID" == "almalinux" ]; then
        if ! rpm -q yum-plugin-priorities >/dev/null; then
            # failure to install yum-plugin-priorities is not fatal
            install_latest_package yum-plugin-priorities || echo "  continuing"
        fi
    elif [ "$ID" == "rhel" ]; then
        if ! rpm -q yum-plugin-priorities >/dev/null; then
            run_with_sudo SPINNER "Installing yum-plugin-priorities" \
                  "yum --enablerepo=rhel-7-server-optional-rpms -y install yum-plugin-priorities > >(tee --append '${LOG_FILE}') 2>&1"
        fi
   else
       cleanup 1 "$ID is not a supported distribution, exiting"
   fi
else
    msg_status "Skipping install of yum-plugin-priorities on ${PRETTY_NAME}"
fi

if [ -n "${CWINSTALLED}" -a -z "${RECONF}" ]; then
    # If updating from CW11 to CW12, then remove ${TICK_OLD}.
    OLD_CW_VER=$(rpm -q --qf "%{VERSION}\n" clusterware)
    if [ -n "${OLD_CW_VER}" ] && [[ "${OLD_CW_VER}" =~ ^"11" ]]; then
        # Remove the old CW11 TICK packages, if they still exist.
        # Otherwise the new influxdb2 will conflict with old influxdb.
        for RPM in ${TICK_OLD}; do
            if rpm -q "${RPM}" >& /dev/null; then
                run_with_sudo SPINNER "Removing ${RPM}" "rpm -e --nodeps ${RPM} > >(tee --append '${LOG_FILE}') 2>&1"
            fi
        done
        if [ -d /var/log/chronograf ]; then
            run_with_sudo "rm -fr /var/log/chronograf"
        fi
        RPM="python3-mod_wsgi"
        if rpm -q ${RPM} >& /dev/null; then
            # If python3-mod_wsgi is installed, then it is the CW11 el8 python3
            #  package and conflicts with CW12 python39.  Just delete.
            run_with_sudo SPINNER "Removing ${RPM}" "rpm -e --nodeps ${RPM} > >(tee --append '${LOG_FILE}') 2>&1"
        fi
        UPDATE_CW11_TO_CW12=yes
    fi
    INSTALL_RPMS=""
    # If updating from an earlier version of cw12 that didn't install
    #  clusterware-grafana, then add that package to the list.
    #  TODO: this can eventually go away when all early cw12 go away.
    if ! rpm -q clusterware-grafana >& /dev/null; then
        INSTALL_RPMS="clusterware-grafana "
    fi

    INSTALL_RPMS+=$(rpm -qa --qf "%{NAME}\n" | grep clusterware | sort; echo ${INFLUXDB2} scyld-nss)
    OBSOLETES=$(yum --assumeyes list obsoletes | grep ^"    .*scyld" | sed 's/    //' | sed 's/\..*$//')
fi

# install or update the top listed RPMs
for RPM in ${INSTALL_RPMS}; do
    ORELSE="cleanup 9"
    if [ "$RPM" == "clusterware-docs" ]; then
        ORELSE="msg_warn ClusterWare documentation was not installed, continuing anyway."
    fi

    # If this RPM is in someone's Obsoletes list, then skip it.
    skip_rpm=
    for obsolete in ${OBSOLETES}; do
        if [ "${RPM}" == "${obsolete}" ]; then
            skip_rpm=yes
            break
        fi
    done

    # if no reason to skip this one, proceed
    if [ -z "${skip_rpm}" ]; then
        install_latest_package "${RPM}" || ${ORELSE}

        # enable scyld-nss at install time
        if [ -z "${CWINSTALLED}" -a "${RPM}" == "scyld-nss" ]; then
            enable_nss=yes
        fi

        # If we've made it this far, then the install/update has
        #  not triggered an error, so stop checking the validity.
        CHECKED_OS_COMPATIBLE=yes
    fi
done

# If this is this an update of a joined head node that changed the major or
#  minor version number, then issue warning to update all joined head nodes.
if [ -n "${CWINSTALLED}" -a $(num_joined_head_nodes) -gt 1 ]; then
    OLD_MAJ=$(cut -f1 -d. <<<$OLD_CW_VER)
    NEW_CW_VER=$(rpm -q --qf "%{VERSION}\n" clusterware)
    NEW_MAJ=$(cut -f1 -d. <<<$NEW_CW_VER)
    if   [ "${OLD_MAJ}" != "${NEW_MAJ}" ]; then
        URGENCY="as soon as possible"
    elif [ "${OLD_CW_VER}" != "${NEW_CW_VER}" ]; then
        URGENCY="soon"
    fi
    if [ -n "${URGENCY}" ]; then
        msg_warn "You have updated ${OLD_CW_VER} to ${NEW_CW_VER}, so please ensure that all head nodes are\n similarly updated to ${NEW_CW_VER} ${URGENCY}."
    # else OLD_CW_VER == NEW_CW_VER
    fi
fi

# Packages can change the rsyslog config so we restart rsyslog post
# install or upgrade, but we could also push that into each rpm's
# %post.
run_with_sudo "systemctl restart rsyslog"

# Packages can change a systemd service, so reload the daemons.
run_with_sudo "systemctl daemon-reload"
if [ "${firewall_cmd}" == "firewall-cmd" ]; then
    run_with_sudo "${firewall_cmd} --reload"
fi

# confirm the clusterware package is installed at this point
if ! rpm -q clusterware &>/dev/null; then
    msg_error "Failed to install the 'clusterware' package. Please check the ~/.scyldcw/logs/\nto help identify the specific error and try again after resolving any issues."
    cleanup 1
fi

# a few steps are only done during the initial install
if [ -z "${CWINSTALLED}" ]; then
    # early chance to replace code for debugging during / just post install
    if [ -n "${DEVEL}" -a -d "${DEVEL}/clusterware/" ]; then
        msg_warn "DEVEL: Copying in new clusterware code"
        run_with_sudo "cp -a ${DEVEL}/clusterware/ /opt/scyld"
        run_with_sudo "restorecon -rF /opt/scyld"
    fi

    # As of CentOS 8 installing mod_ssl no longer generates
    # /etc/pki/tls/certs/localhost.crt so we explicitly trigger that here.
    #   https://bugzilla.redhat.com/show_bug.cgi?id=1764838
    if [ ! -e /etc/pki/tls/certs/localhost.crt ]; then
        msg_warn "Creating missing /etc/pki/tls/certs/localhost.crt"
        run_with_sudo /usr/libexec/httpd-ssl-gencerts
    fi
fi

# Randomize bits of the base.ini on fresh installs
NEW_INSTALL=
if [ -z "${CWINSTALLED}" -o -n "${REINIT}" ]; then
    if [ -z "${DB_PASSWD}" -o -n "${JOIN_CLUSTER}" ]; then
        run_with_sudo /opt/scyld/clusterware/bin/randomize_ini
    else
        run_with_sudo "DB_PASSWD='${DB_PASSWD}' /opt/scyld/clusterware/bin/randomize_ini"
    fi
    NEW_INSTALL=yes

    # TODO: might be able to call headctl for this?
    dbplugin=$(echo "${DB_RPM}" | sed 's/^.*-//')
    msg_status "Selecting '${dbplugin}' as the database backend."
    run_with_sudo "sed --in-place 's/^plugins.database.*/plugins.database\\\\\\ =\\\\\\ db_${dbplugin}/' '/opt/scyld/clusterware/conf/base.ini'"

    # enable https on new installs or joins
    msg_status "Enabling HTTPS communication through Apache."
    run_with_sudo "/opt/scyld/clusterware/bin/headctl --enable-https"
fi

# configure tftp on the head node
if [ -n "${NEW_INSTALL}" -o -n "${RECONF}" ]; then
    if [ "${os_release_major}" == "7" ]; then
        msg_status "Enable tftpd in xinetd."
        run_with_sudo 'sed --in-place "s/^\(\s*disable\s*= \)yes/\1no/" /etc/xinetd.d/tftp'
    fi
fi

# set the user as an admin before the base.ini gets read
if [ -n "${NEW_INSTALL}" -o -n "${RECONF}" ]; then
    if [ -n "${INSTALL_TOOLS}" ]; then
        msg_status "Setting ${USER} as the only auth.tmpadmin in base.ini."
        run_with_sudo \
            "sed --in-place \"s/^[ #]*auth.tmpadmins.*/auth.tmpadmins = ${USER}/\" /opt/scyld/clusterware/conf/base.ini"
    fi

    # start the firewalld service before trying to open ports
    # This may lose uncommitted changes to the running firewall, but is necessary 
    # to force firewalld to re-read the config files so we can enable the service
    enable_start_service firewalld reload

    # database ports must be open before any attempt to join an existing cluster
    if [ "${DB_RPM}" == "clusterware-couchbase" ]; then
        open_port_or_service couchbase
    elif [ "${DB_RPM}" == "clusterware-etcd" ]; then
        open_port_or_service cw-etcd
    else
        msg_warn "No ports opened for backend database ${DB_RPM}"
    fi
fi

# add a --purge argument if --clear was passed
if [ -n "${REINIT}" ]; then
    DBARG="--purge"
fi

# join this head node to an existing cluster
if [ -n "${JOIN_CLUSTER}" ]; then
    log "Joining the cluster with head node ${JOIN_CLUSTER}"
    BASE_INI_OLD="${TMPDIR}/base.ini-old"
    BASE_INI_NEW="${TMPDIR}/base.ini-new"
    # Copy the current base.ini to an admin-owned file for easier access
    run_with_sudo cp ${BASE_INI} ${BASE_INI_OLD}
    run_with_sudo chmod 0666 ${BASE_INI_OLD}
    linenum=$(grep -n "^database.admin_pass" ${BASE_INI_OLD} | cut -f1 -d:)
    cat ${BASE_INI_OLD} | sed "${linenum}cdatabase.admin_pass = ${DB_PASSWD}" > ${BASE_INI_NEW}
    run_with_sudo cp ${BASE_INI_NEW} ${BASE_INI}

    http_proxy= https_proxy= no_proxy= run_with_sudo SPINNER "Joining an existing cluster" \
        "${MANAGEDB} -vv join ${DBARG} '${JOIN_CLUSTER}' \
                                               > >(cat >> '${LOG_FILE}') 2>&1"
    if [ $? -ne 0 ]; then
        # Restore the base.ini back to the original contents
        run_with_sudo cp ${BASE_INI_OLD} ${BASE_INI}
        cleanup 1
    fi
    run_with_sudo LIVE ${MANAGEDB} --wait-ready

# or on new installs configure the database
elif [ -n "${NEW_INSTALL}" ]; then
    # poke the database backend once per loop, up to 10 times, until it is active
    log "Waiting for database backend to become active: $(date)"
    for n in $(seq 1 10); do
        sudo /opt/scyld/clusterware/bin/managedb --heads &> /dev/null
        etcdActive=$(systemctl is-active clusterware-etcd)
        if [ "${etcdActive}" == "active" ]; then
            break
        fi
    done

    # now that the database should be up, initialize the contents
    log "Configuring the database for a new cluster: $(date)"
    http_proxy= https_proxy= no_proxy= run_with_sudo SPINNER "Initializing the key store" \
        "${MANAGEDB} --verbose clear ${DBARG} > >(cat >> '${LOG_FILE}') 2>&1" || cleanup 1

# update the database
elif [ -n "${DO_UPDATE}" ]; then
    log "Updating the database on an existing cluster."
    # switched from SPINNER to LIVE to fix a no_proxy issue that
    # caused a hang when upgrading a cluster with proxies.
    # TODO: change SPINNER to properly handle proxies, but requires a fair bit of testing.
    http_proxy= https_proxy= no_proxy= run_with_sudo LIVE bash -c \
        "echo 'Updating key store...'; ${MANAGEDB} --verbose update > >(cat >> '${LOG_FILE}') 2>&1" || \
        cleanup 1
fi

# stop httpd so we can be sure of a fresh start
run_with_sudo SPINNER "Stopping httpd" "systemctl stop httpd"

# Start default services, except grafana-server which gets started later.
for service in httpd influxdb telegraf; do
    enable_start_service $service
done

# If we installed scyld-nss enable it here now that the clusterware
#   service is recognized (post enabling httpd once)
if [ -n "enable_nss" ]; then
    msg_status "Enabling scyld-nss on the head node"
    run_with_sudo "/opt/scyld/scyld-nss/bin/scyld-nssctl start"
fi

# also start the tftp-server appropriately
if [ "${os_release_major}" == "7" ]; then
    enable_start_service xinetd
else
    enable_start_service tftp
fi

if [ -n "${NEW_INSTALL}" -o -n "${RECONF}" ]; then
    # If the baseurl points to the ${JOIN_CLUSTER} node,
    #  then change that to "localhost".
    if [ -n "${JOIN_CLUSTER}" ]; then
        read -r start end junk < <(awk '/\[scyld-/ { print NR }' ${YUMDIR_CWREPO} | tr '\n' ' ')
        if [ -z "${end}" ]; then
            end=$(cat "${YUMDIR_CWREPO}" | wc --lines)
        fi
        linenum=$(tail -n+${start} ${YUMDIR_CWREPO} | head -n$(( ${end} - ${start} )) | \
                  grep -n ^"baseurl*=*http:\/\/${JOIN_CLUSTER}/api/v" ${YUMDIR_CWREPO})
        if [ -n "${linenum}" ]; then
            linenum=$(echo ${linenum} | sed 's/:.*$//')
            sed "${linenum}s/${JOIN_CLUSTER}/localhost/" "${YUMDIR_CWREPO}" \
                           > "${TMPDIR}/tmp.repo"
            run_with_sudo "cp ${TMPDIR}/tmp.repo ${YUMDIR_CWREPO}"
        fi
    fi

    # Open access to the http service
    for service in http https; do
        open_port_or_service $service
    done

    # Allow public access to tftp, but ignore a specific error
    msg_status "Opening firewall for service tftp"
    open_firewall_for_tftp
    open_firewall_for_tftp --permanent

    # TODO: provide a mechanism to open ports only on appropriate interfaces,
    #       perhaps by reading the from the cluster configuration file?
    #### Open assorted ports
    # 53/udp # dns to expose dnsmasq to the cluster and optionally outside
    # 123/udp # ntp to expose chrony to peer servers and node clients
    # 3260/tcp # iscsi
    # 8094/udp # telegraf from incoming from nodes to local telegraf
    for port in dns ntp iscsi-target telegraf; do
        open_port_or_service $port
    done
fi

if [ -n "${NEW_INSTALL}" -o -n "${RECONF}" -o -n "${UPDATE_CW11_TO_CW12}" ]; then
    # Call python code to configure the influx + grafana subsystem
    run_with_sudo SPINNER "Configuring Grafana + InfluxDB" \
                          "/opt/scyld/clusterware/bin/influx_grafana_setup --purge --tele-env > >(tee --append '${LOG_FILE}') 2>&1"
fi
# trigger a reconfig of telegraf with any enabled plugins
if [ -x "/opt/scyld/clusterware-telegraf/bin/reconfig-telegraf.sh" ]; then
    run_with_sudo /opt/scyld/clusterware-telegraf/bin/reconfig-telegraf.sh
fi

if [ -n "${INSTALL_TOOLS}" ]; then
    if [ -n "${NEW_INSTALL}" ]; then
        msg_info "Installing and configuring the ClusterWare tools."
        install_latest_package clusterware-tools || cleanup 8
    fi

    # for debugging tool problems during install
    if [ -n "${DEVEL}" -a -d "${DEVEL}/clusterware-tools/" ]; then
        msg_warn "DEVEL: Copying in new clusterware-tools"
        run_with_sudo "cp -a ${DEVEL}/clusterware-tools/ /opt/scyld"
        run_with_sudo "restorecon -rF /opt/scyld"
    fi

    # assume the user's .scyldcw is set up unless this is a new install
    if [ -n "${NEW_INSTALL}" ]; then
        msg_status "Creating the user's settings.ini"
        rm -f ~/.scyldcw/settings.ini
        PYTHONUNBUFFERED=1 scyld-tool-config --example > >(tee --append "${LOG_FILE}") 2>&1

        # ensure there is a ClusterWare admin account for the current
        # user unless this is a curl install
        if [ "${SERIAL_NUM_TARGET}" == "=--serial-from-head--" ]; then
            msg_status "No new user account created when installing via the HTTP endpoint"
        else
            ADMIN_LIST=$(scyld-adminctl --json --user "${USER}:" ls 2>/dev/null | grep "\"${USER}\"")
            if [ -n "${ADMIN_LIST}" ]; then
                msg_status "A ClusterWare admin account already exists for ${USER}."
            else
                msg_status "Creating a ClusterWare admin account for ${USER}."
                scyld-adminctl --user "${USER}:" create name="${USER}"
            fi
        fi

        msg_status "Removing ${USER} from auth.tmpadmin in base.ini."
        run_with_sudo "sed --in-place \"s/^auth.tmpadmins.*/#\0/\" /opt/scyld/clusterware/conf/base.ini"
    fi

    # try to detect if we are running from a mounted ISO
    if using_mounted_ISO; then
        # Assume for now that ClusterWare major number begins with "1".
        if [ $(rpm -q clusterware) == $(rpm -qp ${DIR}/ScyldPackages/clusterware-1*.rpm) ]; then
            ver_matched_iso=yes
        fi
    fi
  
    # if we detect the parent directory is a mounted and version matched ISO, then...
    if [ "${path}" == "${MOUNT}" -a -n "${ver_matched_iso}" ]; then
        # optionally copy the device if it cannot already be read
        if [ ! -r "${DEV}" ]; then
            path="${TMPDIR}/scyld.iso"
            sudo cp "${DEV}" "${path}"
            sudo bash -c "chown \${SUDO_USER} '${path}'"
        else
            path="${DEV}"
        fi
        repourl=http://localhost/api/v1/isomount/scyldiso

        # check if the scyldiso already exists and create or update it
        lsout=$(scyld-clusterctl --json repos -iscyldiso ls 2>/dev/null)
        if [ "${lsout}" != '["scyldiso"]' ]; then
            scyld-clusterctl repos create name=scyldiso iso=@"${path}"
        else
            scyld-clusterctl repos --ids scyldiso update iso=@"${path}"
        fi

        # update the baseurl
        repo=${YUMDIR_CWREPO}
        read -r start end junk < <(awk '/\[scyld-/ { print NR }' "${repo}" | tr '\n' ' ')
        if [ -z "${end}" ]; then
            end=$(cat "${repo}" | wc --lines)
        fi
        baseurl=$(tail -n+${start} "${repo}" | head -n$(( ${end} - ${start} )) | \
                awk -F= 'BEGIN{IGNORECASE = 1} /^\s*baseurl\s*=\s*FILE:\/\// { print $2 }')
        sudo sed --in-place "s|${baseurl}|${repourl}|" "${repo}"

        # give the repo a chance to come online
        echo "Waiting for repo URL to come online..."
        for n in $(seq 10); do
            code=$(curl --silent --output /dev/null --write-out "%{http_code}" "${repourl}/scyld-install")
            if [ "${code}" == "200" ]; then
                echo "  ...success"
                break
            fi
            sleep 1
        done
        if [ "${code}" != "200" ]; then
            echo "  ...timeout. Image creation may fail."
        fi
    fi

    # if we're not loading an archive
    if [ -z "${LOAD_ARCHIVE}" ]; then
        # only create the first image on new non-join installs
        if [ -n "${NEW_INSTALL}" -a -z "${JOIN_CLUSTER}" ]; then
            if [ -n "${OSISO_SKIP}" ]; then
                msg_status "Skip creating the first default image etc."
            else
                msg_status "Creating the first default image etc."
                if [ -n "${OSISO}" ]; then
                    isoargs="--iso ${OSISO}"
                fi
                PYTHONUNBUFFERED=1 scyld-add-boot-config --make-defaults ${isoargs} > >(tee --append "${LOG_FILE}") 2>&1
                if [ "$?" != "0" ]; then
                    msg_error "scyld-add-boot-config failed to create DefaultImage and DefaultBoot"
                    ADD_BOOT_CONFIG_FAILED=yes
                fi
            fi
        fi

        if [ -z "${LOAD_CONFIG}" ]; then
            if [ -z "${JOIN_CLUSTER}" -a -n "${NEW_INSTALL}" ]; then
                msg_status "No cluster configuration file was loaded."
                # TODO: check the cluster summary and only same something if it's empty
                msg_status "Be sure to load one using the scyld-cluster-conf tool after installation completes."
            fi
        else
            if [ -n "${LOAD_CONFIG_SKIP}" ]; then
                msg_status "Skip loading a cluster configuration."
            else
                msg_status "Parsing and applying ${LOAD_CONFIG}"
                PYTHONUNBUFFERED=1 scyld-cluster-conf load "${LOAD_CONFIG}" > >(tee --append "${LOG_FILE}") 2>&1
                if [ "$?" != "0" ]; then
                    msg_error "Failed to load cluster configuration. Please correct any errors and"
                    msg_error "then manually load the configuration using scyld-cluster-conf load."
                    CLUSTER_CONF_LOAD_FAILED=yes
                fi
            fi

            # TODO: this shouldn't be necessary but is for now?
            msg_status "Reloading the clusterware service"
            run_with_sudo "systemctl reload httpd"
        fi
        if [ -n "${ADD_BOOT_CONFIG_FAILED}" ]; then
            cleanup 1
        fi
    fi
fi

# now that clusterware is installed load any provided object file
if [ -n "${LOAD_ARCHIVE}" ]; then
    msg_info "Loading ClusterWare database from ${TARGET}"
    run_with_sudo "systemctl stop httpd"
    run_with_sudo "${MANAGEDB} load ${ARCHIVE_ARGS} '${TARGET}'"
    run_with_sudo "systemctl start httpd"
fi

# Has a clusterware update created *.rpmnew or *.rpmsave files?
report_cw_rpmnew_rpmsave

if [ -z "${CLUSTER_CONF_LOAD_FAILED}" ]; then
    INSTALL_RETURN=0
else
    INSTALL_RETURN=1
fi
if [ -n "${JOIN_CLUSTER}" ]; then
    cleanup ${INSTALL_RETURN} "Joining existing cluster complete."
else
    cleanup ${INSTALL_RETURN} "Installation complete."
fi
PAYLOAD:
H4sIAHHhqmUAA+3UYWvaQBgHcF/nUzzgXi65xBkrpUqLy5yUqtjYUcaQ83Jq6CUXLhdSYR9+F63F
OakwpGPw/F6Yi+by5M4nfyaKXHNVUsUdxTNpPhKpuaN5kgmqee0MXKPVbG6OxuHRM2pe0/cbfsv1
W17N9Vqe16iBe47ip5jlUwVQU1Lqt6479ft/qg73bC0i6G374JvpA3ic3sHE9AKEL01g1a06hCsO
CymELON0CXEONAX+TM0VHHbdAnIBhGtG1kWyaafciQg7aLHN3R7iPNaw0jq7JKQsSyfj6bKIUyaT
rNCmgmNGJC+yTCpNzAQtIZJlKiSNYC0LBczcVSYfIVPcnhex0HC00BepgOZ5bP7mlPFL8814Wwl6
u1LwEzy73W7b42DYnw6G5vyl8PWxp7Ks73m1ZzYXVw/B5H4wGs4Gn7s/rJQmvLO/kT2puDWnOS+U
6FRrzc1ir26m4ddgGA56N2E1NRzdBsPudZFFZgPz4/uwtzLiNcjvhcmHqgRVbEUsntK54FHHs5bZ
kq04ezLDai9me+dm+MTXrw/0ZuXJ+M7uj/v2bfBob1e99yxWpmKpYr3ufLL+dSOjv3L4zjhCMirO
Gv+n8t+9aPoH+e9fNHzM//eA+f9e+b+IBa/ifxKMR7PJaBR2j6W1e5DW7i6t/5yO2YwQQgghhBBC
CCGEEEIIIYQQQujVL/t5mk8AKAAA