nfv_install

#!/bin/bash

###################### ECHO FUNCTIONS ######################

# Prints a separation line long $1 (required) characters on stdout
function echo_sep_line() {
    printf '+'
    printf '%0.s-' $(seq 1 "$1")
    printf '+'
    printf '\n'
}

# Prints a colorful section title ($1, required) on stdout
function echo_section() {
    boxsize=80

    str="$1"
    len=$((${#str} + 2)) # Two vertical lines, before and after

    if [ $len -lt $boxsize ]; then
        pad=$((boxsize - len))
        lpad=$((pad / 2))
        rpad=$((pad - lpad))

        if [ $lpad -ne 0 ]; then
            lpad=$(printf '%.0s ' $(seq 1 $lpad))
        else
            lpad=""
        fi

        rpad=$(printf '%.0s ' $(seq 1 $rpad))
    else
        boxsize=$len
        lpad=""
        rpad=""
    fi

    printf "${COLOR}"
    echo_sep_line $boxsize
    printf '| %s%s%s |\n' "$lpad" "$str" "$rpad"
    echo_sep_line $boxsize
    printf "${NC}"
}

# Prints a colorful message ($1, required) on stdout
function echo_message() {
    printf "${COLOR}"
    printf ' ##\n'
    printf '## %s\n' "$1"
    printf '##\n'
    printf "${NC}"
}

# Prints a colorful single action ($1, required) on stdout
function echo_action() {
    printf "${COLOR}  -- $1${NC}\n"
}

# Prints a colorful warning ($1, required) on stdout
function echo_warning() {
    printf "${COLOR_WARN}  !! $1${NC}\n"
}

#################### UTILITY FUNCTIONS #####################

function parse_parameters() {
    args=""
    for opt in "$@"; do
        case $opt in
        -i)
            # Install command, default action
            cmd=i
            ;;
        -d)
            # Delete command
            cmd=d
            ;;
        -y)
            # Accept all default parameters
            default=y
            ;;
        *)
            # Other argument
            args="${args}_${opt}_"
            ;;
        esac
    done

    # NOTICE: underscores are important
    if [ -z "$args" ] || [[ "$args" == *_all_* ]]; then
        args="_all_rootfs_lxc_dpdk_ovs_vpp_snabb_bin_"
    fi
}

# Ask the location of a folder to the user, with an optional
# default value. Repeats the question until a valid location
# is supplied.
# Args:
# - $1 (required) is the prompt
# - $2 (required) is the default value
function asklocation() {
    local response
    response=""

    read -rep "$1 [$2]: " response

    if [ -z "$response" ]; then
        response=$2
    fi

    # Create a directory and at the same time test whether it is a valid
    # potential location
    while [[ $(mkdir -p "$response" 2>/dev/null) = 0 ]]; do
        read -rep $'The specified location does not exist!\n' \
            "$1 [$2]: " response
    done

    echo "$response"
}

# Returns the number of processes to use when running make.
# The behavior depends on the MAKE_AGGRESSIVE environment
# variable (as indicated with warnings).
# Call this function only ONCE per script execution.
function get_make_cpus() {
    local cpunum
    local makecpus

    # To parallelize better make command, let's get the number of CPUs (including hyperthr.)
    cpunum=$(grep -c ^processor /proc/cpuinfo)

    # Set MAKE_AGGRESSIVE environment variable to 1 to fully utilize the system
    # in make commands
    if [ -n "$MAKE_AGGRESSIVE" ] && [ "$MAKE_AGGRESSIVE" -eq 1 ]; then
        echo_warning "WARN: Using AGGRESSIVE MODE for build commands may fully utilize your system during installation!" >/dev/stderr
        makecpus=$(echo "$cpunum*1.5" | bc)
        makecpus=$(printf '%.0f\n' "$makecpus")
    else
        echo_warning "HINT: To build applications faster (using more CPUs) export MAKE_AGGRESSIVE=1 before running the installation script" >/dev/stderr
        makecpus=$(echo "$cpunum*0.7" | bc)
        makecpus=$(printf '%.0f\n' "$makecpus")
    fi

    echo "$makecpus"
}

################## FUNCTIONS ##################

# Generates a run-commands file in the NFV_RCFILE path
function generate_rcfile() {
    cat <<EOF >"$NFV_RCFILE"
#!/bin/bash

THIS_FILE_NAME="\${BASH_SOURCE[0]}"

# This is used to get reliably the original path of the script
# Source: https://stackoverflow.com/a/246128/7030275
SOURCE="\${BASH_SOURCE[0]}"
# Resolve \$SOURCE until the file is no longer a symlink
while [ -h "\$SOURCE" ]; do
    DIR_SCRIPT="\$(cd -P "\$(dirname "\$SOURCE")" >/dev/null 2>&1 && pwd)"
    SOURCE="\$(readlink "\$SOURCE")"
    # If \$SOURCE was a relative symlink, we need to resolve it
    # relative to the path where the symlink file was located
    [[ \$SOURCE != /* ]] && SOURCE="\$DIR_SCRIPT/\$SOURCE"
done
DIR_SCRIPT="\$(cd -P "\$(dirname "\$SOURCE")" >/dev/null 2>&1 && pwd)"

# Framework's base directory
export NFV_PATH=\${DIR_SCRIPT}
export NFV_CONFPATH=\${NFV_PATH}/conf/rc
export RTE_SDK=${DIR_DPDK_INSTALL}/share/dpdk/
export RTE_INSTALL_PATH=${DIR_DPDK_INSTALL}
export RTE_TARGET=x86_64-native-linuxapp-gcc
export PATH="\${NFV_PATH}/bin:\${PATH}"
EOF
    # Base directories
    # export NFV_PATH_SRC="\$NFV_PATH/src"
    # export NFV_PATH_BLD="\$NFV_PATH/build"
    # export NFV_PATH_INS="\$NFV_PATH/install"
    # export NFV_PATH_CNF="\$NFV_PATH/conf"
    # export NFV_PATH_BIN="\$NFV_PATH/bin"
}

# # TODO: move this elsewhere
# # Source sub-configuration files
# if [ -f "\$NFV_PATH_CNF/$DPDKRC_NAME" ] ; then
#     export NFV_DPDK_CONFRC="\$NFV_PATH_CNF/$DPDKRC_NAME"
# fi
#
# # TODO: move this elsewhere
# if [ -f "\$NFV_PATH_CNF/$ROOTFSRC_NAME" ] ; then
#     export NFV_ROOTFS_CONFRC="\$NFV_PATH_CNF/$ROOTFSRC_NAME"
# fi
#
# # Switch directories
# # TODO: move this elsewhere
# export NFV_DIR_OVS="$DIR_OVS_SRC"
# export NFV_DIR_SNABB="$DIR_SNABB_SRC"
#
# # Custom commands
# # TODO: move this elsewhere
# export NFV_SNABB_CMD="$SNABB_PROGRAM_FILE"
# EOF

################### DIRECTORY MANAGEMENT ###################

function directories_install() {
    echo_section "Installing directories"

    echo_action "MKDIR $DIR_BASE"
    mkdir -p "$DIR_BASE"
    echo_action "MKDIR $DIR_SRC"
    mkdir -p "$DIR_SRC"
    echo_action "MKDIR $DIR_BUILD"
    mkdir -p "$DIR_BUILD"
    echo_action "MKDIR $DIR_INSTALL"
    mkdir -p "$DIR_INSTALL"
    echo_action "MKDIR $DIR_BIN_DST"
    mkdir -p "$DIR_BIN_DST"
    echo_action "MKDIR $DIR_CONFIG"
    mkdir -p "$DIR_CONFIG"
    mkdir -p "$DIR_CONFIGRC"
}

# Removes entire framework directory tree
function directories_remove() {
    echo_section "Removing directories"
    echo_action "RM $DIR_BASE"
    rm -rf "$DIR_BASE"
}

###################### APT MANAGEMENT ######################

function packages_install() {
    echo_section "Checking if required packages are installed"

    echo_action "APT INSTALL DEP"
    #sudo apt update # FIXME: re-enable this
    sudo apt install git build-essential libnuma-dev lxc \
        pkg-config python3-pip automake -y
}

#################### ROOTFS MANAGEMENT #####################

function rootfs_install() {
    echo_section "Installing the rootfs"

    if [ -e "$ROOTFS_CHECK_FILE" ]; then
        echo_warning "ALREADY INSTALLED -- SKIPPING" >/dev/stderr
        echo_warning "INFO: if you want to re-install it, " \
            "just remove it and install it once again." >/dev/stderr
        return 0
    fi

    echo_action "CLONING BUILDCORE"
    git clone https://gitlab.retis.santannapisa.it/l.abeni/BuildCore \
        "$DIR_BUILDCORE"

    echo_action "BUILDING BUILDCORE"
    mkdir -p "$DIR_BUILDCORE_BUILD"
    (
        cd "$DIR_BUILDCORE_BUILD"
        rm -rf ./*
        sh "$DIR_BUILDCORE/buildcore.sh" "$DIR_BUILDCORE_BUILD/test.gz"

        echo_action "RM UNUSED ARCHIVES"
        rm "$DIR_BUILDCORE_BUILD/test.gz"
        rm "$DIR_BUILDCORE_BUILD/busybox-1.29.3.tar.bz2"
        rm "$DIR_BUILDCORE_BUILD/sudo-1.8.26.tar.gz"
        # TODO: remove other stuff too ?
    )

    # Copy rootfs into install directory
    echo_action "CP ROOTFS TO INSTALLDIR"
    sudo rm -rf "$DIR_ROOTFS"
    cp -a "$DIR_BUILDCORE_BUILD/bb_build-1.29.3/_install" "$DIR_ROOTFS"

    # Copy libnuma library into container filesystem
    cp /usr/lib/x86_64-linux-gnu/libnuma.so.1 "$DIR_ROOTFS/lib"

    # VERY IMPORTANT: Remove ttyS0 from inittab file, otherwise you get rogue
    # GETTY processes consuming 100% CPU after the container is shut down
    sed -e 's/^ttyS0/#&/g' -i "$DIR_ROOTFS/etc/inittab"

    # Create some necessary directories
    sudo mkdir -p "$DIR_ROOTFS/rootfs/var/run"
    sudo mkdir -p "$DIR_ROOTFS/rootfs/var/run/VIO"
    sudo mkdir -p "$DIR_ROOTFS/rootfs/var/run/hugepages"

    # TODO: a directory where to put user-config for each container

    # This is used to check whether it is installed successfully or not
    touch "$ROOTFS_CHECK_FILE"
}

function rootfs_remove() {
    echo_section "Removing the rootfs"

    echo_action "RM $DIR_BUILDCORE"
    rm -rf "$DIR_BUILDCORE"
    echo_action "RM $DIR_BUILDCORE_BUILD"
    rm -rf "$DIR_BUILDCORE_BUILD"
    echo_action "RM $DIR_ROOTFS"
    sudo rm -rf "$DIR_ROOTFS"

    rm -f "$ROOTFS_CHECK_FILE"
}

###################### LXC MANAGEMENT ######################

# FUTURE IDEA: containers created and destroyed 'on the go' by the testbed

# Requires as input:
# - $1 <container_name>
# - $2 <cpu_set>
# - $3 <name_sriov_vf> (expressed as '????' - unused for now but still required)
# - $4 <mac_address>
# - $5 <ip_address>
# - $6 <netmask>
# - $7 <sriov_dev_name> (expressed as 'uioN')
function lxc_create_container() {
    local container_name=$1
    local cpu_set=$2
    local name_sriov_vf=$3
    local mac_address=$4
    local ip_address=$5
    local netmask=$6
    local sriov_dev_name=$7

    sudo lxc-create "$container_name" -t none || true

    local FILE_BASE_CONFIG=$DIR_INSTALL_DATA/lxcbase.config
    local FILE_CONT_CONFIG=/tmp/${container_name}.config

    cp "$FILE_BASE_CONFIG" "$FILE_CONT_CONFIG"
    sed -i -e "s#<rootfs_absolute_path>#${DIR_ROOTFS}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<container_name>#${container_name}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<cpu_set>#${cpu_set}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<name_sriov_vf>#${name_sriov_vf}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<mac_address>#${mac_address}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<ip_address>#${ip_address}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<netmask>#${netmask}#g" "$FILE_CONT_CONFIG"
    sed -i -e "s#<sriov_dev_name>#${sriov_dev_name}#g" "$FILE_CONT_CONFIG"

    sudo cp "$FILE_CONT_CONFIG" "/var/lib/lxc/${container_name}/config"

    # FIXME: create bridge lxcbr0
}

function lxc_install() {
    echo_section "Creating LXC Containers"

    # FIXME: change to finalize installation part to copy stuff again to rootfs
    echo_action "LXC INSTALL DEP [ROOTFS]"
    [ ! -e "$ROOTFS_CHECK_FILE" ] && rootfs_install

    # File may be non existent, no containers will be created
    source "$DIR_INSTALL_DATA/lxcconfrc.bash" || true

    # If no container is specified, do not install anything
    if [ "${#LXC_CONT_NAMES[@]}" -lt 1 ]; then
        return 0
    fi

    # Now we have some configuration variables that we can iterate to create LXC
    # containers beforehand, to use them later during tests

    # First clear a few configuration files
    truncate -s 0 "$CONF_SRIOV_DEVLIST"
    truncate -s 0 "$CONF_SRIOV_TESTPMD"

    # Create a directory for containers rc configuration files
    rm -rf "$DIR_ROOTFS/conf"
    mkdir -p "$DIR_ROOTFS/conf"

    # Iterate through containers specification
    for i in "${!LXC_CONT_NAMES[@]}"; do
        echo_action "LXC-CREATE ${LXC_CONT_NAMES[i]}"
        lxc_create_container \
            "${LXC_CONT_NAMES[i]}" \
            "${LXC_CONT_CPUS[i]}" \
            "${LXC_CONT_VFS[i]}" \
            "${LXC_CONT_MACS[i]}" \
            "${LXC_CONT_IPS[i]}" \
            "${LXC_CONT_NMASKS[i]}" \
            "${LXC_CONT_VFDEVS[i]}"

        # Create a configuration file with these variables for each container
        cat >"$DIR_ROOTFS/conf/${LXC_CONT_NAMES[i]}_rc.conf" <<EOF
CONT_NAME=${LXC_CONT_NAMES[i]}
CONT_CPU=${LXC_CONT_CPUS[i]}
CONT_VF=${LXC_CONT_VFS[i]}
CONT_MAC=${LXC_CONT_MACS[i]}
CONT_IP=${LXC_CONT_IPS[i]}
CONT_NMASK=${LXC_CONT_NMASKS[i]}
CONT_VFDEV=${LXC_CONT_VFDEVS[i]}
CONT_VFPCI=${LXC_CONT_VFPCIS[i]}
CONT_SOCK=${LXC_CONT_SOCKS[i]}
EOF

        # Add to the two run commands configuration files for SR-IOV the specs
        # of the current container
        # FIXME: check whether a dynamic CONF_SRIOV_TESTPMD file is necessary
        echo "${LXC_CONT_VFPCIS[i]}" >>"$CONF_SRIOV_DEVLIST"
        echo "set vf mac addr 0 $i ${LXC_CONT_VFPCIS[i]}" \
            >>"$CONF_SRIOV_TESTPMD"
    done

    # MAGIC WAIT - DO NOT REMOVE, OTHERWISE NEXT OPERATION MAY FAIL
    # IF IT DOES, JUST REPEAT THE INSTALLATION COMMAND, IT WILL SKIP ALL THE
    # SKIPPABLE STUFF WHILE COMPLETING CORRECTLY
    sleep 4s

    echo_warning "NOTICE: Next sequence of commands may lead to failure of install command" \
        >/dev/stderr
    echo_warning "        If that happens, just execute again install command with the same parameters as before" \
        >/dev/stderr
    echo_warning "        In that case, it is guaranteed to succeed" \
        >/dev/stderr

    mkdir -p /tmp/VIO

    # When all containers are created, change 'myself' user password to no password
    # The command is repeated because usually the first time fails.
    # If the last attempt fails, the installation fails
    sudo lxc-start -n "${LXC_CONT_NAMES[0]}"
    (sudo lxc-attach -n "${LXC_CONT_NAMES[0]}" -- passwd -d myself) || true
    (sudo lxc-attach -n "${LXC_CONT_NAMES[0]}" -- passwd -d myself) || true
    (sudo lxc-attach -n "${LXC_CONT_NAMES[0]}" -- passwd -d myself)
    sudo lxc-stop -n "${LXC_CONT_NAMES[0]}"
}

function lxc_remove() {
    echo_section "Destroying LXC Containers"

    # File may be non existent, no containers will be deleted
    source "$DIR_INSTALL_DATA/lxcconfrc.bash" || true

    # If no container is specified, do not remove anything
    if [ "${#LXC_CONT_NAMES[@]}" -lt 1 ]; then
        return 0
    fi

    # Iterate through containers specification to delete them
    for i in "${!LXC_CONT_NAMES[@]}"; do
        echo_action "LXC-DESTROY ${LXC_CONT_NAMES[i]}"
        local cont_name=${LXC_CONT_NAMES[i]}

        # Create dummy rootfs that will be deleted by the destroy command
        sudo mkdir -p "/var/lib/lxc/${cont_name}/rootfs"

        # Chanhge rootfs in config of each container to point to the dummy one
        sudo sed -i -e \
            "s#${DIR_ROOTFS}#/var/lib/lxc/${cont_name}/rootfs#g" \
            "/var/lib/lxc/${cont_name}/config" || true

        # Stop container if running
        sudo lxc-stop "${cont_name}" || true

        # Destroy container
        sudo lxc-destroy "${cont_name}" || true

        # Remove all container if directory still exists
        sudo rm -rf "/var/lib/lxc/${cont_name}"
    done
}

##################### DPDK MANAGEMENT ######################

function dpdk_export_vars() {
    export RTE_SDK=${DIR_DPDK_INSTALL}/share/dpdk/
    export RTE_TARGET=x86_64-native-linuxapp-gcc
}

function dpdk_install() {
    echo_section "Installing DPDK"

    if [ -e "$DPDK_CHECK_FILE" ]; then
        echo_warning "ALREADY INSTALLED -- SKIPPING" >/dev/stderr
        return 0
    fi

    # Downloading and extracting DPDK sources
    echo_action "DOWNLOAD DPDK $DPDK_VERSION"
    (
        cd "$DIR_BASE"
        wget "http://fast.dpdk.org/rel/dpdk-${DPDK_VERSION}.tar.xz" \
            -O dpdk.tar.xz

        rm -rf "$DIR_DPDK_SRC"
        mkdir -p "$DIR_DPDK_SRC"
        tar xvf dpdk.tar.xz -C "$DIR_DPDK_SRC" --strip 1
        rm dpdk.tar.xz
    )

    # Changing a configuration setting for i40e driver
    sed -i -E "s/CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=n/CONFIG_RTE_LIBRTE_I40E_16BYTE_RX_DESC=y/" "$DIR_DPDK_SRC/config/common_base"

    # Changing a configuration setting for igb_uio driver
    # (which is not built by default)
    sed -i -E "s/CONFIG_RTE_EAL_IGB_UIO=n/CONFIG_RTE_EAL_IGB_UIO=y/" "$DIR_DPDK_SRC/config/common_base"

    rm -rf "$DIR_DPDK_BUILD"
    rm -rf "$DIR_DPDK_INSTALL"

    # Build and install
    echo_action "BUILD AND INSTALL DPDK $DPDK_VERSION (-j $MAKECPUS)"
    make -C "$DIR_DPDK_SRC" -j "$MAKECPUS" install \
        "RTE_SDK=$DIR_DPDK_SRC" \
        "RTE_OUTPUT=$DIR_DPDK_BUILD" \
        "DESTDIR=$DIR_DPDK_INSTALL" \
        T=x86_64-native-linuxapp-gcc # \
    # EXTRA_CFLAGS=-g

    # # Pushing all environment variables for DPDK into a single file
    # # TODO: is there anything else to be pushed into this file?
    #     cat >$CONF_DPDK_RC <<EOF
    # export RTE_SDK=${DIR_DPDK_INSTALL}/share/dpdk/
    # export RTE_TARGET=x86_64-native-linuxapp-gcc
    # export TESTPMD=${DIR_DPDK_INSTALL}/bin/testpmd
    # EOF

    # # Source it # TODO: STANDARD WAY TO FIND THIS FILE
    # source $CONF_DPDK_RC

    # Export necessary variables for make
    dpdk_export_vars

    echo_action "BUILD DPDK EXAMPLES (-j $MAKECPUS)"
    make -C "$DIR_DPDK_INSTALL/share/dpdk/examples" -j "$MAKECPUS"
    # EXTRA_CFLAGS=-g

    touch "$DPDK_CHECK_FILE"
}

function dpdk_remove() {
    echo_section "Removing DPDK $DPDK_VERSION"

    echo_action "RM $DIR_DPDK_SRC"
    rm -rf "$DIR_DPDK_SRC"
    echo_action "RM $DIR_DPDK_BUILD"
    rm -rf "$DIR_DPDK_BUILD"
    echo_action "RM $DIR_DPDK_INSTALL"
    rm -rf "$DIR_DPDK_INSTALL"

    rm -f "$DPDK_CHECK_FILE"
}

###################### VPP MANAGEMENT ######################

function vpp_install() {
    echo_section "Installing VPP from APT [version $VPP_VERSION]"

    # Setting up the repository and adding key
    sudo sh -c "echo 'deb [trusted=yes] https://packagecloud.io/fdio/release/ubuntu bionic main' > /etc/apt/sources.list.d/99fd.io.list"
    curl -L https://packagecloud.io/fdio/release/gpgkey | sudo apt-key add -

    echo_action "APT INSTALL VPP-REQUIRED"

    # Install Mandatory Packages
    sudo apt update
    sudo apt install -y \
        "vpp=$VPP_VERSION" \
        "libvppinfra=$VPP_VERSION" \
        "vpp-plugin-core=$VPP_VERSION" \
        "vpp-plugin-dpdk=$VPP_VERSION"

    echo_action "APT INSTALL VPP-OPTIONAL"

    # Install Optional Packages
    sudo apt install -y \
        "vpp-api-python=$VPP_VERSION" \
        "python3-vpp-api=$VPP_VERSION" \
        "vpp-dbg=$VPP_VERSION" \
        "vpp-dev=$VPP_VERSION" \
        "libvppinfra-dev=$VPP_VERSION"

    # Disable automatic VPP execution as a service
    sudo systemctl stop vpp || true
    sudo systemctl disable vpp

    # Copy startup configuration for VPP in install location
    cp "$DIR_INSTALL_DATA/vppconfig.conf" "$CONF_VPP"
}

function vpp_remove() {
    echo_section "Removing VPP using APT"

    sudo apt autoremove --purge vpp* python3-vpp-api libvppinfra* -y
}

# To run VPP use:
# sudo service vpp start
# Or better, use
# sudo vpp -c startup.conf
# and kill it later with sudo pkill vpp
# TODO: create this startup.conf file somewhere
#
# NOTICE: To check whether VPP is running use
# ps -eaf | grep vpp | grep -v "grep" | wc -l
# the output should be one
#
# To stop VPP use:
# sudo service vpp stop
#

###################### OVS MANAGEMENT ######################

function ovs_install() {
    echo_section "Installing Open vSwitch [version $OVS_VERSION]"

    echo_action "OVS INSTALL DEP [DPDK]"
    [ ! -e "$DPDK_CHECK_FILE" ] && dpdk_install

    if [ -e "$OVS_CHECK_FILE" ]; then
        echo_warning "ALREADY INSTALLED -- SKIPPING" >/dev/stderr
        return 0
    fi

    # Downloading and extracting OVS sources
    echo_action "DOWNLOAD OVS"
    (
        cd "$DIR_BASE"
        wget "https://www.openvswitch.org/releases/openvswitch-${OVS_VERSION}.tar.gz" \
            -O ovs.tar.gz

        rm -rf "$DIR_OVS_SRC"
        mkdir -p "$DIR_OVS_SRC"
        tar xvf ovs.tar.gz -C "$DIR_OVS_SRC" --strip 1
        rm ovs.tar.gz

        # More recent version of DPDK changed the name of a couple of stuff
        # without changing behavior, this "patches" OVS to support newer DPDK
        # versions
        # (important: -print0 must be last argument)
        find "$DIR_OVS_SRC" -type f -print0 | xargs -0 sed -i '
            s/e_RTE_METER_GREEN/RTE_COLOR_GREEN/g;
            s/struct ether_addr/struct rte_ether_addr/g;
            s/struct ether_hdr/struct rte_ether_hdr/g;
            s/ETHER_CRC_LEN/RTE_ETHER_CRC_LEN/g;
            s/ETHER_HDR_LEN/RTE_ETHER_HDR_LEN/g;
            s/ETHER_MIN_MTU/RTE_ETHER_MIN_MTU/g;
            s/ETHER_MTU/RTE_ETHER_MTU/g;
            '
    )

    # echo_action "PIP INSTALL DEP"
    sudo pip install -I ovs-sphinx-theme
    sudo pip install -I docutils

    (
        cd "$DIR_OVS_SRC"
        echo_action "OVS CONFIGURE BUILD"
        ./configure "--with-dpdk=$DIR_DPDK_BUILD"

        echo_action "OVS BUILD (-j $MAKECPUS)"
        make -j "$MAKECPUS"

        echo_action "OVS INSTALL"
        sudo make install # TODO: change to another destination?
    )

    # Configuring OVS

    # Kill potential processes still running
    sudo killall ovsdb-server ovs-vswitchd || true

    # Creating the DB
    echo_action "OVS CREATE DB"
    sudo mkdir -p /usr/local/var/run/openvswitch
    sudo rm -f /usr/local/var/run/openvswitch/conf.db
    sudo ovsdb-tool create /usr/local/var/run/openvswitch/conf.db \
        "$DIR_OVS_SRC/vswitchd/vswitch.ovsschema"

    # Starting the server
    sudo ovsdb-server /usr/local/var/run/openvswitch/conf.db \
        --remote=punix:/usr/local/var/run/openvswitch/db.sock \
        --remote=db:Open_vSwitch,Open_vSwitch,manager_options \
        --pidfile --detach --log-file

    # Initializing the DB
    sudo ovs-vsctl --no-wait init

    # FIXME: SET THIS AT RUNTIME
    # Open vSwitch DPDK options [DPDK can run only on core 0 and 2]
    #sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
    #sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x5
    #sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,1024"

    # Can execute only on cores 0 and 2
    #sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:pmd-cpu-mask=0x5

    # TODO: port configuration at run time (should also ignore PCI devices
    # when running on single host)

    # TODO: THIS SHOULD BE IN A .ovsrc FILE
    export PATH="$PATH:/usr/local/share/openvswitch/scripts"

    # Stop the service after configuration
    # The "which" is necessary because sudo ignores user PATH
    sudo "$(which ovs-ctl)" stop

    touch "$OVS_CHECK_FILE"
}

function ovs_remove() {
    echo_section "Removing OVS $OVS_VERSION"

    if [ ! -e "$OVS_CHECK_FILE" ]; then
        echo_warning "NOT INSTALLED -- SKIPPING" >/dev/stderr
        return 0
    fi

    if [ ! -e "$DPDK_CHECK_FILE" ]; then
        echo_warning "UNINSTALLING OVS REQUIRES DPDK INSTALLED" >/dev/stderr
        echo_warning "INSTALL DPDK BEFORE REMOVING OVS," >/dev/stderr
        echo_warning "THEN UNINSTALL DPDK AGAIN" >/dev/stderr
        return 0
    fi

    # Kill potential processes still running
    sudo killall ovsdb-server ovs-vswitchd || true

    if [ ! -d "$DIR_OVS_SRC" ]; then
        echo_warning "UNINSTALLING OVS REQUIRES ITS SOURCES IN $DIR_OVS_SRC " \
            "DIRECTORY!" >/dev/stderr
        return 0
    fi

    (
        cd "$DIR_OVS_SRC"
        echo_action "OVS CONFIGURE BUILD"
        ./configure "--with-dpdk=$DIR_DPDK_BUILD" # FIXME: DIR_DPDK_INSTALL?

        echo_action "OVS UNINSTALL"
        sudo make uninstall
    )

    echo_action "RM $DIR_OVS_SRC"
    rm -rf "$DIR_OVS_SRC"

    rm -f "$OVS_CHECK_FILE"
}

##################### SNABB MANAGEMENT #####################

function snabb_install() {
    echo_section "Installing Snabb"

    if [ -e "$SNABB_CHECK_FILE" ]; then
        echo_warning "ALREADY INSTALLED -- SKIPPING" >/dev/stderr
        return 0
    fi

    rm -rf "$DIR_SNABB_SRC"
    git clone https://github.com/snabbco/snabb.git "$DIR_SNABB_SRC"

    mkdir -p "$DIR_SNABB_PROGRAM"
    cp "$SNABB_INPUT_FILE" "$SNABB_PROGRAM_FILE"

    echo_action "SNABB BUILD (-j $MAKECPUS)"
    make -C "$DIR_SNABB_SRC" -j "$MAKECPUS"

    touch "$SNABB_CHECK_FILE"
}

function snabb_remove() {
    echo_section "Removing Snabb"

    echo_action "RM $DIR_SNABB_SRC"
    rm -rf "$DIR_SNABB_SRC"

    rm -f "$SNABB_CHECK_FILE"
}

################ CUSTOM PROGRAMS MANAGEMENT ################

function bin_install() {
    echo_section "Installing custom scripts and binaries"

    echo_action "INSTALL TESTAPP DEP [ROOTFS]"
    [ ! -e "$ROOTFS_CHECK_FILE" ] && rootfs_install

    echo_action "INSTALL TESTAPP DEP [DPDK]"
    [ ! -e "$DPDK_CHECK_FILE" ] && dpdk_install

    dpdk_export_vars

    export DIR_ROOTFS=$DIR_ROOTFS

    # Installation is performed automatically by make command
    echo_action "MAKE INSTALL TESTAPP (-j $MAKECPUS)"
    make -C "$DIR_TESTAPP" -j "$MAKECPUS"

    # Additional useful command
    echo_action "MAKE INSTALL CPU_LIST_TO_MASK"
    make -C "$DIR_CPUMASK" -j "$MAKECPUS"

    echo_action "INSTALL BIN IN ROOTFS"
    cp "$DIR_SCRIPT/bin/nfv_run_in_container" "$DIR_ROOTFS"

    if [ -e "$DPDK_CHECK_FILE" ]; then
        cp "$DIR_DPDK_INSTALL/bin/testpmd" "$DIR_ROOTFS"
    fi

    echo_action "INSTALL COMMANDS (SCRIPTS)"
    for f in "$DIR_BIN"/./*; do
        if [[ ! "$f" =~ nfv_run_in_container ]] &&
            [[ ! "$f" =~ README ]]; then
            cp "$f" "$DIR_BIN_DST/"
        fi
    done

    echo_action "INSTALL CONFIGURATIONS"
    cp "${DIR_CONFIGURATIONS}/"* "${DIR_CONFIGRC}"
}

function bin_remove() {
    echo_section "Removing custom scripts and binaries"

    if [ -e "$ROOTFS_CHECK_FILE" ]; then
        echo_action "RM BINS FROM ROOTFS"
        rm -f "$DIR_ROOTFS/testapp" \
            "$DIR_ROOTFS/cpu_list_to_mask" \
            "$DIR_ROOTFS/testpmd" \
            "$DIR_ROOTFS/nfv_run_in_container"
    fi

    echo_action "RM COMMANDS"
    rm -f "$DIR_BIN_DST"/./*
}

################## FRAMEWORK FINALIZATION ##################

# First argument is the file, second argument is the line
function add_line_if_missing() {
    ([ -f "$1" ] && [[ "$(grep -c -e "$3" "$1")" -lt 1 ]] &&
        echo "$2" >>"$1") || true
}

function finalize_install() {
    echo_section "Finalizing Installation"

    # Configure the machine to use hugepages
    echo_action "Configuring HUGEPAGES on Linux kernel options [REBOOT TO APPLY CHANGES]"

    HUGEPAGE_CMDLINE_OPTION="hugepagesz=1G hugepages=32 intel_iommu=on iommu=pt"
    sudo sed -i "s#\(GRUB_CMDLINE_LINUX_DEFAULT\s*=\s*\)\"\(.*\)\"#\1\"\2 $HUGEPAGE_CMDLINE_OPTION\"#g" /etc/default/grub

    # Remove potential duplicates
    sudo sed -i "s#$HUGEPAGE_CMDLINE_OPTION\(.*$HUGEPAGE_CMDLINE_OPTION\)#\1#g" /etc/default/grub

    echo_action "UPDATE-GRUB2"
    sudo update-grub2

    # TODO: FIX, WORKS BUT CAN BE BETTER
    # LINE MUST BE:
    # hugetlbfs /dev/hugepages hugetlbfs rw,relatime,pagesize=1024M 0 0
    HUGELINE="hugetlbfs /dev/hugepages hugetlbfs rw,relatime,pagesize=1G 0 0"
    if [ "$(grep -c /etc/fstab -e 'hugetlbfs')" -lt 1 ]; then
        echo_action "ADD HUGEPAGETLB TO FSTAB"
        sudo sh -c "echo \"$HUGELINE\" >> /etc/fstab"
    fi

    generate_rcfile

    # Add rcfile to user profile (sh/bash/zsh)
    add_line_if_missing "$HOME/.profile" "source \"$NFV_RCFILE\"" \
        "$NFV_RCFILE"
    add_line_if_missing "$HOME/.bash_login" "source \"$NFV_RCFILE\"" \
        "$NFV_RCFILE"
    add_line_if_missing "$HOME/.bash_profile" "source \"$NFV_RCFILE\"" \
        "$NFV_RCFILE"
    add_line_if_missing "$HOME/.zprofile" "source \"$NFV_RCFILE\"" \
        "$NFV_RCFILE"

    # I source the file to have everything available immediately
    source "$NFV_RCFILE"
}

######################### MAIN BODY ########################

function main_f() {
    parse_parameters "$@"

    DIR_INSTALL_DATA="$DIR_SCRIPT/installdata"
    DIR_TESTAPP="$DIR_SCRIPT/src/testapp-rewrite"
    DIR_CPUMASK="$DIR_SCRIPT/src/cpu_list_to_mask"
    DIR_CONFIGURATIONS="$DIR_SCRIPT/configurations"
    DIR_BIN="$DIR_SCRIPT/bin"

    DIR_BASE="$HOME/nfv_testperf"

    if [ $default != y ]; then
        DIR_BASE=$(
            asklocation "Select where the framework should be installed" \
                "$DIR_BASE"
        )
    fi
    DIR_BASE=$(realpath "$DIR_BASE")
    DIR_SRC=$DIR_BASE/src
    DIR_BUILD=$DIR_BASE/build
    DIR_INSTALL=$DIR_BASE/install
    DIR_BIN_DST=$DIR_BASE/bin
    DIR_CONFIG=$DIR_BASE/conf
    DIR_CONFIGRC=$DIR_CONFIG/rc
    NFV_RCFILE=$DIR_BASE/.rcfile

    DIR_BUILDCORE=$DIR_SRC/buildcore
    DIR_BUILDCORE_BUILD=$DIR_BUILD/buildcore
    DIR_ROOTFS=$DIR_INSTALL/rootfs

    DIR_DPDK_SRC=$DIR_SRC/dpdk-${DPDK_VERSION}
    DIR_DPDK_BUILD=$DIR_BUILD/dpdk-${DPDK_VERSION}
    DIR_DPDK_INSTALL=$DIR_INSTALL/dpdk-${DPDK_VERSION}

    DIR_OVS_SRC=$DIR_SRC/ovs
    DIR_SNABB_SRC=$DIR_SRC/snabb

    ROOTFS_CHECK_FILE=$DIR_INSTALL/.rootfs-installed
    DPDK_CHECK_FILE=$DIR_INSTALL/.dpdk-${DPDK_VERSION}-installed
    OVS_CHECK_FILE=$DIR_INSTALL/.ovs-${OVS_VERSION}-installed
    SNABB_CHECK_FILE=$DIR_INSTALL/.snabb-installed

    SNABB_PROGRAM_NAME=custom_switch
    DIR_SNABB_PROGRAM=$DIR_SNABB_SRC/src/program/$SNABB_PROGRAM_NAME
    SNABB_INPUT_FILE=$DIR_INSTALL_DATA/$SNABB_PROGRAM_NAME.lua
    SNABB_PROGRAM_FILE=$DIR_SNABB_PROGRAM/$SNABB_PROGRAM_NAME.lua

    # Configuration files and run commands
    CONF_VPP=$DIR_CONFIG/vpp.conf
    CONF_SRIOV_DEVLIST=$DIR_CONFIG/sriov_devlist.conf
    CONF_SRIOV_TESTPMD=$DIR_CONFIG/sriov_testpmd.conf

    # Get number of threads for make command
    MAKECPUS=$(get_make_cpus)

    case $cmd in
    i)
        # INSTALL COMMAND
        directories_install
        packages_install

        [[ "$args" == *_rootfs_* ]] && rootfs_install
        [[ "$args" == *_lxc_* ]] && lxc_install
        [[ "$args" == *_dpdk_* ]] && dpdk_install
        [[ "$args" == *_ovs_* ]] && ovs_install
        [[ "$args" == *_vpp_* ]] && vpp_install
        [[ "$args" == *_snabb_* ]] && snabb_install
        [[ "$args" == *_bin_* ]] && bin_install

        finalize_install

        ;;
    d)
        # REMOVE COMMANDS
        # NOTICE: ORDER MATTERS

        [[ "$args" == *_bin_* ]] && bin_remove
        [[ "$args" == *_snabb_* ]] && snabb_remove
        [[ "$args" == *_vpp_* ]] && vpp_remove
        [[ "$args" == *_ovs_* ]] && ovs_remove
        [[ "$args" == *_dpdk_* ]] && dpdk_remove
        [[ "$args" == *_lxc_* ]] && lxc_remove
        [[ "$args" == *_rootfs_* ]] && rootfs_remove
        [[ "$args" == *_all_* ]] && directories_remove

        ;;
    esac

    return 0
}

######################### CONSTANTS ########################

COLOR='\033[1;36m'
COLOR_WARN='\033[1;35m'
NC='\033[0m'

DPDK_VERSION=20.02
OVS_VERSION=2.11.1
VPP_VERSION=19.08.1-release

######################### VARIABLES ########################

cmd=i
args=
default=n

set -e

# This is used to get reliably the original path of the script
# Source: https://stackoverflow.com/a/246128/7030275
SOURCE="${BASH_SOURCE[0]}"
# Resolve $SOURCE until the file is no longer a symlink
while [ -h "$SOURCE" ]; do
    DIR_SCRIPT="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)"
    SOURCE="$(readlink "$SOURCE")"
    # If $SOURCE was a relative symlink, we need to resolve it
    # relative to the path where the symlink file was located
    [[ $SOURCE != /* ]] && SOURCE="$DIR_SCRIPT/$SOURCE"
done
DIR_SCRIPT="$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd)"

# Only command to run on global scope
main_f "$@"