#!/bin/bash
#
# Bring up the kernel RDMA stack
#
# This is usually run automatically by systemd after a hardware activation
# event in udev has triggered a start of the rdma.service unit
#

shopt -s nullglob

CONFIG=/etc/rdma/rdma.conf

LOAD_ULP_MODULES=""
LOAD_CORE_USER_MODULES="ib_umad ib_uverbs ib_ucm rdma_ucm"
LOAD_CORE_CM_MODULES="iw_cm ib_cm rdma_cm"
LOAD_CORE_MODULES="ib_core"
LOAD_TECH_PREVIEW_DRIVERS="no"

if [ -f $CONFIG ]; then
    . $CONFIG

    if [ "${RDS_LOAD}" == "yes" ]; then
        IPOIB_LOAD=yes
    fi

    if [ "${IPOIB_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="ib_ipoib"
    fi

    if [ "${RDS_LOAD}" == "yes" -a -f /lib/modules/`uname -r`/kernel/net/rds/rds.ko ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds"
	if [ -f /lib/modules/`uname -r`/kernel/net/rds/rds_tcp.ko ]; then
	    LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds_tcp"
	fi
	if [ -f /lib/modules/`uname -r`/kernel/net/rds/rds_rdma.ko ]; then
	    LOAD_ULP_MODULES="$LOAD_ULP_MODULES rds_rdma"
	fi
    fi

    if [ "${SRP_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_srp"
    fi

    if [ "${SRPT_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_srpt"
    fi

    if [ "${ISER_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_iser"
    fi

    if [ "${ISERT_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES ib_isert"
    fi

    if [ "${XPRTRDMA_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES xprtrdma"
    fi

    if [ "${SVCRDMA_LOAD}" == "yes" ]; then
	LOAD_ULP_MODULES="$LOAD_ULP_MODULES svcrdma"
    fi
    if [ "${TECH_PREVIEW_LOAD}" == "yes" ]; then
        LOAD_TECH_PREVIEW_DRIVERS="$TECH_PREVIEW_LOAD"
    fi
else
    LOAD_ULP_MODULES="ib_ipoib"
fi

# If module $1 is loaded return - 0 else - 1
is_loaded()
{
    /sbin/lsmod | grep -w "$1" > /dev/null 2>&1
    return $?
}

load_modules()
{
    local RC=0

    for module in $*; do
	if ! /sbin/modinfo $module > /dev/null 2>&1; then
	    # do not attempt to load modules which do not exist
	    continue
	fi
	if ! is_loaded $module; then
	    /sbin/modprobe $module
	    res=$?
	    RC=$[ $RC + $res ]
	    if [ $res -ne 0 ]; then
		echo
		echo "Failed to load module $module"
	    fi
	fi
    done
    return $RC
}

load_hardware_modules()
{
    local -i RC=0

    # We match both class NETWORK and class INFINIBAND devices since our
    # iWARP hardware is listed under class NETWORK.  The side effect of
    # this is that we might cause a non-iWARP network driver to be loaded.
    udevadm trigger --subsystem-match=pci --attr-nomatch=driver --attr-match=class=0x020000 --attr-match=class=0x0c0600
    udevadm settle
    if [ -r /proc/device-tree ]; then
	if [ -n "`ls /proc/device-tree | grep lhca`" ]; then
	    if ! is_loaded ib_ehca; then
		load_modules ib_ehca
		RC+=$?
	    fi
	fi
    fi
    if is_loaded mlx4_core -a ! is_loaded mlx4_ib; then
        load_modules mlx4_ib
	RC+=$?
    fi
    if is_loaded mlx4_core -a ! is_loaded mlx4_en; then
        load_modules mlx4_en
	RC+=$?
    fi
    if is_loaded mlx5_core -a ! is_loaded mlx5_ib; then
	load_modules mlx5_ib
	RC+=$?
    fi
    if is_loaded cxgb4 -a ! is_loaded iw_cxgb4; then
	load_modules iw_cxgb4
	RC+=$?
    fi
    if is_loaded be2net -a ! is_loaded ocrdma; then
	load_modules ocrdma
	RC+=$?
    fi
    if is_loaded enic -a ! is_loaded usnic_verbs; then
	load_modules usnic_verbs
	RC+=$?
    fi
    if [ "${LOAD_TECH_PREVIEW_DRIVERS}" == "yes" ]; then
        if is_loaded i40e -a ! is_loaded i40iw; then
	    load_modules i40iw
	    RC+=$?
        fi
    fi
    return $RC
}

errata_58()
{
    # Check AMD chipset issue Errata #58
    if test -x /sbin/lspci && test -x /sbin/setpci; then
	if ( /sbin/lspci -nd 1022:1100 | grep "1100" > /dev/null ) &&
	   ( /sbin/lspci -nd 1022:7450 | grep "7450" > /dev/null ) &&
	   ( /sbin/lspci -nd 15b3:5a46 | grep "5a46" > /dev/null ); then
	    CURVAL=`/sbin/setpci -d 1022:1100 69`
	    for val in $CURVAL
	    do
		if [ "${val}" != "c0" ]; then
		    /sbin/setpci -d 1022:1100 69=c0
		    if [ $? -eq 0 ]; then
			break
		    else
			echo "Failed to apply AMD-8131 Errata #58 workaround"
		    fi
		fi
	    done
	fi
    fi
}

errata_56()
{
    # Check AMD chipset issue Errata #56
    if test -x /sbin/lspci && test -x /sbin/setpci; then
	if ( /sbin/lspci -nd 1022:1100 | grep "1100" > /dev/null ) &&
	   ( /sbin/lspci -nd 1022:7450 | grep "7450" > /dev/null ) &&
	   ( /sbin/lspci -nd 15b3:5a46 | grep "5a46" > /dev/null ); then
	    bus=""
	    # Look for devices AMD-8131
	    for dev in `/sbin/setpci -v -f -d 1022:7450 19 | cut -d':' -f1,2`
	    do
		bus=`/sbin/setpci -s $dev 19`
		rev=`/sbin/setpci -s $dev 8`
		# Look for Tavor attach to secondary bus of this devices
		for device in `/sbin/setpci -f -s $bus: -d 15b3:5a46 19`
		do
		    if [ $rev -lt 13 ]; then
			/sbin/setpci -d 15b3:5a44 72=14
			if [ $? -eq 0 ]; then
			    break
			else
			    echo
			    echo "Failed to apply AMD-8131 Errata #56 workaround"
			fi
		    else
			continue
		    fi
		    # If more than one device is on the bus the issue a
		    # warning
		    num=`/sbin/setpci -f -s $bus: 0 | wc -l |  sed 's/\ *//g'`
		    if [ $num -gt 1 ]; then
			echo "Warning: your current PCI-X configuration might be incorrect."
			echo "see AMD-8131 Errata 56 for more details."
		    fi
		done
	    done
	fi
    fi
}


load_hardware_modules
RC=$[ $RC + $? ]
load_modules $LOAD_CORE_MODULES
RC=$[ $RC + $? ]
load_modules $LOAD_CORE_CM_MODULES
RC=$[ $RC + $? ]
load_modules $LOAD_CORE_USER_MODULES
RC=$[ $RC + $? ]
load_modules $LOAD_ULP_MODULES
RC=$[ $RC + $? ]

errata_58
errata_56

/usr/libexec/rdma-set-sriov-vf

exit $RC