RUN.bsub

#!/bin/bash
# --- specify queue
#BSUB -q hpcintro
# --- asking for number of cores
#BSUB -n 120
# --- reserving N cores on each machine up to the total number cores requested
#BSUB -R "span[block=24]"
# --- Wall time
#BSUB -W 00:40
# --- Specify that we need X GB of memory per core/slot
#BSUB -R "rusage[mem=10GB]"
#BSUB -J profiling
#BSUB -e Error_%J.err

module purge
module load mpi/3.1.3-gcc-8.2.0

# do a clean build to ensure everything is updated
make clean
make CFLAGS="-O3 -march=native"

tol=0

# Loop matrix size
for N_n in 100 500 1000 1500 2000 2500 3000 3500 4000
do

if [[ $N_n -lt 200 ]]
then
    iter=100
elif [[ $N_n -lt 1000 ]]
then
    iter=20
else
    iter=5
fi

# Loop px
all_p="1 4 9 16 25 36 49 64 81 100" # start at 1? end at 100?
if [[ $N_n -lt 2750 ]]
then
    all_p_sqr="1 2 3 4 5 6 7 8 9 10"    # squared number of processes
elif [[ $N_n -lt 3250 ]]
then
    all_p_sqr="7 8 9 10"
elif [[ $N_n -lt 3750 ]]
then
    all_p_sqr="9 10"
else
    all_p_sqr="10"
fi

for p_n in $all_p_sqr ; do

N=$(($N_n-($N_n%$p_n)))
p=$(($p_n*$p_n))

# skip calculations with more processors than available
[ $p -gt $LSB_DJOB_NUMPROC ] && continue

# adjust here for the different algorithms
all_algo="6"
for algo in $all_algo ; do

if [[ $algo -eq 8 ]]
then
    export OMP_NUM_THREADS=4
fi

out="data/${N}_p${p}_${algo}"
[ -e $out ] && continue

# Adjust arguments to your project
echo "mpirun -np $p --map-by dist:span --mca rmaps_dist_device ib0 \
--report-bindings \
./main $N $iter $tol $algo 2>&1 > $out"

mpirun -np $p --map-by dist:span --mca rmaps_dist_device ib0 \
       --report-bindings \
       ./main $N $iter $tol $algo > $out 2>${out}_binding
if [ $? -ne 0 ]; then
    echo "Exitting before done!!!"
    exit 1
fi

done #algo

done #p_n

done #N_n