#!/bin/bash

#SBATCH -t 00:10079:00
#SBATCH -N 36
#SBATCH --ntasks=36 --cpus-per-task=20
#SBATCH -p scec
#SBATCH --mem 0

PBS_NODEFILE="/tmp/${USER}-hostfile-${SLURM_JOBID}"
echo "creating PBS_NODEFILE: $PBS_NODEFILE"
scontrol show hostnames $SLURM_NODELIST > $PBS_NODEFILE

NEW_NODEFILE="/tmp/${USER}-hostfile-fmpj-${PBS_JOBID}"
echo "creating PBS_NODEFILE: $NEW_NODEFILE"
hname=$(hostname)
if [ "$hname" == "" ]
then
  echo "Error getting hostname. Exiting"
  exit 1
else
  cat $PBS_NODEFILE | sort | uniq | fgrep -v $hname > $NEW_NODEFILE
fi

export PBS_NODEFILE=$NEW_NODEFILE
export FMPJ_HOME=/project/scec_608/kmilner/mpj/FastMPJ
export PATH=$PATH:$FMPJ_HOME/bin

if [[ -e $PBS_NODEFILE ]]; then
  #count the number of processors assigned by PBS
  NP=`wc -l < $PBS_NODEFILE`
  echo "Running on $NP processors: "`cat $PBS_NODEFILE`
else
  echo "This script must be submitted to PBS with 'qsub -l nodes=X'"
  exit 1
fi

if [[ $NP -le 0 ]]; then
  echo "invalid NP: $NP"
  exit 1
fi

MAIN_DIR=/project/scec_608/kmilner/nshm23/batch_inversions
DIR=$MAIN_DIR/2023_01_17-nshm23_branches-NSHM23_v2-CoulombRupSet-TotNuclRate-NoRed-ThreshAvgIterRelGR
JVM_MEM_MB=51200

date
echo "RUNNING FMPJ"
fmpjrun_errdetect_wrapper.sh -machinefile $PBS_NODEFILE -np $NP -dev niodev -Djava.library.path=$FMPJ_HOME/lib -Xmx${JVM_MEM_MB}M -cp $DIR/opensha-dev-all.jar  -class org.opensha.sha.earthquake.faultSysSolution.inversion.mpj.MPJ_LogicTreeInversionRunner --logic-tree $DIR/logic_tree.json --output-dir $DIR/results --inversion-factory 'org.opensha.sha.earthquake.rupForecastImpl.nshm23.NSHM23_InvConfigFactory' --annealing-threads 20 --cache-dir $MAIN_DIR/cache --runs-per-bundle 1 --exact-dispatch 1
ret=$?

date
echo "DONE with process 0. EXIT CODE: $ret"

exit $ret