#!/bin/bash

#SBATCH -t 00:2835:00
#SBATCH -N 36
#SBATCH --ntasks=36 --cpus-per-task=20
#SBATCH -p scec_hiprio
#SBATCH --mem 0

PBS_NODEFILE="/tmp/${USER}-hostfile-${SLURM_JOBID}"
echo "creating PBS_NODEFILE: $PBS_NODEFILE"
scontrol show hostnames $SLURM_NODELIST > $PBS_NODEFILE

NEW_NODEFILE="/tmp/${USER}-hostfile-fmpj-${PBS_JOBID}"
echo "creating PBS_NODEFILE: $NEW_NODEFILE"
hname=$(hostname)
if [ "$hname" == "" ]
then
  echo "Error getting hostname. Exiting"
  exit 1
else
  cat $PBS_NODEFILE | sort | uniq | fgrep -v $hname > $NEW_NODEFILE
fi

export PBS_NODEFILE=$NEW_NODEFILE
export FMPJ_HOME=/project/scec_608/kmilner/mpj/FastMPJ
export PATH=$PATH:$FMPJ_HOME/bin

if [[ -e $PBS_NODEFILE ]]; then
  #count the number of processors assigned by PBS
  NP=`wc -l < $PBS_NODEFILE`
  echo "Running on $NP processors: "`cat $PBS_NODEFILE`
else
  echo "This script must be submitted to PBS with 'qsub -l nodes=X'"
  exit 1
fi

if [[ $NP -le 0 ]]; then
  echo "invalid NP: $NP"
  exit 1
fi

MAIN_DIR=/project/scec_608/kmilner/nshm23/batch_inversions
DIR=$MAIN_DIR/2024_02_02-nshm23_branches-WUS_FM_v3
JVM_MEM_MB=51200

date
echo "RUNNING FMPJ"
fmpjrun_errdetect_wrapper.sh -machinefile $PBS_NODEFILE -np $NP -dev niodev -Djava.library.path=$FMPJ_HOME/lib -Xmx${JVM_MEM_MB}M -cp $DIR/opensha-dev-all.jar  -class org.opensha.sha.earthquake.faultSysSolution.hazard.mpj.MPJ_LogicTreeHazardCalc --input-file $DIR/results.zip --output-dir $DIR/results --grid-spacing 0.1 --exact-dispatch 1 --threads 20
ret=$?

date
echo "DONE with process 0. EXIT CODE: $ret"

exit $ret