#!/bin/bash
#---------------------------------------------------------------------------- #
#   University    |   DIFA - Dept of Physics and Astrophysics 
#       of        |   Open Physics Hub
#    Bologna      |   (https://site.unibo.it/openphysicshub/en)
#----------------------------------------------------------------------------
#
# License
#    This is free software: you can redistribute it and/or modify it
#    under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
# Author
#   Carlo Cintolesi
#
# Application
#   slurm workload manager
#
# Usage
#   run a job:         sbatch run.sh
#   check processes:   slurmtop
#   delete a job:      scancel <jobID>   
#
# Description
#   Run job on the new cluster of OPH with SLURM
#
# --------------------------------------------------------------------------- #
# SLURM setup
# --------------------------------------------------------------------------- #

#- (1) [optional] Choose the account of your research group
##SBATCH --account=oph         ## This job must be "billed" to OPH project
##SBATCH --reservation=prj-can ## Use the node reserved for CAN project
##SBATCH --qos=normal          ## Also available 'debug' (max 15', no billing) 
                               ## and 'long' (max 72h, low priority)

#- (2) Select the subcluster partition to work on (optional),
#  the number of tasks to be used (or specify the number of nodes and tasks),
#  and the RAM memory available for each node
#-
#SBATCH --constraint=matrix  ## run on matrix subcluster (parallel computing)
##SBATCH --constraint=blade  ## run on blade subcluster (pre/post-processing)
#SBATCH --ntasks=56          ## total number of tasks
##SBATCH --nodes=2           ## number of nodes to be allocated
##SBATCH --tasks-per-node=28 ## number of tasks per node (multiple of 28)
#SBATCH --mem-per-cpu=2G     ## ram per cpu (to be tuned)

#- (3) Set the name of the job, the log and error files,
#  define the email address for communications (just UniBo)
#-
#SBATCH --job-name="jobName" ## job name in the scheduler
#SBATCH --output=%N_%j.out   ## log file 
#SBATCH --error=%N_%j.err    ## err file
#SBATCH --mail-type=ALL      ## send a message when the job start and end
#SBATCH --mail-user=nome.cognome@unibo.it  ## email address for messages

# --------------------------------------------------------------------------- #
# Modules setup and applications run
# --------------------------------------------------------------------------- #

#- (4) Modules to be load 
#-
module load mpi/openmpi/4.1.4

#- (5) Run the job: just an example.
#  Note that the number of processors "-np 56" must be equal to --ntasks=56
#-
mpirun -np 56 ./executable <params>
  
# ------------------------------------------------------------------------end #