WRF¶

...
module load cray-hdf5   #the netcdf library depends on hdf5
module load cray-netcdf
module load cray-parallel-netcdf
...

export NETCDF_classic=1               #use classic (CDF1) as default
#use 64-bit offset format (CDF2) of netcdf files
export WRFIO_NCD_LARGE_FILE_SUPPORT=1 
#netcdf4 compression (serial) with the hdf5 module can be very slow
export USE_NETCDF4_FEATURES=0

#!/bin/bash
set -e
machname="Perlmutter"
scname=$BASH_SOURCE  #name of this script

echo "loading modules using ${scname}"
echo "target system: ${machname} "

#users may want to unload unnecessary/conflicting modules loaded in .bash_profile. 
#e.g., hugepages. But keep other modules loaded automatically by the system. 
#each user has to edit here:
# module unload craype-hugepages64M

#general modules
module load cpu  
module load PrgEnv-gnu 

#module for WRF file I/O
#order of loading matters!
module load cray-hdf5  #required to load netcdf library
module load cray-netcdf 
module load cray-parallel-netcdf

module list

Please select from among the following Linux x86_64 options:
....
32. (serial)  33. (smpar)  34. (dmpar)  35. (dm+sm)   GNU (gfortran/gcc)
...
Enter selection [1-75] :
Compile for nesting? (0=no nesting, 1=basic,...) [default 0] :

doclean_all=false #true if previously compiled with different configure options

doclean=false

runconf=false

docompile=false

#!/bin/bash -l
set -e
set -o pipefail 

imach="pm"  #target system name. "pm" for Perlmutter.

#change the following boolean variables to run/skip certain compiling steps
doclean_all=true #true if previously compiled with different configure options

doclean=false

runconf=true    #run WRF's configure script; should do this first before compiling

docompile=false  #run WRF's compile script; should do this after configure

debug=false  #true to compile WRF with debug flag (no optimizations, -g flag for debugger, etc.)

# WRF directories
mversion="v4.4"
#WRF-SIG project directory as example; accessible only by WRF-SIG members
wrfroot="/global/cfs/cdirs/m4232/model"  
script_dir="/global/cfs/cdirs/m4232/scripts/build"

export WRF_DIR=${wrfroot}/${imach}/${mversion}/WRF

#Modules --------------------------------------------------------------------
modversion="2023-09"  #use year of major update that module (default) are introduced (INC0182147)
loading_script="${script_dir}/load_modules_${modversion}_wrfsig.sh"
source ${loading_script}

#set environmental variables used by WRF build system, borrowing environmental variables 
#set by modules

export NETCDF_classic=1               #use classic (CDF1) as default
export WRFIO_NCD_LARGE_FILE_SUPPORT=1 #use 64-bit offset format (CDF2) of netcdf files
export USE_NETCDF4_FEATURES=0         #do not use netcdf4 compression (serial), need hdf5 module
#configure says WRF won't use netcdf4 compression, but I still see a flag
#NETCDF4_IO_OPTS = -DUSE_NETCDF4_FEATURES..., confusing..


export HDF5=$HDF5_DIR
export HDF5_LIB="$HDF5_DIR/lib"
export HDF5_BIN="$HDF5_DIR/bin"

export NETCDF=$NETCDF_DIR
export NETCDF_BIN="$NETCDF_DIR/bin"
export NETCDF_LIB="$NETCDF_DIR/lib"

#create PNETCDF environment variable to use the parallel netcdf library
export PNETCDF=$PNETCDF_DIR  

export LD_LIBRARY_PATH="/usr/lib64":${LD_LIBRARY_PATH}
#export PATH=${NETCDF_BIN}:${PATH}
export PATH=${NETCDF_BIN}:${HDF5_BIN}:${PATH}
export LD_LIBRARY_PATH=${NETCDF_LIB}:${LD_LIBRARY_PATH}


#other special flags to test
export PNETCDF_QUILT="0"  #Quilt output is not stable, better not use it

#check environment variables
echo "LD_LIBRARY_PATH: "$LD_LIBRARY_PATH
echo "PATH: "$PATH
echo "MANPATH: "$MANPATH

echo "NETCDF is $NETCDF"
echo "NETCDF_LIB is $NETCDF_LIB"

echo "HDF5 is $HDF5"
echo "HDF5_LIB is $HDF5_LIB"

echo "PNETCDF: ${PNETCDF}"
echo "PNETCDF_QUILT: ${PNETCDF_QUILT}"

##capture starting time for log file name
idate=$(date "+%Y-%m-%d-%H_%M")
#
##run make in the top directory
cd $WRF_DIR

if [ "$doclean_all" = true ]; then
    ./clean -a
    #"The './clean –a' command is required if you have edited the configure.wrf 
    #or any of the Registry files.", but this deletes configure.wrf....

fi

if [ "$doclean" = true ]; then
    ./clean
fi

#echo "running configure"
if [ "$runconf" = true ]; then

    if [ "$debug" = true ]; then
        echo "configure debug mode"
        ./configure -d
    else
        ./configure
    fi

   ##configure options selected are:
   # 32. (serial)  33. (smpar)  34. (dmpar)  35. (dm+sm)   GNU (gfortran/gcc)
   # choose 35 for real (not idealized) cases

    configfile="${WRF_DIR}/configure.wrf"

    #the sed commands below will change the following lines in configure.wrf
    #--- original
    #SFC             =       gfortran
    #SCC             =       gcc
    #CCOMP           =       gcc
    #DM_FC           =       mpif90
    #DM_CC           =       mpicc

    #--- edited (FC and CC with MPI)
    #SFC             =       gfortran
    #SCC             =       gcc
    #CCOMP           =       cc
    #DM_FC           =       ftn
    #DM_CC           =       cc

    if [ -f "$configfile" ]; then
        echo "editing configure.wrf"
        #need to remove -cc=$(SCC) in DM_CC
        sed -i 's/-cc=\$(SCC)/ /' ${configfile}
        sed -i 's/mpif90/ftn/' ${configfile}
        sed -i 's/mpicc/cc/' ${configfile}

        #also user can remove the flag -DWRF_USE_CLM from ARCH_LOCAL if not planning to 
        #use the CLM4 land model to speed up compilation
        #sed -i 's/-DWRF_USE_CLM/ /' ${configfile} 

    fi

fi

if [ "$docompile" = true ]; then
    export J="-j 4"  #build in parallel
    echo "J = $J"

    bldlog=${script_dir}/compile_em_${idate}_${imach}.log
    echo  "compile log file is ${bldlog}"

    #run the compile script 
    ./compile em_real &> ${bldlog}

    #check if there is an error in the compile log
    #grep command exits the script in case of nomatch after the 2022-12 maintenance
    set +e #release the exit flag before grep

    grep "Problems building executables" compile_em_real_${idate}_${imach}.log     
    RESULT=$?

    #set the exit flag again
    set -e  

    if [ $RESULT -eq 0 ]; then
        echo "compile failed, check ${bldlog}"      
    else
        echo "compile success"
        #sometimes renaming executable with descriptive information is useful
        #cp $WRF_DIR/main/ideal.exe $WRF_DIR/main/ideal_${idate}_${imach}.exe
        #cp $WRF_DIR/main/real.exe $WRF_DIR/main/real_${idate}_${imach}.exe
        #cp $WRF_DIR/main/wrf.exe $WRF_DIR/main/wrf_${idate}_${imach}.exe
        #cp $WRF_DIR/main/ndown.exe $WRF_DIR/main/ndown_${idate}_${imach}.exe
    fi

fi

#!/bin/bash 
#SBATCH -N 1
#SBATCH -q debug
#SBATCH -t 00:30:00
#SBATCH -J test
#SBATCH -A <account>   #user needs to change this
#SBATCH --mail-type=END,FAIL
#SBATCH --mail-user=<email address>  #and this
#SBATCH -L scratch,cfs
#SBATCH -C cpu
#SBATCH --tasks-per-node=64

pwd
ntile=4  #number of OpenMP threads per MPI task
#need to set the "numtiles" variable in the wrf namelist (namelist.input) to be the same 

#example using the WRFSIG project CFS directories; 
#files only accesible by the WRFSIG members

bindir="/global/common/software/m4232/bin"
binname="wrf_2023-01-09-15_55_pm.exe"
rundir="/pscratch/sd/e/elvis/simulation/WRF/run" #user needs to change this

#Modules --------------------------------------------------------------------
#use the example module-loading script given above; accessible for WRF-SIG members
modversion="2023-09" 
loading_script="/global/cfs/cdirs/m4232/scripts/build/load_modules_${modversion}_wrfsig.sh"
source ${loading_script}

#OpenMP settings:
export OMP_NUM_THREADS=$ntile
export OMP_PLACES=threads  #"true" when not using multiple OpenMP threads (i.e., ntile=1)
export OMP_PROC_BIND=spread

cd $rundir

#run simulation
srun -n 64 -c 4 --cpu_bind=cores ${bindir}/${binname}

#rename and save the process 0 out and err files
cp rsl.error.0000 rsl.error_0_$SLURM_JOBID
cp rsl.out.0000 rsl.out_0_$SLURM_JOBID