Example LoadLeveler job file for a parallel run

#!/bin/ksh
# @ requirements = (Arch=="p2sc") && 
        (OpSys=="AIX51") && (Memory >= 256)
# @ resources = ADF(1)
# @ error = ADFxenaII.err
# @ output = ADFxenaII.out
# @ class = lhugex
# @ notification = never
# @ notify_user = deumens@qtp.ufl.edu
# @ checkpoint = no
# @ job_type = parallel
# @ node_usage = shared
# @ node = 2,2
# @ tasks_per_node = 1
# @ restart = no
# @ queue
. $HOME/.kshrc

cd adf/chang
# Use global scratch
mkdir /scr_2/tmp/`hostname`.chang.$$
cp adf2.in /scr_2/tmp/`hostname`.chang.$$
cd /scr_2/tmp/`hostname`.chang.$$
./adf2.in
cp TAPE21 $HOME/adf/chang
cd ..
rm -rf `hostname`.chang.$$
echo done

• c98scf
• c98scfdir
• c98integrals
• c98properties
• c98convert
• c98loptcg
• c98bulk

Example LoadLeveler job file for a serial run

#!/bin/ksh
# @ output = pHe.out
# @ error = pHe.err
# @ class = lhugex
# @ notification = never
# @ notify_user = deumens@qtp.ufl.edu
# @ checkpoint = no
# @ restart = no
# @ requirements = (Arch == "p2sc") && 
                       (OpSys == "AIX51")
# @ queue

HOST=`uname -n`
echo $HOST
MOL=pHe
SCR=/scr_1/tmp/$HOST.$MOL_ed.$$
mkdir $SCR
cp $HOME/END/$MOL.inp $SCR/
cd $SCR
echo "running version 2..."
time endyne -e 2.7.6 $MOL.inp > ${MOL}v2.log 2>&1
echo "running version 5..."
time endyne -e 5 $MOL.inp > ${MOL}v5.log 2>&1
echo "========= files ========="
ls -l
echo "========================="
cp ${MOL}*.log $HOME/END/
echo done

You should set the environment variable GAUSS_SCRDIR=/scr_1/tmp/your_directory;export GAUSS_SCRDIR before running the command g98 or g03. By default GAUSS_SCRDIR=/scr_1/tmp and then the 6 am cleaning daemon will remove all integral files. By setting the variable to the correct directory as explained in diskspace instructions, this is avoided.

You must contact Erik Deumens to sign a non-disclosure agreement before you will be allowed to access the program.

Current status of platforms and versions.

Machines	Processor	Version	Command
buddy	SPARC	g03 B.04	g03
simu1-9 atanasoff01-48	power3	g03 B.04	g03
arwen00-1d	IA32	g03 C.02	g03
haku00-1f ra00-3i amun0-f surg1-7	x86_64 or IA32	g03 D.01 or g03 C.02	g03-64 or g03

Note that g03 and g03-64 both work on Intel EM64T and AMD Opteron nodes; g03 works on all Linux nodes.

Example LoadLeveler job file for a 4-way parallel run of g03 on ATANASOFF

#!/bin/ksh
# @ requirements = (Arch=="power3") && 
             (OpSys=="AIX51") && (Memory >= 512)
# @ error = g03par.err
# @ output = g03par.out
# @ class = lhuge
# @ notification = never
# @ notify_user = XXXXX@qtp.ufl.edu
# @ checkpoint = no
# @ node_usage = not_shared
# @ restart = no
# @ queue
. $HOME/.kshrc

# Use huge global scratch
mkdir /scr_2/tmp/`hostname`.g03.$$
# Use smaller local scratch
#mkdir /scr_2/tmp/`hostname`.g03.$$
GAUSS_SCRDIR=/scr_2/tmp/`hostname`.g03.$$
export GAUSS_SCRDIR
cd /scr_2/tmp/`hostname`.g03.$$
# Make sure the job.in file has the Gaussian 
# directive to use 4 CPUs
cp $HOME/job.in .
g03 job.in > job.out
cp job.out $HOME/job.out
cd ..
rm -rf `hostname`.g03.$$
echo done

Example PBS job file for a 4-way parallel run of g03 on SURG

#!/bin/bash
#PBS -N g03par
#PBS -o g03par.out
#PBS -e g03par.err
#PBS -m abe
#PBS -M XXXXX@qtp.ufl.edu
#PBS -q brute
#PBS -l nodes=1:surg:ppn=4

# Use local scratch
mkdir /scr_1/tmp/`hostname`.g03.$$
GAUSS_SCRDIR=/scr_1/tmp/`hostname`.g03.$$
export GAUSS_SCRDIR
cd /scr_1/tmp/`hostname`.g03.$$
# Make sure the job.in file has the Gaussian 
# directive to use 4 CPUs
cp $HOME/job.in .
g03 job.in > job.out
cp job.out $HOME/job.out
cd ..
rm -rf `hostname`.g03.$$
echo done

An example input deck for a run on Xena II is (to be saved as nwchem.job):

#!/bin/ksh
# @ job_type=parallel
# @ class = lhugex
# @ output = nwchem.out
# @ error =  nwchem.err
# @ requirements = (Arch == "p2sc")
# @ notify_user = your_name@qtp.ufl.edu
# @ min_processors = 12
# @ max_processors = 15
# @ network.lapi = css0,not_shared,US
# @ environment = COPY_ALL; MP_PULSE=0; 
             MP_SINGLE_THREAD=yes; 
             MP_WAIT_MODE=yield; restart=no
# @ queue
export CNAME=nwchem
export CALC="/scr_2/tmp/$machine.nwchem.$$"
mkdir $CALC
export PROC=p2sc
export LOCAL="`pwd`"
date
uname -a
cd $CALC
if [ `pwd` != $CALC ]
then 
   echo "In `pwd` instead of $CALC: STOP!"
   exit
fi
echo '******* NWChem will start now *******'
nwchem $LOCAL/$CNAME".nwi" $LOCAL/$CNAME".nwo"
echo '******* NWChem exited *******'
echo "starting clean-up"
cd ..
/bin/rm -r $CALC
echo done
exit

start
title "q0l-02f, Me2N-NO2, Cs, 
         CCSD(T)-fc/cc-pVTZ//B3LYP/6-31G**"
geometry autosym
N -.567554 -.224363 .000000
N .804248 -.034262 .000000
O 1.360973 .024660 1.097057
O 1.360973 .024660 -1.097057
C -1.239866 .053279 -1.263921
C -1.239866 .053279 1.263921
H -.755338 -.502712 -2.064122
H -2.273466 -.281438 -1.163781
H -1.227028 1.121644 -1.516981
H -.755338 -.502712 2.064122
H -1.227028 1.121644 1.516981
H -2.273466 -.281438 1.163781
end
basis spherical nosegment
C library cc-pvtz
H library cc-pvtz
N library cc-pvtz
O library cc-pvtz
end
memory 512 mb
echo
ccsd; maxiter 40; freeze atomic; end
task ccsd(t)

task shell all "/bin/rm -f *"

Example LoadLeveler job file for a parallel run

#!/bin/bash
#@ shell = /bin/bash
#@ job_name = siesta_h2o
#@ output   = siesta.out
#@ error    = siesta.err
#@ job_type = parallel
#@ class    = quick
#@ node_usage = shared
#@ tasks_per_node = 1 
#@ node        = 2
#@ wall_clock_limit = 1:00:00
#@ notification     = never
#@ network.MPI      = css0,shared,us
###@ environment = MPI_SHARED_MEMORY=yes; 
            MP_SAVEHOSTFILE=hf.$(host).$(jobid); 
            COPY_ALL;
#@ queue

# Use global scratch disk
SCR=/scr_2/tmp/`hostname`.my_h2o.$$
mkdir $SCR
cd $SCR
# Copy input to work directory
cp $HOME/h2o.fdf .
# Run
/usr/bin/poe /usr/local/bin/siesta < h2o.fdf 
         > h2o.out
# Copy output to a safe place
cp h2o.out $HOME
# Remove scratch files
cd ..
rm -r $SCR

SystemName          Water molecule
SystemLabel         h2o
NumberOfAtoms       3
NumberOfSpecies     2

%block ChemicalSpeciesLabel
 1  8  O # Species index, atomic number, 
                     species label
 2  1  H
%endblock ChemicalSpeciesLabel

AtomicCoordinatesFormat  Ang
%block AtomicCoordinatesAndAtomicSpecies
 0.000  0.000  0.000  1
 0.757  0.586  0.000  2
-0.757  0.586  0.000  2
%endblock AtomicCoordinatesAndAtomicSpecies

MD.TypeOfRun             Nose # Type of dynamics:
MD.InitialTemperature    300 K
MD.TargetTemperature     300 K
MD.InitialTimeStep         1
MD.FinalTimeStep         100
MD.LengthTimeStep        1.0 fs

The codumentation explains how to create the control file and initial coordinates and other files. The command define is used to create a control file. It may refer to other files with coordinates and basis specifications.

Example LoadLeveler job file for a serial run

#!/bin/ksh 
# @ output = sertur.out 
# @ error = sertur.err 
# @ class = lhugex 
# @ notification = never 
# @ notify_user = greene@qtp.ufl.edu 
# @ checkpoint = no 
# @ restart = no 
# @ requirements = (Arch == "p2sc" && 
                     OpSys == "AIX51") 
# 
# mandatory directive to process the job. 
# @ queue 
# 
# User commands follow.... 
echo "pure serial job:" 
echo master: `uname -n` 
# ==== begin executable statement 
. /usr/local/lib/turbomole/scripts/turbopath
scr=/scr_1/tmp/`uname -n`.sg.$$
mkdir $scr
cd $scr
cp /ufl/chem/nxr/sg/RUB/OXID/SER/* .
# Geometry optimization
jobex
# single DFT SCF
#dscf > dscf.out
cp * /camp/crunch_1/chem/nxr/sg/TURBO/RUBox/SER
# ==== end executable statement 
echo "Job done." 

Example LoadLeveler job file for a parallel run

You must create a control file with define just is in the case of the serial file. Then you must prepare for a parallel run. This must be done on one of the interactive nodes of Xena II. Create a directory for the files for this particular run. Put a file host.list in it with the following content:

xena00
xena10
xena20
xena30
xena40
xena50
xena60
xena70
xena80
xena90
xenaa0
xenab0

The actual SCF or gradient or geometry calculations should be submitted to LoadLeveler with a script like the folowing. Parallel calculations are always prepared with turbo_start -batch_prepare and executed in the LoadLeveler script with turbo_start -batch. Note the following when answering the questions asked by turbo_start -batch_prepare:

• Because Turbomole uses a master-client architecture for parallel execution, it is necessary to assign one more process. Thus to run two-way parallel, you must ask LoadLeveler to assign three processes. When you create the control file as explained in the manual, you must specify that you want N clients, and you must specify N+1 processes in the LoadLeveler job file.
• The directory for scratch files should be a global directory accessible by all nodes. For Xena II this is /scr_2/tmp/.... It is recommended that you create a keep-directory, such as /scr_2/tmp/keep.sg.13-04-03, so that it will not be deleted by the scratch cleaning process and so that you can specify it in your LoadLeveler script (see example below).
• When answered whether the machine is in dedicated use, you answer yes for Xena II and no for Simu.

#!/bin/ksh 
# @ output = partur.out 
# @ error = partur.err 
# @ class = lhugex 
# @ notification = complete 
# @ notify_user = greene@qtp.ufl.edu 
# @ checkpoint = no 
# @ restart = no 
# @ requirements = (Arch == "p2sc" && 
                        OpSys == "AIX51") 
# @ wall_clock_limit = 10:00,9:00 
# 
# ---- begin specification of the kind
# ---- of parallel job 
# @ job_type = parallel 
# @ node_usage = shared 
# @ node = 3,3 
# @ tasks_per_node = 1 
# @ network.MPI = css0,not_shared,US 
# ---- end specification of the kind 
# ---- of parallel job 
# 
# mandatory directive to process the job. 
# @ queue 
# 
# User commands follow.... 
echo "pure MPI job on 2 different nodes:" 
echo master: `uname -n` 
# ==== begin executable statement 
. /usr/local/lib/turbomole/scripts/turbopath
. /usr/local/lib/turbomole/scripts/turbopathmpi
# Need global directory for parallel execution
scr=/scr_2/tmp/keep.sg.13-04-03
mkdir $scr
cd /ufl/chem/nxr/sg/RUB/OXID/PAR/
cp control control_seq basis coord alpha beta $scr/
cp xyzcase xyzproc $scr
cp DSCF-par-stat $scr
cp GRAD-par-stat $scr
cd $scr
turbo_start -batch
cp * /camp/crunch_1/chem/nxr/sg/TURBO/RUBox/PAR
# ==== end executable statement 
echo "Job done."