#!/bin/bash
#PBS -A STF006
#PBS -N genarray
#PBS -j oe
#PBS -l walltime=1:00:00,size=20
## On the Cray, you need at least 3 nodes for 3 separate application runs

cd $PBS_O_WORKDIR

SERVER=/ccs/proj/e2e/dataspaces/sith/pgi/bin/dataspaces_server

WRITEPROC=1
STAGINGPROC=1
READPROC=1
let "PROCALL=WRITEPROC+READPROC"

# clean-up previous run
rm -f log.* core* conf dataspaces.conf 
rm staged.bp genarray.bp

# Prepare config file for DataSpaces
echo "## Config file for DataSpaces
ndim = 3
dimx = 128
dimy = 32768
dimz = 128
max_versions = 10
#max_readers =" $READPROC " 
#lock_type = 2
" > dataspaces.conf

# Run DataSpaces
echo "-- Start DataSpaces server "$SERVER" on $STAGINGPROC PEs, -s$STAGINGPROC -c$PROCALL"
mpirun -np $STAGINGPROC $SERVER -s$STAGINGPROC -c$PROCALL &> log.server &

## Give some time for the servers to load and startup
sleep 1s
while [ ! -f conf ]; do
    echo "-- File conf is not yet available from server. Sleep more"
    sleep 1s
done
sleep 3s  # wait server to fill up the conf file

## Export the main server config to the environment
while read line; do
    export set "${line}"
done < conf

echo "-- DataSpaces IDs: P2TNID = $P2TNID   P2TPID = $P2TPID"

# Start GENARRAY
echo "-- Start GENARRAY on $WRITEPROC PEs"
mpirun -np $WRITEPROC ./genarray_stream genarray.bp $WRITEPROC 1 1 16 32 64 5 10 >& log.genarray &

# Start STAGE_WRITE
echo "-- Start STAGE_WRITE on $READPROC PEs"
mpirun -np $READPROC ./stage_write genarray.bp staged.bp MPI \"\" $READPROC 1 1 >& log.stage_write &

echo "-- Wait until all applications exit. Run ./check.sh to see status"
wait
rm -f conf

