#!/usr/bin/perl
# 
# script for running on a slave node of the cluster.
# called from submit_eod3d.pl and submit_eod3c.pl. 
#                                      - gpg 12/03/02
#
# set up the environment including the node and run 
# number assignments that come from the arguments of the
# script.

@option  = @ARGV;
$options = @option;

$NODE =  $option[0];
$RUNNO =  $option[1];
$delete_data =  $option[2];
$use_existing_data =  $option[3];
$RUNFILE = "/home/gilfoyle/eod/run/files_2.558r/run$RUNNO\_files.dat";
#$RUNFILE = "/home/gilfoyle/eod/run/files_2.558n/run$RUNNO\_files.dat";
#$RUNFILE = "/home/gilfoyle/eod/run/files_4.232/run$RUNNO\_files.dat";
system("setenv ROOTSYS /usr/root/PRO");
system("setenv NO_LOCAL 1");

# the following statement was commented out early on because the CLAS
# software was not ready and it seems we do not need these environment 
# variables.

#system("source /home/clas/builds/PRODUCTION/packages/cms/rich.cshrc PRODUCTION");

# Make a working directory and go to it
# NOTE: Make this on the master as well as a place holder
# no actual data goes to the placeholder on the master.

system("bpsh $NODE mkdir -p /scratch/gilfoyle/e5/$RUNNO/ >| /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
system("mkdir -p /scratch/gilfoyle/e5/$RUNNO/ >> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
system("cd /scratch/gilfoyle/e5/$RUNNO/");

# move the data from the raid disk to the slave node unless you're using files
# that are already on the slave.

if ($use_existing_data == 0) {

# get the data filenames.

    system("date >> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
    system("echo 'Get the data.'>> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
    print "\nrun files: $RUNFILE, run number: $RUNNO.\n";
    open(FILES,"$RUNFILE") || die "Failed to open filename file.\n";
    @filenames=;
    close(FILES);

# make the slave directory and copy the data to it. do NOT use bpcp because it
# apparently copies the data twice according to steven james.

    system("bpsh $NODE mkdir -p /scratch/gilfoyle/e5/$RUNNO/");
    foreach $data_filename (@filenames) {
        chomp($data_filename);
        print "copy /data2/e5/root/2.558r/$data_filename to $NODE:/scratch/gilfoyle/e5/$RUNNO/$data_filename.\n";
        system("bpsh $NODE cp /data2/e5/root/2.558r/$data_filename /scratch/gilfoyle/e5/$RUNNO/$data_filename");
#       print "copy /data2/e5/root/2.558n/$data_filename to $NODE:/scratch/gilfoyle/e5/$RUNNO/$data_filename.\n";
#       system("bpsh $NODE cp /data2/e5/root/2.558n/$data_filename /scratch/gilfoyle/e5/$RUNNO/$data_filename");
#       print "copy /data2/e5/root/4.232/$data_filename to $NODE:/scratch/gilfoyle/e5/$RUNNO/$data_filename.\n";
#       system("bpsh $NODE cp /data2/e5/root/4.232/$data_filename /scratch/gilfoyle/e5/$RUNNO/$data_filename");
    }
}

# copy other necessary files to slave directory. these files are small so 
# bpcp is not a bottleneck (see previous comment).

system("bpcp $RUNFILE $NODE:/scratch/gilfoyle/e5/$RUNNO/data_filenames.dat");
system("bpcp /home/gilfoyle/eod/root/eod3.C $NODE:/scratch/gilfoyle/e5/$RUNNO/");
system("bpcp /home/gilfoyle/eod/root/eod3.h $NODE:/scratch/gilfoyle/e5/$RUNNO/");
system("bpcp /home/gilfoyle/eod/root/user_eod3.h $NODE:/scratch/gilfoyle/e5/$RUNNO/");
system("bpcp /home/gilfoyle/eod/root/run_eod3.C $NODE:/scratch/gilfoyle/e5/$RUNNO/");
 
# More housekeeping and monitoring information.

system("cd /scratch/gilfoyle/e5/$RUNNO/");
chdir("/scratch/gilfoyle/e5/$RUNNO/");
#system("rm /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO}");
system("date >> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
system("pwd  >>  /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");

# run my root.

system("echo 'run my root.'>> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
system("bpsh $NODE /home/gilfoyle/eod/root/gpgroot -b -q /scratch/gilfoyle/e5/$RUNNO/run_eod3.C >> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");
system("date >> /scratch/gilfoyle/e5/eod3_log${NODE}_run${RUNNO} 2>&1");

# Copy histo file back to the master and clean up the slave directory.
# Do this here instead of in submit_eod3c.pl so that root is done before
# we start cleaning up.

system("bpcp ${NODE}:/scratch/gilfoyle/e5/${RUNNO}/eod3_hists.root /home/gilfoyle/eod/run/results/run${RUNNO}_results.root");
chdir("/home/gilfoyle/eod/run/");
if ($delete_data == 1) {
    system("bpsh ${NODE} rm -r /scratch/gilfoyle/e5/$RUNNO/");
    print("Deleting files on slave node.\n");
}