#!/usr/bin/perl
#
# script to analyze E5 root files on the Richmond cluster.
#                                          - gpg
#
# run this be executing 'submit_eod3b.pl' at the 
# command line.
#
# it calls run_root_on_node2.sh.
#
# housekeeping - first set first slave and number of
# nodes, and clean out directory that will
# hold the final histogram files. the transfer_data
# variable is 1-true (transfer root data files) or
# 0- false (don't transfer the data since it is presumably
# already on the cluster).
#
# use the variable data_chunk to pick one of several input
# files that contain a list of the raw data to be analyzed.
# this is a kludge because of the limit on the number of
# processes.
#
$number_of_nodes=48;
$first_node=0;
$transfer_data = 0; 
$data_chunk = 1;
if ($data_chunk == 1) {
    system("rm /home/gilfoyle/eod/run/results/*");
    system("rm /scratch/gilfoyle/e5/eod*");
    system("rm -r /scratch/gilfoyle/e5/24*");
}
#
# read the file with the run numbers and read the numbers into an array.
#
$run_numbers_file="E5_4.232_run_numbers$data_chunk.dat";
open(RUNS,"$run_numbers_file") || die "Failed to open run numbers file.\n";
@runno=<RUNS>;
close(RUNS);
#
# construct the filename with the input data filenames and read that
# file to get the data filenames.
#
$job=0;
$skipped_nodes=0;
foreach $line (@runno) {
    chomp($line);
    $runfile = "files_4.232/run$line\_files.dat";
    print "\nrun files: $runfile, run number: $line.\n";
    open(FILES,"$runfile") || die "Failed to open filename file.\n";
    @filenames=<FILES>;
    close(FILES);
#
# get the node number and be careful about skipping the bad ones.
#
    $node=($job+$skipped_nodes) % $number_of_nodes;
#
# skip bad nodes (these have been fixed, but haven't moved any
# data there yet).
#
    if ($node == 12) {
	$node++;
	$skipped_nodes++;
	print("Skipping node 12.\n");
    }
    if ($node == 45) {
	$node++;
	$skipped_nodes++;
	print("Skipping node 45.\n");
    }
#
# make the slave directory and copy the data to it.
#
    system("bpsh $node mkdir -p /scratch/gilfoyle/e5/$line/");
    if ($transfer_data == 1) {
	foreach $line2 (@filenames) {
	    chomp($line2);
	    print "copy /data2/e5/root/4.232/$line2 to $node:/scratch/gilfoyle/e5/$line/$line2.\n";
	    system("bpcp /data2/e5/root/4.232/$line2 $node:/scratch/gilfoyle/e5/$line/$line2");
	}
    }
#
# copy other stuff to slave directory.
#
    system("rm data_filenames.dat");
    system("cp $runfile data_filenames.dat");
    system("bpcp data_filenames.dat $node:/scratch/gilfoyle/e5/$line/data_filenames.dat");
    system("bpcp /home/gilfoyle/eod/root/eod3.C $node:/scratch/gilfoyle/e5/$line/");
    system("bpcp /home/gilfoyle/eod/root/eod3.h $node:/scratch/gilfoyle/e5/$line/");
    system("bpcp /home/gilfoyle/eod/root/user_eod3.h $node:/scratch/gilfoyle/e5/$line/");
    system("bpcp /home/gilfoyle/eod/root/run_eod3.C $node:/scratch/gilfoyle/e5/$line/");
    $job++;
            }
#
$total_jobs=$job-1;
#
# now loop over the nodes and send the
# data from one run number to one node.
#
$job=0;
$skipped_nodes=0;
while ($job <= $total_jobs) {
    $node=($job + $skipped_nodes) % $number_of_nodes;
#
# skip busted nodes.
#
    if ($node == 12) {
	$node++;
	$skipped_nodes++;
	print("Skipping node 12.\n");
    }
    if ($node == 45) {
	$node++;
	$skipped_nodes++;
	print("Skipping node 45.\n");
    }
#
# get the batch command file ready to submit the job and let're rip. 
# clean up is done in run_root_on_node2.sh since we have to wait for 
# root to get done on the slave.
#
    open(OUT,">run_job");
    print     "./run_root_on_node2.sh $node $runno[$job]\n";
    print OUT "./run_root_on_node2.sh $node $runno[$job]\n";
    close(OUT);
    system("batch -f run_job");
    print "Submit on node $node run $runno[$job] for analysis.\n";
#
# go to the next node and next run number.
#
    $node++;
    $job++;
}