#!/usr/local/bin/perl5.00503
#------------------------------------------------------------
#  automation script to copy cd images files to
#  local filesystem and backup same to ADSM.
#  
#  Author: Dale Posey 
#  Org:    IBM Global Services
#  Date:   June 26, 2000
#
#  INPUT:  name of file for list of cd's to archive
#          i.e. /s1jb0.us  
#          and run level
#-----------------------------------------------------------
use lib "/dfs/.rw/admin/bin";
# use lib "/home/dale";
use File::Basename;
use DBI qw(:sql_types);
require 'ctime.pl';
require "common.pl";


# The first arg is our input file name, eg /s1jb0.us
$ipf = $ARGV[0];

# determines if we go to images1 or 2
$lvl = $ARGV[1];

if ( $lvl eq "" ) {
    die "Gotta have a level number. 1 or 2."
}
if ( $ipf eq "" ) {
    die "Gotta have a filename.\n";
} else {
    @s_if = stat($ipf);
    if ($? != 0){
	 die "Oooops, can't stat the input file....please fix and retry\n";
     }
}

setupDB2();

$collection = "JP";
$server = "adsmpat";

init_log_files();

#
# we will not make it through this list, but we proceed
# as far as we can
#
$ToDo_cd_count = @cd_list;
$Doing_cd_count=0;
while ( $cd = $cd_list[$cd_pointer] ) {

    if ( -e "$base_dir/abort" ) {
	print STDOUT "Aborting by request.\n";
	print DBG "Aborting by request.\n";
	close DBG;
	close REC;
	close LF;
	exit;
    }

    while ( -e "$base_dir/stall" ) {
	# lets wait a few minutes and try again
	# print STDOUT "Stalling for 1 minute.\n";
	sleep(60);
    }

    #
    # wait here until image requests get below 40
    #
    $waiting = 0;
    while ( 1 ) {
	$irs = `/ips/bin/rtebrows -# rtework.que.status`;
	if ( $waiting ) {
	    if ( $irs <= 40 ) {
		$waiting = 0;
		last;
	    }
	    sleep 60;
	} elsif ( $irs >= 50 ) {
	    $waiting = 1;
	    sleep 60;
	} else {
	    last;
	}
    }

#
# Get rid of trailing \n character and any trailing blanks.
# This is only necessary for our initial input file (eg /s1jb0.us),
# not for the checkpoint.log file.
    chomp $cd;
    $cd =~ s/ *//g;

    $Doing_cd_count++;
    print STDOUT "Started processing cd $cd [$Doing_cd_count of $ToDo_cd_count] at ",&ctime(time),"\n";
    print DBG    "Started processing cd $cd [$Doing_cd_count of $ToDo_cd_count] at ",&ctime(time),"\n";

    check_adsm();
    print STDOUT "max_pool=$max_pool\t max_tape=$max_tape\t pct_util_tape=$pct_util_tape\n";
    print DBG    "max_pool=$max_pool\t max_tape=$max_tape\t pct_util_tape=$pct_util_tape\n";

    if ( $pct_util_tape > 95 ) {
	print STDOUT "The tape space is $pct_util_tape percent full.  Can't do any more CDs.\n";
	print DBG "The tape space is $pct_util_tape percent full.  Can't do any more CDs.\n";
	close DBG;
	close REC;
	close LF;
	exit;
    }
    
    # this is the path for S40 type with cd names beginning ESP, EPB, WLD.  MIWO will be another program.
    
    $idx_name = "/cdrom/$cd/exchange.idx";

    print STDOUT "Clearing out ctl files from the '$tmp_dir'.\n";
    if ( $tmp_dir eq "" ) {
	print STDOUT "tmp_dir not set up\n";
	exit;
	}
    @lines = `find $tmp_dir -name '*.ctl' -exec rm {} \\; 2>&1`;
    $rc = $?;
    if ( $rc ) {
	print DBG @lines;
	print STDOUT @lines;
    }

    undef @in_list;
    undef @patents;

    print STDOUT "Getting list of files from '$idx_name' - please wait\n";
    open(CD, $idx_name) || die "Could not open index file '$idx_name' on CD '$cd'\n";
    #                                                                                         >page number
    #      --patn-- ------------ junk ---------------     dir      dir      dir      patn     |  
    #                                                                                             > zeros
    # JPA  11000009 000000010000000000002900000053850\DOC\DOCUJPA\DOC110V1\D11000D1\11000009\00000001.TIF
    # JPA  11000009 SGMLSGML0000000000003200000020810\DOC\DOCUJPA\DOC110V1\D11000D1\11000009\SGML.NRM
    #
    # need to create:
    #     tar -cvf /imagesx/JP/xx/yy/JPnnnnyyxxA2.tar /cdrom/mijpxxxxxx/doc/yyyyyyy/xxxxxxx/zzzzzzzz/00000001.tif \
    #               /cdrom/mijpxxxxxx/doc/yyyyyyy/xxxxxxx/zzzzzzzz/sgml.nrm
    #
    $this_patn = "";
    while (<CD> ) {
	$uc = $_;
	tr/[A-Z]*/[a-z]/;
	if (/(..)(..) (.*) .*\\doc\\(.*)\\(.*)\\(.*)\\(........)\\00000001.tif/) {
	    # per Eric these patents need A2 as the kind.
	    $ep = "JP";
	    $kind = "A2";
	    $patn = $3;
	    $d1 = $4;
	    $d2 = $5;
	    $d3 = $6;
	    $dup_pat = $7;
	    if ( $patn ne $prev_patn ) {
		if ( $prev_patn ne "" ) {
		    # we've changed patents so create an entry to process
		    $cmds{$prev_patn} = $tar_cmd;
		    $tar_files{$prev_patn} = $tar_file;
		    $cmp_files{$prev_patn} = $cmp_cmd;
		    push(@patents, $prev_patn);
		}
		$prev_patn = $patn;
		# create a filename for tar to work with
		$patn =~ /(....)(..)(..)/;
		$cmp_cmd = "compress -f $image_dir/$cd/$3/$2/$ep$patn$kind\.tar";
		$tar_cmd = "tar -cvf $image_dir/$cd/$3/$2/$ep$patn$kind\.tar /cdrom/$cd/doc/$d1/$d2/$d3/$patn/00000001.tif ";
		$tar_file = "$image_dir/$3/$2/$ep$patn$kind\.tar.Z";
	    } else {
		    $tar_cmd = $tar_cmd."/cdrom/$cd/doc/$d1/$d2/$d3/$patn/00000001.tif ";
		}
	    } elsif (/(..)(..) (.*) .*\\doc\\(.*)\\(.*)\\(.*)\\(........)\\sgml.nrm/) {
		$ep = $1;
		$ep =~ tr/[a-z]/[A-Z]/;
		# per Eric these patents need A2 as the kind.
		$kind = "A2";
		$patn = $3;
		$d1 = $4;
		$d2 = $5;
		$d3 = $6;
		$dup_pat = $7;
		if ( $patn ne $prev_patn ) {
		    if ( $prev_patn ne "" ) {
			# we've changed patents so create an entry to process
			$cmds{$prev_patn} = $tar_cmd;
			$tar_files{$prev_patn} = $tar_file;
			$cmp_files{$prev_patn} = $cmp_cmd;
			push(@patents, $prev_patn);
		    }
		    $prev_patn = $patn;
		    # create a beginning tar command
		    $patn =~ /(....)(..)(..)/;
		    $cmp_cmd = "compress $image_dir/$cd/$3/$2/$ep$patn$kind\.tar";
		    $tar_cmd = "tar -cvf $image_dir/$cd/$3/$2/$ep$patn$kind\.tar /cdrom/$cd/doc/$d1/$d2/$d3/$patn/sgml.nrm ";
		} else {
		    $tar_cmd = $tar_cmd."/cdrom/$cd/doc/$d1/$d2/$d3/$patn/sgml.nrm ";
		}
	    } else {
		print STDOUT "Unrecognized line:$_\n";
		print DBG "Unrecognized line:$_\n";
		next;
	    }
    }
    $tar_files{$prev_patn} = $tar_file;
    $cmp_files{$prev_patn} = $cmp_cmd;
    $cmds{$prev_patn} = $tar_cmd;
    push(@patents, $prev_patn);

    print DBG "tar commands for CD:'$cd' created - continuing.\n";
    print STDOUT "tar commands for CD:'$cd' created - continuing.\n";
    
    print DBG "Removing old directories and files\n";
    print STDOUT "Removing old directories and files\n";
    rm_dirs();

    print DBG "Building new directories\n";
    print STDOUT "Building new directories\n";

    # build directories needed
    $dir1_cnt = 0;
    $dir2_cnt = 0;
    if ( ! -d "$image_dir/$cd" ) {
	mkdir "$image_dir/$cd", $mode;
        #  Don't count this high-level directory.  Of course, we have to mkdir it.
        #  $dir1_cnt++;
    }

    # remove the list of db2 items 
    undef @db2list;
    $dir1_cnt = 0;
    $dir2_cnt = 0;
    foreach (@patents) {
	if ( ! /([0-9][0-9])([0-9][0-9])$/ ) {
	    print STDOUT "Can't handle the file $_ on the $cd CD, because the name isn't .*digit-digit-digit-digit.??,\n";
	    print DBG "Can't handle the file $_ on the $cd CD, because the name isn't .*digit-digit-digit-digit.??,\n";
	    next;
	}
	$d1 = $1;
	$d2 = $2;
	$dir1 = "$image_dir/$cd/$d2";
	$dir2 = "$image_dir/$cd/$d2/$d1";

	if ( ! -d $dir1 ) {
	    mkdir($dir1, $mode) ||  die "can't mkdir of $dir1";
	    $dir1_cnt++;
	} 
	if ( ! -d $dir2 ) {
	    mkdir($dir2, $mode) || die "can't mkdir of $dir2";
	    $dir2_cnt++;
	} 
	if ( $dir1_cnt > 99 && $dir2_cnt > 9999 ) {
	    last;
	}
    }
    print STDOUT "Created $dir1_cnt+$dir2_cnt directories to copy $nfiles files.\n";
    print DBG "Created $dir1_cnt+$dir2_cnt directories to copy $nfiles files.\n";

#
#
    $cnt = 1;
    $todo = @patents;

    foreach $p (@patents) {
	print STDOUT "tar for [$cnt of $todo]\n$cmds{$p}";
	print DBG "tar for [$cnt of $todo]\n$cmds{$p}";
	$err_count = 0;
	while ( $err_count < 5 ) {
	    @lines = `$cmds{$p}`;
	    $rc = $?;
	    if ( $rc ) {
		$errno = $!;
		print STDOUT " - failed\nTar command failed - $rc - $errno\n";
		print DBG " - failed\nTar command failed - $rc - $errno\n";
		$err_count++;
		next;
	    } else {
		print STDOUT " - done.\n";
		print STDOUT "compress for [$cnt of $todo]\n$cmp_files{$p}\n";
		print DBG "compress for [$cnt of $todo]\n$cmp_files{$p}\n";
		# now run the compress command
		@lines = `$cmp_files{$p}`;
		$rc = $?;
		if ( $rc ) {
		    $errno = $!;
		    print STDOUT " - failed\nCompress command failed - $rc - $errno\n";
		    print DBG " - failed\nCompress command failed - $rc - $errno\n";
		    exit;
		}
		$cnt ++;
		
		# build a record for this patent.
		# use db2 date format 2000-04-21-10.55.42.000000
		
		@pcs = localtime(time);
		$yr = ($pcs[5] + 1900);
		$smon = ($pcs[4] + 1);
		$mo = sprintf("%02d", $smon);
		$dy = sprintf("%02d", $pcs[3]);
		$hr = sprintf("%02d", $pcs[2]);
		$mn = sprintf("%02d", $pcs[1]);
		$sc = sprintf("%02d", $pcs[0]);
		$rec_date = "$yr-$mo-$dy-$hr\.$mn\.$sc\.000000";
		$page_count = 1;
		$rec = "$cd $p $rec_date $tar_files{$p} $page_count";
		#
		# we only put out this list *after* it has been backed up to ADSM.
		@db2list = (@db2list, $rec);
		last;
	    }
	}
    }
    print STDOUT "Starting adsm backup for $cd at ",&ctime(time),"\n";
    print DBG "Starting adsm backup for $cd at ",&ctime(time),"\n";


    $budir = "$image_dir/$cd/";
    $cmd = "dsmc selective -server=$server -subdir=yes $budir";
    @adsm_out = `$cmd`;
    $rc = $?;
    if ( $rc ) {
	$error_code = $!;
	print STDOUT "adsm incremental failed - rc=$rc and error code=$error_code\n";
	print DBG "adsm incremental failed - rc=$rc and error code=$error_code\n";
	close LF;
	close DBG;
	exit;
    } else {
	# scan through the adsm output and find out how much data we backed up.
	foreach (@adsm_out) {
	    if ( /^Selective Backup processing/ ) {
		print STDOUT $_;
		print DBG $_;
	    }
	}
	#--------------------------------------------
	# add these patents to the database
	#--------------------------------------------
	foreach $rec (@db2list) {
	    #  199901053 5855787 2000-05-28-13.24.30.000000 /images1/US/mepa199901053/87/57/EP05855787A2.pdf 1
	    ($cd_label, $filename, $create_date, $out_put_name, $page_count) = split(/ /, $rec);
	    print STDOUT "new row for: '$cd_label', '$filename', '$create_date', '$out_put_name', '$page_count'.\n";
	    if (! $sti->execute($cd_label, $filename, $create_date, $out_put_name, $page_count)) {
		$ermsg = $DBI::errstr;
		if ( $ermsg =~ /SQL0803N/ ) {
		    print STDOUT "duplicate found for '$cd_label', '$filename', '$create_date', '$out_put_name', '$page_count'.\n";
		    print DUP "duplicate found for '$cd_label', '$filename', '$create_date', '$out_put_name', '$page_count'.\n";
		    if ( ! $stq->execute($filename)) {
			print DUP "Error fetching info for '$filename'\n";
			print DUP $DBI::errstr;
			print STDOUT "Error fetching info for '$filename'\n";
			print STDOUT $DBI::errstr;
			exit;
		    }
		    ($p_cd, $p_patn, $p_date, $p_outname, $p_count) = $stq->fetchrow_array();
		    $stq->finish();
		    print STDOUT "previous='$p_cd, $p_patn, $p_date, $p_outname, $p_count'\n";
		    print DUP "previos='$p_cd, $p_patn, $p_date, $p_outname, $p_count'\n";
		    next;
		} else {
		    print STDOUT "some other error.\n$DBI::errstr\n";
		    print DBG "some other error.\n$DBI::errstr\n";
		    exit;
		}
	    }
	    
	}
    }
    print LF "DONE $cd\n";
    $cd_pointer++;
}