#--emely.pl
#-- "xml-izer" -
#-- Produces prototype XML from House provided composition code data
#-- to be rendered more formally in separate production code
#------ 
#-- For each USC title processed,
#-- The input is one file in HOR "locator code" (we say "bell") format
#-- The output is a directory containing one file per input fragment, where
#-- each fragment contains exactly one USC section OR USC super-section
#-- Other output:
#---- A file for use in loading this data into SIM
#---- Appends to nsgmls.log in pwd all error level messages from nsgmls
#---- (disabled) Label (sub-sectioning) analysis added to (assumed parallel) geezer
#-- 
#-- 
#------ 
#-- 
#-- The COMMAND LINE - 
#-- see help routine at end of file, or
#-- perl emely.pl -help
#-- 
#-- 
#-- The OUTPUT is a directory for each input file (USC title) containing
#-- one XML file for each USC fragment found, with fragmentation level of
#-- (Super-)Section
#-- Each USC Title is given a directory in the current directory,
#-- named uscodeNN, where NN is zero filled title number (with the additional
#-- extension "a" for an appendix).
#-- A pop-up index is contributed to the parallel geezer directory, assumed to
#-- pre-exist at ../geezer/uscodeNN, to be used in the gzr review context.
#-- 
#-- Until someone does a decent job of teasing out the database and validation
#-- stuff into neat rotines:
#---- look for
#-- mySQL --
#-- comment lines for (awkwardly embedded) mySQL activities
#---- look for
#-- NSGMLS --
#-- comment lines for nsgmls related code
#-- 
#-- Table handling is now in USCTable.pm
#-- Xref handling is now in USCXref.pm
#-- Conversion from bell codes to gzr codes is handled in USCRinger.pm
#-- R-code lookup is in USCRinger.pm
#-- Super-Section handling is still here, and needs to move to a library module
#-- Label handling is still here, and needs to move to a library module, probably USCLister.pm
#-- Footnote handling is still here, and needs to move to a library module, probably USCFootnote.pm
#-- Typeface handling is still here, and needs to move to some library module
#-- Fragment detection is still embedded here, and needs to be clarified and externalized
#-- Misc. code detection could be relegated to a USCMisc.pm
#-- 
#-- David Shetland Jan-2002 ff
#******************************************************************************************

use strict;
use lib '..'; 
use LDMS_NG::USCRinger;
use LDMS_NG::USCXrefs;
use LDMS_NG::USCTable;
use Roman;  #-- from CPAN
use File::stat;

#** command line **
#-- titles to process
my $inFileName = "";
my $fileprefix = '../data/uscode-bell/unix/uscode';
my $filesuffix = '.bell';
my $db = 0;
my $sp = 0;
my $la = 0;
my $appendixOnly = 0; 
my $appendixNo   = 0; 
my $chosenFrag   = 0; 
my $nextArg =   shift(@ARGV);
while ($nextArg && $nextArg =~ /^\-/) {
	if ($nextArg =~ /\-[Hh][Ee][Ll][Pp]/) { &helpout(); }
	if ($nextArg =~ /\-[aA][Oo]/) { $appendixOnly = 1; }
	if ($nextArg =~ /\-[aA][Nn]/) { $appendixNo   = 1; }
	if ($nextArg =~ /\-[Dd][Bb]/) { $db   = 1; }
	if ($nextArg =~ /\-[Ss][Pp]/) { $sp   = 1; }
	if ($nextArg =~ /\-[Ll][Aa]/) { $la   = 1; }
	if ($nextArg =~ /\-[Ff][Rr]([0-9]+)/) { $chosenFrag = $1; }
	if ($nextArg =~ /\-[Ff][Ii][Pp]\=?([0-9A-Za-z\.\_\-\/\\]+)/) { $fileprefix = $1; }
	if ($nextArg =~ /\-[Ff][Ii][Ss]\=?([0-9A-Za-z\.\_\/\\]+)/) { $filesuffix = $1; }
	if ($nextArg =~ /\-[Ff][Ii][Ll][Ee]\=?([0-9A-Za-z\.\_\/\\]+)/) { $inFileName = $1; }
	$nextArg =   shift(@ARGV);
	}
if ($chosenFrag) {print("debug: request for fragment $chosenFrag only\n");}

my $titleFirst =   "";
my $titleLast  =   "";
   $titleFirst =   $nextArg;
   $titleLast  =   shift(@ARGV);

if (!$titleFirst) {  &helpout(); }

if (!$titleLast ) {  
$titleLast=$titleFirst; }
#*******************


#*******************
#-- mySQL --
my $opt_user     = "daves";
my $opt_password = "osw3g0";
my $host         = "localhost";
my $usc_db       = "uscode";
my $dbh = "";
if ($db){
use DBI;
$|= 1;				# Autoflush
print "Connection to database $usc_db\n";
$dbh = DBI->connect("DBI:mysql:$usc_db:$host",$opt_user,$opt_password) || die "Can't connect: $DBI::errstr\n";
}else{
print "Not in database mode\n";
}
#*******************


#use Sys::Hostname;
#print (hostname()."\n");

use POSIX qw(uname);
my $kernel   = (uname)[0];
my $hostname = (uname)[1];
my $version  = (uname)[2];
my $release  = (uname)[3];
my $hardware = (uname)[4];
print( "System: [".$hardware."] ".$kernel."-".$version." - ".$release." (".$hostname.")\n");

#*******************
#-- NSGMLS --
#-- open log file for append
if ($sp){
open (NSGMLS_LOG, ">>", "nsgmls.log") || die { "can't open nsgmls log file\n" };
}
my $cntSP = 0; #-- tally for final log entry
#*******************

my $progFile = $0." ".localtime(stat($0)->mtime);
print ($progFile."\n");

#-- stats --
#-- intra-section 
my $statFN_sec = 0;
my $statTF_sec = 0;
my $statTFGRD_sec = 0;
my $statPSEC_sec = 0;
my $statHDSUB_sec = 0;
my $statLblCl_sec = 0;
my $statTable_sec = 0;
#-- total for title
my $statFN_tot    = 0;
my $statTF_tot    = 0;
my $statTFGRD_tot = 0;
my $statHDSUP_tot = 0;
my $statSEC_tot   = 0;
my $statPSEC_tot = 0;
my $statHDSUB_tot = 0;
my $statLblCl_tot = 0;
my $statTable_tot = 0;
#-----------

my $startTime  = "";
my $inFileTime = "";
my $inFileSize = "";
my $inFileStat = "";

my $curfmt = "5800";
my $fmttbl = "5800";
my $curcod = "00";

my $lblTy  = 0;
my $lblTyPrev  = 0;
my $lblLev = 0;
my $lblLevPrev  = 0;
my $levDif = 0;
my %lblTyHist  = ();

my @seqPrev = (0,0,0,0,0,0,0,0,0,0);
my @locPrev = (0,0,0,0,0,0,0,0,0,0);
my @labPrev = ("na","0\.","(0)","(0)","(0)","(0)","(0)","(0)","(0)","(0)");

my $inSupSec  = 0;
my $inSection = 0;
my $inSC      = 0;
my $inText    = 0;
my $inFtNt    = 0;
my $inSecHead = 0;
my $inNotes   = 0;
my $flgHdSup  = 0;

my $nextTmp = "";


my $titlenum = "00";

#-- file fragment counters
my $ctrFrag    = 0;
my $ctrFragStr = "";
my $flgFragStr = sprintf("%05d",$ctrFrag); #-- marker for last flagged fragment

my $chunk = "";

my $tmpkey = "00T00";

my $rCode = 0;

my @headlist = (); #-- expcite style list of super-section heads maintained as a stack 
my @hdtykeys = (); #-- ssdiv keys that match headlist entries

my @supctr   = (1);

my $prevLvl  = 0;
my $pushHead = 0;
my $replHead = 0;
my $popHead  = 0;


#-- xrefs --
my $ctrAREFEntry = 0;
my $ctrAREFEntStr = "";
my @curFrag = ("","","","","","","","","","","","","","","","","","","","");
my @cntFrag = (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);
my $artNum  =0;

  #-- patterns for list item label candidates
  my $pItemCntrNum     = "[\(][0-9]{1,3}[a-zA-Z]?[\)]"; #-- lev 1: numeric with round parens (sometimes with alpha suffix)
  my $pItemCntrAlphLow = "[\(]([a-z]|aa|bb|cc|dd|ee|ff|gg|hh|ii|jj|kk|ll|mm|nn|oo|pp|qq|rr|ss|tt|uu|vv|xx|yy|zz|aaa|bbb|ccc)[\)]"; #-- lev 2: alpha lower case with round parens
  my $pItemCntrAlphCap = "[\(]([A-Z]|AA|BB|CC|DD|EE|FF|GG|HH|II|JJ|KK|LL|MM|NN|OO|PP|QQ|RR|SS|TT|UU|VV|XX|YY|ZZ|AAA|BBB|CCC)[\)]"; #-- lev 3: alpha upper case with round parens
  my $pItemCntrRomLow  = "[\(][ivx]{1,5}[\)]"; #-- lev 4: roman numeral lower case with round parens
  my $pItemCntrRomCap  = "[\(][IVX]{1,5}[\)]"; #-- lev 5: roman numeral upper case with round parens
  my $pItemProto       = "([\(][0-9a-zA-Z]{1,5}[\)])";  #-- generic label prototype

my $uscExists = 0;

my @tableBuffer = ();

my $lcLen = length("[=LC=1234567]");  #-- length of locator code
my $title_head = "";

#**************************************
#-- Process File(s)
#**************************************

#-- first check for single file spec.
if ($inFileName){
	if ($titleFirst =~ /([0-9]{1,2})a/){
	$titlenum = sprintf( "%02da",$1);
	}else{
	$titlenum = sprintf( "%02d",$titleFirst); }
	&process_file(); 
}else{

	for (my $tind=$titleFirst; $tind<=$titleLast; $tind++) {

	#-- try both base name and appendix, as modified by possible parm
	my @titlenumalt = ( sprintf( "%02d",$tind ), sprintf( "%02da",$tind ) );
	if ($appendixNo   ){ @titlenumalt = ( sprintf( "%02d",$tind ) ); }
	if ($appendixOnly ){ @titlenumalt = ( sprintf( "%02da",$tind ) ); }

	foreach my $titlenumalt ( @titlenumalt ) {
	$titlenum = $titlenumalt;

	$inFileName = $fileprefix.$titlenum.$filesuffix;

	&process_file();

} #-- try both base name and appendix

} #-- end of loop through files

} #-- check for single file

#-------------------------------------------------------
#-- mySQL --
if ($db){
#-- disconnect from database
my $rc = $dbh->disconnect;
}
#-------------------------------------------------------

#-- end emely (xml-izer) main (subs below) --


#**************************************
sub process_file() {
#**************************************

#-- file open US Code data in - 
$uscExists = 1;
open (USCODE, "< $inFileName") || do { $uscExists = 0; };

#-- tell gizer about the new $titlenum, and reset the gizer level tracker
&set_titlenum($titlenum);
&reset_prevLvl();

if ($uscExists) {
$inFileStat = $inFileName." ".localtime(stat($inFileName)->mtime);
print ("found ".$inFileStat."\n");

#*******************
#-- NSGMLS --
$cntSP = 0; #-- tally for final log entry
#*******************

#-- run time stats 
$startTime = time();
$inFileTime = stat($inFileName)->mtime;
$inFileSize = stat($inFileName)->size;
print ("debug: file size: ".$inFileSize."\n");

#-- make new directory, if needed, for this title
if(mkdir "uscode$titlenum"){print("Started new directory: uscode$titlenum\n");};

#-- open the SIM loader file
open (SIMLOAD, "> uscode".$titlenum."\/T".$titlenum.".blu") || die("\ncan't open output file uscode".$titlenum."\/T".$titlenum.".blu\n");


#-- reset file fragment counters
$ctrFragStr = "";
$ctrFrag    = 0;
$flgFragStr = sprintf("%05d",$ctrFrag); #-- marker for last flagged fragment

$rCode = 0;

@headlist = (); #-- expcite style list of super-section heads maintained as a stack 
@hdtykeys = (); #-- ssdiv keys that match headlist entries

@supctr   = (1);

$prevLvl  = 0;
$pushHead = 0;
$replHead = 0;
$popHead  = 0;

&genHeaderPage();

&startDataPage("null"); #-- leader DataPage to collect file header oddments

my $lineCntr	= 0;

my $dataRecTot	= 0;

# $ta	= "T";  #-- Title vs Appendix flag, used in document level hash key 
&reset_ta();  #-- (USCRinger.pm) Title vs Appendix flag, used in document level hash key 

my $cur_expcite;
my $cur_head;

# my $rCodeChunk = "";

   $curfmt = "5800";
   $fmttbl = "5800";
my $inTable = 0;
my $centering = 0;

#-- debug flags for label analysis...
if ($la) {
 &startLBLPage();
}

#-- init. stats - total for title
$statFN_tot    = 0;
$statTF_tot    = 0;
$statTFGRD_tot = 0;
$statHDSUP_tot = 0;
$statSEC_tot   = 0;
$statPSEC_tot = 0;
$statHDSUB_tot = 0;
$statLblCl_tot = 0;
$statTable_tot = 0;

$dataRecTot	= 0;

#**************************************
#-- Loop through the lines of the file
#-- chunking at each newline
#-- (Nominally, each data record is terminated by a double newline, but we
#-- do not assume that)
#**************************************

while (<USCODE>) {

$lineCntr++;
$chunk	= $_;

#-------------------------------------
#-- TRANSLATE to readable codes --
#-- NB: set_titlenum is called once per title, above
#-------------------------------------
my $outChunk = "";
   $curcod = "00";
my $curlab = "";

#-- empty line (new record)
if (length($chunk)==1) { $chunk = "[=newrec]"; }
#-- otherwise, geezerize
else { $chunk  = &gizer($chunk); #-- first translate, then
       #-- sync with various initial-scan-related flags and variables --
       $rCode  = &get_rCode();
       $curcod = &get_curcod();
       $curfmt = &get_curfmt();
       ($pushHead, $replHead, $popHead) = &get_head_flags();
       $tmpkey = &get_tmpkey();
       $inTable = &get_inTable();

#-- heading capture, etc. before output

#-- Save title head
if ( ($curfmt =~ /5800/) and ( $curcod =~ /04/ ) and ($rCode) ) {
  $title_head = $chunk;
}

#-- Super-Section head update
if ( ( ($curfmt =~ /5800/) and ( $curcod =~ /(81)|(52)|(04)/ ) ) ||
     ( ($curfmt =~ /5802/) and ( $curcod =~ /(52)|(04)/ ) ) || 
     ( ($curfmt =~ /5804/) and ( $curcod =~ /(83)/ ) ) ) { 
	if ( $rCode ) {
	my $hdchunk = $chunk;
	chomp($hdchunk);

#----------------------------------------------------
#-- Stack maintenance for headings
#-- >>> this needs to disappear into USCRinger or a library file of its own
#-- NB: @hdtykeys, @headlist and @supctr are parallel arrays - must stay in sync.
	if ($replHead) { $supctr[$#supctr]++; }               #-- increment the counter currently on top
	if ($pushHead) { push(@supctr, 1); }                  #-- start a new counter at 1
	if ($popHead)  { pop(@supctr); $supctr[$#supctr]++; } #-- increment top counter after pop

	#-- construct new extid
	
#-----------save	
	#-- modify the stack in one of three ways, all but the final push(@headlist, $hdchunk) -
#	if ($pushHead) { push ( @headlist, $hdchunk); push ( @hdtykeys, $tmpkey); $pushHead=0; &reset_pushHead(); }
#	if ($replHead) { pop  ( @headlist ); pop (@hdtykeys); push ( @headlist, $hdchunk); push ( @hdtykeys, $tmpkey); $replHead=0; &reset_replHead(); }
#	if ($popHead)  {
	#-- pop back to the current value of $tmpkey, then that one, too; at most back to the first (title level)
#	until ( (scalar(@hdtykeys)<3) || ($tmpkey eq pop(@hdtykeys)) ) {
#		pop(@headlist);  }
#	pop(@headlist);
#	push(@headlist, $hdchunk ); push ( @hdtykeys, $tmpkey); $popHead=0; &reset_popHead(); }
#-----------end-save	
	
	#-- modify the stack in one of three ways, 
	#-- Note>> all but the final push(@headlist, $hdchunk) -
	if ($pushHead) {  push ( @hdtykeys, $tmpkey); $pushHead=0; &reset_pushHead(); }
	if ($replHead) { pop  ( @headlist ); pop (@hdtykeys); push ( @hdtykeys, $tmpkey); $replHead=0; &reset_replHead(); }
	if ($popHead)  {
	#-- pop back to the current value of $tmpkey, then that one, too; at most back to the first (title level)
	until ( (scalar(@hdtykeys)<3) || ($tmpkey eq pop(@hdtykeys)) ) {
		pop(@headlist);  }
	pop(@headlist);
	push ( @hdtykeys, $tmpkey); $popHead=0; &reset_popHead(); }

	#-- embellish $hdchunk with xml tagging -
	#-- and do the final push(@headlist, $hdchunk) -

#-- build external sup-sec ID in tmpextid out of Rcode_# pairs, where # comes from head text if available, else auto sequence number
#-- each cycle builds on the extid of the containing super-section (the one which at this point should be top of the @headlist stack)
	my $tmpextid = "usc_sup";
	my $tmphead  = "just_starting";
	if ( $headlist[$#headlist] ) { $tmphead  = $headlist[$#headlist]; }
	my $tmpRcode = "00";
	my $tmpnum   = 0;

#	print("debug, headlist-top-befor: $headlist[$#headlist]\n");	
	
	if ($tmphead =~ /extid=\"([0-9A-Za-z\-\_]*)\"/ ) {
	$tmpextid = $1;
	}
	if ($hdchunk =~ /\[?(?:SUB)?(?:(?:TITLE)|(?:APPENDIX)|(?:CHAPTER)|(?:PART)|(?:DIVISION))[\ ]([0-9a-zA-Z\-]{1,9})/i ) {
		$tmpnum   = $1;
		}
	else {
		$tmpnum   = 'sq'.$supctr[$#supctr];
	}
	if ($tmpkey =~ /([0-9]{2})a?[AT]([0-9]{2})/ ) {
	$tmpRcode = $2;
	}
	
	$tmpextid = $tmpextid."_".$tmpRcode."_".$tmpnum;

	$hdchunk = "<hdsupnest extid=\"".$tmpextid."\" ty=\"".$tmpkey."\" seq=\"".$supctr[$#supctr]."\">".$hdchunk."</hdsupnest>";
	push(@headlist, $hdchunk);

#	print("debug, headlist-top-after: $headlist[$#headlist]\n\n");	
	
#-- construct an underscore indent for use in the TOC (not the place for this)
	my $headsize = @headlist;
	if ($headsize>0) {$headsize--;}
	my $headIndent = "";
	if ( "________________" =~ /((_){$headsize})/  ) { $headIndent = $1; }


#----- OLD CODE --------------
#-- build external sup-sec ID in keyglob out of Rcode_# pairs, where # comes from head text if available, else auto sequence number
#	my $keyglob = "usc_sup";                   #-- init with prefix
#	my $level = 0;
#	#-- loop through the current headlist, pulling out pieces for the ID
#	foreach my $onehead ( @headlist ) {
#	$level++;
#	$onehead  =~ s/\[=md\]/\ -\ /g; #-- em-dash
#	$onehead  =~ s/\[=nd\]/-/g;     #-- en-dash
#	$onehead  =~ s/\[=.*?\]//g;     #-- forsaking all others
#	my $tmp_title = '';
#	my $tmp_rCode = '';
#	my $tmp_seq   = '';
#	if ($onehead =~ /ty=\"([0-9]{2})a?[AT]([0-9]{2})\"\ seq=\"(?:[0-9]{1,3})\"\>\[?(?:SUB)?(?:(?:TITLE)|(?:APPENDIX)|(?:CHAPTER)|(?:PART)|(?:DIVISION))[\ ]([0-9a-zA-Z\-]{1,9})/i ) {
#		$tmp_title = $1;
#		$tmp_rCode = $2;
#		$tmp_seq   = $3;
#		if ($level==1) {$tmp_seq = $titlenum;}
#		}
#	elsif ($onehead =~ /ty=\"([0-9]{2})a?[AT]([0-9]{2})\"\ seq=\"([0-9]{1,3})\"/ ) {
#		$tmp_title = $1;
#		$tmp_rCode = $2;
#		$tmp_seq   = 'sq'.$3;
#	}
#	#-- accumulate the pieces -
#	$keyglob .= '_'.$tmp_rCode.'_'.$tmp_seq;
#	}
#----- end OLD CODE --------------


#-- MYSQL --------------
	if($db){
	my $head  = $chunk;
	$ctrFragStr = sprintf("%05d",$ctrFrag);    #-- stringify frag counter
	my $frag = 'T'.$titlenum.'F'.$ctrFragStr;  #-- construct frag ID
	&sql_supsecmap($frag, $tmpextid, "$head");
	} #-- end if db
#-- end MYSQL --------------
	
#-- Super-Section starts a new page
	&startDataPage($tmpextid);

	}
	}

#-- Table
#---- build buffer for handleTable (do not include newrec lines)
if ( $inTable ) { push ( @tableBuffer, $chunk ); }
#---- filter and output table when we get to the endtable
if ( $chunk =~ m/\[=endtable\]/ ) {
 push ( @tableBuffer, $chunk ); #-- endtable has already turned off inTable
 $statTable_sec++;
 @tableBuffer = &handleTable( @tableBuffer );
print ( XMLOUT "\n" );
foreach my $tableLine ( @tableBuffer ) { &outRecord($tableLine); }
@tableBuffer = ();
}

#-- can't find where this is used -- delete if emely works without it
# if ( $rCode ) { $rCodeChunk = $chunk; }


#---------------------------------------------
#---------- OUTPUT ---------------------------
#---------------------------------------------

#-- Section starts a new page
if ( ($curfmt =~ /(5800)|(5802)/) and ( $curcod =~ /(80)/ ) ) {

	&startDataPage("usc_sec_");

	}

#-------------------------------------------------------
#-- output every chunk (except tables output above) ----
if ( !$inTable ) {
if (length($chunk) > 2) { &outDataChunk($chunk); }
}
#-------------------------------------------------------

} #-- end empty line check

} #-- end (while more lines in file)

#-- tidy up --
&lastDataPage();
close (SIMLOAD);

if ($la) {
close (HTMLBL);
}

close (USCODE);

#-------------------------------------------------------
#-- mySQL --
if ($db){
#-- summary stats to mySQL
&sql_summary;
}
#-------------------------------------------------------

#*******************
#-- NSGMLS --
#-- summary stats to SP
if ($sp){
&sp_summary($cntSP);
}
#*******************

} #-- end file existence check

} #-- end process a file

#####################################################################################################
#####################################################################################################
#---------------------------------------------------
sub genProcessData{
#-- Should this be one tag with attributes?
#-- Maybe with content that is a composite of the attributes?
#---------------------------------------------------

print(XMLOUT "<procdata>\n");
print(XMLOUT "<proctime>".localtime(time())."</proctime>\n");
print(XMLOUT "<procsys>[".$hardware."] ".$kernel."-".$version." - ".$release." (".$hostname.")</procsys>\n");
print(XMLOUT "<procprog>".$progFile."</procprog>\n");
print(XMLOUT "<sourcedata>".$inFileStat."</sourcedata>\n");
print(XMLOUT "</procdata>\n");

}
#---------------------------------------------------

#---------------------------------------------------
sub genHeaderPage{
#---------------------------------------------------
$ctrFragStr = sprintf("%05d",$ctrFrag);
open (XMLOUT, "> uscode".$titlenum."\/T".$titlenum."F".$ctrFragStr.".XML") || die("\ncan't open output file uscode".$titlenum."\\T".$titlenum."F".$ctrFragStr.".XML\n");
print(XMLOUT "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");

$ctrFragStr = sprintf("%05d",$ctrFrag);
print(XMLOUT "<uscfrag id=\"T".$titlenum."F".$ctrFragStr."\">\n");

&genProcessData();

#-- let next startDataPage do the close (XMLOUT);
}
#---------------------------------------------------


#---------------------------------------------------
sub startDataPage{
#-- starts a fresh data fragment, currently for
#-- (1) an Rcode, or
#-- (2) a Section
#---------------------------------------------------
#-- close any open XMLOUT
&finishPage();
#--
my $extid = shift;

$ctrFrag++;
$ctrFragStr = sprintf("%05d",$ctrFrag);
open (XMLOUT, "> uscode".$titlenum."\/T".$titlenum."F".$ctrFragStr.".XML") || die("\ncan't open output file uscode".$titlenum."\\T".$titlenum."F".$ctrFragStr.".XML\n");

print(XMLOUT "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE uscfrag SYSTEM \"uscbell.dtd\" []>\n");

#-- make entry in SIM loader
print(SIMLOAD "\"R_USC\" (\n  \"uscfrag\"  UNISGML include \"uscode".$titlenum."\/T".$titlenum."F".$ctrFragStr.".XML\");\n");

$ctrFragStr = sprintf("%05d",$ctrFrag);
print(XMLOUT "<uscfrag id=\"T".$titlenum."F".$ctrFragStr."\">\n");

&genProcessData();

#-- The ouput of the hdnestgrp -------------------------
   my $tmphead = join("\n",@headlist);
   $tmphead =~ s/\[=md\]/\ \-\ /g;
   $tmphead =~ s/\[=nd\]/\-/g;
   $tmphead =~ s/\[=fn\][0-9]{1,2}\[=fn\]//g;     #-- discard footnote numbers
   $tmphead =~ s/\[[a-zA-Z0-9\ \=\.]+\]//g;
if ($rCode) { #-- trim the last entry, since it duplicates the current code head
   $tmphead =~ s/.+?$//; }
if ($tmphead ne "\n" ) { print(XMLOUT "\n<hdnestgrp>\n".$tmphead."<\/hdnestgrp>\n"); }
#-- End ouput of the hdnestgrp -------------------------

if ($rCode) {
#-- get the supsec started by looking up the Rcode nomenclature and opening the tag
my $RcodeNomenTmp = &getRcodeNomen("$tmpkey");
if ($RcodeNomenTmp) {
   print(XMLOUT "<supsec key=\"".$tmpkey."\" nomen=\"".$RcodeNomenTmp."\" extid=\"".$extid."\">\n"); 
   } else {
   print(XMLOUT "<supsec key=\"".$tmpkey."\" nomen=\"NOT FOUND\" extid=\"".$extid."\">\n"); 
   }
   $inSupSec = 1;
   $rCode = 0;
   &reset_rCode();
   }

#-- reset metasec scope counters, etc.

   @seqPrev = (0,0,0,0,0,0,0,0,0,0);
   @locPrev = (0,0,0,0,0,0,0,0,0,0);
   @labPrev = ("na","0\.","(0)","(0)","(0)","(0)","(0)","(0)","(0)","(0)");

 $lblTy  = 0;
 $lblTyPrev  = 0;
 $lblLev = 0;
 $lblLevPrev  = 0;
 $levDif = 0;
 %lblTyHist  = ();

 $inNotes   = 0;
 $flgHdSup  = 0;

#-- stats --
$statFN_sec = 0;
$statTF_sec = 0;
$statTFGRD_sec = 0;
$statPSEC_sec = 0;
$statHDSUB_sec = 0;
$statLblCl_sec = 0;
$statTable_sec = 0;
#-----------

}
#---------------------------------------------------


#---------------------------------------------------
sub lastDataPage{
#---------------------------------------------------
#-- close any open XMLOUT
&finishPage();
#--
$ctrFrag++;
$ctrFragStr = sprintf("%05d",$ctrFrag);
open (XMLOUT, "> uscode".$titlenum."\/T".$titlenum."F".$ctrFragStr.".XML") || die("\ncan't open output file uscode".$titlenum."\\T".$titlenum."F".$ctrFragStr.".XML\n");
print(XMLOUT "<xml>\n<head><title>T".$titlenum."F".$ctrFragStr."</title>\n");
print(XMLOUT "</head>\n<body>\nT".$titlenum."F".$ctrFragStr."\n");

print(XMLOUT "---".localtime(time())."\n");

print(XMLOUT "<data>End of File<\/data>\n</body>\n</xml>\n"); 

close( XMLOUT );

}
#---------------------------------------------------



#---------------------------------------------------
sub finishPage{
#---------------------------------------------------

#-- total the stats
$statFN_tot    += $statFN_sec;
$statTF_tot    += $statTF_sec;
$statTFGRD_tot += $statTFGRD_sec;
if ($inSupSec) { 
$statHDSUP_tot ++;
}else{
$statSEC_tot   ++;
}
$statPSEC_tot  += $statPSEC_sec;
$statHDSUB_tot += $statHDSUB_sec;
$statLblCl_tot += $statLblCl_sec;
$statTable_tot += $statTable_sec;
#-- stats debug --
# print ("debug: stats: $statHDSUP_tot $statSEC_tot $statPSEC_tot $statHDSUB_tot $statTable_tot $statLblCl_tot $statFN_tot\n");


#-- close any open XMLOUT
if ($inSupSec) { 
  #-- Close Notes
  if ($inNotes) { print(XMLOUT "\n</notes>\n"); $inNotes=0; }
  #-- close super-section
  print(XMLOUT "\n</supsec>\n"); $inSupSec=0; 
  }

if ($inSection) { 
#-- reset running label level and close previous subsections to match
  $lblLev = 0;
  while ($lblLev lt $lblLevPrev) { 
    print(XMLOUT "\n</psection>");
    $lblLevPrev--;
    }

#-- Close Notes
  if ($inNotes) { print(XMLOUT "\n</notes>\n"); $inNotes=0; }

#-- Close SECTIONCONTENT if there were no notes
  if ($inSC)      { print(XMLOUT "</sectioncontent>" ); $inSC=0; }

#-- close section
  print(XMLOUT "\n</section>\n"); $inSection=0; 
  }

  #-- Close Notes, if not caught above
  if ($inNotes) { print(XMLOUT "\n</notes>\n"); $inNotes=0; }


#-- close fragment
print( XMLOUT "\n<stats>" );
if ($statFN_sec)    {print( XMLOUT " FN($statFN_sec) "       );}
if ($statTF_sec)    {print( XMLOUT " TF($statTF_sec) "       );}
if ($statTFGRD_sec) {print( XMLOUT " TFGRD($statTFGRD_sec) " );}
if ($statPSEC_sec)  {print( XMLOUT " PSEC($statPSEC_sec) "  );}
if ($statHDSUB_sec) {print( XMLOUT " HDSUB($statHDSUB_sec) " );}
if ($statLblCl_sec) {print( XMLOUT " LblCl($statLblCl_sec) " );}
if ($statTable_sec) {print( XMLOUT " TABLE($statTable_sec) " );}
print( XMLOUT "</stats>\n" );
print( XMLOUT "\n</uscfrag>\n" );
close( XMLOUT );

#*******************
#-- NSGMLS --
if ( $ctrFrag > 0 ) {
   $cntSP++; #-- tally for final log entry
   if ($sp){
   &spout("uscode".$titlenum."\/T".$titlenum."F".$ctrFragStr.".XML");
   }
   }
#*******************

}


############## XML Data Out ###########################################
#---------------------------------------------------
sub outDataChunk($){
#-- (1) Filters a chunk for XML and writes it
#-- (2) Routines, if not objects, should be isolated for major parts, such as table
#---------------------------------------------------
#-- Local copy of the output record is $nextOut
#-- At this point, the records are exactly the raw input with bell codes and
#-- special characters converted to the printable [=...] syntax

our $nextOut = shift;

#-- initialize standard locator code length (possible system constant?)
my $locLen = length( "\[=LC=5800=80\]" );

#-----------------------
#--- super-section heading
if (!$inSection) { #-- alas, we seem to have some (rare) re-use of the LC within section
if ( ( ($curfmt =~ /5800/) and ( $curcod =~ /(81)|(52)|(04)/ ) ) ||
     ( ($curfmt =~ /5802/) and ( $curcod =~ /(52)|(04)/ ) ) ||
     ( ($curfmt =~ /5804/) and ( $curcod =~ /83/ ) ) ) {          #-- see T26F00489
   $nextOut = "\n<head lc=\"".$curfmt."-".$curcod."\">".$nextOut."<\/head>\n"; 
#--output and return
&filterTypeface();
&outRecord($nextOut);
#-- flag the first one to generate notes next trip (if there is another record)
$flgHdSup=1;
return;
}
}

#-----------------------
#-- leader - first occurence starts "Notes," which is a state maintained until the end of the section
if ( ($curfmt =~ /(5800)|(5802)/) and ( $curcod =~ /(06)|(53)|(73)|(74)|(75)|(76)|(78)|(84)|(85)|(86)|(87)/ ) ) { 
  
  #-- Open Notes signalled by first leader, unless already open; we close any subsecs and open the Notes
  if (!$inNotes) {
  $lblLev = 0;
  while ($lblLev lt $lblLevPrev) { 
    print(XMLOUT "\n</psection>");
    $lblLevPrev--;
    }
    if ($inSC)      { print(XMLOUT "</sectioncontent>" ); $inSC=0; }
    print(XMLOUT "\n\n<notes>\n");
    $inNotes=1;
    }
    
#-- Leaders:
#-- Record contains only heading
#----"Source Credit"
if ( $curcod =~ /(53)/ ) { print(XMLOUT "\n<leader ty=\"SOURCE\"></leader>\n"); }
#-- Record contains data
#----"Running Head"
if ( $curcod =~ /(06)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"RUNNINGHEAD\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Referred to in other"
if ( $curcod =~ /(73)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"SECREF\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Amendments"
if ( $curcod =~ /(74)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"AMENDMENTS\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"References in text"
if ( $curcod =~ /(75)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"REFTEXT\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Codification"
if ( $curcod =~ /(76)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"COD\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Change of Name"
if ( $curcod =~ /(78)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"CHANGE\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Italic Amendment Note"
if ( $curcod =~ /(84)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"STATAMEND\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Transfer of functions"
if ( $curcod =~ /(85)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"TRANS\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Ex, Ord. No. etc."
if ( $curcod =~ /(86)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"EXEC\">".substr($nextOut,$locLen)."</leader>\n"; }
#----"Federal Rules, cross refs"
if ( $curcod =~ /(87)/ ) { chomp($nextOut); $nextOut = "\n<leader ty=\"CROSS\">".substr($nextOut,$locLen)."</leader>\n"; }
    
    }


if ($inSecHead) {$inSecHead++;}

my $curfmt = "0000";
my $curcod = "00";
if ( $nextOut =~ m/\[=LC=([0-9]{4})=([0-9]{2})\]/ ) { $curfmt = $1; $curcod = $2; }

#-----------------------
#-- Section head --
if (
#  $nextOut =~ /\[=LC=5800=80\](?:\[)?((?:\[=st\])?(?:\[=st\]\[=ts\])([a-zA-Z0-9\-]+(?:\[=nd\][a-zA-Z0-9\-]+)?))/
  $nextOut =~ /\[=LC=5800=80\](?:\[)?((?:\[=st\])?(?:\[=st\]\[=ts\])(.*$))/
   ) { 
# my $secnum = &sectionNumber($chunk); #-- Ringer extract and filter of secnum
#    $nextOut = "<section num=\"$secnum\">\n";

my $seclin = $1;
my $secnum = $2;

   #-- sad workaround for 33 USC 702a-1 inserts
   $secnum =~ s/\[=fn\]1\/2\[=fn\]/m/g;
   $secnum =~ s/\[=fn\]3\/4\[=fn\]/r/g;
   
   #-- cleanup of geezer codes
   $secnum =~ s/\[=md\]/-/g;
   $secnum =~ s/\[=nd\]/-/g;
   $secnum =~ s/\[=ensp\]/_/g;
   $secnum =~ s/\[=emsp\]/_/g;
   $secnum =~ s/\[=.+?\]//g;
   #-- filter blanks to underscore
   $secnum =~ s/\ /_/g;

   $inSection = 1; 
   $inSecHead = 1; 
   my $extid = &sectionExtID($secnum);
   $nextOut = "<section num=\"$secnum\" extid=\"$extid\">\n<head>$seclin";
   }
#-- Rule (or Form) is a flavor of section, but a little different --
elsif (
  $nextOut =~ /\[=LC=580[02]=80\](?:\[)?((?:Rule(?:s)?\ )|(?:Form(?:s)?\ ))([a-zA-Z0-9\ \-]+)/
   ) { 
   my $extid = &sectionExtID($2);
   print(XMLOUT "<section num=\"$2\" extid=\"$extid\">\n"); 
   $inSection = 1; 
   $inSecHead = 1; 
   $nextOut = "<head>".$1.$2; }
#-- Expect the next record to be the section (or rule) heading --
if (
  $nextOut =~ /\[=LC=580[02]=89\]/
   ) { $inSecHead = 0; 
       $nextOut = substr($nextOut,$locLen)."</head>";
       #-- Start the section content container - ends when notes begin
       $nextOut = $nextOut."\n<sectioncontent>\n";
       $inSC = 1; 
       }


#-----------------------
#-- label
#-- Assume a label is perhaps clustered, with zero or more having heading text.
#-- If detected by $curcod, it consumes exactly one source data record

my $lab1 = "";
my $curHd  = "";
#-- Label Category
my $labCat = 0;
my $tfpp = "(?:\[=[0-9A-Za-z=\.]+\])";

#-- do not look for subsections in notes; we might use similar code to look for lists.
if ($inSection && !$inNotes) {
#-- (1of2) start with special category where locator code indicates whole record is a single label and head
if ( ($curcod == 18) || ($curcod == 19) || ($curcod == 79) ) 
     { $lab1 = substr( $nextOut, $locLen );  $labCat=1; }
#-- (2of2) possible cluster of label-heading pairs, terminated by <<\.\[=md\]>>
elsif ( 
#  $nextOut =~ /\[=loc\ CD=[0-9\.]{7}\]((((?:[A-Z0-9]{1,5}\.)|(?:\[?\([A-Za-z0-9]{1,5}\)))(\ \[=tf[0-9]\][a-zA-Z0-9\ \[\]\=\/\,]+?\.\[=md\])?){1,4})/
 ( $nextOut =~ /\[=LC=[0-9=]{7}]((?:\[=tf[0-9]\])?((?:\[?\((?:$tfpp)*[A-Za-z0-9]{1,5}(?:$tfpp)*\))(\ (?:\[=tf[0-9]\])?[a-zA-Z0-9\-\ \[\]\=\/\,]+?(?:(\.\[=md\])|(\:\[=egr\]\ )))?){1,4})/ )
   ) { $lab1 = $1; $labCat=2; }
}

if    ( $lab1 ) { 
$nextTmp = substr($nextOut,(length($lab1)+$locLen));
#$nextOut = "<newrec/>";
$nextOut = "";
my $clusterNum = 0;
   $lblTy  = 0;
while ( #-- another label
#-- *** notes on regex below ***
#-- any leading typeface is discarded (this needs to be fixed)
#-- find and save label, number-dot or round paren formats
#---- ((?:[A-Z0-9]{1,5}\.)|(?:\[?\([A-Za-z0-9]{1,5}\)))
#-- Heading - optional leading typeface, ends with either dot-emdash or colon-endgrid
#-- (still need to deal with typeface, which can start either above or here)
###-- following in old format, need to review comment - das 6/7/02
#---- (\ (?:\[=typeface)?[a-zA-Z0-9\-\ \[\]\=\/\,]+?(?:(\.\[=md\])|(\:\[=egr\])))?

#-- the letter-dot format is still too ambiguous with the likes of -- Wash, D.C. --
#  $lab1 =~ /(?:\[=tf[0-9]\])?((?:[A-Z0-9]{1,5}\.)|(?:\[?\([A-Za-z0-9]{1,5}\)))(\ (?:\[=tf[0-9]\])?[a-zA-Z0-9\-\ \[\]\=\/\,]+?(?:(\.\[=md\])|(\:\[=egr\])))?/g )
  $lab1 =~ /(?:\[=tf[0-9]\])?((?:[0-9]{1,5}\.)|(?:\[?\((?:$tfpp)*[A-Za-z0-9]{1,5}(?:$tfpp)*\)))(\ (?:\[=tf[0-9]\])?[a-zA-Z0-9\-\ \[\]\=\/\,]+?(?:(\.\[=md\])|(\:\[=egr\])))?/g )
#-- still experimenting - see title 10a (following made no difference) das --
#  $lab1 =~ /(?:\[=typeface\ CD=2\])?((?:[A-Z0-9]{1,5}\.)|(?:\[?\([A-Za-z0-9]{1,5}\)))(\ (?:\[=typeface)?[a-zA-Z0-9\-\ \[\]\=\/\,]+?(?:(\.\[=md\])|([\:]?\[=egr\])))?/g )
  {
  #-- label classification
  #-- type# - <<seq. #1>> - description
  #-- 1 - <<1.>> ---- not currently handled
  #-- 2 - <<(a)>> --- lower case alpha
  #-- 3 - <<(1)>> --- arabic numeral
  #-- 4 - <<(A)>> --- upper case alpha
  #-- 5 - <<(i)>> --- roman numeral (lower case), ambiguous with corresponding chars. of type 2
  #-- 6 - <<(I)>> --- roman numeral (upper case) ambiguous with corresponding chars. of type 4
  #-- 7 - <<(aa)>> -- double char. lower case alpha (ambiguous with occasional use to extend type 2)
  #-- 8 - <<(aaa)>> - triple char. lower case alpha
       $clusterNum++;

       $lblTy  = 0;
    my $lblSeq = 0;
    my $curLbl = $1;
    if ( $labCat==2 ) { $curHd = $2; }
      else            { $curHd = substr ( $lab1, length($curLbl) ); }
    my $savLbl = $curLbl;

#-- Filter the following type of bells for, at least, (o) and (l) - das
#-- eg, where (l) is "[=LC=5800=19]([=GRD=uGRD.742][=tf3]l[=egr]) Use of financial..." 
    #-- for some reason, just filtering globally for ($tfpp)* (eg $curLbl =~ s/($tfpp)*//g  )
    #-- means we wind up with () instead of (l) or (o)
    #-- This very awkward version is a workaround
    if (  $curLbl =~ s/(?:$tfpp)*([lo])(?:$tfpp)/$1/ ) {
#      print ("debug: $savLbl --> $curLbl\n");
      }

       $curLbl =~ s/\[//;
    my $lblLen = length($curLbl);
    my $ctrLabel = substr ( $curLbl, 1, length($curLbl)-2 );  #-- trim parens
    if ($curLbl =~ /$pItemCntrAlphLow/) {$lblTy = 2; $lblSeq = ord($ctrLabel)-96;  }
    if ( ($curLbl =~ /$pItemCntrAlphLow/) && ($lblLen==4) ) {$lblTy = 7; $lblSeq = ord($ctrLabel)-96; }
    if ( ($curLbl =~ /$pItemCntrAlphLow/) && ($lblLen==5) ) {$lblTy = 8; $lblSeq = ord($ctrLabel)-96; }
    if ($curLbl =~ /$pItemCntrNum/)     {$lblTy = 3; if ($ctrLabel =~ /[0-9]+/) {$lblSeq =$&;}   }
    if ($curLbl =~ /$pItemCntrAlphCap/) {$lblTy = 4; $lblSeq = ord($ctrLabel)-64;  }
    if ($curLbl =~ /$pItemCntrRomLow/)  {$lblTy = 5; $lblSeq = arabic($ctrLabel) if isroman($ctrLabel);  }
    if ($curLbl =~ /$pItemCntrRomCap/)  {$lblTy = 6; $lblSeq = arabic($ctrLabel) if isroman($ctrLabel);  }

    #-- Correction for "I" problems (I after H, i after h, V after U, etc.
    #-- Upper I
    if ( ($lblSeq==1)&&($lblTy==6)&&($seqPrev[4]==8)&&($locPrev[4]==$curcod) ) { $lblTy = 4; $lblSeq = 9; }
    #-- Lower i
    if ( ($lblSeq==1)&&($lblTy==5)&&($seqPrev[2]==8)&&($locPrev[2]==$curcod) ) { $lblTy = 2; $lblSeq = 9; }
    #-- Upper V
    if ( ($lblSeq==5)&&($lblTy==6)&&($seqPrev[4]==21)&&($locPrev[4]==$curcod) ) { $lblTy = 4; $lblSeq = 22; }
    #-- Lower v
    if ( ($lblSeq==5)&&($lblTy==5)&&($seqPrev[2]==21)&&($locPrev[2]==$curcod) ) { $lblTy = 2; $lblSeq = 22; }
    #-- Upper X
    if ( ($lblSeq==10)&&($lblTy==6)&&($seqPrev[4]==23)&&($locPrev[4]==$curcod) ) { $lblTy = 4; $lblSeq = 24; }
    #-- Lower x
    if ( ($lblSeq==10)&&($lblTy==5)&&($seqPrev[2]==23)&&($locPrev[2]==$curcod) ) { $lblTy = 2; $lblSeq = 24; }
    #-- Reset previous trackers - 
    $locPrev[$lblTy] = $curcod;
    
  #-- begin debug ---------------------
   my $lblSeqDif = $lblSeq - $seqPrev[$lblTy];
#  if ( $lblSeqDif > 1 ) { print ("debug: lblSeg increased by $lblSeqDif in T".$titlenum."F".$ctrFragStr."\n" ); }
  #-- end   debug ---------------------

  #-- CLUSTERED LABEL handling
  #-- be *very* conservative about what we let through
  if ($clusterNum > 1) {
     if ( $lblSeq != 1 ) { next; }
     if ( $lab1 =~ /(Repealed)|((sub)?section(s)?)|((sub)?paragraph(s)?)/i ) { next; }
     }
  if ($clusterNum==2) {
     $statLblCl_sec++; #** STATS **
     # print( "CLUSTERED LABEL in T".$titlenum."F".$ctrFragStr.": ".$lab1."\n" );
	if ($la) {
	&addLBLEntry( "CLUSTERED LABEL in T".$titlenum."F".$ctrFragStr.": ".$lab1 );
	}
     }

  #-- reset "previous"
    $seqPrev[$lblTy] = $lblSeq;
    $labPrev[$lblTy] = $curLbl;
  #-- end label classification --
  
  #-------------------------------------------------
  #-- label level adjuster - 
  #-- if the type of current label is found in history hash
  #-- then
  #---- examine and tune running level ($lblLev)
  #------ Establish a current differential between new label's level (from the hash) and the current running level
  #------ The differential cannot be negative.  If it is positive, then there is at least one entry in the hash that needs to be removed.
  #------ Therefore, if ( $levDif > 0 ), traverse the entire hash, as a list, to find all types with higher level number (more deeply nested), and remove them.
  #------ For each one removed, we adjust the running level number downward (one level less nested)
  #------ The corresp. adjustment of $levDif is for completeness - it is not as of this writing used for anything outside of this module.
  #-- else
  #---- start a new level for this type and add it to the hash
  #-------------------------------------------------
  my $lblTyTemp  = 0;
  my $lblLevTemp = 0;
  if ( exists $lblTyHist{$lblTy} ) {
  $levDif = $lblLev - $lblTyHist{$lblTy};
  if ( $levDif > 0 ) {
    while ( ($lblTyTemp, $lblLevTemp) = each (%lblTyHist) ) {
      if ($lblLevTemp > $lblTyHist{$lblTy}) { 
      delete $lblTyHist{$lblTyTemp}; $lblLev--; $levDif--; }
      }
    }
  } else {
  $levDif = 1;
  $lblLev++;
  $lblTyHist{$lblTy} = $lblLev; }

  #-- close nested subsection tags -
  if ($lblLev gt $lblLevPrev) { 
  }
  while ($lblLev le $lblLevPrev) { 
  $nextOut .= "\n</psection>";
  $lblLevPrev--;
  }

  #-- Make Sub-Section ID by arranging the current labels of various types
  #-- into current level order, then concatenating and filtering
  my @curIDArray = ("na","0\.","(0)","(0)","(0)","(0)","(0)","(0)","(0)","(0)"); #-- for use in constructing the subsection ID
  while ( ($lblTyTemp, $lblLevTemp) = each (%lblTyHist) ) {
  $curIDArray[$lblLevTemp] = $labPrev[$lblTyTemp];  }
  my $curID =  join("",@curIDArray[1..($lblLev)]);  #-- for use in constructing the subsection ID
     $curID =~ s/(\)\()|\./\_/g; #-- switch internal parens to underscores
     $curID =~ s/[\)\(]//g; #-- leading and trailing parens are deleted

 #-- begin debug ---------------------
 if ( $lblSeqDif > 1 ) { 
# print (" ($curID)\n" );
	if ($la) {
	&addLBLEntry( "lblSeg increased by ".$lblSeqDif." in T".$titlenum."F".$ctrFragStr." (".$curID.")" );
	}
 }
 #-- end   debug ---------------------

  $nextOut .= "\n<psection lev=\"".$lblLev."\" id=\"".$curID."\">";

  if ($curHd) { #-- include the heading
  $curHd =~ s/^[\ ]+//; #-- trim any retained leading blank
  $nextOut .= "\n<enum ty=\"".$lblTy."\" seq=\"".$lblSeq."\" lc=\"".$curfmt."-".$curcod."\" cln=\"".$clusterNum."\">".$savLbl."</enum>"."\n<head>".$curHd."</head>"; 
  $statHDSUB_sec++ }
  else {        #-- headless version
  $nextOut .= "\n<enum ty=\"".$lblTy."\" seq=\"".$lblSeq."\" lc=\"".$curfmt."-".$curcod."\" cln=\"".$clusterNum."\">".$savLbl."</enum>"; }

  $statPSEC_sec++;

  $lblTyPrev = $lblTy;
  $lblLevPrev = $lblLev;

  } #-- end while label
$nextTmp =~ s/^[\ ]+//; #-- trim any retained leading blank
#-- output the rest as text with format and loc code implied from label
if ( $nextTmp ){ $nextOut .= "\n<text>".$nextTmp."</text>"; } 
}

elsif ( #-- use default text element
   $nextOut =~ s/\[=LC=([0-9SGL]{4})=([0-9]{2})\]/\n<text lc=\"$1-$2\">/) 
   {$inText=1;}


#-----------------------
#-- NB: filterTypeface is sequence-dependent - other transforms that rely on the TF gzr
&filterTypeface();
#-----------------------


#-----------------------
#-- Footnotes
if (
  ($nextOut =~ s/\[=fnMv\]\[=fn\]([0-9]{1,2})\[=fn\]/<footnote  fnumber=\"$1\" eid=\"FN\-$1\">/)
  || ($nextOut =~ s/\[=fn\]([0-9]{1,2})\[=fn\]\[=ts\]/<footnote  fnumber=\"$1\" eid=\"FN\-$1\">\[=ts\]/)
  ) {
  $inFtNt=1;
  $statFN_sec++; #**stats**
  }
  $nextOut =~ s/\[=fn\]([0-9]{1,2})\[=fn\](?:\[=fnMv\])?/<footref target=\"FN\-$1\">$1<\/footref>/g;
#-- an alternate use that seems to be Fractional Number instead of FootNote
  $nextOut =~ s/\[=fn\]([0-9]+\/[0-9]+)\[=fn\]/<frac>$1<\/frac>/g;
#-- another alternate use - superscripted numeric (rare - noticed in 42USCfrag2760) - das 15apr2002
  $nextOut =~ s/\[=fn\]([0-9]+)\[=fn\]/<sup>$1<\/sup>/g;
if (
  $nextOut =~ /\[=fn\]/g
  ) {
  print ("debug: leftover footnote found in Fragment: $ctrFragStr\n");
  }


#-----------------------
#-- reference detection 
$nextOut = &markArefs($nextOut);


#-----------------------
#-- Out with the record, closing any local elements...

#-- for super-secs, start notes right away  if (!$inNotes)
if ($flgHdSup && !$inNotes) {
    print(XMLOUT "\n<notes>\n");
    $inNotes=1; $flgHdSup=0;
    }

#-- for the odd [=bL], use raw text element
   if (!$inText && $nextOut =~ s/\[=bL\]/\n<text>/ )
   {$inText=1;}

#-- for the very odd record starting with no tag, use raw text element and complain
   if ( $nextOut =~ s/^[a-zA-Z0-9]/\n<text>$nextOut/ ) 
   {$inText=1;
   print ("debug: data problem: bell record with no start tag\n"); }


&outRecord($nextOut);  #-- see sub below

#-- close text last - others might be inside --
if ($inFtNt)      { print(XMLOUT "<\/footnote>" ); $inFtNt=0; }
if ($inText)      { print(XMLOUT "<\/text>" );      $inText=0; }
if ($inSecHead>1) { $inSecHead = 0;   }

#####################################################################


#---------------------------------------------------
sub filterTypeface{
#---------------------------------------------------
#-- typeface transform
#-- Assumes that there is no nesting of typeface codes
#-- NB: currently, this is very sequence-dependent - other transforms that rely on the TF gzr

#-- dbl quote substituton with braces (so we have single chars to search for) -
#-- operates on the assumption of non-existence of braces, so we start with a check
if ( $nextOut =~ /([\{\}])/ ) { print ("debug: yikes! we do have a <<$1>> after all - Fragment: $ctrFragStr\n") }
   $nextOut =~ s/\[=ldq\]/\{/g;
   $nextOut =~ s/\[=rdq\]/\}/g;

#-- initialization for typeface transform 
$nextTmp = "";        #-- piecemeal re-build of $nextOut
my $tmpGzr     = "";  #-- gzr codes, found one at a time
my $tmpGzr_len = "";
my $tmpGzr_pos = "";
my $curTF      = "";
my $flgTF      = 0;  #-- flag whether tf work was done
my $segBeg     = 0;
my $segEnd     = 0;
my $segSize    = 0;
my $curGrd     = "";
my $inTF      = 0;

#-- loop through all gzr codes in this record, translating TF related ones and leaving others
while ( 
$nextOut =~ /(\[=.+?\])/g 
) {
$tmpGzr     = $&;
$tmpGzr_len = length($tmpGzr);
$tmpGzr_pos = pos ($nextOut);
$segEnd     = $tmpGzr_pos-$tmpGzr_len;
$segSize    = $segEnd-$segBeg;
$nextTmp   .= substr($nextOut,$segBeg,$segSize);
$segBeg     = $tmpGzr_pos;

#-- the GRD gzr resets current grid (locally, for use with TF)
if ( $tmpGzr =~ /\[=GRD=(uGRD\.[0-9]{3})\]/ ) 
   { $curGrd = $1; 
#     print ("debug: Microgrd tag in Fragment: $ctrFragStr tokbeg,size=$segEnd,$tmpGzr_len\n");
   }
elsif ( $tmpGzr =~ /\[=GRD=(INVGRD\.[0-9]{3})\]/ ) 
   { $curGrd = $1; 
#     print ("debug: INVGRD tag in Fragment: $ctrFragStr tokbeg,size=$segEnd,$tmpGzr_len\n");
   }
elsif ( $tmpGzr =~ /\[=tf([0-9])\]/ ) 
   { 
     if ($inTF) {$nextTmp .= "<etf$curTF\/>";}
     $curTF = $1;
     $inTF  =  1;
     $flgTF =  1;
     if ( $curGrd ) {$nextTmp .= "<tf$curTF grd=\"$curGrd\"\/>";
                     $curGrd = "";
                     $statTFGRD_sec++; } #**stats**
     else           {$nextTmp .= "<tf$curTF\/>";}
#     print ("debug: typeface tag in Fragment: $ctrFragStr tokbeg,size=$segEnd,$tmpGzr_len\n");
#     print ($nextTmp."\n");
   }
#-- and ($curTF) in following is a workaround - leading tf's still ignored - see t26a,title III,Rule 21 (a)
elsif ( ($tmpGzr =~ /\[=egr\]/) and ($curTF) ) 
   { 
   $nextTmp .= "<etf$curTF\/>";
   $inTF = 0;
#   print ("\ndebug: endgrid tag in Fragment: $ctrFragStr tokbeg,size=$segEnd,$tmpGzr_len\n"); 
#   print ($nextTmp."\n");
   }
elsif ( ($inTF) && ($tmpGzr =~ /\[=boxhead CD=([0-9])\]/) ) 
   { 
   $nextTmp .= "<etf$curTF>[=boxhead CD=$1]";
   $inTF = 0;
   }

else { $nextTmp .= $tmpGzr; }

}

if ( $flgTF ) {  #-- if there was any TF activity, cleanup and update nextOut
$statTF_sec++; #**stats**
$nextTmp .= substr($nextOut,$segBeg);  #-- add the rest of the record
#-- and in closing --
if ( $inTF  ) {  #-- finish any active TF (nesting properly in open containers
     $inTF=0;
     if (    $nextTmp =~ s/<\/head>/<etf$curTF\/><\/head>/g ) {}
#     elsif ( $nextTmp =~ s/<\/head>/<etf$curTF\/><\/head>/ ) {}
#     elsif ( $nextTmp =~ s/<\/head>/<etf$curTF\/><\/head>/g ) {}
     elsif ( $nextTmp =~ s/<\/leader>$/<etf$curTF\/><\/leader>/ ) {}
     elsif ( $nextTmp =~ s/<\/text>$/<etf$curTF\/><\/text>/ ) {}
     else     { $nextTmp   .= "<etf$curTF\/>"; }
#   print ("\ndebug: TF EOL in Fragment: $ctrFragStr tokbeg,size=$segEnd,$tmpGzr_len\n"); 
#     print ($nextTmp."\n");
}
#-- replace all of nextOut with our working buffer
$nextOut = $nextTmp;
}

#-- dbl quote substituton switchback -
   $nextOut =~ s/\{/\[=ldq\]/g;
   $nextOut =~ s/\}/\[=rdq\]/g;
if (
   $nextOut =~ /\[=tf[0-9]\]/g ) {
#-- the following needs to be turned into a toc...
 print ("debug: typeface still found in Fragment: $ctrFragStr\n");
}
}

}
#---------------------------------------------------

#---------------------------------------------------
sub outRecord($){
#---------------------------------------------------
#-- final cleanup and output
   our $nextOut = shift;
#-----------------------
#-- format changes (should be PI's?) - just drop for now
   $nextOut =~ s/\[=Format.+?\]//g;

#-----------------------
#-- misc
   $nextOut =~ s/\[=newrec\]//g;
   $nextOut =~ s/\&/\&amp;/g;
   $nextOut =~ s/\[=copy\]/&copy;/g;
   $nextOut =~ s/\[=reg\]/&reg;/g;
   $nextOut =~ s/\[=deg\]/&deg;/g;
   $nextOut =~ s/\[=ts\]/&thinsp;/g;
   $nextOut =~ s/\[=st\]/&sect;/g; #-- should get or start an entity table for these
   $nextOut =~ s/\[=md\]/&mdash;/g;
   $nextOut =~ s/\[=nd\]/&ndash;/g;
   $nextOut =~ s/\[=ensp\]/&ensp;/g;
   $nextOut =~ s/\[=emsp\]/&emsp;/g;
   $nextOut =~ s/\[=Rcode CD=([0-9\.]{5})\]//g;
   #-- temp. debug mode - use same quote characters fore and aft --
   $nextOut =~ s/\[=ldq\]/&ldquo;/g;
   $nextOut =~ s/\[=rdq\]/&rdquo;/g;
   $nextOut =~ s/\[=lsq\]/&lsquo;/g;
   $nextOut =~ s/\[=rsq\]/&rsquo;/g;
   $nextOut =~ s/\[=shy\]/&shy;/g;
   $nextOut =~ s/\[=vSp SP=([0-9]{2})\]/\n<etc vertspace=\"$1\"><\/etc>/g;

#-- remaining bells go to etc.
#   $nextOut =~ s/\[=bell-([a-zA-Z])\]/<etc bellcd=\"$1\"><\/etc>/g;

#-- remaining gezers (not still bells, above) go to etc.
#   $nextOut =~ s/\[=([a-zA-Z0-9\ \=\.]+)\]/<etc gzr=\"$1\"><\/etc>/g;

#-- final cleanup of geezer codes
   $nextOut =~ s/\[=.+?\]//g;

chomp($nextOut);
print(XMLOUT $nextOut );

}
#---------------------------------------------------

###### Analysis pages ##############################
#---------------------------------------------------
sub startLBLPage{
#---------------------------------------------------
###### NB: Analysis pages are put over in the geezer set ###
open (HTMLBL, "> ../geezer/uscode".$titlenum."\/T".$titlenum."LBL.HTML") || die("\ncan't open output file\n");
print(HTMLBL "<HTML>\n<HEAD><TITLE>T".$titlenum."LBL</TITLE>\n");
print(HTMLBL q~
<script>
function pager(url){
window.opener.location=url
}
</script>
~);
print(HTMLBL "</HEAD>\n<BODY>\nT".$titlenum."LBL\n");
print(HTMLBL "---".localtime(time())."\n");
print(HTMLBL "<FONT COLOR=\"PURPLE\" SIZE=+1>Flags on Labels, etc.<BR><HR><\/FONT>\n"); 
}
#---------------------------------------------------

#---------------------------------------------------
sub addLBLEntry($){
#---------------------------------------------------
###### NB: Analysis pages are put over in the geezer set ###
my $ctrFragStr = sprintf("%05d",$ctrFrag);
my $nextEntry = shift;
   $nextEntry =~ s/\[[a-zA-Z0-9\ \=\.]+\]//g;  #-- discard all other tags
my $fnp = "T".$titlenum."F".$ctrFragStr.".HTML";
print(HTMLBL "<BR><A NAME=\"$ctrFragStr\" HREF=\"#".$ctrFragStr."\" onClick=\"pager(\'".$fnp."\')\" onFocus=\"pager(\'".$fnp."\')\"><FONT SIZE=-1>".$nextEntry."<\/FONT></A>\n");
}
#---------------------------------------------------


#*******************
#---------------------------------------------------
sub sql_summary{
#---------------------------------------------------
#-- mySQL ----------------------
# print ("debug: update DB: $statHDSUP_tot $statSEC_tot $statPSEC_tot $statHDSUB_tot $statTable_tot $statLblCl_tot $statFN_tot\n");
  my $table="title_summary";
  my $tmp1 = "\'".$titlenum."\'";
  my $tmp2 = "\'".$title_head."\'";
  $tmp2 =~ s/\[=md\]/\ \-\ /g;         #-- translate mdash
  $tmp2 =~ s/\[=rsq\]/\'\'/g;          #-- [=rsq] to std. single quote
  $tmp2 =~ s/\[=ensp\]/\ /g;           #-- [=ensp] to std. blank
  $tmp2 =~ s/\[=.*?\]/\ /g;            #-- erase all others
  my $tmp3 = "\'".$statHDSUP_tot."\'";
  my $tmp4 = "\'".$statSEC_tot."\'";
  my $tmp5 = "\'".$statPSEC_tot."\'";
  my $tmp6 = "\'".$statTable_tot."\'";
  my $tmp7 = "\'".$statFN_tot."\'";
  my $tmp8 = "\'".$statLblCl_tot."\'";
  my $sec=0;
  my $min=0;
  my $hr=0;
  my $mday=0;
  my $mth=0;
  my $year=0;
  my $wday=0;
  my $yday=0;
  my $isdst=0;
($sec, $min, $hr, $mday, $mth , $year, $wday, $yday, $isdst) = localtime($inFileTime);
  my $tmp9 = sprintf("\'%04d-%02d-%02d %02d:%02d:%02d\'", $year+1900, $mth+1, $mday, $hr, $min, $sec );

($sec, $min, $hr, $mday, $mth , $year, $wday, $yday, $isdst) = localtime($startTime);
  my $tmp10 = sprintf("\'%04d-%02d-%02d %02d:%02d:%02d\'", $year+1900, $mth+1, $mday, $hr, $min, $sec );

  my $tmpTime = time() - $startTime;
  my $tmpHr  = int($tmpTime/3600);
     $tmpTime -= $tmpHr*3600;
  my $tmpMin = int($tmpTime/60);
     $tmpTime -= $tmpMin*60;
  my $tmpSec = $tmpTime;
  my $tmp11 = sprintf("\'%02d:%02d:%02d\'", $tmpHr, $tmpMin, $tmpSec);

  my $no_update = 0;
  my $udp_ret_status = $dbh->do("update $table set titlenam=$tmp2, cnt_supsec=$tmp3, cnt_sec=$tmp4, cnt_subsec=$tmp5, cnt_table=$tmp6, cnt_fn=$tmp7, source_dtime=$tmp9, source_bytes=$inFileSize, proc_start=$tmp10, proc_delta=$tmp11 where titlenum=$tmp1" ) or $no_update=1;
  print("debug: update return: $udp_ret_status\n");
  if ($udp_ret_status != 1) {
  print("debug: update didn't work, trying insert\n");
  $dbh->do("insert into $table (titlenum, titlenam, cnt_supsec, cnt_sec, cnt_subsec, cnt_table, cnt_fn, source_dtime, source_bytes, proc_start, proc_delta) values ($tmp1, $tmp2, $tmp3,  $tmp4, $tmp5, $tmp6, $tmp7, $tmp9, $inFileSize, $tmp10, $tmp11 )") or die $DBI::errstr;
  }
}
#*******************


#*******************
#---------------------------------------------------
sub sql_supsecmap(@){
#---------------------------------------------------
#-- mySQL ----------------------
#-- Table supsecmap holds relations needed for convenient mapping between external IDs for super-sections and internal fragment IDs
  my $table="supsecmap";
  my $frag_id     = shift;
  my $norm_ext_id = shift;
  my $head_txt    = shift;
#-- fragment identifier (currently of the form T01F00002, but may change)
  my $tmp1 = "\'".$frag_id."\'";
#-- normallized external identification information
  my $tmp2 = "\'".$norm_ext_id."\'";
#-- heading for the supsec
  my $tmp3 = $head_txt;
  $tmp3 =~ s/\[=md\]/\ \-\ /g;         #-- translate mdash
  $tmp3 =~ s/\[=nd\]/\-/g;             #-- translate ndash
  $tmp3 =~ s/\[=rsq\]/\'\'/g;          #-- [=rsq] to std. single quote
  $tmp3 =~ s/\[=ensp\]/\ /g;           #-- [=ensp] to std. blank
  $tmp3 =~ s/\[=.*?\]//g;              #-- erase all others
#  $tmp3 =~ s/$\ *//;                  #-- trim leading blanks
  $tmp3 = "\'".$tmp3."\'";

  print ("debug: update DB: $tmp1 $tmp2 $tmp3\n");

#-- if $norm_ext_id is already here, update, else insert
  my $no_update = 0;
  my $udp_ret_status = $dbh->do("update $table set frag_id=$tmp1, head_txt=$tmp3 where norm_ext_id=$tmp2" ) or $no_update=1;
  print("debug: update return: $udp_ret_status\n");
  if ($udp_ret_status != 1) { print("debug: update didn't work, trying insert\n");
  $dbh->do("insert into $table (frag_id, norm_ext_id, head_txt) values ($tmp1, $tmp2, $tmp3)") or die $DBI::errstr;
  }
}
#*******************


#*******************
#-- NSGMLS --
#---------------------------------------------------
sub spout($){
#---------------------------------------------------
my $xml_in = shift;
#-- use -s to suppress warning level diagnostics
my $spout = `nsgmls -s $xml_in 2>&1`;
if ($spout) { 
	my @spouts = split(/\n/, $spout);
	foreach my $spoutone (@spouts) {
	#-- don't worry about the old favorite "is not a function name"
	unless ($spoutone =~ /is\ not\ a\ function\ name/) { 
	print (NSGMLS_LOG $spoutone."\n"); 
	}
	}
	}
}
#---------------------------------------------------
sub sp_summary($){
#---------------------------------------------------
my $cntSP = shift;
print (NSGMLS_LOG "total of $cntSP XML files checked for $inFileName\n"); 
}
#*******************

sub helpout(){
  print( q~
  Command line:
  perl emely.pl [switches] <start_num> [<end_num>]
  or
  perl emely.pl -file=<path><filename> <title number>

  switches
  -db database interaction
  -sp nsgmls checking
  -la label analysis

  eg 0: to process local file f.red as title 50a
  perl emely.pl -file=f.red 50a
 
  examples 1 to 4 assuming source data of the form 
  ../data/uscode01.bell (for title number 1)

  eg 1: to process title 11 and 11a
  perl emely.pl 11 
  eg 2: to process all titles, including appendices
  perl emely.pl 1 50
  eg 3: to process title 50, appendix only
  perl emely.pl -ao 50
  eg 4: to process title 50 without appendix
  perl emely.pl -an 50
  eg 5: to process all titles in ../dir123/source
        with file names like xyz01.dat
        use file prefix and suffix parameters
  perl emely.pl -fip../dir123/source/xyz -fis.dat 1 50
~);
  print("  default fileprefix: \"$fileprefix\"\n");
  print("  default filesuffix: \"$filesuffix\"\n");
  exit; 
   }


1;
__END__;
