kewensis-collect.pl

   1 #! /usr/bin/perl
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd;
   7 use Data::Dumper;
   8 use DBI;
   9
  10 my(%DB,%OWNS,$debugmatch,$D,$key,$owner,$ref,$filename,$doimport,$import_xlate);
  11
  12 %DB=();
  13 $D=0;
  14 $debugmatch=0;
  15 $doimport=1;
  16 $import_xlate=1;
  17
  18 sub extract_year
  19 {
  20         ($_)=@_;
  21         if (defined($_)) {
  22                 s/\(([0-9]{4}) publ. [0-9]{4}\)/($1)/g;
  23                 while (/\(([0-9]{4})(-[0-9]+)?\)/) {
  24                         return $1 if ($1>=1700 && $1<=2010);
  25                         $_=$';
  26                         }
  27                 }
  28         return -1;
  29 }
  30
  31 sub failed
  32 {
  33 my($file)=@_;
  34
  35         print("-$file----------------$'--------------\n");
  36         warn "Unable to match file \"".getcwd()."/$file\".";
  37 }
  38
  39 sub process_file
  40 {
  41 my($file)=@_;
  42
  43         if (!open(FI,$file)) {
  44                 warn("Unable to open file \"$file\" in dir \"".getcwd()."\": $!");
  45                 return;
  46                 }
  47         undef $/;
  48         my($fi)=<FI>;
  49         close(FI);
  50         my($word)='[^<]*';
  51   my($bigword)='[^<]*(?:<IT>)?[^<]*(?:<RO>)?[^<]*(?:<1000 m)?[^<]*';
  52         my($idone)='[-\d]';
  53         my($id)="$idone+";
  54         my($any)='[\x00-\xFF]*';
  55         my($ipniservletword)="<a href=\"\\./IpniServlet\\?id=($id)&query_type=by_id\">($word)</a>";
  56         my($idquoted)=$id; $idquoted=~s%\W%\\$&%g;
  57         my($ipniservletwordthree)=$ipniservletword; $ipniservletwordthree=~s%\($idquoted\)%((?:$idone){3})$&%os or die;
  58         my($attrpat)="<p>nomenclatural synonym(\\(Main Record\\))?:$ipniservletword</p>|<p>basionym(\\(\\d+\\))?:$ipniservletword</p>|<p>basionym:($word)|<p>replaced synonym:$ipniservletword</p>|<p>replaced synonym:($word)|<p>Is a replaced synonym of:$ipniservletword</p>|<p>Is a basionym of:$ipniservletword</p>|<p>later publication of(\\(\\d+\\))?:$ipniservletword</p>|<p>Is a later publication of of:$ipniservletword</p>";
  59         $|=1;
  60         "<undef>"=~/^/;
  61         if ($fi!~m%^<html>
  62 <head>
  63 <title>IPNI Query Results</title>
  64 </head>
  65 <body bgcolor="#ffffff" text="#000000" link="#006666" vlink="#008080" alink="#008080">
  66 <HR><b><i>Orchidaceae</i> ($word)</b> ($word) <br>
  67 ((?:<a href="\./PublicationServlet\?id=($id)&query_type=by_id"> ($word)</a> ($word)|$bigword(?:<br>\n$word)*)?<p>($bigword)?</p><p>($word)?</p>(?:
  68 remarks: .*)?(<HR><h4>Type</h4>)?(<table $any</table>)?(?:
  69 <h4>Linked Records</h2>
  70 ((?:$attrpat)*))?(<br>
  71
  72 <h4>Original Data</h2>
  73 (?:basionym: ($word)<br>
  74 )?(?:hybrid parentage: ($word)<br>
  75 )?(?:replaced synonym: ($word)<br>
  76 )?(?:distribution: ($word)<br>
  77 )?(?:Notes: ($word))?)?)(?:<p><a href="\./query_ipni.html">Back to Search Page</a></p>
  78 </body>
  79 </html>)$%os) {
  80                 failed($file);
  81                 return;
  82                 }
  83         my(%rec);
  84         $rec{"name"}=$1;
  85         $rec{"Publ. Author"}=$2;
  86         $rec{"Publication"}="$5 $6" if defined($5) && defined($6);
  87         $rec{"html"}=$3 if defined $3;
  88         my($attrsbody)=$11;
  89         ($rec{"id"}=$file)=~s#^($id)\.html$#$1#os or failed($file);
  90         $rec{"html"}=~s#$ipniservletwordthree#<a href="$1/$1$2.html">$3</a>#osg if $import_xlate && exists $rec{"html"};
  91         my $score=0; # -: upper, +: lower
  92         while (defined($attrsbody) && $attrsbody=~s%^(?:$attrpat)%%os) {
  93                 # nomenclatural synonym: id=$2
  94                 # basionym: id=$5
  95                 # replaced synonym: id=$8
  96                 # Is a replaced synonym: id=$11
  97                 # Is a basionym: id=$13
  98                 # later publication: id=$16
  99                 # Is a later publication: id=$18
 100                 my(@refs)=($2,$5,$8,$11,$13,$16,$18);
 101                 $rec{"refs"}=[];
 102                 while (@refs) {
 103                         push(@{$rec{"refs"}},$_) if defined ($_=shift @refs);
 104                         }
 105                 $score+=-10 if defined  $5; # basionym: id=$5
 106                 $score+=+10 if defined $13; # Is a basionym: id=$13
 107                 $score+=+10 if defined $16; # later publication: id=$16
 108                 $score+=-10 if defined $18; # Is a later publication: id=$18
 109                 }
 110         $rec{"score"}=$score;
 111         if ($attrsbody) {
 112                 failed($file);
 113                 return;
 114                 }
 115         $DB{$rec{"id"}}=\%rec;
 116 }
 117
 118
 119 sub process_dir
 120 {
 121 my( $dir )=@_;
 122 my( $old )=getcwd();
 123
 124         if (!chdir($dir)) {
 125                 warn("Unable to change to dir \"$dir\" from dir \"$old\": $!");
 126                 return;
 127                 }
 128         opendir(DIR,".") or die("Cannot open . in \"".getcwd()."\": $!");
 129         foreach (sort readdir(DIR)) {
 130                 process_entry($_) if ($_!~/^\./);
 131                 }
 132         closedir(DIR);
 133         chdir($old) or warn("Unable to retreat to dir \"$old\": $!");
 134 }
 135
 136 sub process_entry
 137 {
 138 my( $entry )=@_;
 139
 140         process_dir ($entry) if -d $entry;
 141         process_file($entry) if -f $entry;
 142 }
 143
 144 %OWNS=();
 145
 146 foreach (@ARGV)
 147         { process_entry($_); }
 148
 149 my($id);
 150 for $id (keys %DB) {
 151         my($refid);
 152         my(@refs);
 153         for $refid (@{$DB{$id}{"refs"}}) {
 154                 if (!exists $DB{$refid}) {
 155                         warn "Undefined ref id \"$refid\" from id \"$id\"" if $D;
 156                         next;
 157                         }
 158                 next if $id eq $refid;  # self-ref
 159 #               push(@refs,$DB{$refid});
 160                 push(@refs,$refid);
 161                 }
 162         $DB{$id}{"refs"}=\@refs;
 163         }
 164
 165 print Data::Dumper->Dump([\%DB],["%DB"]) if $D;
 166
 167 %OWNS=map { $_=>[] } keys(%DB);
 168
 169 for $id (keys %OWNS) {
 170         next if !exists $OWNS{$id};
 171         for my $refid (@{$DB{$id}{"refs"}}) {
 172                 next if !exists $OWNS{$refid};
 173                 push(@{$OWNS{$id}},$refid,@{$OWNS{$refid}});
 174                 delete $OWNS{$refid};
 175                 print "connected: id=$id,refid=$refid\n" if $D;
 176                 }
 177         }
 178
 179 print Data::Dumper->Dump([\%OWNS],["%OWNS"]) if $D;
 180
 181 foreach $key (keys %OWNS) {
 182         my(@keys)=@{$OWNS{$key}};
 183         delete($OWNS{$key});
 184         unshift(@keys,$key);
 185         @keys=sort { $DB{$a}{"score"} <=> $DB{$b}{"score"}
 186                         || extract_year($DB{$b}{"Publication"}) <=> extract_year($DB{$a}{"Publication"}); } @keys;
 187         my($pkey)=shift(@keys);
 188         $OWNS{$pkey}=\@keys;
 189         }
 190
 191 my($db_driver,$db_host,$db_user,$db_pwd,$DB_PWD,$db_name,$db);
 192 my($tb_tree);
 193
 194 $db_driver="mysql";
 195 $db_host="";
 196 $db_user="short";
 197 $DB_PWD=$ENV{"HOME"}."/priv/mysql.${db_user}.pwd";
 198 $db_name="short";
 199 $tb_tree="kewensis_tree";
 200
 201 $db=DBI->connect("DBI:$db_driver:database=$db_name;host=$db_host",$db_user,$db_pwd) or die "Database open fail: $!";
 202
 203 sub db_do
 204 {
 205 my( $cmd )=@_;
 206
 207         $db->do($cmd) or die("SQL command \"$cmd\" failed: $!");
 208 }
 209
 210 eval { &db_do("drop table $tb_tree") };
 211
 212 &db_do("create table $tb_tree ("
 213                 ."id char(10) not null,"
 214                 ."family_id char(10) not null,"
 215                 ."family_order int not null,"
 216                 ."name varchar(100) not null,"
 217                 ."PublAuthor text null,"
 218                 ."Publication text null,"
 219                 ."Notes text null,"
 220                 ."html text null"
 221                 .")");
 222
 223 &db_do("alter table $tb_tree add unique (id)");
 224 &db_do("alter table $tb_tree add index (name)");
 225 &db_do("alter table $tb_tree add unique (family_id,family_order)");
 226
 227 my $insert_tb_tree=$db->prepare("insert into $tb_tree (id,family_id,family_order,name,PublAuthor,Publication,Notes,html) values (?,?,?,?,?,?,?)")
 228                 or die "Prepare fail: $!";
 229
 230 foreach $owner (sort { $DB{$b}{"name"} cmp $DB{$a}{"name"}; } keys %OWNS) {
 231         my @family=@{$OWNS{$owner}};
 232         unshift(@family,$owner);
 233         my $family_id=$DB{$owner}{"id"};
 234         for my $family_order (0..$#family) {
 235                 my $id=$family[$family_order];
 236                 print "insert:$id,".$DB{$id}{"name"}."\n" if $D;
 237                 $insert_tb_tree->execute($id,$family_id,$family_order,
 238                                 $DB{$id}{"name"},$DB{$id}{"Publ. Author"},$DB{$id}{"Publication"},$DB{$id}{"Notes"},$DB{$id}{"html"}
 239                                 ) or die "SQL insert failure: $!";
 240                 }
 241         }