- added CLI option "--continue"

- improved transactions around calls of write_addsql() for each IE

perl/exit_strategy.pl

@@ -43,6 +43,7 @@ no warnings "experimental::signatures";
 use utf8; # for debugging output
 use Carp;
 use Path::Tiny;
+use File::Basename qw(basename);
 use File::Find;
 use XML::LibXML;
 use Time::Progress;
@@ -79,8 +80,12 @@ SQL_PRAGMA_WAL
   my $sql2=<<"SQL_PRAGMA_ASYNC";
 PRAGMA synchronous = OFF;
 SQL_PRAGMA_ASYNC
+  my $sql3=<<"SQL_PRAGMA_TEMP";
+PRAGMA temp_store = OFF;
+SQL_PRAGMA_TEMP
   my $sth1 = $dbh->prepare($sql1); $sth1->execute() or die "sql problem detected", $dbh->errstr;
   my $sth2 = $dbh->prepare($sql2); $sth2->execute() or die "sql problem detected", $dbh->errstr;
+  my $sth3 = $dbh->prepare($sql3); $sth3->execute() or die "sql problem detected", $dbh->errstr;
   return;
 }
 
@@ -212,21 +217,21 @@ SQL_DC_PLAN
 #     $ret{"dcrecords"} = \@dcrecords;
 ###############################################################################
 sub write_addsql ($dbh, $plans, $refhash) {
-  my $iefile = path($refhash->{"filename"})->basename();
+  my $iefile = basename($refhash->{"filename"});
   my ($ieid,$ieversion);
   if ($iefile =~ m/^V(\d+)-(IE\d*)\.xml$/) {
     $ieversion=$1; $ieid=$2;
   } else {
     die "Could not detect PID and Version from IEFile '$iefile'\n";
   }
-  $dbh->begin_work;
+
   # start SQL insert
   $plans->{aip}->execute($ieid, $ieversion)  or die "sql problem detected", $dbh->errstr;
   # FIXME if multiple locations exists
 
   $plans->{ie}->execute( $ieid, $ieversion, $iefile, $sourcetype)  or die "sql problem detected", $dbh->errstr;
   foreach my $location (@{$refhash->{"files"}}) {
-    my $file = path($location)->basename(); # FIXME if multiple locations
+    my $file = basename($location); # FIXME if multiple locations
     $plans->{file}->execute($ieid, $ieversion, $file)  or die "sql problem detected", $dbh->errstr;
     $plans->{locat}->execute($ieid, $ieversion, $file, $location, $sourcetype)  or die "sql problem detected", $dbh->errstr;
   }
@@ -236,7 +241,6 @@ sub write_addsql ($dbh, $plans, $refhash) {
     $dcvalue=~tr/'/"/;
     $plans->{dc}->execute($ieid, $ieversion, $dckey, $dcvalue)  or die "sql problem detected", $dbh->errstr;
   }
-  $dbh->commit;
   return 1;
 }
 
@@ -477,16 +481,18 @@ my $tmp_ies_dir = Path::Tiny->tempdir(TEMPLATE => "exitstrategy_XXXXXXXXXXX", CL
 my $tmp_ies_unsorted_file = $tmp_ies_dir->child("unsorted_ies");
 my $flag_recovery = undef;
 my $flag_sqldump = undef;
+my $flag_continue = undef;
 my $db_filename = $db_name . ".db";
 my @ARGV_tail;
 GetOptions(
     "help|?"          => sub {
       say <<"HELP";
 call $0 with following options
-        --help | this help
-        --recovery | set special recovery flag
-        --sqlitedb-file=FILE | set database to file FILE
-        --enable-sqldump | dumps a given database as SQL to STDOUT
+        --help ............... this help
+        --recovery ........... set special recovery flag
+        --continue ........... tries to add IEs to existing database, ignores IEs which already exists (dangerous!)
+        --sqlitedb-file=FILE . set database to file FILE
+        --enable-sqldump ..... dumps a given database as SQL to STDOUT
 
         create an exit-DB:
         $0 [--recovery] [--sqlitedb-file=FILE] DIR [DIR…]
@@ -496,6 +502,7 @@ call $0 with following options
 HELP
       exit;
     },
+    "continue"        => \$flag_continue,
     "recovery"        => \$flag_recovery,
     "sqlitedb-file=s" => \$db_filename,
     "enable-sqldump"  => \$flag_sqldump,
@@ -512,17 +519,23 @@ if ($#ARGV_tail < 0) {
   die "you need at least a directory as argument\n";
 }
 
-say "preparing SQL";
-my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
-    RaiseError     => 1,
-    sqlite_unicode => 1,
-}) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
-write_database_creation($dbh);
-write_tables_creation($dbh);
-write_prepare_insert($dbh);
-write_index_creation($dbh);
-$dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
-
+if (
+    (! defined $flag_continue)
+        || (! -e $db_filename)
+) {
+  say "preparing SQL";
+  my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
+      RaiseError     => 1,
+      sqlite_unicode => 1,
+  }) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
+  write_database_creation($dbh);
+  write_tables_creation($dbh);
+  write_prepare_insert($dbh);
+  write_index_creation($dbh);
+  $dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
+} else {
+  say "using existing DB (continue mode!)";
+}
 if (0==@ARGV_tail ){
     die "no directory given on commandline"
 }
@@ -541,7 +554,7 @@ my $cnt_unsorted_files = searching_ie_files(\@dirs, $tmp_ies_unsorted_file);
 my $fh_unsorted_IEs = $tmp_ies_unsorted_file->openr();
 my $count = 0;
 my $progressbar = Time::Progress->new(min => 0, max => $cnt_unsorted_files, smoothing => 1);
-$dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
+my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
     RaiseError     => 1,
     sqlite_unicode => 1,
 }) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
@@ -552,8 +565,18 @@ while (<$fh_unsorted_IEs>) {
   print $progressbar->report("parse IE files:       %40b  ETA: %E   \r", $count++);
   s/V0*(\d+-IE)/V$1/; # revert fake version
   my $ret = parse_iexml($_, $flag_recovery);
-  write_addsql($dbh, $plans, $ret);
+  $dbh->begin_work;
+  eval {
+    write_addsql($dbh, $plans, $ret);
+  };
+  if ($@) {
+    $dbh->rollback;
+    say "Rollback, because $@";
+  } else {
+    $dbh->commit;
+  }
 }
+
 $dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
 say "\rprocessed $count uniq IEs                                                                                      ";
 say "";