- changed to use SQLite

perl/exit_strategy.pl

@@ -51,6 +51,8 @@ use File::Sort qw(sort_file);
 use XML::LibXML;
 use Time::Progress;
 use XML::LibXML::XPathContext;
+use DBD::SQLite;
+use DBI;
 use Getopt::Long;
 use constant DEBUG => 0; # no debug
 
@@ -78,83 +80,68 @@ sub write_database_creation ($fh) {
 }
 
 # write tables creation;:
-sub write_tables_creation ($fh) {
-  # Transactions for tables creation
-  say $fh "BEGIN;";
-
-  # SEQUENCE
-  say $fh "/* create SEQUENCE generator */";
-  say $fh "CREATE SEQUENCE serial START 1;";
+sub write_tables_creation ($dbh) {
+  my $sql1=<<"SQL_CREATE1";
+  /* create AIP table */
+  CREATE TABLE aip (
+	id	INTEGER,
+	ie_id	TEXT NOT NULL UNIQUE,
+	PRIMARY KEY(id AUTOINCREMENT)
+  );
+SQL_CREATE1
+  my $sql2=<<"SQL_CREATE2";
+  /* create IEFILE table */
+  CREATE TABLE metadatafile (
+    id INTEGER,
+    aip_id INTEGER NOT NULL REFERENCES aip (id),
+    location TEXT NOT NULL,
+    sourcetype TEXT NOT NULL,
+    PRIMARY KEY(id AUTOINCREMENT)
+  );
+SQL_CREATE2
+  my $sql3=<<"SQL_CREATE3";
+  /* create DC table */
+  CREATE TABLE dc (
+    id INTEGER,
+    aip_id INTEGER NOT NULL REFERENCES aip (id),
+    element TEXT NOT NULL,
+    value TEXT NOT NULL,
+    PRIMARY KEY(id AUTOINCREMENT)
+  );
+SQL_CREATE3
+  my $sql4=<<"SQL_CREATE4";
+  /* create FILE table */
+  CREATE TABLE sourcedatafile (
+    id INTEGER,
+    aip_id INTEGER NOT NULL REFERENCES aip (id),
+    name TEXT NOT NULL,
+    PRIMARY KEY(id AUTOINCREMENT)
+  );
+SQL_CREATE4
+  my $sql5=<<"SQL_CREATE5";
+  /* create LOCAT table */
+  CREATE TABLE sourcedatalocat (
+    id INTEGER,
+    file_id INTEGER NOT NULL REFERENCES sourcedatafile (id),
+    location TEXT NOT NULL,
+    sourcetype TEXT NOT NULL,
+    PRIMARY KEY(id AUTOINCREMENT)
+  );
+SQL_CREATE5
+  my $sth1 = $dbh->prepare($sql1); $sth1->execute() or die "sql problem detected", $dbh->errstr;
+  my $sth2 = $dbh->prepare($sql2); $sth2->execute() or die "sql problem detected", $dbh->errstr;
+  my $sth3 = $dbh->prepare($sql3); $sth3->execute() or die "sql problem detected", $dbh->errstr;
+  my $sth4 = $dbh->prepare($sql4); $sth4->execute() or die "sql problem detected", $dbh->errstr;
+  my $sth5 = $dbh->prepare($sql5); $sth5->execute() or die "sql problem detected", $dbh->errstr;
 
-  # AIP
-  say $fh "/* create AIP table */";
-  say $fh "CREATE TABLE aip (";
-  say $fh "\tid INT PRIMARY KEY DEFAULT nextval('serial'),";
-  say $fh "\tie_id VARCHAR(30) NOT NULL UNIQUE";
-  say $fh ");";
-  # IEFILE
-  say $fh "/* create IEFILE table */";
-  say $fh "CREATE TABLE metadatafile (";
-  say $fh "\tid INT PRIMARY KEY DEFAULT nextval('serial'),";
-  say $fh "\taip_id INT NOT NULL REFERENCES aip (id),";
-  say $fh "\tlocation VARCHAR(1024) NOT NULL,";
-  say $fh "\tsourcetype VARCHAR(30) NOT NULL";
-  say $fh ");";
-  # DC
-  say $fh "/* create DC table */";
-  say $fh "CREATE TABLE dc (";
-  say $fh "\tid INT PRIMARY KEY DEFAULT nextval('serial'),";
-  say $fh "\taip_id INT NOT NULL REFERENCES aip (id),";
-  say $fh "\telement VARCHAR(30) NOT NULL,";
-  say $fh "\tvalue VARCHAR(8192) NOT NULL";
-  say $fh ");";
-  # FILE
-  say $fh "/* create FILE table */";
-  say $fh "CREATE TABLE sourcedatafile (";
-  say $fh "\tid INT PRIMARY KEY DEFAULT nextval('serial'), ";
-  say $fh "\taip_id INT NOT NULL REFERENCES aip (id),";
-  say $fh "\tname VARCHAR(1024) NOT NULL";
-  say $fh ");";
-  # LOCAT
-  say $fh "/* create LOCAT table */";
-  say $fh "CREATE TABLE sourcedatalocat (";
-  say $fh "\tid INT PRIMARY KEY DEFAULT nextval('serial'),";
-  say $fh "\tfile_id INT NOT NULL REFERENCES sourcedatafile (id),";
-  say $fh "\tlocation VARCHAR(1024) NOT NULL,";
-  say $fh "\tsourcetype VARCHAR(30) NOT NULL";
-  say $fh ");";
-  #end transaction
-  say $fh "COMMIT;";
-  return;
+  return 1;
 }
 
 ###############################################################################
 # Prepare SQL INSERT Statements for AIPs
 ###############################################################################
-sub write_prepare_insert ($fh) {
-  say $fh "BEGIN;";
-  say $fh "PREPARE aip_plan (varchar) AS";
-  say $fh "  INSERT INTO aip (ie_id) VALUES (\$1);";
-  say $fh "PREPARE ie_plan (varchar, varchar, varchar) AS";
-  say $fh "  INSERT INTO metadatafile (aip_id, location, sourcetype) VALUES (";
-  say $fh "    (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2, \$3";
-  say $fh "  );";
-  say $fh "PREPARE file_plan (varchar, varchar) AS";
-  say $fh "  INSERT INTO sourcedatafile (aip_id, name) VALUES (";
-  say $fh "    (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2";
-  say $fh "  );";
-  say $fh "PREPARE locat_plan (varchar, varchar, varchar, varchar) AS";
-  say $fh "  INSERT INTO sourcedatalocat (file_id, location, sourcetype) VALUES (";
-  say $fh "    (SELECT sourcedatafile.id FROM sourcedatafile,aip WHERE";
-  say $fh "    sourcedatafile.aip_id=aip.id AND aip.ie_id=\$1 AND";
-  say $fh "    sourcedatafile.name=\$2), \$3, \$4";
-  say $fh "  );";
-  say $fh "PREPARE dc_plan (varchar, varchar, varchar) AS";
-  say $fh "  INSERT INTO dc (aip_id, element, value) VALUES (";
-  say $fh "    (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2, \$3";
-  say $fh "  );";
-  say $fh "COMMIT;";
-  return;
+sub write_prepare_insert ($dbh) {
+  return 1;
 }
 
 
@@ -179,27 +166,55 @@ sub write_prepare_insert ($fh) {
 #     $ret{"files"} = \@files;
 #     $ret{"dcrecords"} = \@dcrecords;
 ###############################################################################
-sub write_addsql ($fh, $refhash) {
+sub write_addsql ($dbh, $refhash) {
   my $ieid = path($refhash->{"filename"})->basename(qw/.xml/);
-  say $fh "BEGIN;";
-  say $fh "EXECUTE aip_plan ('$ieid');";
+  my $sql_aip_plan=<<"SQL_AIP_PLAN";
+  INSERT INTO aip (ie_id) VALUES (\$1);
+SQL_AIP_PLAN
+  my $sql_ie_plan=<<"SQL_IE_PLAN";
+  INSERT INTO metadatafile (aip_id, location, sourcetype) VALUES (
+      (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2, \$3
+    );
+SQL_IE_PLAN
+  my $sql_file_plan=<<"SQL_FILE_PLAN";
+    INSERT INTO sourcedatafile (aip_id, name) VALUES (
+      (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2
+    );
+SQL_FILE_PLAN
+  my $sql_locat_plan=<<"SQL_LOCAT_PLAN";
+    INSERT INTO sourcedatalocat (file_id, location, sourcetype) VALUES (
+      (SELECT sourcedatafile.id FROM sourcedatafile,aip WHERE
+      sourcedatafile.aip_id=aip.id AND aip.ie_id=\$1 AND
+      sourcedatafile.name=\$2), \$3, \$4
+    );
+SQL_LOCAT_PLAN
+  my $sql_dc_pan=<<"SQL_DC_PLAN";
+    INSERT INTO dc (aip_id, element, value) VALUES (
+      (SELECT id FROM aip WHERE aip.ie_id=\$1), \$2, \$3
+    );
+SQL_DC_PLAN
+  my $sth_aip_plan = $dbh->prepare($sql_aip_plan);
+  my $sth_ie_plan = $dbh->prepare($sql_ie_plan);
+  my $sth_file_plan = $dbh->prepare($sql_file_plan);
+  my $sth_locat_plan = $dbh->prepare($sql_locat_plan);
+  my $sth_dc_plan = $dbh->prepare($sql_dc_pan);
+  # start SQL insert
+  $sth_aip_plan->execute($ieid)  or die "sql problem detected", $dbh->errstr;
   # FIXME if multiple locations exists
   my $iefile = path($refhash->{"filename"})->basename();
-  say $fh "EXECUTE ie_plan ('$ieid', '$iefile', '$sourcetype');";
+  $sth_ie_plan->execute( $ieid, $iefile, $sourcetype)  or die "sql problem detected", $dbh->errstr;
   foreach my $location (@{$refhash->{"files"}}) {
     my $file = path($location)->basename(); # FIXME if multiple locations
-    say $fh "EXECUTE file_plan ('$ieid', '$file');";
-    say $fh "EXECUTE locat_plan ('$ieid', '$file', '$location', '$sourcetype' );";
+    $sth_file_plan->execute($ieid, $file)  or die "sql problem detected", $dbh->errstr;
+    $sth_locat_plan->execute($ieid, $file, $location, $sourcetype)  or die "sql problem detected", $dbh->errstr;
   }
   foreach my $dcpair   (@{$refhash->{"dcrecords"}}) {
     my ($dckey,$dcvalue) = @{$dcpair};
     # quote ' in dcvalue
     $dcvalue=~tr/'/"/;
-    say $fh "EXECUTE dc_plan ( '$ieid', '$dckey', '$dcvalue');";
+    $sth_dc_plan->execute($ieid, $dckey, $dcvalue)  or die "sql problem detected", $dbh->errstr;
   }
-  say $fh "COMMIT;";
-  say $fh "\n";
-  return;
+  return 1;
 }
 
 
@@ -207,11 +222,15 @@ sub write_addsql ($fh, $refhash) {
 ###############################################################################
 # add INDEX and other TRICKs to increase performance
 ###############################################################################
-sub write_index_creation($fh) {
-  say $fh "-- BEGIN;";
-  say $fh "-- CREATE UNIQUE INDEX aip_index on aip (ie_id);";
-  say $fh "-- COMMIT;";
-  return;
+sub write_index_creation($dbh) {
+  my $sql=<<"SQL_INDEX";
+  -- BEGIN;
+  -- CREATE UNIQUE INDEX aip_index on aip (ie_id);
+  -- COMMIT;
+SQL_INDEX
+  my $sth = $dbh->prepare($sql);
+  $sth->execute() or die "sql problem detected", $dbh->errstr;
+  return 1;
 }
 
 ###############################################################################
@@ -456,18 +475,21 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) {
 ############# main ############################################################
 ###############################################################################
 ###############################################################################
-  my $recovery = undef;
+  my $flag_recovery = undef;
+  my $flag_sqldump = undef;
+  my $db_filename = $db_name.".db";
   my @ARGV_tail;
   GetOptions(
-      "recovery" => \$recovery,
+      "recovery" => \$flag_recovery,
+      "sqlitedb-file=s" => \$db_filename,
+      "enable_sqldump" => \$flag_sqldump,
       '<>' => sub {push @ARGV_tail, @_;}
   );
-  if ($#ARGV_tail != 1) {
-      die "you need  a SQL-file and a directory as argument\n";
+  if ($#ARGV_tail < 0) {
+      die "you need  a directory as argument\n";
   }
-  if (defined $recovery) { warn "recovery enabled for XML processing\n"; }
-  my $sqlfile = shift @ARGV_tail;
-  if($sqlfile !~ m/[A-Za-z0-9]+\.sql$/) {die "SQL file should be named like 'foo.sql', but was '$sqlfile'\n";}
+  if (defined $flag_recovery) { warn "recovery enabled for XML processing\n"; }
+
   my $dir = shift @ARGV_tail;
 
   if (defined $dir && -d "$dir") {
@@ -487,20 +509,20 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) {
     my $fh_truncated_IEs = $tmp_ies_truncated_file->openr();
     my $count=0;
     my $progressbar =Time::Progress->new(min => 0, max => $cnt_truncated_files, smoothing => 1);
-    open(my $fh, ">:encoding(UTF-8)", "$sqlfile") || die "could not open file '$sqlfile' for writing, $!";
-    write_database_creation($fh);
-    write_tables_creation($fh);
-    write_prepare_insert($fh);
+    my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "") or die "could not connect to database (file '$db_filename')", $DBI::errstr;
+    write_database_creation($dbh);
+    write_tables_creation($dbh);
+    write_prepare_insert($dbh);
     while( <$fh_truncated_IEs> ) {
       chomp;
       print $progressbar->report("parse IE files:       %40b  ETA: %E   \r", $count++);
       s/V0*(\d+-IE)/V$1/; # revert fake version
-      my $ret = parse_iexml($_, $recovery);
-      write_addsql($fh, $ret);
+      my $ret = parse_iexml($_, $flag_recovery);
+      write_addsql($dbh, $ret);
     }
     say "";
-    write_index_creation($fh);
-    close ($fh);
+    write_index_creation($dbh);
+    $dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
     say "processed $count uniq IEs";
   } else {
     die "no directory given on commandline"