Skip to content
Snippets Groups Projects
Commit 55142eae authored by Andreas Romeyke's avatar Andreas Romeyke
Browse files

- added support for multiple search dirs

parent 63e18fa0
No related branches found
No related tags found
No related merge requests found
...@@ -394,8 +394,13 @@ sub check_if_db_conform ($string, $filename) { ...@@ -394,8 +394,13 @@ sub check_if_db_conform ($string, $filename) {
} }
} }
sub searching_ie_files ($dir, $tmp_ies_unsorted_file) { sub searching_ie_files ($dirs_ref, $tmp_ies_unsorted_file) {
my $cnt_unsorted_files = 0; my $cnt_unsorted_files = 0;
my $pass = 0;
my @dirs = @{ $dirs_ref };
my $maxpass=scalar @dirs;
foreach my $dir (@dirs) {
$pass++;
my $first_two_levels_of_dirs = 0; my $first_two_levels_of_dirs = 0;
### ###
my $wanted_twolevel_dircount = sub { my $wanted_twolevel_dircount = sub {
...@@ -422,12 +427,13 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) { ...@@ -422,12 +427,13 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) {
$tmp_ies_unsorted_file->append($file . "\n"); $tmp_ies_unsorted_file->append($file . "\n");
$cnt_unsorted_files++; $cnt_unsorted_files++;
$File::Find::prune = 1; $File::Find::prune = 1;
} elsif (-d ) { }
elsif (-d) {
my $relpath = $File::Find::name; my $relpath = $File::Find::name;
$relpath =~ s{^\Q$dir\E/?}{}; $relpath =~ s{^\Q$dir\E/?}{};
my $depth = File::Spec->splitdir($relpath); my $depth = File::Spec->splitdir($relpath);
if ($depth <= 2) { if ($depth <= 2) {
print $progressbar->report("find IE files: %40b ETA: %E \r", $dircount++); print $progressbar->report("find IE files (pass $pass/$maxpass: %40b ETA: %E \r", $dircount++);
} }
} }
return; return;
...@@ -435,6 +441,8 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) { ...@@ -435,6 +441,8 @@ sub searching_ie_files ($dir, $tmp_ies_unsorted_file) {
### ###
find($wanted_process_sip, $dir); find($wanted_process_sip, $dir);
say ""; say "";
}
return $cnt_unsorted_files; return $cnt_unsorted_files;
} }
...@@ -482,40 +490,51 @@ if ($#ARGV_tail < 0) { ...@@ -482,40 +490,51 @@ if ($#ARGV_tail < 0) {
die "you need a directory as argument\n"; die "you need a directory as argument\n";
} }
say "preparing SQL";
my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
RaiseError => 1,
sqlite_unicode => 1,
}) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
write_database_creation($dbh);
write_tables_creation($dbh);
write_prepare_insert($dbh);
write_index_creation($dbh);
$dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
if (0==@ARGV_tail ){
die "no directory given on commandline"
}
my @dirs;
while (@ARGV_tail > 0) {
my $dir = shift @ARGV_tail; my $dir = shift @ARGV_tail;
if (defined $dir && -d "$dir") { if (defined $dir && -d "$dir") {
say "preparing SQL"; push @dirs, $dir;
}
}
$tmp_ies_unsorted_file->touch(); $tmp_ies_unsorted_file->touch();
say "searching IE files"; say "searching IE files";
my $cnt_unsorted_files = searching_ie_files($dir, $tmp_ies_unsorted_file); my $cnt_unsorted_files = searching_ie_files(\@dirs, $tmp_ies_unsorted_file);
# /permanent_storage/2020/04/02/IE201080/V1-FL201091.xml # /permanent_storage/2020/04/02/IE201080/V1-FL201091.xml
# /permanent_storage/2020/04/02/IE201080/V2-FL201091.xml # /permanent_storage/2020/04/02/IE201080/V2-FL201091.xml
my $fh_unsorted_IEs = $tmp_ies_unsorted_file->openr(); my $fh_unsorted_IEs = $tmp_ies_unsorted_file->openr();
my $count = 0; my $count = 0;
my $progressbar = Time::Progress->new(min => 0, max => $cnt_unsorted_files, smoothing => 1); my $progressbar = Time::Progress->new(min => 0, max => $cnt_unsorted_files, smoothing => 1);
my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
RaiseError => 1,
sqlite_unicode => 1,
}) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
write_database_creation($dbh);
write_tables_creation($dbh);
write_prepare_insert($dbh);
write_index_creation($dbh);
while (<$fh_unsorted_IEs>) { while (<$fh_unsorted_IEs>) {
chomp; chomp;
print $progressbar->report("parse IE files: %40b ETA: %E \r", $count++); print $progressbar->report("parse IE files: %40b ETA: %E \r", $count++);
s/V0*(\d+-IE)/V$1/; # revert fake version s/V0*(\d+-IE)/V$1/; # revert fake version
my $ret = parse_iexml($_, $flag_recovery); my $ret = parse_iexml($_, $flag_recovery);
my $dbh = DBI->connect("dbi:SQLite:dbname=$db_filename", "", "", {
RaiseError => 1,
sqlite_unicode => 1,
}) or die "could not connect to database (file '$db_filename')", $DBI::errstr;
write_addsql($dbh, $ret); write_addsql($dbh, $ret);
$dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
} }
say ""; say "";
$dbh->disconnect or warn("disconnecting problems, ", $dbh->errstr);
say "processed $count uniq IEs"; say "processed $count uniq IEs";
}
else {
die "no directory given on commandline"
}
say ""; say "";
1; 1;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment