- autoflush for stderr enabled

- in check_file_fixities() using Path::Tiny::openr()_raw instead open_r() to avoid extra binmode() call
- extracted common stuff from while-loop in stage2() to stage2_for_ie(),
  this could reduce the memory footprint because reduced scope
- in stage2() instead seek we close and reopen fh now
HINT: no real reason for "out of memory"-problem found

deep_fixitycheck.pl

@@ -37,6 +37,7 @@ use IO::Handle;
 use POSIX qw(strftime);
 use Fcntl qw(SEEK_END SEEK_SET);
 STDOUT->autoflush(1);
+STDERR->autoflush(1);
 # guarantee, that output will be UTF8
 binmode(STDOUT, ":encoding(UTF-8)");
 
@@ -253,6 +254,7 @@ sub check_file_seekable($$) {
   if (!$result->{seekable}) {
     add_error($fileobj, $result, "is not seekable, $!");
   }
+  close($fh);
   return $result;
 }
 
@@ -267,8 +269,7 @@ sub check_file_fixities($$) {
       'SHA256' => Digest::SHA->new(256),
       'SHA512' => Digest::SHA->new(512)
   );
-  my $fh = path($fileobj->{file_mounted})->openr();
-  binmode($fh);
+  my $fh = path($fileobj->{file_mounted})->openr_raw();
   my $buffer;
   my $block=0;
   my $blocksize = 128*1024; # 128kB blocks
@@ -327,46 +328,12 @@ sub stage1 ($$$) {
   return 1;
 }
 
-sub stage2 ($$$$) {
-  my $tmp_ies_unsorted_path = shift;
-  my $report_path = shift;
+sub stage2_for_ie {
+  my $unseen = shift;
+  my $bf = shift;
   my $map_path = shift;
-  my $recovery = shift;
-  my $fh_unsorted_file = $tmp_ies_unsorted_path->openr();
-  my $cnt_unsorted_files = 0;
-  while (<$fh_unsorted_file>) {
-    $cnt_unsorted_files++;
-  }
-  seek $fh_unsorted_file, 0, 0; # seek to first byte
-  my $count = 0;
-  my $bf = Bloom::Filter->new(
-      capacity   => $capacity, # we cache n last seen files
-      error_rate => 0.0001,
-  );
-  say "using bloomfilter of length ".int($bf->length/8)." bytes";
-  say "checking IEs";
-  my $progressbar = Time::Progress->new(min => 0, max => $cnt_unsorted_files, smoothing => 1);
-  my $stat;
-  $stat->{IEs} = 0;
-  $stat->{files} = 0;
-  $stat->{errors} = 0;
-  $stat->{scansize} = 0;
-  $stat->{begin} = time;
-  my $bfreset="⏸";
-  my $prev_ie = "";
-  while (<$fh_unsorted_file>) {
-    # scan each IE
-    $stat->{IEs}++;
-    chomp;
-    my $transferrate_in_MBs = sprintf("%0.2f", $stat->{scansize} / (time - $stat->{begin} + 1) / 1024 / 1024);
-    my $bfusage = int($bf->key_count()*100/$capacity );
-    print $progressbar->report("parse IE files:       %40b $bfreset running: %L ETA: %E ($count/$cnt_unsorted_files IEs, tfr=$transferrate_in_MBs MB/s, bfu=$bfusage%)               \r", ++$count);
-    my $timestamp = strftime("%Y-%m-%d %H:%M:%S %z (%Z)", localtime(time));
-    $report_path->append_utf8("$timestamp, IE $_ with following errors:\n");
-    my $ret = parse_iexml($_, $recovery);
-    my $unseen = bloomfilter_to_unseen($bf,  $ret->{files} );
-    if (scalar @{$unseen} == 0 ) { $report_path->append_utf8("skipped because files already checked using IE $prev_ie\n"); }
-    $prev_ie = $_;
+  my $report_path = shift;
+  my $stat = shift;
   foreach my $fileobj (@{ $unseen }) {
     if ($bf->key_count() >= (0.8 * $capacity)) { # reset Bloom filter if 80% filled
       #print "reset bloomfilter\n";
@@ -374,8 +341,7 @@ sub stage2 ($$$$) {
           capacity   => $capacity, # we cache n last seen files
           error_rate => 0.0001,
       );
-        $bfreset = "⏮";
-      } else { $bfreset = "▶";}
+    }
     $bf->add( $fileobj->{fileid} );
     $fileobj->{file_mounted} = map_file($map_path, $fileobj->{filepath});
     $stat->{files}++;
@@ -404,6 +370,51 @@ sub stage2 ($$$$) {
   }
   # write report
 }
+
+sub stage2 ($$$$) {
+  my $tmp_ies_unsorted_path = shift;
+  my $report_path = shift;
+  my $map_path = shift;
+  my $recovery = shift;
+  my $fh_unsorted_file = $tmp_ies_unsorted_path->openr();
+  my $cnt_unsorted_files = 0;
+  while (<$fh_unsorted_file>) {
+    $cnt_unsorted_files++;
+  }
+  close($fh_unsorted_file);
+  my $count = 0;
+  my $bf = Bloom::Filter->new(
+      capacity   => $capacity, # we cache n last seen files
+      error_rate => 0.0001,
+  );
+  say "using bloomfilter of length ".int($bf->length/8)." bytes";
+  say "checking IEs";
+  my $progressbar = Time::Progress->new(min => 0, max => $cnt_unsorted_files, smoothing => 1);
+  my $stat;
+  $stat->{IEs} = 0;
+  $stat->{files} = 0;
+  $stat->{errors} = 0;
+  $stat->{scansize} = 0;
+  $stat->{begin} = time;
+  my $prev_ie = "";
+  $fh_unsorted_file = $tmp_ies_unsorted_path->openr();
+  while (<$fh_unsorted_file>) {
+    my $actual_ie = $_;
+    # scan each IE
+    $stat->{IEs}++;
+    chomp;
+    my $transferrate_in_MBs = sprintf("%0.2f", $stat->{scansize} / (time - $stat->{begin} + 1) / 1024 / 1024);
+    my $bfusage = int($bf->key_count() * 100 / $capacity);
+    print $progressbar->report("parse IE files:       %40b running: %L ETA: %E ($count/$cnt_unsorted_files IEs, tfr=$transferrate_in_MBs MB/s, bfu=$bfusage%)               \r", ++$count);
+    my $timestamp = strftime("%Y-%m-%d %H:%M:%S %z (%Z)", localtime(time));
+    $report_path->append_utf8("$timestamp, IE $actual_ie with following errors:\n");
+    my $ret = parse_iexml($actual_ie, $recovery);
+    my $unseen = bloomfilter_to_unseen($bf, $ret->{files});
+    if (scalar @{$unseen} == 0) {$report_path->append_utf8("skipped because files already checked using IE $prev_ie\n");}
+    $prev_ie = $actual_ie;
+    ###
+    stage2_for_ie($unseen, $bf, $map_path, $report_path, $stat);
+  }
   say "";
   $stat->{end} = time;
   $stat->{duration} = $stat->{end} - $stat->{begin};