Skip to content
Snippets Groups Projects
Commit 3b2aea51 authored by Andreas Romeyke's avatar Andreas Romeyke
Browse files

- add cli option --complete

- refactored report execution and generator
- added support for LDP projects
parent 1cd26db2
No related branches found
No related tags found
No related merge requests found
...@@ -50,7 +50,8 @@ sub opt_spec { ...@@ -50,7 +50,8 @@ sub opt_spec {
[ 'daily|d' => 'report based on last day'], [ 'daily|d' => 'report based on last day'],
[ 'weekly|W' => 'report based on last week'], [ 'weekly|W' => 'report based on last week'],
[ 'monthly|m' => 'report based on last month (default)' ], [ 'monthly|m' => 'report based on last month (default)' ],
[ 'yearly|y' => 'report based on last year' ] [ 'yearly|y' => 'report based on last year' ],
[ 'complete|C'=> 'report based on all AIPs'],
], ],
}, },
], ],
...@@ -77,10 +78,12 @@ sub validate_args { ...@@ -77,10 +78,12 @@ sub validate_args {
} elsif (exists $opt->{monthly}) { } elsif (exists $opt->{monthly}) {
($from_year, $from_month, $from_day) = Add_Delta_YM($cyear, $cmonth, 1, 0, -1); ($from_year, $from_month, $from_day) = Add_Delta_YM($cyear, $cmonth, 1, 0, -1);
($to_year, $to_month, $to_day) = Add_Delta_YMD($from_year, $from_month, $from_day, 0, 1, -1); ($to_year, $to_month, $to_day) = Add_Delta_YMD($from_year, $from_month, $from_day, 0, 1, -1);
} } elsif (exists $opt->{yearly}) {
elsif (exists $opt->{yearly}) {
($from_year, $from_month, $from_day) = Add_Delta_YM($cyear, 1, 1, -1, 0); ($from_year, $from_month, $from_day) = Add_Delta_YM($cyear, 1, 1, -1, 0);
($to_year, $to_month, $to_day) = Add_Delta_YMD($from_year, $from_month, $from_day, 1, 0, -1); ($to_year, $to_month, $to_day) = Add_Delta_YMD($from_year, $from_month, $from_day, 1, 0, -1);
} elsif (exists $opt->{complete}) {
$from_year=2015; $from_month=1; $from_day=1;
$to_year=$cyear; $to_month=$cmonth; $to_day=$cday;
} }
printf STDERR "reporting for period %04u-%02u-%02u … %04u-%02u-%02u\n", $from_year, $from_month, $from_day, $to_year, $to_month, $to_day; printf STDERR "reporting for period %04u-%02u-%02u … %04u-%02u-%02u\n", $from_year, $from_month, $from_day, $to_year, $to_month, $to_day;
my $from_epoch = Date_to_Time($from_year, $from_month, $from_day, 0, 0, 0); my $from_epoch = Date_to_Time($from_year, $from_month, $from_day, 0, 0, 0);
...@@ -134,137 +137,136 @@ sub _execute { ...@@ -134,137 +137,136 @@ sub _execute {
#p($aips_response); #p($aips_response);
return $aips_response; return $aips_response;
} }
sub execute_aip_total {
return _execute(@_);
}; sub get_ldp_projects {
sub execute_aip_new {
my ($self, $opt, $args) = @_;
my %newhash = %{$opt};
$newhash{only_new}=1;
return _execute($self, \%newhash, $args);
};
sub execute_aip_updated {
my ($self, $opt, $args) = @_;
my %newhash = %{$opt};
$newhash{only_updated}=1;
return _execute($self, \%newhash, $args);
};
sub execute_ldp_funder_saxon_new {
my ($self, $opt, $args) = @_;
my %newhash = %{$opt};
$newhash{only_ldp_saxon}=1;
$newhash{only_new}=1;
return _execute($self, \%newhash, $args);
};
sub execute_ldp_funder_saxon_updated {
my ($self, $opt, $args) = @_;
my %newhash = %{$opt};
$newhash{only_ldp_saxon}=1;
$newhash{only_updated}=1;
return _execute($self, \%newhash, $args);
};
sub execute_ldp_funder_misc_new {
my ($self, $opt, $args) = @_;
my %newhash = %{$opt};
$newhash{only_ldp_without_saxon}=1;
$newhash{only_new}=1;
return _execute($self, \%newhash, $args);
};
sub execute_ldp_funder_misc_updated {
my ($self, $opt, $args) = @_; my ($self, $opt, $args) = @_;
my %newhash = %{$opt}; my $query = SLUB::LZA::TA::Archivematica::Elasticsearch::PrepareQuery::prepare_ldpprojects_query($opt);
$newhash{only_ldp_without_saxon}=1; if (exists $opt->{debug}) {
$newhash{only_updated}=1; use Data::Printer;
return _execute($self, \%newhash, $args); say STDERR "query:";
}; say STDERR "------------------------";
sub execute_no_ldp_new { say STDERR np($query);
say STDERR "------------------------";
}
#p($aips_query);
my $response = SLUB::LZA::TA::Archivematica::Elasticsearch::query_elasticsearch(
$SLUB::LZA::TA::config{elasticsearchprotocol},
$SLUB::LZA::TA::config{elasticsearchhost},
$SLUB::LZA::TA::config{elasticsearchport},
'aips', # indexname
$query, # query_hash ref
);
my %projects;
foreach my $match (@{$response->{hits}->{hits}}) {
foreach my $tme (@{ $match->{_source}->{transferMetadata} }) {
my $dict = $tme->{'bim:bag-info_dict'};
if (ref $dict eq 'ARRAY') {
foreach my $e (@{ $dict }) {
my $project = $e->{'bim:LDP-project'};
$projects{$project} = 1;
}
} elsif (ref $dict eq 'HASH') {
my $project = $dict->{'bim:LDP-project'};
$projects{$project} = 1;
}
}
}
return keys %projects;
}
sub execute {
my ($self, $opt, $args) = @_; my ($self, $opt, $args) = @_;
my %results;
$results{date} = sprintf("%04u-%02u-%02u", Today());
$results{package} = __PACKAGE__;
$results{from} = $opt->{creationdate_epochs}->{from_string};
$results{to} = $opt->{creationdate_epochs}->{to_string};
my @ldp_projects= get_ldp_projects(@_);
foreach my $aip_state (qw(total new updated)) {
my %newhash = %{$opt}; my %newhash = %{$opt};
$newhash{no_ldp}= 1; if ($aip_state eq 'new') {$newhash{only_new} = 1;}
$newhash{only_new}=1; elsif ($aip_state eq 'updated') {$newhash{only_updated} = 1;}
return _execute($self, \%newhash, $args); foreach my $ldp_funder (qw(saxony misc none full)) {
}; delete $newhash{only_ldp_saxon};
sub execute_no_ldp_updated { delete $newhash{only_ldp_without_saxon};
my ($self, $opt, $args) = @_; delete $newhash{no_ldp};
if ($ldp_funder eq 'saxony') {$newhash{only_ldp_saxon} = 1}
elsif ($ldp_funder eq 'misc') {$newhash{only_ldp_without_saxon} = 1;}
elsif ($ldp_funder eq 'none') {$newhash{no_ldp} = 1;}
else { # full, do not filter for ldp
}
my $res = _execute($self, \%newhash, $args);
$results{flavour}->{count}->{$aip_state}->{$ldp_funder} = $res->{hits}->{total};
$results{flavour}->{size}->{$aip_state}->{$ldp_funder} = $res->{aggregations}->{total_aip_size}->{value};
$results{flavour}->{files}->{$aip_state}->{$ldp_funder} = $res->{aggregations}->{total_file_count}->{value};
}
foreach my $ldp_project (@ldp_projects) {
my %newhash = %{$opt}; my %newhash = %{$opt};
$newhash{no_ldp}= 1; if ($aip_state eq 'new') {$newhash{only_new} = 1;}
$newhash{only_updated}=1; elsif ($aip_state eq 'updated') {$newhash{only_updated} = 1; }
return _execute($self, \%newhash, $args); $newhash{only_ldp_project}=$ldp_project;
}; my $res = _execute($self, \%newhash, $args);
sub execute { $results{ldp_project}->{$ldp_project}->{flavour}->{count}->{$aip_state} = $res->{hits}->{total};
my ($self, $opt, $args) = @_; $results{ldp_project}->{$ldp_project}->{flavour}->{size}->{$aip_state} = $res->{aggregations}->{total_aip_size}->{value};
my $res_total = execute_aip_total(@_); $results{ldp_project}->{$ldp_project}->{flavour}->{files}->{$aip_state} = $res->{aggregations}->{total_file_count}->{value};
my $res_new = execute_aip_new(@_); }
my $res_update = execute_aip_updated(@_); }
my $res_ldpsaxony_new = execute_ldp_funder_saxon_new(@_); print_humanreadable_report(\%results);
my $res_ldpsaxony_updated = execute_ldp_funder_saxon_updated(@_); say STDERR "report is already sent to STDOUT.";
my $res_ldpmisc_new = execute_ldp_funder_misc_new(@_); }
my $res_ldpmisc_updated = execute_ldp_funder_misc_updated(@_);
my $res_noldp_new = execute_no_ldp_new(@_);
my $res_noldp_updated = execute_no_ldp_updated(@_); sub print_humanreadable_report ($result) {
my $date = sprintf("%04u-%02u-%02u", Today()); say <<"RPTHEADER";
my $package = __PACKAGE__;
say <<"REPORT";
:lang: en :lang: en
:date: $date :date: $result->{date}
:generator: $0 ($package) :generator: $0 ($result->{package})
== Report from $opt->{creationdate_epochs}->{from_string} to $opt->{creationdate_epochs}->{to_string} == Report from $result->{from} to $result->{to}
=== count RPTHEADER
-- foreach my $flavour (sort keys %{$result->{flavour}}) {
first ingest: $res_new->{hits}->{total} say <<"REPORT";
(ldp saxony: $res_ldpsaxony_new->{hits}->{total})
(ldp misc: $res_ldpmisc_new->{hits}->{total})
(no ldp: $res_noldp_new->{hits}->{total})
aip update: $res_update->{hits}->{total} === $flavour
(ldp saxony: $res_ldpsaxony_updated->{hits}->{total})
(ldp misc: $res_ldpmisc_updated->{hits}->{total})
(no ldp: $res_noldp_updated->{hits}->{total})
total: $res_total->{hits}->{total} first ingest: $result->{flavour}->{$flavour}->{new}->{full}
(ldp saxony: $result->{flavour}->{$flavour}->{new}->{saxony})
(ldp misc: $result->{flavour}->{$flavour}->{new}->{misc})
(no ldp: $result->{flavour}->{$flavour}->{new}->{none})
-- aip update: $result->{flavour}->{$flavour}->{updated}->{full}
(ldp saxony: $result->{flavour}->{$flavour}->{updated}->{saxony})
(ldp misc: $result->{flavour}->{$flavour}->{updated}->{misc})
(no ldp: $result->{flavour}->{$flavour}->{updated}->{none})
=== size total: $result->{flavour}->{$flavour}->{total}->{full}
-- REPORT
first ingest: $res_new->{aggregations}->{total_aip_size}->{value} }
(ldp saxony: $res_ldpsaxony_new->{aggregations}->{total_aip_size}->{value})
(ldp misc: $res_ldpmisc_new->{aggregations}->{total_aip_size}->{value})
(no ldp: $res_noldp_new->{aggregations}->{total_aip_size}->{value})
aip update: $res_update->{aggregations}->{total_aip_size}->{value} foreach my $ldp_project (sort keys %{$result->{ldp_project}}) {
(ldp saxony: $res_ldpsaxony_updated->{aggregations}->{total_aip_size}->{value}) say <<"LDPREPORT_HEADER";
(ldp misc: $res_ldpmisc_updated->{aggregations}->{total_aip_size}->{value})
(no ldp: $res_noldp_updated->{aggregations}->{total_aip_size}->{value})
total: $res_total->{aggregations}->{total_aip_size}->{value} === LDP project '$ldp_project'
-- LDPREPORT_HEADER
=== file count foreach my $flavour (sort keys %{$result->{ldp_project}->{$ldp_project}->{flavour}}) {
say <<"LDPREPORT";
-- ==== $flavour
first ingest: $res_new->{aggregations}->{total_file_count}->{value}
(ldp saxony: $res_ldpsaxony_new->{aggregations}->{total_file_count}->{value})
(ldp misc: $res_ldpmisc_new->{aggregations}->{total_file_count}->{value})
(no ldp: $res_noldp_new->{aggregations}->{total_file_count}->{value})
aip update: $res_update->{aggregations}->{total_file_count}->{value} first ingest: $result->{ldp_project}->{$ldp_project}->{flavour}->{$flavour}->{new}
(ldp saxony: $res_ldpsaxony_updated->{aggregations}->{total_file_count}->{value})
(ldp misc: $res_ldpmisc_updated->{aggregations}->{total_file_count}->{value})
(no ldp: $res_noldp_updated->{aggregations}->{total_file_count}->{value})
total: $res_total->{aggregations}->{total_file_count}->{value} aip update: $result->{ldp_project}->{$ldp_project}->{flavour}->{$flavour}->{updated}
-- total: $result->{ldp_project}->{$ldp_project}->{flavour}->{$flavour}->{total}
REPORT LDPREPORT
say STDERR "report is already sent to STDOUT."; }
}
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment