Skip to content
Snippets Groups Projects

Wegfall ZIP und Erweiterung Testsuite

Merged Wegfall ZIP und Erweiterung Testsuite
2-Anpassung_einer_Testsuite into master
Merged Andreas Romeyke requested to merge 2-Anpassung_einer_Testsuite into master
Compare
and
2 files
+ 579
− 108
Compare changes
  • Side-by-side
  • Inline

Files

bin/slubsipbuilder.pl
+ 202
− 101
@@ -10,7 +10,7 @@
# REQUIREMENTS: ---
# BUGS: ---
# NOTES: related to official document
# "SIP Spezifikation (v1.4.2)"
# "SIP Spezifikation (v1.5)"
# AUTHOR: Andreas Romeyke (romeyke@slub-dresden.de)
# ORGANIZATION: SLUB
# VERSION: 1.1
@@ -18,7 +18,6 @@
#===============================================================================
use strict;
use warnings;
use Carp;
@@ -34,6 +33,8 @@ package SLUB::LZA::SIPBuilder;
use XML::LibXSLT;
use XML::XPath;
use Carp;
use Encode;
my $marc_mods_url = 'http://www.loc.gov/standards/mods/v3/MARC21slim2MODS3-6.xsl';
my $marc_utils_url = 'http://www.loc.gov/standards/marcxml/xslt/MARC21slimUtils.xsl';
my $swb_url = 'https://sru.bsz-bw.de/swb';
@@ -45,6 +46,11 @@ package SLUB::LZA::SIPBuilder;
# write data to file (UTF-8)
sub write_file($$) {
if(! defined $_[0]) { croak "filename not defined!"; }
if(! defined $_[1]) { croak "value not defined!"; }
if($_[0] eq "" || $_[1] eq ""){
die "invalid parameters.";
}
my $filename = $_[0];
my $value = $_[1];
open(my $fh, '>:encoding(UTF-8)', $filename) || (croak "Can't open '$filename', $!");
@@ -55,6 +61,8 @@ package SLUB::LZA::SIPBuilder;
# this will patch the mods-xml as a workaround for bugs in LOCs xslt files
sub patch_mods($) {
if(! defined $_[0]) { croak "modsobject not defined!"; }
if($_[0] eq ""){ die "invalid parameters."; }
my $modsobj = shift; # mods expected as XML Parser object
# TODO: Bugfix for /mets:mets/mets:dmdSec[1]/mets:mdWrap[1]/mets:xmlData[1]/mods:modsCollection[1]/mods:mods[1]/mods:relatedItem[2]/mods:internetMediaType[1]
my $xslt_patch_string = <<'PATCH';
@@ -86,6 +94,8 @@ PATCH
}
sub patch_marc_response($) {
if(! defined $_[0]) { croak "marcobject not defined!"; }
if($_[0] eq ""){ die "invalid parameters."; }
my $marcobj = shift; # marcobj expected as XML Parser object
my $xslt_patch_string = <<'PATCH2';
<?xml version="1.0" encoding="UTF-8"?>
@@ -121,6 +131,11 @@ PATCH2
# check MARC21 utility xsl
sub check_marc21_utility {
if(! defined $_[0]) { croak "xsl directory not defined!"; }
if(! defined $_[1]) { croak "user agent not defined!"; }
if($_[0] eq "" || $_[1] eq ""){
die "invalid parameters.";
}
my $xsl_dir = shift;
my $ua = shift;
my $marc_utils_basename = path($marc_utils_url)->basename;
@@ -140,6 +155,11 @@ PATCH2
# check MARC21->MODS xsl
sub check_marc21_mods_xsl {
if(! defined $_[0]) { croak "xsl directory not defined!"; }
if(! defined $_[1]) { croak "user agent not defined!"; }
if($_[0] eq "" || $_[1] eq ""){
die "invalid parameters.";
}
my $xsl_dir = shift;
my $ua = shift;
my $marc_mods_basename = path($marc_mods_url)->basename;
@@ -176,18 +196,21 @@ PATCH2
# specification SRU/SRW BSZ: https://wiki.k10plus.de/pages/viewpage.action?pageId=132874251
sub get_mods_from($$$$) {
if(! defined $_[0]) { croak "url not defined!"; }
if(! defined $_[1]) { croak "ppn not defined!"; }
if(! defined $_[2]) { croak "key not defined!"; }
if(! defined $_[3]) { croak "schema not defined!"; }
if($_[0] eq "" || $_[1] eq "" || $_[2] eq "" || $_[3] eq ""){
die "invalid parameters.";
}
# $mods = ($url, $ppn, $searchkey, $recordschema)
my $url = shift;
my $ppn = shift; # example: "457035137" for "Der Fichtelberg"
my $key = shift;
my $schema = shift;
#### where to find XSLT
my $ua = LWP::UserAgent->new;
$ua->agent("MyApp/0.1 ");
$ua->timeout(3600); #1h
@@ -196,7 +219,6 @@ PATCH2
check_marc21_utility($xsl_dir, $ua);
check_marc21_mods_xsl($xsl_dir, $ua);
my $srubase = $url; # host
my $srusearchkey = $key; # SRU search key
my $sruvalue = $ppn;
@@ -226,6 +248,7 @@ PATCH2
say "write DEBUG_${ppn}_marc.xml";
write_file("DEBUG_${ppn}_marc.xml", $marcblob_patched);
}
my $marc_mods_patched_path = check_marc21_mods_xsl($xsl_dir, $ua);
my $xslt = XML::LibXSLT->new();
my $marcmods = XML::LibXML->load_xml(location => $marc_mods_patched_path, no_cdata => 1);
@@ -236,7 +259,6 @@ PATCH2
say "write DEBUG_${ppn}_unpatched_mods.xml";
write_file("DEBUG_${ppn}_unpatched_mods.xml", $stylesheet->output_string($result));
}
$result = patch_mods($result);
my $result_string = $stylesheet->output_string($result);
return $result_string;
@@ -248,6 +270,11 @@ PATCH2
}
sub create_filecopyhash {
if(! defined $_[0]) { croak "directory not defined!"; }
if(! defined $_[1]) { croak "content not defined!"; }
if($_[0] eq "" || $_[1] eq ""){
die "invalid parameters.";
}
my $directory = shift;
my $content = shift;
my %filecopyhash;
@@ -257,6 +284,7 @@ sub create_filecopyhash {
();
} else {
my $file=$File::Find::name;
# system unabhängige pfade
if ($file !~ m#^[-A-Za-z0-9_\.:\\/]+$#) {
confess("file '$file' does not match regex '^[-A-Za-z0-9_\.:\\/]+\$'");
}
@@ -275,12 +303,12 @@ sub create_filecopyhash {
$filecopyhash{$source}{'md5sum'}=$md5;
}
};
finddepth($wanted, $directory);
return \%filecopyhash;
}
sub prepare_dmd_section_with_ppn($) {
if(! defined $_[0]) { croak "ppn not defined!"; }
my $ppn = shift;
my $mods = SLUB::LZA::SIPBuilder::get_mods_from($swb_url, $ppn, $searchkey, $recordschema);
if ($with_debug) {
@@ -302,6 +330,7 @@ DMD
}
sub prepare_dmd_section_with_noppn($) {
if(! defined $_[0]) { croak "noppn not defined!"; }
my $noppn = shift;
my $mods =<<"MODS";
<mods version="3.6"
@@ -325,19 +354,50 @@ DMD
}
sub prepare_amd_section($$$$$$) {
if(! defined $_[0]) { croak "export to archive date not defined!"; }
if(! defined $_[1]) { croak "external workflow not defined!"; }
if(! defined $_[2]) { croak "external id not defined!"; }
if(! defined $_[3]) { croak "external conservation flag not defined!"; }
if(! defined $_[4]) { croak "external isil not defined!"; }
if(! defined $_[5]) { croak "external value description not defined!"; }
if($_[0] eq "" || $_[1] eq "" || $_[2] eq "" || $_[3] eq "" || $_[5] eq ""){
die "invalid parameters.";
}
my $export_to_archive_date = shift;
my $external_workflow = shift;
my $external_id = shift;
my $external_conservation_flag = shift;
my $external_isil = shift;
my $external_value_descr = shift;
my $amd =<<"AMD";
my $external_value_descr = decode_utf8(shift);
my $amd;
if ($external_isil eq '') {
$amd =<<"AMD";
<mets:amdSec ID="AMD">
<!-- SIP metadata for automated processing by submission application -->
<mets:techMD ID="ARCHIVE">
<mets:mdWrap MDTYPE="OTHER" MIMETYPE="text/xml" OTHERMDTYPE="ARCHIVE">
<mets:xmlData>
<archive:record version="v2019.2" xmlns:archive="http://slub-dresden.de/slubarchiv">
<archive:exportToArchiveDate>$export_to_archive_date</archive:exportToArchiveDate>
<archive:externalId>$external_id</archive:externalId>
<archive:externalWorkflow>$external_workflow</archive:externalWorkflow>
<archive:hasConservationReason>$external_conservation_flag</archive:hasConservationReason>
<archive:archivalValueDescription>$external_value_descr</archive:archivalValueDescription>
</archive:record>
</mets:xmlData>
</mets:mdWrap>
</mets:techMD>
</mets:amdSec>
AMD
} else {
$amd =<<"AMD";
<mets:amdSec ID="AMD">
<!-- SIP metadata for automated processing by submission application -->
<mets:techMD ID="ARCHIVE">
<mets:mdWrap MDTYPE="OTHER" MIMETYPE="text/xml" OTHERMDTYPE="ARCHIVE">
<mets:xmlData>
<archive:record version="v2017.1" xmlns:archive="http://slub-dresden.de/slubarchiv">
<archive:record version="v2019.2" xmlns:archive="http://slub-dresden.de/slubarchiv">
<archive:exportToArchiveDate>$export_to_archive_date</archive:exportToArchiveDate>
<archive:externalId>$external_id</archive:externalId>
<archive:externalWorkflow>$external_workflow</archive:externalWorkflow>
@@ -350,10 +410,12 @@ DMD
</mets:techMD>
</mets:amdSec>
AMD
}
return $amd;
}
sub prepare_files_sections($) {
if(! defined $_[0]) { croak "filecopyhash not defined!"; }
my $filecopyhash = shift;
my @fsec;
my $i=0;
@@ -375,6 +437,7 @@ FILESEC
}
sub prepare_struct_map($) {
if(! defined $_[0]) { croak "filecopyhash not defined!"; }
my $filecopyhash = shift;
my @ssec;
my $i=0;
@@ -395,6 +458,62 @@ STRUCTMAP
return $structmap;
}
sub check_directory($$) {
if(! defined $_[0]) { croak "fullname not defined!"; }
if(! defined $_[1]) { croak "entry not defined!"; }
if($_[0] eq "" || $_[1] eq ""){
die "invalid parameters.";
}
my $fullname = shift;
my $entry = shift;
my $dir_err = '';
# Die Gesamtlänge der relativen Pfade darf jeweils 255 Zeichen nicht übersteigen.
if (length($fullname) > 255){
return $dir_err = "path to file: $fullname is too long, expected maximum 255 characters";
}
# Als Dateinamen und Verzeichnisnamen sind für die zu archivierenden Dateien die Zeichen A-Za-z0-9_.- sowie / für Verzeichnistrenner erlaubt.
if ($entry !~ m#^[A-Za-z0-9_.-]+$#){
return $dir_err = "you need to specify a valid file or directory (name: $entry) to (^[A-Za-z0-9_.-]+\$)";
}
# Relative Pfade mit Bestandteilen der Form "../" sind nicht erlaubt.
if ($fullname =~ /\.\.\//){
return $dir_err = "relativ path($fullname) in form '../' is not allowed";
}
return $dir_err;
}
sub validate_directory($) {
if(! defined $_[0]) { croak "directory not defined!"; }
if(! -d $_[0]){
die "Could not find directory";
}
my $directory = shift;
my @dirs = ($directory);
my $dir_err = '';
while (@dirs && ($dir_err eq '')) {
my $thisdir = shift @dirs;
if (!($thisdir)) {
# dir, do nothing
return $dir_err='';
}
opendir (my $dh, $thisdir) or die "Could not find $thisdir";
while (my $entry = readdir $dh && ($dir_err eq '')) {
next if $entry eq '.';
next if $entry eq '..';
my $fullname = "$thisdir/$entry";
$dir_err = check_directory($fullname, $entry);
if (-d $fullname) {
push @dirs, $fullname;
}
}
closedir $dh;
}
return $dir_err;
}
# end package
package main;
@@ -405,7 +524,6 @@ BEGIN{
}
return 1 if caller; # avoids main code running if module stuff is needed
use SLUB::LZA::SIPBuilder;
use Archive::Zip::SimpleZip qw($SimpleZipError);
use Getopt::Long;
use Path::Tiny;
use Digest::MD5;
@@ -418,7 +536,6 @@ my $directory;
my $ppn;
my $noppn;
my $output;
my $as_zip;
my $external_id;
my $external_workflow;
my $external_isil="";
@@ -428,13 +545,11 @@ my $external_conservation_flag;
my $help;
my $man;
GetOptions(
"IE_directory=s" => \$directory, # required
"ppn=s" => \$ppn, # semi-optional (choice 1 of 2)
"noppn=s" => \$noppn, # semi-optional (choice 2 of 2)
"SIP_output_path=s" => \$output, # required
"as_zip" => \$as_zip, # optional, default: do not zip
"external_id=s" => \$external_id, # required
"external_workflow=s" => \$external_workflow, # required
"external_ISIL=s" => \$external_isil, # optional, default: no ISIL
@@ -445,35 +560,38 @@ GetOptions(
"man" => \$man, # optional
) or pod2usage(2);
my $dir_err = SLUB::LZA::SIPBuilder::validate_directory($directory);
if ($help) { pod2usage(1); }
if ($man) { pod2usage(-exitval => 0, -verbose => 2); }
if (!defined $directory) { confess("you need to specify an IE directory, which needs to be archived"); }
if ((defined $directory) && ($dir_err ne '')) { confess($dir_err); }
if ((defined $ppn) && (defined $noppn)) { confess("you can only specify either -ppn or -noppn"); }
if ((!defined $ppn) && (!defined $noppn)) { confess("you need to specify a PPN with -ppn or use --noppn"); }
if (!defined $output) { confess("you need to specify an output path, where the SIP will be stored"); }
if (!defined $external_id) { confess("you need to specify external ID"); }
if ((defined $external_id) && ($external_id !~ m#^[a-z0-9_-]+$#)) { confess("you need to specify a valid external ID (^[a-z0-9_-]+\$)"); }
if (!defined $external_workflow) { confess("you need to specify external workflow"); }
if ((defined $external_workflow) && ($external_workflow !~ m#^[a-z0-9_-]+$#)) { confess("you need to specify a valid external workflow (^[a-z0-9_-]+\$)"); }
if (!$external_value_descr) { confess("you need to specify an external value description (reason for archiving)"); }
if (!defined $external_conservation_flag) { $external_conservation_flag="false"; } else { $external_conservation_flag="true"; }
if (! -d $directory) { confess("you need to specify an IE directory, which needs to be archived, $!"); }
$directory = path($directory)->realpath->stringify;
path($output)->mkpath;
$output = path($output)->realpath->stringify;
if ($external_id !~ m#^[a-z0-9]+$#) { confess("you need to specify a valid external ID (^[a-z0-9]+\$)"); }
if ($external_workflow !~ m#^[a-z0-9]+$#) { confess("you need to specify a valid external workflow (^[a-z0-9]+\$)"); }
if (!$external_value_descr) { confess("you need to specify an external value description (reason for archiving)"); }
#===============================================================================
sub main {
# get date
my $export_to_archive_date = DateTime->now->iso8601();
my $export_to_archive_date = DateTime->now(time_zone=>'local')->iso8601();
my $file_date = $export_to_archive_date;
$file_date =~ s/T/_/g; # replace 'T' with '_'
$file_date =~ s/:/-/g; # replace ':' with '-'
# prepare dirs
my $sip_root_dir = (defined $ppn)? "PPN-${ppn}_${file_date}" : "ID-${noppn}_${file_date}";
my $content = path($output)->child($sip_root_dir)->child("data")->stringify;
if (!defined $as_zip) {
path($content)->mkpath;
}
my $filecopyhash = SLUB::LZA::SIPBuilder::create_filecopyhash($directory, $content);
# prepare dmd-sec
@@ -505,7 +623,6 @@ sub main {
METS
# write stuff out
if (!defined $as_zip) {
SLUB::LZA::SIPBuilder::write_file( path($output)->child($sip_root_dir)->child("sip.xml")->stringify, $sip );
# copy source to target
foreach my $source (sort keys (%{$filecopyhash})) {
@@ -517,22 +634,7 @@ METS
cp($source, $target, buffer) || confess ("could not copy from '$source' to '$target', $!");
}
say "SIP '$sip_root_dir' build successfully in '$output'";
} else {
# compress it
my $zip_file_path = path($output)->child("$sip_root_dir.zip")->stringify;
my $zip = Archive::Zip::SimpleZip->new( $zip_file_path, Zip64=>1 );
$zip->addString($sip, Name=>path($sip_root_dir)->child("sip.xml")->stringify);
# copy source to target
foreach my $source (sort keys (%{$filecopyhash})) {
my $target = path($sip_root_dir)->child($filecopyhash->{$source}->{"relative"})->stringify; # CHECK ON WINDOWS
my $basename = path($target)->parent->stringify;
$zip->add( $source, Name=> $target) || confess ("could not zip copy from '$source' to '$target', $!");
}
unless ( $zip->close()) {
confess "write error to '$zip_file_path', $SimpleZipError, $!";
}
say "SIP '$sip_root_dir' build successfully in '$zip_file_path'";
}
return;
}
@@ -563,7 +665,6 @@ slubsipbuilder.pl [options]
-IE_directory=<IE dir> existing IE directory (absolute path!)
-ppn=<ppn>|-noppn=<noppn> SWB-PPN or any identifier (uses minimalistic MODS)
-SIP_output_path=<target dir> where to put the SIP dir (absolute path!)
-as_zip optional, if set a ZIP will be created
-external_id=<id> mandatory, should be uniqe ID
-external_workflow=<workflow> mandatory, should be uniqe workflow name
-external_ISIL=<isil> optional, ISIL number of library

DE: Impressum · Datenschutzerklärung | EN: Legal Notice · Privacy Policy