- refactoring, the most probable case first

- fixed throughput calculation
- added own copyFile implementation

PLUGIN-INF/metadata_SLUBStoragePlugin.xml

@@ -70,7 +70,7 @@
 		</fr:x_form>
 	</pl:initParameters>
 	<pl:description>SLUB Storage Plugin</pl:description>
-	<pl:version>2.836</pl:version>
+	<pl:version>2.839</pl:version>
 	<pl:materialType>DIGITAL</pl:materialType>
 	<pl:module>Repository</pl:module>
 	<pl:generalType>TASK</pl:generalType>

java/org/slub/rosetta/dps/repository/plugin/storage/nfs/SLUBStoragePlugin.java

@@ -26,6 +26,7 @@ import com.exlibris.core.infra.svc.api.scriptRunner.ExecExternalProcess;
 import com.exlibris.core.sdk.storage.containers.StoredEntityMetaData;
 import com.exlibris.core.sdk.storage.handler.AbstractStorageHandler;
 import com.exlibris.core.sdk.storage.handler.StorageUtil;
+import com.exlibris.core.sdk.utils.FileTransferUtil;
 import com.exlibris.core.sdk.utils.FileUtil;
 import com.exlibris.digitool.common.dnx.DnxDocument;
 import com.exlibris.digitool.common.dnx.DnxSection;
@@ -35,11 +36,14 @@ import com.exlibris.digitool.infrastructure.utils.Checksummer;
 
 import java.io.BufferedInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.security.NoSuchAlgorithmException;
@@ -84,14 +88,14 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
     private int getBlockSize () {
         int blockSize = 32*1024;
         String blocksizeStr = this.parameters.get("BLOCK_SIZE");
-        if (null == blocksizeStr) {
-            log.error("Could not retrieve BLOCK_SIZE");
-        }
+        if (null != blocksizeStr) {
             try {
                 blockSize = Integer.parseInt(blocksizeStr);
             } catch (NumberFormatException e) {
                 log.error("Could not convert BLOCK_SIZE string to int, " + e.getMessage());
-            blockSize = 32*1024;
+            }
+        } else {
+            log.error("Could not retrieve BLOCK_SIZE");
         }
         return blockSize;
     }
@@ -115,8 +119,22 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
     }
     private String throughput(long starttime_in_ms, long endtime_in_ms, long fsize) {
         long duration_in_ms = endtime_in_ms - starttime_in_ms;
-        if (fsize > 0 && duration_in_ms >= 0) {
-            return (duration_in_ms / fsize) + " kBytes/s"; /* kBytes because ms */
+        log.info("SLUBStoragePlugin.throughput duration=" + duration_in_ms + " filesize=" + fsize);
+        if ((fsize >= 0) && (duration_in_ms) > 0) {
+            /* fsize in Bytes
+               duration in ms = duration in s / 1000
+               ->
+               fsize_B/duration_in_s = fsize_B*1000/duration_in_ms
+               ->
+               fsize_MB/duration_in_s = fsize_B*1000/(1024*1024*duration_in_s)
+
+               Example:
+               fsize_B=31561376
+               duration_in_ms = 86
+               ->
+               386MB/s
+             */
+            return (fsize * 1000) / (1024 * 1024 * duration_in_ms) + " MBytes/s";
         }
         return "(unknown) Bytes/s";
     }
@@ -346,8 +364,6 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
     {
         log.info("SLUBStoragePlugin.retrieveEntity() with storedEntityIdentifier '" + storedEntityIdentifier + "'");
         contractAssertIsRelativePath(storedEntityIdentifier);
-        // TODO: 64k (1MB) buffer set
-        // OLD, without buffering, but working: InputStream foo =  java.nio.file.Files.newInputStream( Paths.get( pathname));
         var absolute_filename = getFullFilePath(storedEntityIdentifier);
         contractAssertIsAbsolutePath(absolute_filename);
         var absolute_path = Paths.get(absolute_filename);
@@ -460,6 +476,31 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
         return storedEntityIdentifier;
     }
 
+    /**
+     * copyFile, specialised function derived from FileUtil.copyFile using improved filecopy
+     * @param source
+     * @param destination
+     * @throws IOException
+     */
+    private void copyFile(String source, String destination) throws IOException {
+        long starttime = System.currentTimeMillis();
+        if (!source.toLowerCase().trim().startsWith("http")) { /* simpleFilecopy, see https://www.digitalocean.com/community/tutorials/java-copy-file */
+            try (
+                    FileChannel sourceChannel = new FileInputStream(source).getChannel();
+                    FileChannel destChannel = new FileOutputStream(destination).getChannel()
+            ) {
+                destChannel.transferFrom(sourceChannel, 0, sourceChannel.size());
+            }
+        } else {
+            InputStream input = FileTransferUtil.getUrlContent(source);
+            FileOutputStream output = new FileOutputStream(destination);
+            IOUtil.copy(input, output, getBlockSize());
+            IOUtil.shutdownStream(input);
+            IOUtil.shutdownStream(output);
+        }
+        long endtime = System.currentTimeMillis();
+        log.info("SLUBStoragePlugin.copyFile (" + source + ", " + destination + ") was successfull (" + throughput(starttime, endtime, destination) + ")");
+    }
     protected void copyStream(StoredEntityMetaData storedEntityMetadata, String destPath)
             throws IOException
     {
@@ -473,31 +514,31 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
         log.debug("SLUBStoragePlugin.copyStream() pid='" + pid + "'");
         String iePid = storedEntityMetadata.getIePid();
         log.debug("SLUBStoragePlugin.copyStream() iePid='" + iePid + "'");
-        if ("move".equalsIgnoreCase(filesHandlingMethod))
-        {
+        String attr = "('\"+srcPath+\"','\" + destPath + \"')";
+        if ("copy".equalsIgnoreCase(filesHandlingMethod)) {
+            // FileUtil.copyFile(srcPath, destPath);
+            copyFile(srcPath, destPath);
+            saveDestPathsTmpFile(iePid, pid, destPath);
+            log.info("SLUBStoragePlugin.copyStream(), copy"+attr + " was successful");
+        } else if ("move".equalsIgnoreCase(filesHandlingMethod)) {
             File canonicalSrcFile = getCanonicalFile(srcPath);
             assert canonicalSrcFile != null;
             FileUtil.moveFile(canonicalSrcFile, new File(destPath));
             saveDestPathsTmpFile(iePid, pid, destPath);
-            log.info("SLUBStoragePlugin.copyStream(), move was successful");
+            log.info("SLUBStoragePlugin.copyStream(), move"+attr+" was successful");
         }
-        else if ("soft_link".equalsIgnoreCase(filesHandlingMethod))
-        {
+        else if ("soft_link".equalsIgnoreCase(filesHandlingMethod)) {
             softLink(srcPath, destPath);
-            log.info("SLUBStoragePlugin.copyStream(), softlink was successful");
+            log.info("SLUBStoragePlugin.copyStream(), softlink"+attr+" was successful");
         }
-        else if ("hard_link".equalsIgnoreCase(filesHandlingMethod))
-        {
+        else if ("hard_link".equalsIgnoreCase(filesHandlingMethod)) {
             hardLink(srcPath, destPath);
-            log.info("SLUBStoragePlugin.copyStream(), hardlink was successful");
+            log.info("SLUBStoragePlugin.copyStream(), hardlink"+attr+" was successful");
         }
-        else
-        {
-            long starttime = System.currentTimeMillis();
-            FileUtil.copyFile(srcPath, destPath);
-            saveDestPathsTmpFile(iePid, pid, destPath);
-            long endtime = System.currentTimeMillis();
-            log.info("SLUBStoragePlugin.copyStream(), copyFile('"+srcPath+"','" + destPath + "') was successful (" + throughput(starttime, endtime, srcPath) + ")");
+        else {
+            String msg = "SLUBStoragePlugin.copyStream(), unknown filehandling method detected: " + filesHandlingMethod;
+            log.fatal(msg);
+            throw new IOException(msg);
         }
     }
     protected String getFilePathInDescIfExists(StoredEntityMetaData storedEntityMetadata)