From d0c8abb9cb7dc5482b9bff7ac8697f738c9992fe Mon Sep 17 00:00:00 2001
From: Andreas Romeyke <andreas.romeyke@slub-dresden.de>
Date: Tue, 24 Jan 2023 17:26:58 +0100
Subject: [PATCH] - added new defaults - added init() to correct input
parameters - added throughput() functions - use BufferedInputStream in
retrieveEntity() to (hopefully) increase throughput
---
.../plugin/storage/nfs/SLUBStoragePlugin.java | 82 +++++++++++++++----
1 file changed, 64 insertions(+), 18 deletions(-)
diff --git a/java/org/slub/rosetta/dps/repository/plugin/storage/nfs/SLUBStoragePlugin.java b/java/org/slub/rosetta/dps/repository/plugin/storage/nfs/SLUBStoragePlugin.java
index 46f2a9b..8fa9efa 100644
--- a/java/org/slub/rosetta/dps/repository/plugin/storage/nfs/SLUBStoragePlugin.java
+++ b/java/org/slub/rosetta/dps/repository/plugin/storage/nfs/SLUBStoragePlugin.java
@@ -33,12 +33,14 @@ import com.exlibris.digitool.common.dnx.DnxSectionRecord;
import com.exlibris.digitool.common.storage.Fixity;
import com.exlibris.digitool.infrastructure.utils.Checksummer;
+import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
@@ -46,7 +48,6 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -72,13 +73,43 @@ import java.util.Map;
*/
public class SLUBStoragePlugin extends AbstractStorageHandler {
private static final ExLogger log = ExLogger.getExLogger(SLUBStoragePlugin.class);
- private static final String DIR_ROOT = "DIR_ROOT"; /** {@inheritDoc} */
- private static final int default_buffer = 32*1024;
+ private static String DIR_ROOT = "DIR_ROOT"; /** {@inheritDoc} */
+ private static String FILES_HANDLING_METHOD = "move";
+ private static int BLOCK_SIZE = 32*1024;
- public SLUBStoragePlugin() {
- log.info("SLUBStoragePlugin instantiated");
+ public SLUBStoragePlugin() {}
+
+ @Override
+ public void init(Map<String, String> params) {
+ super.init(params);
+ DIR_ROOT = parameters.get("DIR_ROOT");
+ try {
+ BLOCK_SIZE = Integer.parseInt(this.parameters.get("BLOCK_SIZE"));
+ } catch (NumberFormatException e) {
+ log.error("Could not convert BLOCK_SIZE string to int, " + e);
+ BLOCK_SIZE = 32*1024;
+ }
+ FILES_HANDLING_METHOD = this.parameters.get("FILES_HANDLING_METHOD");
+ log.info("SLUBStoragePlugin instantiated (using DIR_ROOT="+DIR_ROOT+" BLOCK_SIZE="+BLOCK_SIZE);
}
+ private String throughput(long start, long end, String filename) {
+ try {
+ long fsize = Files.size(Paths.get(filename));
+ if (fsize > 0) {
+ return ((end - start) / fsize) + " Bytes/s";
+ }
+ } catch (IOException e) {
+ /* do nothing */
+ }
+ return "(unknown) Bytes/s";
+ }
+ private String throughput(long start, long end, long fsize) {
+ if (fsize > 0) {
+ return ((end-start)/fsize) + " Bytes/s";
+ }
+ return "(unknown) Bytes/s";
+ }
public boolean checkFixity(List<Fixity> fixities, String storedEntityIdentifier) throws Exception {
log.info("SLUBStoragePlugin.checkFixity()");
return checkFixity(fixities, storedEntityIdentifier, true);
@@ -150,14 +181,17 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
String oldValue = fixity.getValue();
log.info("SLUBStoragePlugin.checkFixity() getAlgorithm (2)=" + algorithm);
log.info("SLUBStoragePlugin.checkFixity() oldvalue=" + oldValue);
+ long starttime = System.currentTimeMillis();
InputStream is = retrieveEntity(storedEntityIdentifier, isRelativePath);
Checksummer checksummer = new Checksummer(is, true, true, true, true);
fixity.setValue(checksummer.getChecksum(algorithm));
log.info("SLUBStoragePlugin.checkFixity() newvalue=" + fixity.getValue());
fixity.setResult((oldValue == null) || (oldValue.equalsIgnoreCase(fixity.getValue())));
boolean result = fixity.getResult();
- log.info("SLUBStoragePlugin.checkFixity() result=" + result);
is.close();
+ long endtime = System.currentTimeMillis();
+ var pathname = (isRelativePath ? DIR_ROOT : "") + storedEntityIdentifier;
+ log.info("SLUBStoragePlugin.checkFixity() result=" + result + " (builtin " + throughput(starttime, endtime, pathname)+ ")");
return result;
}
@@ -188,6 +222,7 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
}
String oldValue = fixity.getValue();
log.info("SLUBStoragePlugin.checkFixityByPlugin() oldvalue=" + oldValue);
+ long starttime = System.currentTimeMillis();
fixity.setValue(getChecksumUsingPlugin(isRelativePath ? getLocalFilePath(storedEntityIdentifier) : storedEntityIdentifier, pluginname, oldValue));
/* HINT: if plugin name is still empty a java.lang.NullPointerException gets thrown
Rosetta will handle it and let us know (creates technical issue in workbench)
@@ -195,14 +230,16 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
fixity.setResult((oldValue == null) || (oldValue.equals(fixity.getValue())));
log.info("SLUBStoragePlugin.checkFixityByPlugin() newvalue=" + fixity.getValue());
boolean result = fixity.getResult();
- log.info("SLUBStoragePlugin.checkFixityByPlugin() result=" + result);
+ long endtime = System.currentTimeMillis();
+ var pathname = (isRelativePath ? DIR_ROOT : "") + storedEntityIdentifier;
+ log.info("SLUBStoragePlugin.checkFixityByPlugin() result=" + result + " (plugins " + throughput(starttime, endtime, pathname)+ ")");
return result;
}
public boolean deleteEntity(String storedEntityIdentifier)
{
log.info("SLUBStoragePlugin.deleteEntity() storedEntityIdentifier='" + storedEntityIdentifier + "'");
- File file = new File(this.parameters.get("DIR_ROOT") + storedEntityIdentifier);
+ File file = new File(DIR_ROOT + storedEntityIdentifier);
try
{
return file.delete();
@@ -223,7 +260,7 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
public String getFullFilePath(String storedEntityIdentifier)
{
log.info("SLUBStoragePlugin.getFullFilePath() with '" + storedEntityIdentifier + "'");
- return this.parameters.get("DIR_ROOT") + storedEntityIdentifier;
+ return DIR_ROOT + storedEntityIdentifier;
}
public InputStream retrieveEntity(String storedEntityIdentifier)
@@ -236,19 +273,23 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
throws IOException
{
log.info("SLUBStoragePlugin.retrieveEntity() with '" + storedEntityIdentifier + "' isrelative=" + isRelative);
- var pathname = (isRelative ? this.parameters.get("DIR_ROOT") : "") + storedEntityIdentifier;
-
- return java.nio.file.Files.newInputStream( Paths.get( pathname));
+ var pathname = (isRelative ? DIR_ROOT : "") + storedEntityIdentifier;
+ // TODO: 64k (1MB) buffer set
+ // OLD, without buffering, but working: InputStream foo = java.nio.file.Files.newInputStream( Paths.get( pathname));
+ return new BufferedInputStream( java.nio.file.Files.newInputStream(Paths.get( pathname)), BLOCK_SIZE);
}
public byte[] retrieveEntityByRange(String storedEntityIdentifier, long start, long end) throws Exception
{
log.info("SLUBStoragePlugin.retrieveEntitybyRange() with '" + storedEntityIdentifier + "' start=" + start + " end=" + end);
/* try-with-ressource block, no close if failed needed */
- try (RandomAccessFile file = new RandomAccessFile(this.parameters.get("DIR_ROOT") + storedEntityIdentifier, "r"))
+ try (RandomAccessFile file = new RandomAccessFile(DIR_ROOT + storedEntityIdentifier, "r"))
{
+ long starttime = System.currentTimeMillis();
file.seek(start);
byte[] bytes = new byte[(int)(end - start + 1L)];
file.readFully(bytes, 0, (int)(end - start + 1L));
+ long endtime = System.currentTimeMillis();
+ log.info("SLUBStoragePlugin.retrieveEntitybyRange() sucessfull (" + throughput(starttime, endtime, file.length())+ ")");
return bytes;
}
catch (FileNotFoundException e) {
@@ -307,8 +348,10 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
}
try (OutputStream output = java.nio.file.Files.newOutputStream( Paths.get( destFilePath)))
{
- IOUtil.copy(is, output, default_buffer);
- log.info("SLUBStoragePlugin.storeEntity() try copy (IOUtil.copy) was successfull");
+ long starttime = System.currentTimeMillis();
+ IOUtil.copy(is, output, BLOCK_SIZE);
+ long endtime = System.currentTimeMillis();
+ log.info("SLUBStoragePlugin.storeEntity() try copy (IOUtil.copy) was successfull (" + throughput(starttime, endtime, destFilePath)+ ")" );
}
}
if (!checkFixity(storedEntityMetadata.getFixities(), storedEntityIdentifier)) {
@@ -324,7 +367,7 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
throws IOException
{
log.info("SLUBStoragePlugin.copyStream()");
- String filesHandlingMethod = this.parameters.get("FILES_HANDLING_METHOD");
+ String filesHandlingMethod = FILES_HANDLING_METHOD;
String srcPath = storedEntityMetadata.getCurrentFilePath();
log.debug("SLUBStoragePlugin.copyStream() destPath='" + destPath + "'");
log.debug("SLUBStoragePlugin.copyStream() srcPath='" + srcPath + "'");
@@ -353,9 +396,12 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
}
else
{
+
+ long starttime = System.currentTimeMillis();
FileUtil.copyFile(srcPath, destPath);
saveDestPathsTmpFile(iePid, pid, destPath);
- log.info("SLUBStoragePlugin.copyStream(), copyFile was successful");
+ long endtime = System.currentTimeMillis();
+ log.info("SLUBStoragePlugin.copyStream(), copyFile was successful (" + throughput(starttime, endtime, srcPath) + ")");
}
}
protected String getFilePathInDescIfExists(StoredEntityMetaData storedEntityMetadata)
@@ -446,7 +492,7 @@ public class SLUBStoragePlugin extends AbstractStorageHandler {
*/
File getStreamDirectory(String path, String fileName) {
log.info("SLUBStoragePlugin.getStreamDirectory path='" + path + "' fileName='" + fileName + "'");
- File newDir = new File(parameters.get(DIR_ROOT) + File.separator + path);
+ File newDir = new File(DIR_ROOT + File.separator + path);
log.debug("SLUBStoragePlugin.getStreamDirectory newDir.getAbsolutePath()=" + newDir.getAbsolutePath());
boolean arecreated = newDir.mkdirs();
log.debug("SLUBStoragePlugin.getStreamDirectory newDir.mkdirs(), directories are created:" + arecreated);
--
GitLab