102-BE/cannot-read-big-file-by-readAllBytes

dspace-api/src/main/java/org/dspace/content/ClarinBitstreamServiceImpl.java

@@ -7,11 +7,13 @@
  */
 package org.dspace.content;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Objects;
 
+import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.dspace.authorize.AuthorizeException;
@@ -24,6 +26,7 @@
 import org.dspace.core.Context;
 import org.dspace.event.Event;
 import org.dspace.storage.bitstore.DSBitStoreService;
+import org.dspace.storage.bitstore.service.BitstreamStorageService;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
@@ -54,6 +57,8 @@ public class ClarinBitstreamServiceImpl implements ClarinBitstreamService {
     protected BundleService bundleService;
     @Autowired
     protected BitstreamService bitstreamService;
+    @Autowired
+    private BitstreamStorageService bitstreamStorageService;
 
     protected ClarinBitstreamServiceImpl() {
     }
@@ -82,8 +87,7 @@ public Bitstream create(Context context, Bundle bundle) throws SQLException, Aut
     }
 
     @Override
-    public boolean addExistingFile(Context context, Bitstream bitstream, Long expectedSizeBytes,
-                                   String expectedCheckSum, String expectedChecksumAlgorithm)
+    public boolean validation(Context context, Bitstream bitstream)
             throws IOException, SQLException, AuthorizeException {
         if (!authorizeService.isAdmin(context)) {
             throw new AuthorizeException(
@@ -95,9 +99,13 @@ public boolean addExistingFile(Context context, Bitstream bitstream, Long expect
         }
         //get file from assetstore based on internal_id
         //recalculate check fields
-        storeService.put(bitstream, new ByteArrayInputStream(storeService.get(bitstream).readAllBytes()));
+        Map wantedMetadata = new HashMap();
+        wantedMetadata.put("size_bytes", null);
+        wantedMetadata.put("checksum", null);
+        wantedMetadata.put("checksum_algorithm", null);
+        Map checksumMap = storeService.about(bitstream, wantedMetadata);
         //check that new calculated values match the expected values
-        if (!valid(bitstream, expectedSizeBytes, expectedCheckSum, expectedChecksumAlgorithm)) {
+        if (MapUtils.isEmpty(checksumMap) || !valid(bitstream, checksumMap)) {
             //an error occurred - expected and calculated values do not match
             //delete all created data
             bitstreamService.delete(context, bitstream);
@@ -113,16 +121,20 @@ public boolean addExistingFile(Context context, Bitstream bitstream, Long expect
 
     /**
      * Validation control.
-     * Control that expected values (method attributes) match with bitstream calculated values.
+     * Control that expected values (bitstream attributes) match with calculated values.
      * @param bitstream bitstream
-     * @param expectedSizeBytes expected size bytes
-     * @param expectedCheckSum expected checksum
-     * @param expectedChecksumAlgorithm expected checksum algorithm
+     * @param checksumMap calculated values
      * @return bitstream values match with expected values
      */
-    private boolean valid(Bitstream bitstream, long expectedSizeBytes,
-                                     String expectedCheckSum, String expectedChecksumAlgorithm) {
-        return bitstream.getSizeBytes() == expectedSizeBytes && bitstream.getChecksum().equals(expectedCheckSum) &&
-                bitstream.getChecksumAlgorithm().equals(expectedChecksumAlgorithm);
+    private boolean valid(Bitstream bitstream, Map checksumMap) {
+        if (!checksumMap.containsKey("checksum") || !checksumMap.containsKey("checksum_algorithm") ||
+                !checksumMap.containsKey("size_bytes")) {
+            log.error("Cannot validate of bitstream with id: " + bitstream.getID() +
+                    ", because there were no calculated all required fields.");
+            return false;
+        }
+        return bitstream.getSizeBytes() == Long.valueOf(checksumMap.get("size_bytes").toString()) &&
+                bitstream.getChecksum().equals(checksumMap.get("checksum").toString()) &&
+                bitstream.getChecksumAlgorithm().equals(checksumMap.get("checksum_algorithm").toString());
     }
 }

dspace-api/src/main/java/org/dspace/content/service/clarin/ClarinBitstreamService.java

@@ -36,23 +36,20 @@ public interface ClarinBitstreamService {
     public Bitstream create(Context context, Bundle bundle) throws SQLException, AuthorizeException;
 
     /**
-     * Add an existing file to bitstream.
+     * Validation between expected values and calculated values based on existing file.
      * The file must be stored in assetstore under internal_id. Internal_id must be specified in input bitstream.
-     * Method transforms data from assetstore back into a stream of bits and fills the bitstream
+     * Method finds data in assetstore and calculates the bitstream
      * check fields (checksum, sizeBytes, checksum algorithm).
+     * These calculated values are compared with expected values from input bitstream.
      * The bitstream is stored into database only if the error was not occur:
      * calculated and expected check fields values match.
      * @param context context
-     * @param bitstream bitstream with entered internal_id
-     * @param expectedSizeBytes expected size
-     * @param expectedCheckSum expected checksum
-     * @param expectedChecksumAlgorithm expected checksum algorithm
-     * @return existing file was successfully added to the bitstream
+     * @param bitstream bitstream
+     * @return validation was successfully
      * @throws IOException If a problem occurs while storing the bits
      * @throws SQLException if database error
      * @throws AuthorizeException if authorization error
      */
-    public boolean addExistingFile(Context context, Bitstream bitstream, Long expectedSizeBytes,
-                                  String expectedCheckSum, String expectedChecksumAlgorithm)
+    public boolean validation(Context context, Bitstream bitstream)
             throws IOException, SQLException, AuthorizeException ;
 }

dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinBitstreamImportController.java

@@ -131,9 +131,14 @@ public BitstreamRest importBitstreamForExistingFile(HttpServletRequest request)
             }
             bitstream.setFormat(context, bitstreamFormat);
             String deletedString = request.getParameter("deleted");
-            //join created bitstream with file stored in assetstore
-            if (!clarinBitstreamService.addExistingFile(context, bitstream, bitstreamRest.getSizeBytes(),
-                    bitstreamRest.getCheckSum().getValue(), bitstreamRest.getCheckSum().getCheckSumAlgorithm())) {
+            //set size bytes
+            bitstream.setSizeBytes(bitstreamRest.getSizeBytes());
+            //set checksum
+            bitstream.setChecksum(bitstreamRest.getCheckSum().getValue());
+            //set checksum algorithm
+            bitstream.setChecksumAlgorithm(bitstreamRest.getCheckSum().getCheckSumAlgorithm());
+            //do validation between input fields and calculated fields based on file from assetstore
+            if (!clarinBitstreamService.validation(context, bitstream)) {
                 return null;
             }
 

dspace-server-webapp/src/test/java/org/dspace/app/rest/ClarinBitstreamImportControllerIT.java

@@ -24,6 +24,7 @@
 import org.dspace.app.rest.test.AbstractEntityIntegrationTest;
 import org.dspace.authorize.service.AuthorizeService;
 import org.dspace.builder.BitstreamBuilder;
+import org.dspace.builder.BitstreamFormatBuilder;
 import org.dspace.builder.BundleBuilder;
 import org.dspace.builder.CollectionBuilder;
 import org.dspace.builder.CommunityBuilder;
@@ -94,7 +95,7 @@ public void setup() throws Exception {
                 .withName("TESTINGBUNDLE")
                 .build();
         token = getAuthToken(admin.getEmail(), password);
-        bitstreamFormat = bitstreamFormatService.create(context);
+        bitstreamFormat = BitstreamFormatBuilder.createBitstreamFormat(context).build();
         String input = "Hello, World!";
         MockMultipartFile file = new MockMultipartFile("file", "hello.txt",
                 MediaType.TEXT_PLAIN_VALUE,