Retrieve MD5 from object metadata (#414)

* Retrieve MD5 from object metadata

* Decode Base64 formatted contentMD5 value

Author: joneubank

score-server/src/main/java/bio/overture/score/server/config/S3Config.java

@@ -39,6 +39,9 @@ public class S3Config {
   private boolean sigV4Enabled;
   private String masterEncryptionKeyId;
 
+  @Value("md5chksum")
+  private String customMd5Property;
+
   @Value("${upload.connection.timeout}")
   private int connectionTimeout;
 

score-server/src/main/java/bio/overture/score/server/repository/s3/S3DownloadService.java

@@ -23,8 +23,10 @@
 import bio.overture.score.core.model.ObjectKey;
 import bio.overture.score.core.model.ObjectSpecification;
 import bio.overture.score.core.model.Part;
+import bio.overture.score.core.util.MD5s;
 import bio.overture.score.core.util.ObjectKeys;
 import bio.overture.score.core.util.PartCalculator;
+import bio.overture.score.server.config.S3Config;
 import bio.overture.score.server.exception.IdNotFoundException;
 import bio.overture.score.server.exception.InternalUnrecoverableError;
 import bio.overture.score.server.exception.NotRetryableException;
@@ -89,6 +91,7 @@ public class S3DownloadService implements DownloadService {
   @Autowired private URLGenerator urlGenerator;
   @Autowired private PartCalculator partCalculator;
   @Autowired private MetadataService metadataService;
+  @Autowired private S3Config s3config;
 
   @Override
   public ObjectSpecification download(
@@ -118,14 +121,17 @@ public ObjectSpecification download(
           parts = partCalculator.divide(offset, length);
         }
         fillPartUrls(objectKey, parts, false, forExternalUse);
+
+        val md5 = getObjectMd5(metadata);
+
         objectSpec =
             new ObjectSpecification(
                 objectKey.getKey(),
                 objectId,
                 objectId,
                 parts,
                 metadata.getContentLength(),
-                metadata.getETag(),
+                md5,
                 false);
       }
 
@@ -189,6 +195,35 @@ public ObjectSpecification download(
     }
   }
 
+  /**
+   * Looks for MD5 hash in the object metadata. This is used as part of the fallback behaviour when
+   * the .meta file cannot be found. To find the MD5, we will look for a value using the built in S3
+   * getContendMD5(), and if no value is found there we will check in a configurable user meta data
+   * property. The name of this property is configurable via the S3 Config. If no MD5 value can be
+   * found in either of these locations then it will be returned null.
+   *
+   * <p>A user can still download files with the MD5 set to null, but they will always fail to
+   * validate through the CLI. To complete a download of a file in this state, the user should add
+   * the argument to their CLI download command: --validate false
+   *
+   * @param metadata
+   * @return
+   */
+  private String getObjectMd5(ObjectMetadata metadata) {
+    val contentMd5 = metadata.getContentMD5();
+    if (contentMd5 != null) {
+      return  MD5s.toHex(contentMd5);
+    }
+    val userMetadataMd5 =
+        metadata.getUserMetaDataOf(s3config.getCustomMd5Property()); // get literal from config
+    if (userMetadataMd5 != null) {
+      return MD5s.toHex(userMetadataMd5);
+    }
+
+    // No value found, returning null.
+    return null;
+  }
+
   private static ObjectSpecification removeUrls(ObjectSpecification spec) {
     spec.getParts().forEach(x -> x.setUrl(null));
     return spec;

score-server/src/main/resources/application.yml

@@ -34,6 +34,9 @@ s3:
   secured: true
   sigV4Enabled: true
 
+  # custom meta property with md5 hash, unused when upload state files are available (default behaviour)
+  # customMd5Property: md5chksum
+
   #amazon
   endpoint: s3-external-1.amazonaws.com