add create timestamps for attributes in obj create

Author: jreadey

admin/config/config.yml

@@ -88,6 +88,7 @@ allow_any_bucket_read: true  # enable reads to buckets other than default bucket
 allow_any_bucket_write: true # enable writes to buckets other than default bucket
 bit_shuffle_default_blocksize: 2048 # default blocksize for bitshuffle filter
 max_rangeget_gap: 1024 # max gap in byte for intelligent range get requests
+predate_maxtime: 10.0 # max delta between object created timestamp in request and actual time
 # DEPRECATED - the remaining config values are not used in currently but kept for backward compatibility with older container images
 aws_lambda_chunkread_function: null # name of aws lambda function for chunk reading
 aws_lambda_threshold: 4 # number of chunks per node per request to reach before using lambda

hsds/servicenode_lib.py

@@ -15,6 +15,7 @@
 
 import asyncio
 import json
+import time
 import numpy as np
 
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPGone, HTTPConflict
@@ -39,6 +40,7 @@
 from .basenode import getVersion
 
 from . import hsds_logger as log
+from . import config
 
 
 async def getDomainJson(app, domain, reload=False):
@@ -1010,6 +1012,18 @@ async def getAttributeFromRequest(app, req_json, obj_id=None, bucket=None):
     else:
         attr_item["value"] = None
 
+    now = time.time()
+    if "created" in req_json:
+        created = req_json["created"]
+        # allow "pre-dated" attributes if the timestamp is within the last 10 seconds
+        predate_max_time = config.get("predate_max_time", default=10.0)
+        if now - created > predate_max_time:
+            attr_item["created"] = created
+        else:
+            log.warn("stale created timestamp for attribute, ignoring")
+    if "created" not in attr_item:
+        attr_item["created"] = now
+
     return attr_item
 
 

tests/integ/dataset_test.py

@@ -333,7 +333,19 @@ def testPostDatasetWithAttributes(self):
         self.assertEqual(rsp.status_code, 201)
         rspJson = json.loads(rsp.text)
         self.assertEqual(rspJson["attributeCount"], 4)
-        self.assertTrue(helper.validateId(rspJson["id"]))
+        dset_id = rspJson["id"]
+        self.assertTrue(helper.validateId(dset_id))
+
+        # fetch the attributes
+        req = f"{helper.getEndpoint()}/datasets/{dset_id}/attributes"
+        rsp = self.session.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        self.assertTrue("hrefs" in rspJson)
+        self.assertFalse("type" in rspJson)
+        self.assertFalse("shape" in rspJson)
+        self.assertTrue("attributes") in rspJson
+        self.assertEqual(len(rspJson["attributes"]), attr_count)
 
     def testScalarEmptyDimsDataset(self):
         # Test creation/deletion of scalar dataset obj

tests/integ/datatype_test.py

@@ -201,12 +201,24 @@ def testPostWithAttributes(self):
         rsp = self.session.post(req, data=json.dumps(payload), headers=headers)
         self.assertEqual(rsp.status_code, 201)
         rspJson = json.loads(rsp.text)
-        self.assertTrue(helper.validateId(rspJson["id"]))
+        ctype_id = rspJson["id"]
+        self.assertTrue(helper.validateId(ctype_id))
         self.assertTrue("type" in rspJson)
         type_json = rspJson["type"]
         self.assertEqual(type_json["class"], "H5T_FLOAT")
         self.assertEqual(type_json["base"], "H5T_IEEE_F32LE")
-        self.assertEqual(rspJson["attributeCount"], 4)
+        self.assertEqual(rspJson["attributeCount"], attr_count)
+
+        # fetch the attributes, check count
+        req = f"{helper.getEndpoint()}/datatypes/{ctype_id}/attributes"
+        rsp = self.session.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        self.assertTrue("hrefs" in rspJson)
+        self.assertFalse("type" in rspJson)
+        self.assertFalse("shape" in rspJson)
+        self.assertTrue("attributes") in rspJson
+        self.assertEqual(len(rspJson["attributes"]), attr_count)
 
     def testPostTypes(self):
         # Test creation with all primitive types

tests/integ/group_test.py

@@ -403,8 +403,20 @@ def testPostWithAttributes(self):
         self.assertEqual(rsp.status_code, 201)
         rspJson = json.loads(rsp.text)
         self.assertEqual(rspJson["linkCount"], 0)
-        self.assertEqual(rspJson["attributeCount"], 4)
-        self.assertTrue(helper.validateId(rspJson["id"]))
+        self.assertEqual(rspJson["attributeCount"], attr_count)
+        grp_id = rspJson["id"]
+        self.assertTrue(helper.validateId(grp_id))
+
+        # fetch the attributes, check count
+        req = f"{helper.getEndpoint()}/groups/{grp_id}/attributes"
+        rsp = self.session.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        self.assertTrue("hrefs" in rspJson)
+        self.assertFalse("type" in rspJson)
+        self.assertFalse("shape" in rspJson)
+        self.assertTrue("attributes") in rspJson
+        self.assertEqual(len(rspJson["attributes"]), attr_count)
 
     def testPostWithPath(self):
         # test POST with implicit parent group creation