Add blocksizes to data buffer

Author: njgheorghita

ethpm/utils/ipfs.py

@@ -9,7 +9,7 @@
 from eth_utils import to_text
 from google.protobuf.descriptor import Descriptor
 
-from ethpm._utils.protobuf.ipfs_file_pb2 import Data, PBNode
+from ethpm._utils.protobuf.ipfs_file_pb2 import Data, PBNode, PBLink
 from ethpm._utils.protobuf.unixfs_pb2 import Data as uData
 from ethpm._utils.protobuf.unixfs_pb2 import Metadata as uMetadata
 from ethpm._utils.protobuf.merkledag_pb2 import PBNode as uPBNode
@@ -110,11 +110,10 @@ def generate_file_hash(content_bytes: bytes) -> str:
 
 def serialize_chunks(file_bytes: bytes) -> Descriptor:
     file_size = len(file_bytes)
-    links = generate_links(file_bytes)
+    links, block_sizes = generate_links(file_bytes)
     data_protobuf = uData(
         Type=uData.DataType.Value("File"),
-        Data=b'',
-        filesize=file_size,
+        blocksizes=block_sizes,
     )
     data_protobuf_bytes = data_protobuf.SerializeToString()
     file_protobuf = uPBNode(Links=links, Data=data_protobuf_bytes)
@@ -131,11 +130,11 @@ def generate_links(content_bytes):
         gen_single_link(content_bytes, i)
         for i in range(0, len(content_bytes), IPFS_CHUNK_SIZE)
     ]
-    return links
+    block_sizes = [link.Tsize - 14 for link in links]
+    return links, block_sizes
 
 
 def gen_single_link(content_bytes, i):
     chunk = content_bytes[i : i + IPFS_CHUNK_SIZE]
     chunk_hash = generate_file_hash(chunk)
-    # not sure why adding 14 is necessary, but is required to make links match ls output size
-    return uPBLink(Hash=to_bytes(text=chunk_hash), Name="", Tsize=len(chunk) + 14)
+    return uPBLink(Hash=to_bytes(text=chunk_hash), Name=None, Tsize=len(chunk) + 14)

tests/ethpm/utils/test_ipfs_utils.py

@@ -119,29 +119,49 @@ def test_generate_file_hash(tmpdir, file_name, file_contents, expected):
 
 
 def test_generate_file_hash_for_large_files(tmp_path):
-    expected_links = (
-        (b'QmZ5RgT3jJhRNMEgLSEsez9uz1oDnNeAysLLxRco8jz5Be', 262158),
-        (b'QmUZvm5TertyZagJfoaw5E5DRvH6Ssu4Wsdfw69NHaNRTc', 262158),
-        (b'QmTA3tDxTZn5DGaDshGTu9XHon3kcRt17dgyoomwbJkxvJ', 262158),
-        (b'QmXRkS2AtimY2gujGJDXjSSkpt2Xmgog6FjtmEzt2PwcsA', 262158),
-        (b'QmVuqvYLEo76hJVE9c5h9KP2MbQuTxSFyntV22qdz6F1Dr', 262158),
-        (b'QmbsEhRqFwKAUoc6ivZyPa1vGUxFKBT4ciH79gVszPcFEG', 262158),
-        (b'QmegS44oDgNU2hnD3j8r1WH8xZ2RWfe3Z5eb6aJRHXwJsw', 262158),
-        (b'QmbC1ZyGUoxZrmTTjgmiB3KSRRXJFkhpnyKYkiVC6PUMzf', 262158),
-        (b'QmZvpEyzP7C8BABesRvpYWPec2HGuzgnTg4VSPiTpQWGpy', 262158),
-        (b'QmZhzU2QJF4rUpRSWZxjutWz22CpFELmcNXkGAB1GVb26H', 262158),
-        (b'QmZeXvgS1NTxtVv9AeHMpA9oGCRrnVTa9bSCSDgAt52iyT', 262158),
-        (b'QmPy1wpe1mACVrXRBtyxriT2T5AffZ1SUkE7xxnAHo4Dvs', 262158),
-        (b'QmcHbhgwAVddCyFVigt2DLSg8FGaQ1GLqkyy5M3U5DvTc6', 262158),
-        (b'QmNsx32qEiEcHRL1TFcy2bPvwqjHZGp62mbcVa9FUpY9Z5', 262158),
-        (b'QmVx2NfXEvHaS8uaRTYaF4ExeLaCSGpTSDhhYBEAembdbk', 69716),
+    pic_expected_links = (
+        (b'QmZ5RgT3jJhRNMEgLSEsez9uz1oDnNeAysLLxRco8jz5Be', 262144),
+        (b'QmUZvm5TertyZagJfoaw5E5DRvH6Ssu4Wsdfw69NHaNRTc', 262144),
+        (b'QmTA3tDxTZn5DGaDshGTu9XHon3kcRt17dgyoomwbJkxvJ', 262144),
+        (b'QmXRkS2AtimY2gujGJDXjSSkpt2Xmgog6FjtmEzt2PwcsA', 262144),
+        (b'QmVuqvYLEo76hJVE9c5h9KP2MbQuTxSFyntV22qdz6F1Dr', 262144),
+        (b'QmbsEhRqFwKAUoc6ivZyPa1vGUxFKBT4ciH79gVszPcFEG', 262144),
+        (b'QmegS44oDgNU2hnD3j8r1WH8xZ2RWfe3Z5eb6aJRHXwJsw', 262144),
+        (b'QmbC1ZyGUoxZrmTTjgmiB3KSRRXJFkhpnyKYkiVC6PUMzf', 262144),
+        (b'QmZvpEyzP7C8BABesRvpYWPec2HGuzgnTg4VSPiTpQWGpy', 262144),
+        (b'QmZhzU2QJF4rUpRSWZxjutWz22CpFELmcNXkGAB1GVb26H', 262144),
+        (b'QmZeXvgS1NTxtVv9AeHMpA9oGCRrnVTa9bSCSDgAt52iyT', 262144),
+        (b'QmPy1wpe1mACVrXRBtyxriT2T5AffZ1SUkE7xxnAHo4Dvs', 262144),
+        (b'QmcHbhgwAVddCyFVigt2DLSg8FGaQ1GLqkyy5M3U5DvTc6', 262144),
+        (b'QmNsx32qEiEcHRL1TFcy2bPvwqjHZGp62mbcVa9FUpY9Z5', 262144),
+        (b'QmVx2NfXEvHaS8uaRTYaF4ExeLaCSGpTSDhhYBEAembdbk', 69702),
     )
-    expected_root_hash = 'QmQgQUbBeMTnH1j3QWwNw9LkXjpWDJrjyGYfZpnPp8x5Lu'
-    test = Path(__file__).parent / 'pic.jpg'
-    ipfs_multihash = generate_links(test.read_bytes())
-    hashes = [(x.Hash, x.Tsize) for x in ipfs_multihash]
-    for x in expected_links:
-        assert x in hashes
+    txt_expected_links = (
+        (b'QmbYfXF7A7hgXUzEZSifji3D6Nf856kdK8Edu8n7knQSHr', 262144),
+        (b'QmZw2JEKQu8n5zisuU7JDAeWSUm2XvqjqV72bizCdDQUDM', 19264),
+    )
+    pic_expected_root_hash = 'QmQgQUbBeMTnH1j3QWwNw9LkXjpWDJrjyGYfZpnPp8x5Lu'
+    txt_exp_hash = 'QmegXsU7EopJ9EVj9ELwgHGE75Xke6FYBP84PaygiAsqB5'
+    pic_path = Path(__file__).parent / 'pic.jpg'
+    txt_path = Path(__file__).parent / 'txt.txt'
+    txt_multihash, txt_blocksizes = generate_links(txt_path.read_bytes())
+    pic_multihash, pic_blocksizes = generate_links(pic_path.read_bytes())
+    txt_hashes = [(x.Hash, x.Tsize) for x in txt_multihash]
+    pic_hashes = [(x.Hash, x.Tsize) for x in pic_multihash]
+    # for x in txt_expected_links:
+        # assert x in txt_hashes
+    # for y in pic_expected_links:
+        # assert y in pic_hashes
     # test works until here, where it has to calculate the root hash
-    root_hash = generate_file_hash(test.read_bytes())
-    assert root_hash == expected_root_hash
+    txt_root_hash = generate_file_hash(txt_path.read_bytes())
+    pic_root_hash = generate_file_hash(pic_path.read_bytes())
+    print("txt: actual / expected")
+    print(txt_root_hash)
+    print(txt_exp_hash)
+    print(f'txt blocksizes: count: {len(txt_blocksizes)}, set: {set(txt_blocksizes)}')
+    print("pic: actual / expected")
+    print(pic_root_hash)
+    print(pic_expected_root_hash)
+    print(f'pic blocksizes: count: {len(pic_blocksizes)}, set: {set(pic_blocksizes)}')
+    assert txt_root_hash == txt_exp_hash	
+    assert pic_root_hash == pic_expected_root_hash