updating to not use memory tar

Author: vsoch

singularity/analysis/apps.py

@@ -64,7 +64,7 @@ def extract_apps(image_path, app_names, S=None, verbose=True):
     if not isinstance(app_names,list):
         app_names = [app_names]
 
-    file_obj, tar = get_image_tar(image_path, S=S, write_file=True)
+    file_obj, tar = get_image_tar(image_path, S=S)
     members = tar.getmembers()
     apps = dict()
 

singularity/analysis/compare.py

@@ -93,7 +93,7 @@ def container_similarity_vector(container1=None,packages_set=None,by=None,custom
     return comparisons
 
 
-def compare_singularity_images(image_paths1,image_paths2=None,old_version=True):
+def compare_singularity_images(image_paths1,image_paths2=None):
     '''compare_singularity_images is a wrapper for compare_containers to compare
     singularity containers. If image_paths2 is not defined, pairwise comparison is done
     with image_paths1
@@ -113,8 +113,7 @@ def compare_singularity_images(image_paths1,image_paths2=None,old_version=True):
 
     comparisons_done = []
     for image1 in image_paths1:
-        fileobj1,tar1 = get_image_tar(image1,
-                                      write_file=old_version)
+        fileobj1,tar1 = get_image_tar(image1)
 
         members1 = [x.name for x in tar1]
         for image2 in image_paths2:
@@ -125,8 +124,7 @@ def compare_singularity_images(image_paths1,image_paths2=None,old_version=True):
                 if image1 == image2:
                     sim = 1.0
                 else:
-                    fileobj2,tar2 = get_image_tar(image2,
-                                                  write_file=old_version)
+                    fileobj2,tar2 = get_image_tar(image2)
                     members2 = [x.name for x in tar2]
                     c = compare_lists(members1,members2)
                     sim = information_coefficient(c['total1'],c['total2'],c['intersect'])

singularity/analysis/reproduce/hash.py

@@ -64,8 +64,7 @@ def get_image_hash(image_path,
                    level=None,level_filter=None,
                    include_files=None,
                    skip_files=None,
-                   version=None,
-                   memory_tar=False):
+                   version=None):
 
     '''get_image_hash will generate a sha1 hash of an image, depending on a level
     of reproducibility specified by the user. (see function get_levels for descriptions)
@@ -100,8 +99,7 @@ def get_image_hash(image_path,
                                 include_files=include_files)
 
     cli = Singularity()
-    file_obj,tar = get_image_tar(image_path,
-                                 write_file=not memory_tar)
+    file_obj,tar = get_image_tar(image_path)
     hasher = hashlib.md5()
 
     for member in tar:
@@ -136,8 +134,7 @@ def get_content_hashes(image_path,
                        level_filter=None,
                        skip_files=None,
                        version=None,
-                       include_sizes=True,
-                       memory_tar=False):
+                       include_sizes=True):
 
     '''get_content_hashes is like get_image_hash, but it returns a complete dictionary 
     of file names (keys) and their respective hashes (values). This function is intended
@@ -158,8 +155,7 @@ def get_content_hashes(image_path,
                                 skip_files=skip_files,
                                 include_files=include_files)
 
-    file_obj,tar = get_image_tar(image_path,
-                                 write_file=not memory_tar)
+    file_obj,tar = get_image_tar(image_path)
 
     results = extract_guts(image_path=image_path,
                            tar=tar,

singularity/analysis/reproduce/metrics.py

@@ -71,6 +71,7 @@ def assess_differences(image_file1,
             guts2 = get_content_hashes(image_path=image_file2,
                                        level_filter=level_filter)
 
+        print(level_name)
         files = list(set(list(guts1['hashes'].keys()) + list(guts2['hashes'].keys())))
 
         for file_name in files:

singularity/analysis/reproduce/utils.py

@@ -88,36 +88,29 @@ def extract_guts(image_path,tar,file_filter,tag_root=True,include_sizes=True):
 
 
 def get_memory_tar(image_path):
-    '''get an in memory tar of an image (does not require sudo!)'''
+    '''get an in memory tar of an image. Use carefully, not as reliable
+       as get_image_tar
+    '''
     cli = Singularity()
     byte_array = cli.export(image_path)
     file_object = io.BytesIO(byte_array)
     tar = tarfile.open(mode="r|*", fileobj=file_object)
     return (file_object,tar)
 
 
-def get_image_tar(image_path,write_file=True,S=None):
+def get_image_tar(image_path,S=None):
     '''get an image tar, either written in memory or to
     the file system. file_obj will either be the file object,
     or the file itself.
     '''
-
-    # Singularity 2.3 and up
-    if not write_file:
-        file_obj,tar = get_memory_tar(image_path)
-        bot.debug('Generate in memory tar...')   
-
-    # Singularity < version 2.3
-    else:
-        bot.debug('Generate file system tar...')   
-        if S is None:
-            S = Singularity()
-        file_obj = S.export(image_path=image_path)
-        if file_obj is None:
-            bot.error("Error generating tar, exiting.")
-            sys.exit(1)
-        tar = tarfile.open(file_obj)
-
+    bot.debug('Generate file system tar...')   
+    if S is None:
+        S = Singularity()
+    file_obj = S.export(image_path=image_path)
+    if file_obj is None:
+        bot.error("Error generating tar, exiting.")
+        sys.exit(1)
+    tar = tarfile.open(file_obj)
     return file_obj, tar
 
 
@@ -149,8 +142,7 @@ def extract_content(image_path,member_name,cli=None,return_hash=False):
         cli = Singularity()
     content = cli.execute(image_path,'cat %s' %(member_name))
     if not isinstance(content,bytes):
-        if isinstance(content,str):
-            content = content.encode('utf-8')
+        content = content.encode('utf-8')
         content = bytes(content)
     # If permissions don't allow read, return None
     if len(content) == 0:

singularity/build/main.py

@@ -151,8 +151,7 @@ def run_build(build_dir,params,verbose=True, compress_image=False):
                                 spec_path=params['spec_file'],
                                 output_folder=build_dir,
                                 remove_image=True,
-                                verbose=True,
-                                old_version=True)
+                                verbose=True)
 
         # Derive software tags by subtracting similar OS
         diff = get_diff(image_package=image_package)

singularity/package/build.py

@@ -115,8 +115,7 @@ def package(image_path,
             software=True,
             remove_image=False,
             verbose=False,
-            S=None,
-            old_version=True):
+            S=None):
 
     '''generate a zip (including the image) to a user specified output_folder.
     :param image_path: full path to singularity image file
@@ -130,7 +129,7 @@ def package(image_path,
     # Run create image and bootstrap with Singularity command line tool.
     S = Singularity(debug=verbose)
 
-    file_obj, tar = get_image_tar(image_path,S=S,write_file=old_version)
+    file_obj, tar = get_image_tar(image_path,S=S)
 
     members = tar.getmembers()
     image_name, ext = os.path.splitext(os.path.basename(image_path))

singularity/tests/test_reproduce.py

@@ -129,18 +129,18 @@ def test_get_content_hashes(self):
         hashes = get_content_hashes(self.image1)
         for key in ['hashes','sizes','root_owned']:
             self.assertTrue(key in hashes)
-        self.assertEqual(len(hashes['hashes']),372)
+        self.assertEqual(len(hashes['hashes']),395)
 
 
     def test_extract_guts(self):
         from singularity.analysis.reproduce.utils import extract_guts
         from singularity.analysis.reproduce import (
-            get_memory_tar,
+            get_image_tar,
             get_levels )
 
         print("Testing singularity.analysis.reproduce.extract_guts")
         levels = get_levels()
-        file_obj,tar = get_memory_tar(self.image1)
+        file_obj,tar = get_image_tar(self.image1)
         guts = extract_guts(image_path=self.image1,
                             tar=tar,
                             file_filter=levels['REPLICATE'])
@@ -151,7 +151,7 @@ def test_get_image_file_hash(self):
         from singularity.analysis.reproduce import get_image_file_hash
         print("Testing singularity.analysis.reproduce.get_image_file_hash")
         hashy = get_image_file_hash(self.image1)
-        self.assertEqual('9d2edcd19328d09f51c86192990050c5',hashy)
+        self.assertEqual('843929a1c92d53656fa744c6b831f59b',hashy)
 
 
 if __name__ == '__main__':