update utils

Author: zmaalick

notebooks/utils.py

@@ -11,7 +11,7 @@
 def _fetch_s3_file(s3_uri, save_to):
 
     bucket_name, key = _split_s3_uri(s3_uri)
-    #print(f"Fetching s3 object {key} from bucket {bucket_name}")
+    # print(f"Fetching s3 object {key} from bucket {bucket_name}")
 
     client = boto3.client("s3")
     obj = client.get_object(
@@ -26,7 +26,7 @@ def _fetch_s3_file(s3_uri, save_to):
 def _save_s3_file(s3_uri, out_filename, file_to_save="/tmp/tmp"):
     bucket, folder = _split_s3_uri(s3_uri)
     out_filepath = os.path.join(folder, out_filename)
-    #print(f"Save s3 object {out_filepath} to bucket {bucket}")
+    # print(f"Save s3 object {out_filepath} to bucket {bucket}")
     client = boto3.client("s3")
     client.upload_file(
         Filename=file_to_save,
@@ -81,7 +81,6 @@ def copy_s3_files(in_fileglob, out_folder):
     in_fileglob: s3 uri of flies (wild card can be used)
     out_folder: local path where data will be stored
     '''
-   
     matching_keys = find_matching_s3_keys(in_fileglob)
     in_bucket_name = _split_s3_uri(in_fileglob)[0]
     out_scheme = urlparse(out_folder).scheme
@@ -103,44 +102,43 @@ def copy_s3_files(in_fileglob, out_folder):
         os.remove(temp_filename)
 
 
-        
 def load_data(inpath):
 
     if inpath.startswith('s3'):
         keys = find_matching_s3_keys(inpath)
         s3dir = _get_directory(inpath)
-        temp_path = '/scratch/zmaalick'
-        
+        temp_path = 'data/'
+        if os.path.exists(temp_path) == 0:
+            os.mkdir(temp_path)
+
         for key in keys:
             file = key.split('/')[-1]
             if os.path.exists(os.path.join(temp_path, file)) == 0:
                 print(os.path.join(s3dir, file))
                 copy_s3_files(os.path.join(s3dir, file), temp_path)
             else:
                 print(key, ' already exist')
-        
         files = inpath.split('/')[-1]
+        data = iris.load(os.path.join(temp_path, files))
 
-        data = iris.load(os.path.join(temp_path,files))
-        
         return data
 
 
 def _get_directory(inpath):
     path = inpath.split('/')
-    dirpath='s3://'
+    dirpath = 's3://'
     for p in path[2:-1]:
-        dirpath = os.path.join(dirpath,p)
+        dirpath = os.path.join(dirpath, p)
     return dirpath
-        
+
+
 def flush_data():
     import glob
     files = glob.glob('/tmp/*.nc')
     for file in files:
         os.remove(file)
 
-  
-        
+
 def main():
     in_fileglob = 's3://ias-pyprecis/data/cmip5/.nc'
     out_folder = '/home/h01/zmaalick/myprojs/PyPRECIS/aws-scripts'