v1.4

file generation performance issues

Author: seanhaythorne

PaleoclimateToolDataFileHelper.py

@@ -1,6 +1,6 @@
 # BUILD FLAG:   Set to True when creating executable packages to ensure dependency work-arounds
 #               Set to False for code releases to ensure Linux installation is easier to achieve
-EXECUTABLE_BUILD_INCLUSION = True
+EXECUTABLE_BUILD_INCLUSION = False
 
 # Python modules
 import re
@@ -830,17 +830,23 @@ def loadClimateDataGrid(self, parameter, year_ad, month_index) : # TODO: load fr
                 # Check current subgroup
                 sub_interval_ad = self.convertDataIntervalLabelToAD(self.cached_netCdf_data[parameter]['sub_interval_str'])
                 if sub_interval_ad['from_year_ad'] <= year_ad <= sub_interval_ad['until_year_ad'] :
-                    subgroup = rootgrp.groups[self.cached_netCdf_data[parameter]['sub_interval_str']]
-                    return subgroup.variables[year_str][month_index]
-
-##                # Check current rootgrp
-##                for sub_interval_str in rootgrp.groups.keys() :
-##                    sub_interval_ad = self.convertDataIntervalLabelToAD(sub_interval_str)
-##                    if sub_interval_ad['from_year_ad'] <= year_ad <= sub_interval_ad['until_year_ad'] :
-##                        self.cached_netCdf_data[parameter]['sub_interval_str'] = sub_interval_str
-##                        return rootgrp.groups[sub_interval_str].variables[year_str][month_index]
-
-                # Not found - clear cache for parameter
+                    try :
+                        subgroup = rootgrp.groups[self.cached_netCdf_data[parameter]['sub_interval_str']]
+                        return subgroup.variables[year_str][month_index]
+                    except :
+                        ignore = None # re-open netCDF file (as below)
+
+                # Check current rootgrp
+                for sub_interval_str in rootgrp.groups.keys() :
+                    sub_interval_ad = self.convertDataIntervalLabelToAD(sub_interval_str)
+                    if sub_interval_ad['from_year_ad'] <= year_ad <= sub_interval_ad['until_year_ad'] :
+                        self.cached_netCdf_data[parameter]['sub_interval_str'] = sub_interval_str
+                        try :
+                            return rootgrp.groups[sub_interval_str].variables[year_str][month_index]
+                        except :
+                            ignore = None # re-open netCDF file (as below)
+
+                # Not found or fail - clear cache for parameter
                 rootgrp.close()
                 self.cached_netCdf_data.pop(parameter)
 

installer.iss

@@ -5,16 +5,16 @@
 ; NOTE: The value of AppId uniquely identifies this application.
 ; Do not use the same AppId value in installers for other applications.
 ; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
-AppId={{B6F86A19-6567-42F1-B8A1-7632B7016A81}
+AppId={{91C97072-E776-4B4E-8F5A-E6B3F2AC7993}
 AppName=PaleoView
-AppVersion=1.1
-AppVerName=PaleoView v1.1
+AppVersion=1.4
+AppVerName=PaleoView v1.4
 AppPublisher=Global Ecology Lab
-AppPublisherURL=http://www.example.com/
-AppSupportURL=http://www.example.com/
-AppUpdatesURL=http://www.example.com/
-DefaultDirName={pf}\PaleoView v1.1
-DefaultGroupName=PaleoView v1.1
+AppPublisherURL=https://github.com/GlobalEcologyLab/PaleoView
+AppSupportURL=https://github.com/GlobalEcologyLab/PaleoView
+AppUpdatesURL=https://github.com/GlobalEcologyLab/PaleoView
+DefaultDirName={pf}\PaleoView v1.4
+DefaultGroupName=PaleoView v1.4
 OutputBaseFilename=setup
 Compression=lzma
 SolidCompression=yes
@@ -26,18 +26,18 @@ Name: "english"; MessagesFile: "compiler:Default.isl"
 Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
 
 [Files]
-Source: "C:\Users\shaythorne\Dropbox\GlobalEcology\PythonResources\vcredist_x86.exe"; DestDir: {tmp}
+Source: "C:\Users\Sean\GoogleDrive\GEL-Archive\PythonResources\vcredist_x86.exe"; DestDir: {tmp}
 Source: "dist\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
 Source: "paleo_view_config.txt"; DestDir: "{app}"
 Source: "Map Data/*"; DestDir: "{app}/Map Data"; Flags: recursesubdirs createallsubdirs
 Source: "Bias Corrections/*"; DestDir: "{app}/Bias Corrections"; Flags: recursesubdirs createallsubdirs
 ; NOTE: Don't use "Flags: ignoreversion" on any shared system files vcredist_x86.exe
 
 [Icons]
-Name: "{group}\PaleoView v1.1"; Filename: "{app}\paleo_view_v1_1.exe"
-Name: "{commondesktop}\PaleoView v1.1"; Filename: "{app}\paleo_view_v1_1.exe"; Tasks: desktopicon
+Name: "{group}\PaleoView v1.4"; Filename: "{app}\paleo_view_v1_4.exe"
+Name: "{commondesktop}\PaleoView v1.4"; Filename: "{app}\paleo_view_v1_4.exe"; Tasks: desktopicon
 
 [Run]
 Filename: "{tmp}\vcredist_x86.exe"; Parameters: "/qu"; StatusMsg: "Installing Microsoft Visual C++ 2008 Redistributable Package..."
 Filename: "{tmp}\vcredist_x86.exe"; Parameters: ""; StatusMsg: "Installing Microsoft Visual C++ 2008 Redistributable Package..."
-Filename: "{app}\paleo_view_v1_1.exe"; Description: "{cm:LaunchProgram,PaleoView}"; Flags: nowait postinstall skipifsilent
+Filename: "{app}\paleo_view_v1_4.exe"; Description: "{cm:LaunchProgram,PaleoView}"; Flags: nowait postinstall skipifsilent

paleo_view_v1_3.py paleo_view_v1_4.pypaleo_view_v1_3.py renamed to paleo_view_v1_4.py

@@ -41,7 +41,7 @@
 from matplotlib import rcParams
 
 # Mac version?
-MAC_VERSION = False # Make True for releases to enable Linux compatibility
+MAC_VERSION = True # Make True for releases to enable Linux compatibility
 
 # Tool library modules
 from PaleoclimateToolDataFileHelper import PaleoclimateToolDataFileHelper
@@ -57,6 +57,9 @@
 DecodeAES = lambda c, e: c.decrypt(base64.b64decode(e)).rstrip(PADDING)
 PASSWORDKEY = '\xe7Z\xb2\xc9\x03\xb4\x8c\x04A\xfe\x97\xc3\xf9\xc7\xd2\x1c\xb9\x1eh\xd0\x91E/\xdb\xc9\x9f\xf0\xda\x8e\x06\xae\xd5' # set using os.urandom(BLOCK_SIZE)
 
+# Allow open interval step and size (set to False for releases)
+OPEN_INTERVALS = False
+
 ## Application GUI
 ## * Constructs tool GUI components
 ## * Performs tool operations utilising climate data helper module
@@ -799,9 +802,12 @@ def createPeriodIntervalFrame(self) :
         for i, selection in enumerate(self.period_postfix_keys) :
             self.period_postfix_menu['from']['menu'].entryconfigure(i, command=(select_period_postfix, 'from', selection))
             self.period_postfix_menu['until']['menu'].entryconfigure(i, command=(select_period_postfix, 'until', selection))
-        self.period_postfix_menu['until']['menu'].entryconfigure(0, state=tk.DISABLED)
+        #self.period_postfix_menu['until']['menu'].entryconfigure(0, state=tk.DISABLED) # use if initial from is AD
 
-        self.interval_step_range = { 'min' : 10, 'max' : 10000 }
+        if OPEN_INTERVALS :
+            self.interval_step_range = { 'min' : 1, 'max' : 10000 }
+        else :
+            self.interval_step_range = { 'min' : 10, 'max' : 10000 }
         self.interval_step_values = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000]
         self.current_interval_step_values = [10, 20, 50, 100]
         self.current_valid_interval_step_value = 20
@@ -811,7 +817,10 @@ def createPeriodIntervalFrame(self) :
         self.interval_step_entry.config(command=(interval_step_spinbox_arrow_press))
         self.current_interval_steps = 8
 
-        self.interval_size_range = { 'min' : 10, 'max' : 100 }
+        if OPEN_INTERVALS :
+            self.interval_size_range = { 'min' : 1, 'max' : 10000 }
+        else :
+            self.interval_size_range = { 'min' : 10, 'max' : 100 }
         self.interval_size_values = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
         self.current_interval_size_values = [10, 20, 30, 40, 50, 60, 70, 80]
         self.current_valid_interval_size_value = 30
@@ -5845,7 +5854,6 @@ def generateDataFiles(self) :
                                                                                  all_months=self.time_unit_is_all_months,
                                                                                  correct_bias=self.utilise_bias_correction.get())
                     self.parameter_data = parameter_data[0].copy()
-                    self.data_file_helper.clearNetCdfDataCache()
                 except Exception, e :
                     self.data_file_helper.clearNetCdfDataCache()
                     showerror('Data extraction error', str(e))
@@ -5893,6 +5901,9 @@ def generateDataFiles(self) :
                 self.updateToolGenerationLogEntry(grids_generated={ 'number' : generated_file_count, 'expected' : len(period_years_ad) })
                 self.updateToolGenerationLogs()
 
+            # Clear netCDF cache
+            self.data_file_helper.clearNetCdfDataCache()
+
         else : # Collect series data then generate file
 
             # Collect series data
@@ -6409,7 +6420,7 @@ def isNonNegativetiveFloat(self, string_value) :
 
 ## Main program
 
-application_name = 'PaleoView v1.3'
+application_name = 'PaleoView v1.4'
 
 # Set user application data directory
 if MAC_VERSION :

setup_mac.py

@@ -13,7 +13,7 @@
 
 # python setup_mac.py py2app
 
-APP = ['paleo_view_v1_1.py']
+APP = ['paleo_view_v1_2.py']
 DATA_FILES= [(u'Map Data', glob(u'/Users/seanhaythorne/Dropbox/GlobalEcology/PaleoView/code/Map Data/*'))]
 DATA_FILES.append((u'Bias Corrections', glob(u'//Users/seanhaythorne/Dropbox/GlobalEcology/PaleoView/code/Bias Corrections/*')))
 DATA_FILES.append((u'.', glob(u'/Users/seanhaythorne/Dropbox/GlobalEcology/PaleoView/code/paleo_view_config.txt')))
@@ -23,7 +23,7 @@
 
 setup(
     app=APP,
-    name="PaleoView v1.1",
+    name="PaleoView v1.4",
     data_files=DATA_FILES,
     options={'py2app': OPTIONS},
 #    py_modules=[u'lib/PaleoclimateToolDataFileHelper'],

setup_pc.py

@@ -1,22 +1,23 @@
 from distutils.core import setup
 from glob import glob
 import matplotlib
-import statsmodels.api
+#import statsmodels.api
 import sys
 import py2exe
 
 # >python setup_pc.py py2exe
+# >py -2 setup_pc.py py2exe
 
-sys.path.append("C:\Users\shaythorne\Dropbox\GlobalEcology\PythonResources\Microsoft.VC90.CRT")
+sys.path.append("C:\Users\Sean\GoogleDrive\GEL-Archive\PythonResources\Microsoft.VC90.CRT")
 
 data_files = matplotlib.get_py2exe_datafiles()
-#data_files.append(("Microsoft.VC90.CRT", glob(r'C:\Users\shaythorne\Dropbox\GlobalEcology\PythonResources\Microsoft.VC90.CRT\*.*')))
+#data_files.append(("Microsoft.VC90.CRT", glob(r'C:\Users\Sean\GoogleDrive\GEL-Archive\PythonResources\Microsoft.VC90.CRT\*.*')))
 data_files.append((r'mpl-data\basemap-data', glob(r'C:\Python27\Lib\site-packages\mpl_toolkits\basemap\data\*')))
 data_files.append((r'docx-data\templates', glob(r'C:\Python27\Lib\site-packages\docx\templates\*')))
 #data_files.append((r'Map Data', glob(r'C:\afat32\Dropbox\GlobalEcologyGroup\ProjectCode\PaleoclimateTool\v0.1\Map Data\*')))
 setup(
     data_files=data_files,
-    windows=['paleo_view_v1_1.py'],
+    windows=['paleo_view_v1_4.py'],
 )
 #    options={'py2exe': { 'includes': 'patsy' }}
 #    options={'py2exe': { 'excludes': ['_gtkagg', '_qt4agg', '_agg2', '_cairo', '_cocoaagg', '_fltkagg', '_gtk', '_gtkcairo'] }}