improved input_merged from StarAniso mtz files

core/CachedLookups.json

@@ -2952,7 +2952,7 @@
             "null": {
                 "DESCRIPTION": null,
                 "INTERRUPTLABEL": null,
-                "MAINTAINER": "liz.potterton@york.ac.uk",
+                "MAINTAINER": "pre@mrc-lmb.cam.ac.uk",
                 "TASKTITLE": null,
                 "class": "pipelines.import_merged.script.import_mergedimport_merged",
                 "clsModule": "pipelines.import_merged.script.import_merged",
@@ -3053,7 +3053,7 @@
             }
         },
         "metalCoord": {
-            "0.2": {
+            "0.3": {
                 "DESCRIPTION": null,
                 "INTERRUPTLABEL": null,
                 "MAINTAINER": "martin.maly@mrc-lmb.cam.ac.uk",

pipelines/aimless_pipe/script/CTaskaimless_pipe.py

@@ -311,7 +311,7 @@ def latticeCentreOptions(self):
                          'than the THRESHOLD set here or by default. KEEP and REMOVE options are unconditional'\
                          ])
         self.createLine(['widget','LATTICE_CENTERING_THRESHOLD',
-                         'label','Probability threshold for removing centred lattice reflections'],
+                         'label','Score threshold for removing centred lattice reflections'],
                         toggleFunction=[self.displayThreshold,
                                         ['KEEP_LATTICE_CENTERING', 'REMOVE_LATTICE_CENTERING']])
 

pipelines/aimless_pipe/script/aimless_pipe.def.xml

@@ -1128,9 +1128,9 @@
       <content id="LATTICE_CENTERING_THRESHOLD">
         <className>CFloat</className>
         <qualifiers>
-          <default>0.35</default>
+          <default>0.1</default>
           <guiLabel>None</guiLabel>
-          <toolTip>Probility threshold for removing centred lattice reflections </toolTip>
+          <toolTip>Score threshold for removing centred lattice reflections </toolTip>
           <charWidth>7</charWidth>
         </qualifiers>
       </content>

pipelines/import_merged/script/Ctaskimport_merged.py

@@ -103,7 +103,7 @@ def drawContents(self):
                      toggle=['STARANISO_DATA', 'closed', [False]])
     self.createLine (['label',
                       'By default the FreeR flag will be imported unchanged without completion'],
-                     toggle=['STARANISO_DATA', 'closed', [False]])
+                     toggleFunction=[self.toggleSkip, ['STARANISO_DATA', 'HASFREER']])
 
     # Optional resolution cutoff
     line = self.createLine( ['label','Select resolution range (\xc5)',
@@ -115,15 +115,32 @@ def drawContents(self):
     self.maximum_resolution_label = line.layout().itemAt(3).widget()
     self.container.inputData.RESOLUTION_RANGE.dataChanged.connect(self.resorangeset)
 
+    # Option to exclude reflections where all selected columns are missing
+    # Default for StarAniso data, allow for any MTZ file
+    self.createLine (['widget', 'EXCLUDE_MISSING',
+                      'label','Exclude reflections with all items missing (recommended for StarAniso data)'],
+                     toggleFunction=[self.toggleExcludeMissing, ['STARANISO_DATA', 'HKLINISMTZ']])
+    self.createLine (['widget', 'EXCLUDE_MISSING',
+                      'label','Exclude reflections with all items missing'],
+                     toggleFunction=[self.toggleExcludeMissing2, ['STARANISO_DATA', 'HKLINISMTZ']])
+
+    self.createLine (['label',''])
+
+    #  FreeR present in file
+    self.createLine (['label', 'The input file contains FreeR data'],
+                     toggle=['HASFREER','open',[True]])
     self.createLine (['label',
-       'Any FreeR in the input data will be automatically read and completed'],
-                     toggle=['STARANISO_DATA', 'open', [False]])
+                      'You can choose a pre-existing FreeR set below to override a FreeR set from the input data'],
+                     toggle=['HASFREER','open',[True]])
+
+    # No FreeR in file
+    self.createLine (['label', 'No FreeR data in the input file'],
+                     toggle=['HASFREER','open',[False]])
     self.createLine (['label',
-       'If no FreeR is present, a new set will be generated, or an existing FreeR set may be defined below'],
-                     toggle=['STARANISO_DATA', 'open', [False]])
+                      'A new set FreeR will be generated, or an existing FreeR set may be chosen below'],
+                     toggle=['HASFREER','open',[False]])
+
     self.createLine (['label',''])
-    self.createLine (['label',
-       'You can define a pre-existing FreeR set below to override a FreeR set from the input data'])
     self.createLine (['subtitle','If a FreeR set is neither present in the input, nor given explicitly, then one will be generated'])
 
     self.createLine( ['widget', 'FREERFLAG'] )
@@ -133,15 +150,21 @@ def drawContents(self):
                      'label',
                      'Cut resolution of FreeR set if necessary to match the data'],
                     toggleFunction=[self.toggleCutResolution, ['FREERFLAG']])
+
+    # No display if  FREERFLAG True (previous set) or HASFREER (in input file)
     self.createLine(['label', 'Fraction of reflections in generated freeR set',
                      'widget','FREER_FRACTION',
-                     'advice', 'Default fraction is 0.05',
-                     'advice',
-                     'Potential twinning operations will be taken into account'],
+                     'advice', 'Default fraction is 0.05'],
+                    toggleFunction=[self.toggleFraction, ['FREERFLAG', 'HASFREER']])
+    self.createLine(['advice',
+                     'Potential twinning operations will be taken into account for the FreeR set'],
                     toggleFunction=[self.toggleFraction, ['FREERFLAG', 'HASFREER']])
+
+    # not FREERFLAG (chosen set) and HASFREER and not StarAniso
     self.createLine(['widget', 'SKIP_FREER',
                      'label', 'Leave input FreeR set unchanged'],
                     toggleFunction=[self.toggleSkip, ['FREERFLAG', 'HASFREER', 'STARANISO_DATA']])
+    # not FREERFLAG (chosen set) and HASFREER and is StarAniso
     self.createLine(['widget', 'SKIP_FREER',
                      'label', 'Leave input FreeR set unchanged (recommended for StarAniso data)'],
                     toggleFunction=[self.toggleSkip2, ['FREERFLAG', 'HASFREER', 'STARANISO_DATA']])
@@ -292,18 +315,33 @@ def updateFromFile(self,force=True):
         self.container.controlParameters.SKIP_FREER.set(False)
         self.container.controlParameters.STARANISO_DATA.set(False)
         self.container.inputData.HASFREER.set(False)
+        self.container.controlParameters.EXCLUDE_MISSING.set(False)
+
+        # MTZ file from StarAniso may have columns labelled:
+        #   SA_flag, SIGNAL_value, SIGNAL_class
+        isStarAniso = False
+
         for column in fileContent.listOfColumns:
           if 'FREE' in str(column.columnLabel).upper():
           # if 'FREER' in str(column.columnLabel).upper():
             self.container.inputData.HASFREER.set(True)
-          if str(column.columnLabel) == 'SA_flag':
-            # Data comes from StarAniso
-            #print('>>** found SA_FLAG')
-            self.container.controlParameters.STARANISO_DATA.set(True)
-            self.container.controlParameters.SKIP_FREER.set(True)
-            self.container.controlParameters.COMPLETE.set(False)
-            self.container.inputData.FREERFLAG.unSet()
-            self.container.inputData.HASFREER.unSet()
+          scl = str(column.columnLabel)
+          if (scl == 'SA_flag') or (scl == 'SIGNAL_value') or (scl == 'SIGNAL_class'):
+            isStarAniso = True
+
+        if isStarAniso:
+          # Data comes from StarAniso
+          self.container.controlParameters.STARANISO_DATA.set(True)
+          # Default to Exclude Missing
+          self.container.controlParameters.EXCLUDE_MISSING.set(True)
+
+          if self.container.inputData.HASFREER:
+            # If there is a freeR column in StarAniso file,
+            #   then set defaults to leave it as is
+            self.container.controlParameters.SKIP_FREER.set(True)  # Skip generation
+            self.container.controlParameters.COMPLETE.set(False)   # don't complete
+            self.container.inputData.FREERFLAG.unSet()             # no separate FreeR file
+
 
         # MTZ
         self.selectObsColumns()
@@ -450,13 +488,35 @@ def toggleCutResolution(self):
     return False
   # -------------------------------------------------------------
   def toggleFraction(self):
+    # False if FREERFLAG True (previous set) or HASFREER (in input file)
     #print('>> toggleFraction FRF', self.container.inputData.FREERFLAG.isSet(),
     #      "HF", self.container.inputData.HASFREER)
     if self.container.inputData.FREERFLAG.isSet():
       return False
     if self.container.inputData.HASFREER:
       return False
     return True
+  # -------------------------------------------------------------
+  def toggleExcludeMissing(self):
+    # True if StarAniso file and is MTZ
+    if self.container.controlParameters.STARANISO_DATA:
+      if self.container.guiParameters.HKLINISMTZ:
+        return True
+    return False
+  # -------------------------------------------------------------
+  def toggleExcludeMissing2(self):
+    # True if not StarAniso file and is MTZ
+    if not self.container.controlParameters.STARANISO_DATA:
+      if self.container.guiParameters.HKLINISMTZ:
+        return True
+    return False
+  # -------------------------------------------------------------
+  def toggleSAfreeR(self):
+    # True if StarAniso file and has FreeR
+    if self.container.controlParameters.STARANISO_DATA:
+      if self.container.inputData.HASFREER:
+        return True
+    return False
 
   # -------------------------------------------------------------
   def toggleSkip2(self):
@@ -955,7 +1015,7 @@ def setCIFblocklist(self):
 
   # -------------------------------------------------------------
   def checkForStarAniso(self):
-    # current check loop for _software.name == STARANISO in 1st block
+    # current check mmcif loop for _software.name == STARANISO in 1st block
     self.container.controlParameters.STARANISO_DATA.set(False)
 
     block = self.rblocks[0].block

pipelines/import_merged/script/import_merged.def.xml

@@ -167,6 +167,13 @@
           <toolTip>Do not or do extend imported FreeR flags</toolTip>
         </qualifiers>
       </content>
+      <content id="EXCLUDE_MISSING">
+        <className>CBoolean</className>
+        <qualifiers>
+          <default>False</default>
+          <toolTip>Exclude reflections with all items missing</toolTip>
+        </qualifiers>
+      </content>
       <content id="STARANISO_DATA">
         <className>CBoolean</className>
         <qualifiers>

pipelines/import_merged/script/import_merged.py

@@ -36,7 +36,8 @@
 class import_merged(CPluginScript):
 
     TASKNAME = 'import_merged'
-    MAINTAINER = 'liz.potterton@york.ac.uk'
+    ## MAINTAINER = 'liz.potterton@york.ac.uk'  # written by Liz
+    MAINTAINER = 'pre@mrc-lmb.cam.ac.uk'  # Phil Evans
     WHATNEXT = []
     ERROR_CODES = { 301 : { 'description' : 'No output file found after conversion program' },
                     302 : { 'description' : 'Output from conversion does not contain recognised reflection or FreeR set data' }
@@ -61,16 +62,15 @@ def process(self):
 
       self.x2mtz = None
       self.mmcifXML = None
-      self.resolutioncutoff = False
-      #print("IDRR", self.container.inputData.RESOLUTION_RANGE_SET)
-      if self.container.inputData.RESOLUTION_RANGE_SET:
-          self.resolutioncutoff = True
+      useimportMtz = True  #  always for mtz file
 
       if str(self.fformat) == 'mtz':
-          # MTZ file: if there is a resolution cutoff specified,
-          # cannot use x2mtz to run cmtzsplit
-          if not self.resolutioncutoff:
-              self.x2mtz = self.makePluginObject('x2mtz')
+          # MTZ file: now always use mtzimport.py script instead (4/4/2026)
+          #  of cmtzsplit in x2mtz
+          pass
+          ##if not useimportMtz:  
+          #  print(">> use x2mtz")
+          # self.x2mtz = self.makePluginObject('x2mtz')
       #elif str(self.fformat) == 'mmcif':
       #    pass
       # # self.x2mtz = self.makePluginObject('cif2mtz')  # not needed now
@@ -88,9 +88,9 @@ def process(self):
       self.importXML = None
       self.freeout = None
       if self.fformat == 'mtz':
-        if self.resolutioncutoff:
+        if useimportMtz:
             self.importXML = etree.Element('IMPORT_LOG')  # information about the import step
-            status = self.importmtz()
+            status = self.importmtz()  # Import the MTZ file
             self.makeReportXML(self.importXML)  # add initial stuff for XML into self.importXML
             self.process1(status)
             return  # Probably doesnt get here
@@ -564,8 +564,10 @@ def makeResoRange(self):
     def importmtz(self):
         # Import an mtz file to a data file (OBSOUT) and
         #  (if present) a FreeR file, with optional resolution cutoffs
+        #  and optional removal of refecltions with all itmes missing
         # Uses Gemmi
 
+        print(">> importmtz")
         outputData = self.container.outputData
         filename = str(self.container.inputData.HKLIN)  # Input file
         outfile = str(outputData.OBSOUT)
@@ -593,10 +595,14 @@ def importmtz(self):
         reducehkl = True  # for now
         resorange = self.makeResoRange()
 
+        excludemissing = False
+        if self.container.controlParameters.EXCLUDE_MISSING:
+            excludemissing = True
+
         mtzimport = ImportMTZ(filename, outfile, freerfile,
                               obsColLabels, int(self.contentFlag),
                               freeRcolumnLabel,
-                              resorange)
+                              resorange, excludemissing)
 
         self.mtzXML = mtzimport.getXML()
         self.importXML.append(self.mtzXML)

pipelines/import_merged/script/import_merged_report.py

@@ -178,7 +178,10 @@ def importReport(self, parent):
       message = self.mtzreport()  # for X2MTZ block from Gemmi import
       #importDiv.append("<br/>")
       for line in message:
-        importDiv.addText(text=line)
+        if 'NOTE' in line:
+          importDiv.addText(text=line, style='color:darkorange')
+        else:
+          importDiv.addText(text=line)
         #importDiv.append("<br/>")
     elif fformat == 'mmcif':
       mmcifDiv = importDiv.addDiv(style="padding:5px;background-color:#CCEEFF;")
@@ -645,8 +648,11 @@ def finalReport(self, parent=None):
 
     # ctruncate
     if self.ctruncatereports != None:
-      self.ctruncatereports[0].keyText(summaryfold, phasertncs=self.phasertNCS)
-
+      if len(self.ctruncatereports) > 0:
+        self.ctruncatereports[0].keyText(summaryfold, phasertncs=self.phasertNCS)
+      else:
+        self.ctruncatereports = None
+
     # . . . .  . . . .  . . . .  . . . .  . . . .  . . . .  . . . .  . . . .
     # Main summary
     overallsummaryDiv = parent.addDiv(\
@@ -834,8 +840,12 @@ def reportResolution(self, rfindall):
       if 'id' in rrp:
         attr = rrp['id']
         if attr == "cutresolution":  # just checking
-          resorange = self.formatresorange(rrpaths[1])
-          s += ", trimmed to " + resorange
+          # is maximum cut resolution higher than main one?
+          maxr = rrpaths[0].findall('max')[0].text
+          maxt = rrpaths[1].findall('max')[0].text
+          if (float(maxt) - float(maxr)) > 0.001:
+            resorange = self.formatresorange(rrpaths[1])
+            s += ", trimmed to " + resorange
     else:
       # Just file resolution
       resorange = self.formatresorange(rrpaths[0])
@@ -877,6 +887,34 @@ def mtzreport(self):
       text.append(s)
       nreflections = self.importxml.findall("X2MTZ/nrefoutput")[0].text
       text.append("Number of reflections output: " + nreflections)
+
+      nrefin = 0
+      nrefexcluded = 0
+      nr = self.importxml.findall("X2MTZ/nrefinput")
+      if len(nr) > 0:
+        nrefin = nr[0].text
+      nr = self.importxml.findall("X2MTZ/nrefexcluded")
+      if len(nr) > 0:
+        nrefexcluded = nr[0].text
+      if int(nrefin) > 0:
+        text.append("Number of reflections input: " + nrefin)
+        if int(nrefexcluded) > 0:
+          text.append("Number of reflections excluded as missing: " + nrefexcluded)
+      nr = self.importxml.findall("X2MTZ/nrefflagged")
+      if len(nr) > 0:
+        mess = 'NOTE: number of reflections flagged as all missing (not removed): ' + nr[0].text
+        text.append(mess)
+      nr = self.importxml.findall("X2MTZ/maxResolutionAccepted")
+      if len(nr) > 0:
+        resaccepted = nr[0].text
+        rr = self.importxml.findall('X2MTZ/ResolutionRange/max')
+        if len(rr)>0:
+          resmax = rr[0].text
+        # Report if larger than maximum resolution
+        if (float(resaccepted) - float(resmax)) > 0.002:
+          mess = 'NOTE: highest resolution for accepted (not flagged) reflections: ' + resaccepted
+          text.append(mess)
+
       if len(self.importxml.findall("X2MTZ/freercolumnname"))>0:
         freercolumnname = self.importxml.findall("X2MTZ/freercolumnname")[0].text
         text.append("FreeR imported from column " + freercolumnname)

pipelines/import_merged/script/importutils.py

@@ -9,7 +9,7 @@ def addXMLelement(containerXML, elementname, elementtext):
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
 
 class ReflectionDataTypes():
-    # Information about data types for I2 converion from mmcif or MTZ to MTZ
+    # Information about data types for I2 conversion from mmcif or MTZ to MTZ
 
     # Dictionary of data types: for each type
     #   List of [mmcif_code, MTZ column name, column type, datasetID]