From 29aeb19fc95a13a09631db9621e0e9df1566f96c Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 14 Aug 2020 10:05:18 +0200 Subject: [PATCH 01/74] ClassMcaTheory -> LegacyMcaTheory --- PyMca5/PyMcaGui/physics/xrf/McaAdvancedFit.py | 8 ++++---- PyMca5/PyMcaGui/physics/xrf/XRFMCPyMca.py | 4 ++-- PyMca5/PyMcaPhysics/xrf/FastXRFLinearFit.py | 8 ++++---- PyMca5/PyMcaPhysics/xrf/LegacyFastXRFLinearFit.py | 8 ++++---- PyMca5/PyMcaPhysics/xrf/LegacyMcaAdvancedFitBatch.py | 12 ++++++------ .../xrf/{ClassMcaTheory.py => LegacyMcaTheory.py} | 10 +++++----- PyMca5/PyMcaPhysics/xrf/McaAdvancedFitBatch.py | 10 +++++----- PyMca5/PyMcaPhysics/xrf/XRFMC/XRFMCHelper.py | 4 ++-- PyMca5/tests/McaAdvancedFitWidgetTest.py | 2 +- PyMca5/tests/XrfTest.py | 10 +++++----- 10 files changed, 38 insertions(+), 38 deletions(-) rename PyMca5/PyMcaPhysics/xrf/{ClassMcaTheory.py => LegacyMcaTheory.py} (99%) diff --git a/PyMca5/PyMcaGui/physics/xrf/McaAdvancedFit.py b/PyMca5/PyMcaGui/physics/xrf/McaAdvancedFit.py index 0dc5a4002..3f16cb20b 100644 --- a/PyMca5/PyMcaGui/physics/xrf/McaAdvancedFit.py +++ b/PyMca5/PyMcaGui/physics/xrf/McaAdvancedFit.py @@ -58,10 +58,10 @@ #not a big problem pass -from PyMca5.PyMcaPhysics.xrf import ClassMcaTheory -FISX = ClassMcaTheory.FISX +from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory +FISX = LegacyMcaTheory.FISX if FISX: - FisxHelper = ClassMcaTheory.FisxHelper + FisxHelper = LegacyMcaTheory.FisxHelper from . import FitParam from . import McaAdvancedTable from . import QtMcaAdvancedFitReport @@ -291,7 +291,7 @@ def __init__(self, parent=None, name="PyMca - McaAdvancedFit",fl=0, self.matrixXRFMCSpectrumButton.setToolTip('Calculate Matrix Spectrum Using Monte Carlo') self.peaksSpectrumButton.setToolTip('Toggle Individual Peaks Spectrum Calculation On/Off') - self.mcafit = ClassMcaTheory.McaTheory() + self.mcafit = LegacyMcaTheory.McaTheory() self.fitButton.clicked.connect(self.fit) self.printButton.clicked.connect(self.printActiveTab) diff --git a/PyMca5/PyMcaGui/physics/xrf/XRFMCPyMca.py b/PyMca5/PyMcaGui/physics/xrf/XRFMCPyMca.py index d19a8eb0f..a0cfc09fd 100644 --- a/PyMca5/PyMcaGui/physics/xrf/XRFMCPyMca.py +++ b/PyMca5/PyMcaGui/physics/xrf/XRFMCPyMca.py @@ -805,7 +805,7 @@ def _start(self): #perform a dummy fit till xmimsim-pymca is upgraded if 0: import numpy - from PyMca import ClassMcaTheory + from PyMca import LegacyMcaTheory newFile['fit']['linearfitflag']=1 newFile['fit']['stripflag']=0 newFile['fit']['stripiterations']=0 @@ -814,7 +814,7 @@ def _start(self): #xdata = numpy.arange(xmin, xmax + 1) * 1.0 xdata = numpy.arange(0, xmax + 1) * 1.0 ydata = 0.0 + 0.1 * xdata - mcaFit = ClassMcaTheory.McaTheory() + mcaFit = LegacyMcaTheory.McaTheory() mcaFit.configure(newFile) mcaFit.setData(x=xdata, y=ydata, xmin=xmin, xmax=xmax) mcaFit.estimate() diff --git a/PyMca5/PyMcaPhysics/xrf/FastXRFLinearFit.py b/PyMca5/PyMcaPhysics/xrf/FastXRFLinearFit.py index e2e408111..9e6b2eb02 100644 --- a/PyMca5/PyMcaPhysics/xrf/FastXRFLinearFit.py +++ b/PyMca5/PyMcaPhysics/xrf/FastXRFLinearFit.py @@ -39,7 +39,7 @@ import time import h5py import collections -from . import ClassMcaTheory +from . import LegacyMcaTheory from . import ConcentrationsTool from PyMca5.PyMcaMath.linalg import lstsq from PyMca5.PyMcaMath.fitting import Gefit @@ -55,7 +55,7 @@ class FastXRFLinearFit(object): def __init__(self, mcafit=None): self._config = None if mcafit is None: - self._mcaTheory = ClassMcaTheory.McaTheory() + self._mcaTheory = LegacyMcaTheory.McaTheory() else: self._mcaTheory = mcafit @@ -518,7 +518,7 @@ def _fitCreateModel(self, dtype=None): freeNames = [] nFreeBkg = 0 for iParam, param in enumerate(self._mcaTheory.PARAMETERS): - if self._mcaTheory.codes[0][iParam] != ClassMcaTheory.Gefit.CFIXED: + if self._mcaTheory.codes[0][iParam] != LegacyMcaTheory.Gefit.CFIXED: nFree += 1 freeNames.append(param) if iParam < self._mcaTheory.NGLOBAL: @@ -532,7 +532,7 @@ def _fitCreateModel(self, dtype=None): derivatives = None idx = 0 for iParam, param in enumerate(self._mcaTheory.PARAMETERS): - if self._mcaTheory.codes[0][iParam] == ClassMcaTheory.Gefit.CFIXED: + if self._mcaTheory.codes[0][iParam] == LegacyMcaTheory.Gefit.CFIXED: continue deriv = self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters, iParam, diff --git a/PyMca5/PyMcaPhysics/xrf/LegacyFastXRFLinearFit.py b/PyMca5/PyMcaPhysics/xrf/LegacyFastXRFLinearFit.py index 05621853f..a3c6fbf97 100644 --- a/PyMca5/PyMcaPhysics/xrf/LegacyFastXRFLinearFit.py +++ b/PyMca5/PyMcaPhysics/xrf/LegacyFastXRFLinearFit.py @@ -37,7 +37,7 @@ import numpy import logging from PyMca5.PyMcaMath.linalg import lstsq -from . import ClassMcaTheory +from . import LegacyMcaTheory from PyMca5.PyMcaMath.fitting import Gefit from . import ConcentrationsTool from PyMca5.PyMcaMath.fitting import SpecfitFuns @@ -51,7 +51,7 @@ class FastXRFLinearFit(object): def __init__(self, mcafit=None): self._config = None if mcafit is None: - self._mcaTheory = ClassMcaTheory.McaTheory() + self._mcaTheory = LegacyMcaTheory.McaTheory() else: self._mcaTheory = mcafit @@ -261,7 +261,7 @@ def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None, freeNames = [] nFreeBackgroundParameters = 0 for i, param in enumerate(self._mcaTheory.PARAMETERS): - if self._mcaTheory.codes[0][i] != ClassMcaTheory.Gefit.CFIXED: + if self._mcaTheory.codes[0][i] != LegacyMcaTheory.Gefit.CFIXED: nFree += 1 freeNames.append(param) if i < self._mcaTheory.NGLOBAL: @@ -275,7 +275,7 @@ def fitMultipleSpectra(self, x=None, y=None, xmin=None, xmax=None, derivatives = None idx = 0 for i, param in enumerate(self._mcaTheory.PARAMETERS): - if self._mcaTheory.codes[0][i] == ClassMcaTheory.Gefit.CFIXED: + if self._mcaTheory.codes[0][i] == LegacyMcaTheory.Gefit.CFIXED: continue deriv= self._mcaTheory.linearMcaTheoryDerivative(self._mcaTheory.parameters, i, diff --git a/PyMca5/PyMcaPhysics/xrf/LegacyMcaAdvancedFitBatch.py b/PyMca5/PyMcaPhysics/xrf/LegacyMcaAdvancedFitBatch.py index 3233d9ba7..b67964ae9 100644 --- a/PyMca5/PyMcaPhysics/xrf/LegacyMcaAdvancedFitBatch.py +++ b/PyMca5/PyMcaPhysics/xrf/LegacyMcaAdvancedFitBatch.py @@ -33,7 +33,7 @@ import sys import os import numpy -from . import ClassMcaTheory +from . import LegacyMcaTheory from PyMca5.PyMcaCore import SpecFileLayer from PyMca5.PyMcaCore import EdfFileLayer from PyMca5.PyMcaIO import EdfFile @@ -71,13 +71,13 @@ def __init__(self,initdict,filelist=None,outputdir=None, self.fitFiles = fitfiles self._concentrations = concentrations if type(initdict) == type([]): - self.mcafit = ClassMcaTheory.McaTheory(initdict[mcaoffset]) + self.mcafit = LegacyMcaTheory.McaTheory(initdict[mcaoffset]) self.__configList = initdict self.__currentConfig = mcaoffset else: self.__configList = [initdict] self.__currentConfig = 0 - self.mcafit = ClassMcaTheory.McaTheory(initdict) + self.mcafit = LegacyMcaTheory.McaTheory(initdict) self.__concentrationsKeys = [] if self._concentrations: self._tool = ConcentrationsTool.ConcentrationsTool() @@ -172,7 +172,7 @@ def processList(self): if not self.roiFit: if len(self.__configList) > 1: if i != 0: - self.mcafit = ClassMcaTheory.McaTheory(self.__configList[i]) + self.mcafit = LegacyMcaTheory.McaTheory(self.__configList[i]) self.__currentConfig = i self.mcafit.enableOptimizedLinearFit() @@ -574,7 +574,7 @@ def __processOneMca(self,x,y,filename,key,info=None): if self.mcafit.config['fit'].get("strategyflag", False): config = self.__configList[self.__currentConfig] print("Restoring fitconfiguration") - self.mcafit = ClassMcaTheory.McaTheory(config) + self.mcafit = LegacyMcaTheory.McaTheory(config) self.mcafit.enableOptimizedLinearFit() return try: @@ -614,7 +614,7 @@ def __processOneMca(self,x,y,filename,key,info=None): if self.mcafit.config['fit'].get("strategyflag", False): config = self.__configList[self.__currentConfig] print("Restoring fitconfiguration") - self.mcafit = ClassMcaTheory.McaTheory(config) + self.mcafit = LegacyMcaTheory.McaTheory(config) self.mcafit.enableOptimizedLinearFit() return if self._concentrations: diff --git a/PyMca5/PyMcaPhysics/xrf/ClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/LegacyMcaTheory.py similarity index 99% rename from PyMca5/PyMcaPhysics/xrf/ClassMcaTheory.py rename to PyMca5/PyMcaPhysics/xrf/LegacyMcaTheory.py index 616d25612..4294bd7e2 100644 --- a/PyMca5/PyMcaPhysics/xrf/ClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/LegacyMcaTheory.py @@ -46,7 +46,7 @@ from PyMca5.PyMcaMath.fitting import Gefit from PyMca5 import PyMcaDataDir _logger = logging.getLogger(__name__) -#"python ClassMcaTheory.py -s1.1 --file=03novs060sum.mca --pkm=McaTheory.dat --continuum=0 --strip=1 --sumflag=1 --maxiter=4" +#"python LegacyMcaTheory.py -s1.1 --file=03novs060sum.mca --pkm=McaTheory.dat --continuum=0 --strip=1 --sumflag=1 --maxiter=4" CONTINUUM_LIST = [None,'Constant','Linear','Parabolic','Linear Polynomial','Exp. Polynomial'] OLDESCAPE = 0 MAX_ATTENUATION = 1.0E-300 @@ -1034,7 +1034,7 @@ def __configure(self): self.laststrip = 0 def setdata(self, *var, **kw): - print("ClassMcaTheory.setdata deprecated, please use setData") + print("LegacyMcaTheory.setdata deprecated, please use setData") return self.setData(*var, **kw) def setData(self,*var,**kw): @@ -2847,7 +2847,7 @@ def estimateexppol(self,x,y,z,xscaling=1.0,yscaling=1.0): return fittedpar,numpy.zeros((3,len(fittedpar)),numpy.float64) -class ClassMcaTheory(McaTheory): +class LegacyMcaTheory(McaTheory): pass """ @@ -2895,8 +2895,8 @@ def test(inputfile=None,scankey=None,pkm=None, if inputfile is None: print("USAGE") - print("python -m PyMca5.PyMcaPhysics.xrf.ClassMcaTheory.py -s1.1 --file=filename --cfg=cfgfile [--plotflag=1]") - #python ClassMcaTheory.py -s2.1 --file=ch09__mca_0005.mca --pkm=TEST.cfg --continuum=0 --stripflag=1 --sumflag=1 --maxiter=4 + print("python -m PyMca5.PyMcaPhysics.xrf.LegacyMcaTheory.py -s1.1 --file=filename --cfg=cfgfile [--plotflag=1]") + #python LegacyMcaTheory.py -s2.1 --file=ch09__mca_0005.mca --pkm=TEST.cfg --continuum=0 --stripflag=1 --sumflag=1 --maxiter=4 sys.exit(0) print("assuming is a specfile ...") sf=specfile.Specfile(inputfile) diff --git a/PyMca5/PyMcaPhysics/xrf/McaAdvancedFitBatch.py b/PyMca5/PyMcaPhysics/xrf/McaAdvancedFitBatch.py index 51b5f801d..c76e6e9f1 100644 --- a/PyMca5/PyMcaPhysics/xrf/McaAdvancedFitBatch.py +++ b/PyMca5/PyMcaPhysics/xrf/McaAdvancedFitBatch.py @@ -35,7 +35,7 @@ import time import logging import numpy -from . import ClassMcaTheory +from . import LegacyMcaTheory from PyMca5.PyMcaCore import SpecFileLayer from PyMca5.PyMcaCore import EdfFileLayer from PyMca5.PyMcaIO import EdfFile @@ -159,13 +159,13 @@ def __init__(self, initdict, filelist=None, outputdir=None, self.chunk = chunk if isinstance(initdict, list): - self.mcafit = ClassMcaTheory.McaTheory(initdict[mcaoffset]) + self.mcafit = LegacyMcaTheory.McaTheory(initdict[mcaoffset]) self.__configList = initdict self.__currentConfig = mcaoffset else: self.__configList = [initdict] self.__currentConfig = 0 - self.mcafit = ClassMcaTheory.McaTheory(initdict) + self.mcafit = LegacyMcaTheory.McaTheory(initdict) self.__concentrationsKeys = [] if self._concentrations: self._tool = ConcentrationsTool.ConcentrationsTool() @@ -272,7 +272,7 @@ def _processList(self): if not self.roiFit: if len(self.__configList) > 1: if i != 0: - self.mcafit = ClassMcaTheory.McaTheory(self.__configList[i]) + self.mcafit = LegacyMcaTheory.McaTheory(self.__configList[i]) self.__currentConfig = i # TODO: outbuffer does not support multiple configurations # Only the first one is saved. @@ -760,7 +760,7 @@ def _restoreFitConfig(self, filename, task): if self.mcafit.config['fit'].get("strategyflag", False): config = self.__configList[self.__currentConfig] _logger.info("Restoring fitconfiguration") - self.mcafit = ClassMcaTheory.McaTheory(config) + self.mcafit = LegacyMcaTheory.McaTheory(config) self.mcafit.enableOptimizedLinearFit() # TODO: why??? def _fitMca(self, filename): diff --git a/PyMca5/PyMcaPhysics/xrf/XRFMC/XRFMCHelper.py b/PyMca5/PyMcaPhysics/xrf/XRFMC/XRFMCHelper.py index 1c3387862..ae722fed0 100644 --- a/PyMca5/PyMcaPhysics/xrf/XRFMC/XRFMCHelper.py +++ b/PyMca5/PyMcaPhysics/xrf/XRFMC/XRFMCHelper.py @@ -232,7 +232,7 @@ def getXRFMCCorrectionFactors(fitConfiguration, xmimsim_pymca=None, verbose=Fals else: # for the time being we have to build a "fit-like" file with the information import numpy - from PyMca5.PyMca import ClassMcaTheory + from PyMca5.PyMca import LegacyMcaTheory fitConfiguration['fit']['linearfitflag']=1 fitConfiguration['fit']['stripflag']=0 fitConfiguration['fit']['stripiterations']=0 @@ -241,7 +241,7 @@ def getXRFMCCorrectionFactors(fitConfiguration, xmimsim_pymca=None, verbose=Fals #xdata = numpy.arange(xmin, xmax + 1) * 1.0 xdata = numpy.arange(0, xmax + 1) * 1.0 ydata = 0.0 + 0.1 * xdata - mcaFit = ClassMcaTheory.McaTheory() + mcaFit = LegacyMcaTheory.McaTheory() mcaFit.configure(fitConfiguration) #a dummy time dummyTime = 1.0 diff --git a/PyMca5/tests/McaAdvancedFitWidgetTest.py b/PyMca5/tests/McaAdvancedFitWidgetTest.py index 612ea565b..52b12317e 100644 --- a/PyMca5/tests/McaAdvancedFitWidgetTest.py +++ b/PyMca5/tests/McaAdvancedFitWidgetTest.py @@ -79,7 +79,7 @@ def _workOnBackend(self, backend): from PyMca5 import PyMcaDataDir dataDir = PyMcaDataDir.PYMCA_DATA_DIR from PyMca5.PyMcaIO import specfilewrapper as specfile - from PyMca5.PyMcaPhysics.xrf import ClassMcaTheory + from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool from PyMca5.PyMcaIO import ConfigDict diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 2008d3f66..46c3a67c8 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -269,7 +269,7 @@ def testTrainingDataFilePresence(self): def testTrainingDataFit(self): from PyMca5.PyMcaIO import specfilewrapper as specfile - from PyMca5.PyMcaPhysics.xrf import ClassMcaTheory + from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool from PyMca5.PyMcaIO import ConfigDict trainingDataFile = os.path.join(self.dataDir, "XRFSpectrum.mca") @@ -289,7 +289,7 @@ def testTrainingDataFit(self): # perform the actual XRF analysis configuration = ConfigDict.ConfigDict() configuration.readfp(StringIO(cfg)) - mcaFit = ClassMcaTheory.ClassMcaTheory() + mcaFit = LegacyMcaTheory.LegacyMcaTheory() configuration=mcaFit.configure(configuration) x = numpy.arange(y.size).astype(numpy.float64) mcaFit.setData(x, y, @@ -351,7 +351,7 @@ def testTrainingDataFit(self): def testStainlessSteelDataFit(self): from PyMca5.PyMcaIO import specfilewrapper as specfile - from PyMca5.PyMcaPhysics.xrf import ClassMcaTheory + from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool from PyMca5.PyMcaIO import ConfigDict @@ -376,7 +376,7 @@ def testStainlessSteelDataFit(self): # configure the fit # make sure no secondary excitations are used configuration["concentrations"]["usemultilayersecondary"] = 0 - mcaFit = ClassMcaTheory.ClassMcaTheory() + mcaFit = LegacyMcaTheory.LegacyMcaTheory() configuration=mcaFit.configure(configuration) mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], @@ -510,7 +510,7 @@ def testStainlessSteelDataFit(self): "Mn", "Fe", "Ni", "-", "-", "-","-","-"] - mcaFit = ClassMcaTheory.ClassMcaTheory() + mcaFit = LegacyMcaTheory.LegacyMcaTheory() configuration=mcaFit.configure(configuration) mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], From 913db54fb71cc135156bed48d1bb473ece302eab Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 19 Aug 2020 16:39:43 +0200 Subject: [PATCH 02/74] Add features to profiling utilities --- PyMca5/PyMcaMisc/ProfilingUtils.py | 168 ++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 41 deletions(-) diff --git a/PyMca5/PyMcaMisc/ProfilingUtils.py b/PyMca5/PyMcaMisc/ProfilingUtils.py index 09ad1ff78..31d16a91f 100644 --- a/PyMca5/PyMcaMisc/ProfilingUtils.py +++ b/PyMca5/PyMcaMisc/ProfilingUtils.py @@ -1,4 +1,4 @@ -#/*########################################################################## +# /*########################################################################## # # The PyMca X-Ray Fluorescence Toolkit # @@ -30,6 +30,7 @@ __contact__ = "wout.de_nolf@esrf.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + try: import tracemalloc import linecache @@ -37,11 +38,11 @@ tracemalloc = None import cProfile import pstats + try: from StringIO import StringIO except ImportError: from io import StringIO -import os import logging from contextlib import contextmanager @@ -49,44 +50,82 @@ logger = logging.getLogger(__name__) -def print_malloc_snapshot(snapshot, key_type='lineno', limit=10, units='KB'): +class bcolors: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +def durationfmtcolor(x): + if x < 0.0005: + return bcolors.OKGREEN + ("%6dµs" % (x * 1000000)) + bcolors.ENDC + elif x < 0.001: + return bcolors.WARNING + ("%6dµs" % (x * 1000000)) + bcolors.ENDC + elif x < 0.1: + return bcolors.WARNING + ("%6dms" % (x * 1000)) + bcolors.ENDC + else: + return bcolors.FAIL + ("%8.3f" % x) + bcolors.ENDC + + +def durationfmt(x): + if x < 0.001: + return "%6dµs" % (x * 1000000) + elif x < 0.1: + return "%6dms" % (x * 1000) + else: + return "%8.3f" % x + + +def print_malloc_snapshot(snapshot, key_type="lineno", limit=10, units="KB"): """ :param tracemalloc.Snapshot snapshot: :param str key_type: :param int limit: limit number of lines :param str units: B, KB, MB, GB """ - n = ['b', 'kb', 'mb', 'gb'].index(units.lower()) - sunits, units = units, 1024**n + n = ["b", "kb", "mb", "gb"].index(units.lower()) + sunits, units = units, 1024 ** n - snapshot = snapshot.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - )) + snapshot = snapshot.filter_traces( + ( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + ) + ) top_stats = snapshot.statistics(key_type) total = sum(stat.size for stat in top_stats) - print('================Memory profile================') + logger.info("================Memory profile================") + out = "" for index, stat in enumerate(top_stats, 1): frame = stat.traceback[0] # replace "/path/to/module/file.py" with "module/file.py" - #filename = os.sep.join(frame.filename.split(os.sep)[-2:]) + # filename = os.sep.join(frame.filename.split(os.sep)[-2:]) filename = frame.filename - print("#%s: %s:%s: %.1f %s" - % (index, filename, frame.lineno, stat.size / units, sunits)) + out += "\n#%s: %s:%s: %.1f %s" % ( + index, + filename, + frame.lineno, + stat.size / units, + sunits, + ) line = linecache.getline(frame.filename, frame.lineno).strip() if line: - print(' %s' % line) + out += "\n %s" % line if index >= limit: break - other = top_stats[index:] if other: size = sum(stat.size for stat in other) - print("%s other: %.1f %s" % (len(other), size / units, sunits)) - - print("Total allocated size: %.1f %s" % (total / units, sunits)) - print('============================================') + out += "\n%s other: %.1f %s" % (len(other), size / units, sunits) + out += "\nTotal allocated size: %.1f %s" % (total / units, sunits) + logger.info(out) + logger.info("============================================") @contextmanager @@ -95,42 +134,89 @@ def print_malloc_context(**kwargs): :param **kwargs: see print_malloc_snapshot """ if tracemalloc is None: - logger.error('tracemalloc required') + logger.info("tracemalloc required") return tracemalloc.start() - yield - snapshot = tracemalloc.take_snapshot() - print_malloc_snapshot(snapshot, **kwargs) + try: + yield + finally: + snapshot = tracemalloc.take_snapshot() + print_malloc_snapshot(snapshot, **kwargs) @contextmanager -def print_time_context(restrictions=None, sortby='cumtime'): +def print_time_context(timelimit=None, sortby="cumtime", color=False, filename=None): + """ + :param int or float timelimit: number of lines or fraction (float between 0 and 1) + :param str sortby: sort time profile + :param bool color: + :param str filename: + """ pr = cProfile.Profile() pr.enable() - yield - pr.disable() - s = StringIO() - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - if restrictions is None: - restrictions = (0.1,) - ps.print_stats(*restrictions) - print('================Time profile================') - print(s.getvalue()) - print('============================================') + try: + yield + finally: + pr.disable() + if isinstance(sortby, str): + sortby = [sortby] + if color: + pstats.f8 = durationfmtcolor + else: + pstats.f8 = durationfmt + for i, sortmethod in enumerate(sortby): + s = StringIO() + ps = pstats.Stats(pr, stream=s) + if sortmethod: + ps = ps.sort_stats(sortmethod) + if timelimit is None: + timelimit = (0.1,) + elif not isinstance(timelimit, tuple): + timelimit = (timelimit,) + ps.print_stats(*timelimit) + if filename and i == 0: + ps.dump_stats(filename) + logger.info("================Time profile================") + msg = "\n" + s.getvalue() + msg += "\n Saved as {}".format(repr(filename)) + logger.info(msg) + logger.info("============================================") @contextmanager -def profile(memory=True, time=True, memlimit=10, - restrictions=None, sortby='cumtime'): +def profile( + memory=True, + time=True, + memlimit=10, + timelimit=None, + sortby="cumtime", + color=False, + filename=None, + units="KB", +): + """ + :param bool memory: profile memory usage + :param bool time: execution time + :param int memlimit: number of lines + :param int or float timelimit: number of lines or fraction (float between 0 and 1) + :param str sortby: sort time profile + :param bool color: + :param str filename: dump for visual tools + :param str units: memory units + """ if not memory and not time: - yield + return elif memory and time: - with print_time_context(restrictions=restrictions, sortby=sortby): - with print_malloc_context(limit=memlimit): + with print_time_context( + timelimit=timelimit, sortby=sortby, color=color, filename=filename + ): + with print_malloc_context(limit=memlimit, units=units): yield elif memory: - with print_malloc_context(limit=memlimit): + with print_malloc_context(limit=memlimit, units=units): yield else: - with print_time_context(restrictions=restrictions, sortby=sortby): + with print_time_context( + timelimit=timelimit, sortby=sortby, color=color, filename=filename + ): yield From de8ce31167c8dea5edcfdc560085deca8e8008f9 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 11 Aug 2020 13:58:57 +0200 Subject: [PATCH 03/74] Concatenated fitmodel (WIP) --- PyMca5/PyMcaMath/fitting/Model.py | 842 ++++++++++++++++++++++++++++++ PyMca5/tests/FitModelTest.py | 333 ++++++++++++ PyMca5/tests/SimpleModel.py | 282 ++++++++++ 3 files changed, 1457 insertions(+) create mode 100644 PyMca5/PyMcaMath/fitting/Model.py create mode 100644 PyMca5/tests/FitModelTest.py create mode 100644 PyMca5/tests/SimpleModel.py diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py new file mode 100644 index 000000000..b66178f16 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -0,0 +1,842 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + + +import functools +from collections.abc import Sequence, MutableMapping +import numpy +from contextlib import contextmanager, ExitStack +from PyMca5.PyMcaMath.linalg import lstsq +from PyMca5.PyMcaMath.fitting import Gefit + + +def enable_caching(method): + @functools.wraps(method) + def cache_wrapper(self, *args, **kw): + with self.caching_context(): + return method(self, *args, **kw) + + return cache_wrapper + + +def memoize(method): + @functools.wraps(method) + def cache_wrapper(self): + if self.caching_enabled: + name = method.__qualname__ + if name in self._cache: + return self._cache[name] + else: + r = self._cache[name] = method(self) + return r + + return cache_wrapper + + +def memoize_property(method): + return property(memoize(method)) + + +class Cashed(object): + def __init__(self): + self._cache = None + + @contextmanager + def caching_context(self): + reset = self._cache is None + if reset: + self._cache = {} + try: + yield + finally: + if reset: + self._cache = None + + @property + def caching_enabled(self): + return self._cache is not None + + +class Model(Cashed): + """Evaluation and derivatives of a model to be used in least-squares fitting.""" + + def __init__(self): + self.included_parameters = None + self.excluded_parameters = None + super(Model, self).__init__() + + @property + def xdata(self): + raise AttributeError from NotImplementedError + + @property + def ydata(self): + raise AttributeError from NotImplementedError + + @property + def ystd(self): + raise AttributeError from NotImplementedError + + @property + def ymodel(self): + return self.evaluate() + + @property + def nchannels(self): + raise AttributeError from NotImplementedError + + @property + def parameters(self): + return self._get_parameters() + + @parameters.setter + def parameters(self, values): + return self._set_parameters(values) + + @property + def nparameters(self): + return sum(tpl[1] for tpl in self._parameter_groups()) + + @property + def parameter_names(self): + return list(self._iter_parameter_names()) + + @property + def parameter_group_names(self): + return self._filter_parameter_names(self._parameter_group_names) + + @property + def _parameter_group_names(self): + raise AttributeError from NotImplementedError + + @property + def linear_parameters(self): + return self._get_parameters(linear_only=True) + + @linear_parameters.setter + def linear_parameters(self, params): + return self._set_parameters(params, linear_only=True) + + @property + def nlinear_parameters(self): + return sum(tpl[1] for tpl in self._parameter_groups(linear_only=True)) + + @property + def linear_parameter_names(self): + return list(self._iter_parameter_names(linear_only=True)) + + @property + def linear_parameter_group_names(self): + return self._filter_parameter_names(self._linear_parameter_group_names) + + @property + def _linear_parameter_group_names(self): + raise AttributeError from NotImplementedError + + def _get_parameters(self, linear_only=False): + """ + :param bool linear_only: + :returns array: + """ + i = 0 + if linear_only: + params = numpy.zeros(self.nlinear_parameters) + else: + params = numpy.zeros(self.nparameters) + for name, n in self._parameter_groups(linear_only=linear_only): + params[i : i + n] = getattr(self, name) + i += n + return params + + def _set_parameters(self, params, linear_only=False): + """ + :returns values: + :param bool linear_only: + """ + i = 0 + for name, n in self._parameter_groups(linear_only=linear_only): + if n > 1: + getattr(self, name)[:] = params[i : i + n] + else: + setattr(self, name, params[i]) + i += n + + def _filter_parameter_names(self, names): + included = self.included_parameters + excluded = self.excluded_parameters + if included is None: + included = names + if excluded is None: + excluded = [] + return [name for name in names if name in included and name not in excluded] + + def evaluate(self, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def evaluate_linear(self, xdata=None): + """Derivate to a specific parameter + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + derivatives = self.linear_derivatives(xdata=xdata) + return self.linear_parameters.dot(derivatives) + + def linear_decomposition(self, xdata=None): + """Linear decomposition + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + derivatives = self.linear_derivatives(xdata=xdata) + return self.linear_parameters[:, numpy.newaxis] * derivatives + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def derivatives(self, xdata=None): + """Derivates to all parameters + + :param array xdata: length nxdata + :returns list(array): nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + return [self.derivative(i, xdata=xdata) for i in range(self.nparameters)] + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + raise NotImplementedError + + @property + def linear(self): + raise AttributeError from NotImplementedError + + def _iter_parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + raise NotImplementedError + + def _parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :returns iterable(str, int): group name, nb. parameters in the group + """ + if self.caching_enabled: + cache = self._cache.setdefault("all_parameter_groups", {}) + a = self.included_parameters + b = self.excluded_parameters + if a is not None: + a = tuple(sorted(a)) + if b is not None: + b = tuple(sorted(b)) + key = a, b + it = cache.get(key) + if it is None: + it = cache[key] = list( + self._iter_parameter_groups(linear_only=linear_only) + ) + else: + it = self._iter_parameter_groups(linear_only=linear_only) + return it + + def _iter_parameter_names(self, linear_only=False): + """ + :param bool linear_only: + :yields str: + """ + for name, n in self._parameter_groups(linear_only=linear_only): + if n > 1: + for i in range(n): + yield name + str(i) + else: + yield name + + def _parameter_name_from_index(self, idx, linear_only=False): + """Parameter index to group name and group index + + :returns str, int: group name, index in parameter group + """ + i = 0 + for name, n in self._parameter_groups(linear_only=linear_only): + if idx >= i and idx < (i + n): + return name, idx - i + i += n + + def fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + if self.linear: + return self.linear_fit(full_output=full_output) + else: + return self.nonlinear_fit(full_output=full_output) + + @enable_caching + def linear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + if self.niter_non_leastsquares: + initial = self.linear_parameters + try: + for i in range(max(self.niter_non_leastsquares, 1)): + A = self.linear_derivatives().T # nchannels, npeaks + b = self.ydata # nchannels + result = lstsq(A, b, digested_output=full_output) + if self.niter_non_leastsquares: + self.linear_parameters = result[0] + self.non_leastsquares_increment() + finally: + if self.niter_non_leastsquares: + self.linear_parameters = initial + result = {"linear": True, "parameters": result[0], "uncertainties": result[1]} + return result + + @enable_caching + def nonlinear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + initial = self.parameters + try: + for i in range(max(self.niter_non_leastsquares, 1)): + result = Gefit.LeastSquaresFit( + self._evaluate, + initial, + model_deriv=self._derivative, + xdata=self.xdata, + ydata=self.ydata, + sigmadata=self.ystd, + fulloutput=full_output, + ) + if self.niter_non_leastsquares: + self.parameters = result[0] + self.non_leastsquares_increment() + finally: + self.parameters = initial + ret = { + "linear": False, + "parameters": result[0], + "uncertainties": result[2], + "chi2_red": result[1], + } + if full_output: + ret["niter"] = result[3] + ret["lastdeltachi"] = result[4] + return ret + + def _evaluate(self, parameters, xdata): + """Update parameters and evaluate model + + :param array parameters: length nparams + :param array xdata: length nxdata + :returns array: nxdata + """ + self.parameters = parameters + return self.evaluate(xdata=xdata) + + def _derivative(self, parameters, param_idx, xdata): + """Update parameters and return derivate to a specific parameter + + :param array parameters: length nparams + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + self.parameters = parameters + return self.derivative(param_idx, xdata=xdata) + + def use_fit_result(self, result): + """ + :param dict result: + """ + if result["linear"]: + self.linear_parameters = result["parameters"] + else: + self.parameters = result["parameters"] + + @property + def niter_non_leastsquares(self): + return 0 + + def non_leastsquares_increment(self): + raise NotImplementedError + + +class ConcatModel(Model): + """Concatenated model with shared parameters""" + + def __init__(self, models, shared_attributes=None): + if not isinstance(models, Sequence): + models = [models] + for model in models: + if not isinstance(model, Model): + raise ValueError("'models' must be a list of type 'Model'") + self._models = models + self.__fixed_shared_attributes = { + "linear", + "included_parameters", + "excluded_parameters", + } + self.shared_attributes = shared_attributes + super(ConcatModel, self).__init__() + + @property + def model(self): + """Model used to get/set shared attributes""" + return self._models[0] + + def __getattr__(self, name): + """Get shared attribute""" + if self.nmodels and name in self.shared_attributes: + return getattr(self.model, name) + raise AttributeError(name) + + def __setattr__(self, name, value): + """Set shared attribute""" + if ( + name != "_models" + and self.nmodels + and hasattr(self.model, name) + and name in self.shared_attributes + ): + for m in self._models: + setattr(m, name, value) + else: + super(ConcatModel, self).__setattr__(name, value) + + @property + def shared_attributes(self): + """Attributes shared between the fit models (they should have the same value)""" + return self._shared_attributes + + @shared_attributes.setter + def shared_attributes(self, shared_attributes): + """ + :param Sequence(str) shared_attributes: + """ + if shared_attributes is None: + shared_attributes = set() + else: + shared_attributes = set(shared_attributes) + shared_attributes |= self.__fixed_shared_attributes + if self.nmodels <= 1: + self._shared_attributes = shared_attributes + return + self.share_attributes(shared_attributes) + self.validate_shared_attributes(shared_attributes) + self._shared_attributes = shared_attributes + + def validate_shared_attributes(self, shared_attributes=None): + """Check whether attributes are shared + + :param Sequence(str) shared_attributes: + :raises AssertionError: + """ + if self.nmodels <= 1: + return + if shared_attributes is None: + shared_attributes = self._shared_attributes + for name in shared_attributes: + value = getattr(self.model, name) + if isinstance(value, (Sequence, MutableMapping, numpy.ndarray)): + for m in self._models[1:]: + assert id(value) == id(getattr(m, name)), name + else: + for m in self._models[1:]: + assert value == getattr(m, name), name + + def share_attributes(self, shared_attributes=None): + """Ensure attributes are shared + + :param Sequence(str) shared_attributes: + """ + if self.nmodels <= 1: + return + if shared_attributes is None: + shared_attributes = self._shared_attributes + model = self.model + adict = {name: getattr(model, name) for name in shared_attributes} + for model in self._models[1:]: + for name, value in adict.items(): + setattr(model, name, value) + + @property + def nmodels(self): + return len(self._models) + + @property + def nchannels(self): + nmodels = self.nmodels + if nmodels == 0: + return 0 + else: + return sum([m.nchannels for m in self._models]) + + @property + def xdata(self): + return self._get_data("xdata") + + @xdata.setter + def xdata(self, values): + self._set_data("xdata", values) + + @property + def ydata(self): + return self._get_data("ydata") + + @ydata.setter + def ydata(self, values): + self._set_data("ydata", values) + + @property + def ystd(self): + return self._get_data("ystd") + + @ystd.setter + def ystd(self, values): + self._set_data("ystd", values) + + def _get_data(self, attr): + """ + :param str attr: + :returns array: + """ + nmodels = self.nmodels + if nmodels == 0: + return None + elif nmodels == 1: + return getattr(self.model, attr) + elif getattr(self.model, attr) is None: + return None + else: + return numpy.concatenate([getattr(m, attr) for m in self._models]) + + def _set_data(self, attr, values): + """ + :param str attr: + :param array values: + """ + if len(values) != self.nchannels: + raise ValueError("Not the expected number of channels") + for idx, model in self._iter_models(values): + setattr(model, attr, values[idx]) + + @contextmanager + def _filter_parameter_context(self, shared=True): + keepex = self.excluded_parameters + keepinc = self.included_parameters + try: + if shared: + if keepinc: + self.included_parameters = list( + set(keepinc) - set(self.shared_attributes) + ) + else: + self.included_parameters = self.shared_attributes + else: + if keepex: + self.excluded_parameters.extend(self.shared_attributes) + else: + self.excluded_parameters = self.shared_attributes + yield + finally: + self.excluded_parameters = keepex + self.included_parameters = keepinc + + @property + def nparameters(self): + return sum(m.nparameters for m in self._iter_parameter_models()) + + @property + def nlinear_parameters(self): + return sum(m.nlinear_parameters for m in self._iter_parameter_models()) + + @property + def nshared_parameters(self): + with self._filter_parameter_context(shared=True): + return self.model.nparameters + + @property + def nshared_linear_parameters(self): + with self._filter_parameter_context(shared=True): + return self.model.nlinear_parameters + + def _get_parameters(self, linear_only=False): + """ + :param bool linear_only: + :returns array: + """ + return numpy.concatenate( + [ + m._get_parameters(linear_only=linear_only) + for m in self._iter_parameter_models() + ] + ) + + def _set_parameters(self, values, linear_only=False): + """ + :returns values: + :param bool linear_only: + """ + i = 0 + for m in self._iter_parameter_models(): + if linear_only: + n = m.nlinear_parameters + else: + n = m.nparameters + if n: + m._set_parameters(values[i : i + n], linear_only=linear_only) + i += n + + def _iter_parameter_models(self): + """Iterate over models which are temporarily configured so that + after iterations, all parameters provided. + :yields Model: + """ + with self._filter_parameter_context(shared=True): + yield self.model + with self._filter_parameter_context(shared=False): + for m in self._models: + yield m + + def _parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + with self._filter_parameter_context(shared=True): + for item in self.model._parameter_groups(linear_only=linear_only): + yield item + with self._filter_parameter_context(shared=False): + for i, m in enumerate(self._models): + for name, n in self.model._parameter_groups(linear_only=linear_only): + yield name + str(i), n + + def _parameter_model_index(self, idx, linear_only=False): + """Convert parameter index of ConcatModel to a parameter indices + of the underlying models (only one when parameter is not shared). + + :param bool linear_only: + :param int idx: + :returns iterable(tuple): model index, parameter index in this model + """ + if self.caching_enabled: + cache = self._cache.setdefault("parameter_model_index", {}) + it = cache.get(idx) + if it is None: + it = cache[idx] = list( + self._iter_parameter_index(idx, linear_only=linear_only) + ) + else: + it = self._iter_parameter_index(idx, linear_only=linear_only) + return it + + def _iter_parameter_index(self, idx, linear_only=False): + """Convert parameter index of ConcatModel to a parameter indices + of the underlying models (only one when parameter is not shared). + + :param bool linear_only: + :param int idx: + :yields (int, int): model index, parameter index in this model + """ + if linear_only: + nshared = self.nshared_linear_parameters + else: + nshared = self.nshared_parameters + shared_attributes = self.shared_attributes + if idx < nshared: + for i, m in enumerate(self._models): + iglobal = 0 + imodel = 0 + for name, n in m._parameter_groups(linear_only=linear_only): + if name in shared_attributes: + if idx >= iglobal and idx < (iglobal + n): + yield i, imodel + idx - iglobal + iglobal += n + imodel += n + else: + iglobal = nshared + for i, m in enumerate(self._models): + imodel = 0 + for name, n in m._parameter_groups(linear_only=linear_only): + if name not in shared_attributes: + if idx >= iglobal and idx < (iglobal + n): + yield i, imodel + idx - iglobal + return + iglobal += n + imodel += n + + @property + def shared_parameters(self): + with self._filter_parameter_context(shared=True): + return self.model.parameters + + @shared_parameters.setter + def shared_parameters(self, values): + with self._filter_parameter_context(shared=True): + self.model.parameters = values + + @property + def shared_linear_parameters(self): + with self._filter_parameter_context(shared=True): + return self.model.linear_parameters + + @shared_linear_parameters.setter + def shared_linear_parameters(self, values): + with self._filter_parameter_context(shared=True): + self.model.linear_parameters = values + + def evaluate(self, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + ret = xdata * 0.0 + for idx, model in self._iter_models(xdata): + ret[idx] = model.evaluate(xdata=xdata[idx]) + return ret + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + ret = xdata * 0.0 + idx_channels = self._idx_channels(len(xdata)) + for model_idx, param_idx in self._parameter_model_index(param_idx): + idx = idx_channels[model_idx] + model = self._models[model_idx] + ret[idx] = model.derivative(param_idx, xdata=xdata[idx]) + return ret + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + ret = numpy.empty((self.nlinear_parameters, xdata.size)) + for idx, model in self._iter_models(xdata): + ret[:, idx] = model.linear_derivatives(xdata=xdata[idx]) + return ret + + def _iter_models(self, xdata): + """Loop over the models and yield xdata slice + + :param array xdata: + :yields (slice, Model): + """ + for item in zip(self._idx_channels(len(xdata)), self._models): + yield item + + def _idx_channels(self, nconcat): + """Index of each model in the concatenated data + + :param int nconcat: + :returns list(slice): + """ + if self.caching_enabled: + cache = self._cache.setdefault("idx_channels", {}) + if nconcat != cache.get("nconcat"): + cache["idx"] = list(self._generate_idx_channels(nconcat)) + cache["nconcat"] = nconcat + return cache["idx"] + else: + return list(self._generate_idx_channels(nconcat)) + + def _generate_idx_channels(self, nconcat, stride=None): + """Yield slice of the concatenated data for each model. + The concatenated data could be sliced as `xdata[::stride]`. + """ + nchannels = [m.nchannels for m in self._models] + if not stride: + stride, remain = divmod(sum(nchannels), nconcat) + stride += remain > 0 + start = 0 + offset = 0 + i = 0 + for n in nchannels: + # Index of model in concatenated xdata due to slicing + stop = start + n + lst = list(range(start + offset, stop, stride)) + nlst = len(lst) + # Index of model in concatenated xdata after slicing + idx = slice(i, i + nlst) + i += nlst + # Prepare for next model + if lst: + offset = lst[-1] + stride - stop + else: + offset -= n + start = stop + yield idx + + @contextmanager + def caching_context(self): + with ExitStack() as stack: + ctx = super(ConcatModel, self).caching_context() + stack.enter_context(ctx) + for m in self._models: + stack.enter_context(m.caching_context()) + yield diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py new file mode 100644 index 000000000..66bd7245e --- /dev/null +++ b/PyMca5/tests/FitModelTest.py @@ -0,0 +1,333 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + +import unittest +import numpy +from PyMca5.tests import SimpleModel + + +def with_model(nmodels): + def inner1(method): + def inner2(self, *args, **kw): + self.create_model(nmodels=nmodels) + try: + return method(self, *args, **kw) + finally: + self.validate_model() + + return inner2 + + return inner1 + + +class testFitModel(unittest.TestCase): + def setUp(self): + self.random_state = numpy.random.RandomState(seed=0) + + def create_model(self, nmodels): + if nmodels == 1: + self.fitmodel = SimpleModel.SimpleModel() + else: + self.fitmodel = SimpleModel.SimpleConcatModel(ndetectors=nmodels) + assert not self.fitmodel.linear + self.init_random() + self.fitmodel.ydata = self.fitmodel.ymodel + numpy.testing.assert_array_equal(self.fitmodel.ydata, self.fitmodel.ymodel) + self.validate_model() + + def init_random(self, **kw): + if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): + for model in self.fitmodel._models: + self._init_random(model, **kw) + self.fitmodel.shared_attributes = self.fitmodel.shared_attributes + else: + self._init_random(self.fitmodel, **kw) + + def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): + """Peaks close to the border will cause the nlls to fail""" + self.nnonglobals = 4 # zero, gain, wzero, wgain + self.nglobals = npeaks # concentrations + model.xdata_raw = numpy.arange(nchannels) + model.ydata_raw = numpy.full(nchannels, numpy.nan) + model.xmin = self.random_state.randint(low=0, high=10) + model.xmax = self.random_state.randint(low=nchannels - 10, high=nchannels) + model.zero = self.random_state.uniform(low=1, high=1.5) + model.gain = self.random_state.uniform(low=10e-3, high=11e-3) + a = model.zero + b = model.zero + model.gain * nchannels + border = border * (b - a) + a += border + b -= border + model.positions = numpy.linspace(a, b, npeaks) + model.wzero = self.random_state.uniform(low=0.0, high=0.01) + model.wgain = self.random_state.uniform(low=0.05, high=0.1) + model.concentrations = self.random_state.uniform(low=0.5, high=1, size=npeaks) + model.efficiency = self.random_state.uniform(low=5000, high=6000, size=npeaks) + + def modify_random(self, only_linear=False): + if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): + self._modify_random_concat(only_linear=only_linear) + else: + self._modify_random(only_linear=only_linear) + self.validate_model() + # self.plot() + + def _modify_random(self, only_linear=False): + if only_linear: + p = self.fitmodel.parameters + plin = self.fitmodel.linear_parameters + plin *= numpy.random.uniform(0.5, 0.8, len(plin)) + else: + p = self.fitmodel.parameters + plin = self.fitmodel.linear_parameters + p *= numpy.random.uniform(0.95, 1, len(p)) + plin *= numpy.random.uniform(0.5, 0.8, len(plin)) + self.fitmodel.parameters = p + assert numpy.array_equal(self.fitmodel.parameters, p) + self.fitmodel.linear_parameters = plin + assert numpy.array_equal(self.fitmodel.linear_parameters, plin) + return p + + def _modify_random_concat(self, only_linear=False): + p = self._modify_random(only_linear=only_linear) + assert not numpy.array_equal( + self.fitmodel.parameters[: self.nglobals], p[: self.nglobals] + ) + assert numpy.array_equal( + self.fitmodel.parameters[self.nglobals :], p[self.nglobals :] + ) + + def validate_model(self): + self._validate_model(self.fitmodel) + if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): + for model in self.fitmodel._models: + self._validate_model(model) + + def _validate_model(self, model): + is_concat = isinstance(model, SimpleModel.SimpleConcatModel) + + assert not model.excluded_parameters + assert not model.included_parameters + assert model.nchannels == len(model.xdata) + assert model.nparameters == len(model.parameters) + assert model.nlinear_parameters == len(model.linear_parameters) + arr1 = model.evaluate() + arr2 = model.evaluate_linear() + arr3 = sum(model.linear_decomposition()) + arr4 = model.ymodel + numpy.testing.assert_allclose(arr1, arr2) + numpy.testing.assert_allclose(arr1, arr3) + numpy.testing.assert_allclose(arr1, arr4) + + nonlin_names = ["zero", "gain", "wzero", "wgain"] + lin_names = ["concentrations" + str(i) for i in range(self.nglobals)] + names = nonlin_names + lin_names + if is_concat: + model.validate_shared_attributes() + assert model.nshared_parameters == self.nglobals + assert model.nshared_linear_parameters == self.nglobals + nmodels = model.nmodels + nonglobal_names = [ + name + str(i) for i in range(nmodels) for name in nonlin_names + ] + global_names = lin_names + names = global_names + nonglobal_names + n = self.nglobals + self.nnonglobals * nmodels + assert model.nparameters == n + assert model.nlinear_parameters == self.nglobals + assert model.parameter_names == names + assert model.linear_parameter_names == lin_names + else: + assert model.nparameters == self.nglobals + self.nnonglobals + assert model.nlinear_parameters == self.nglobals + assert model.parameter_names == names + assert model.linear_parameter_names == lin_names + + def plot(self): + import matplotlib.pyplot as plt + + m = self.fitmodel + derivatives = m.derivatives() + names = m.parameter_names + plt.figure() + plt.plot(m.ydata, label="data") + plt.plot(m.ymodel, label="model") + plt.legend() + plt.figure() + for y, name in zip(derivatives, names): + plt.plot(y, label=name) + plt.title("Derivatives") + plt.legend() + plt.show() + + @with_model(1) + def testLinearFit(self): + self._testLinearFit() + + @with_model(8) + def testLinearFitConcat(self): + self._testLinearFit() + + def _testLinearFit(self): + self.fitmodel.linear = True + expected = self.fitmodel.linear_parameters.copy() + self.modify_random(only_linear=True) + + result = self.fitmodel.fit() + self.assert_result(result, expected) + assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + assert not numpy.allclose(self.fitmodel.linear_parameters, expected) + + self.fitmodel.use_fit_result(result) + numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + numpy.testing.assert_allclose(self.fitmodel.linear_parameters, expected) + + @with_model(1) + def testNonLinearFit(self): + self._testNonLinearFit() + + @with_model(8) + def testNonLinearFitConcat(self): + self._testNonLinearFit() + + def _testNonLinearFit(self): + self.fitmodel.linear = False + expected1 = self.fitmodel.parameters.copy() + expected2 = self.fitmodel.linear_parameters.copy() + self.modify_random(only_linear=False) + + # from PyMca5.PyMcaMisc.ProfilingUtils import profile + # with profile(memory=False, filename="testNonLinearFit.pyprof"): + result = self.fitmodel.fit(full_output=True) + + # TODO: non-linear parameters not precise + # self.assert_result(result, expected1) + assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + assert not numpy.allclose(self.fitmodel.parameters, expected1) + assert not numpy.allclose(self.fitmodel.linear_parameters, expected2) + + self.fitmodel.use_fit_result(result) + # self.plot() + self.assert_ymodel() + # TODO: non-linear parameters not precise + # numpy.testing.assert_allclose(self.fitmodel.parameters, expected1) + numpy.testing.assert_allclose( + self.fitmodel.linear_parameters, expected2, rtol=1e-6 + ) + + def assert_result(self, result, expected): + p = numpy.asarray(result["parameters"]) + pstd = numpy.asarray(result["uncertainties"]) + ll = p - 3 * pstd + ul = p + 3 * pstd + assert all((expected >= ll) & (expected <= ul)) + + def assert_ymodel(self): + a = self.fitmodel.ydata + b = self.fitmodel.ymodel + mask = (a > 1) & (b > 1) + assert mask.any() + numpy.testing.assert_allclose(a[mask], b[mask], rtol=1e-3) + + @with_model(8) + def testParameterIndex(self): + # Test parameter index conversion from concatenated model to single model + nmodels = self.fitmodel.nmodels + nglobals = self.nglobals + for linear in [False, True]: + self.fitmodel.linear = linear + if linear: + nnonglobals = 0 + else: + nnonglobals = self.nnonglobals + imodels = [] + iparams = [] + for param_idx in range(self.fitmodel.nparameters): + lst = list( + self.fitmodel._parameter_model_index(param_idx, linear_only=linear) + ) + if lst: + imodel, iparam = list(zip(*lst)) + else: + imodel, iparam = tuple(), tuple() + if param_idx < nglobals: + assert imodel == tuple(range(nmodels)) + assert iparam == tuple([nnonglobals + param_idx] * nmodels) + else: + imodels.extend(imodel) + iparams.extend(iparam) + assert len(imodels) == nnonglobals * nmodels + expected = numpy.repeat(list(range(nmodels)), nnonglobals) + assert imodels == expected.tolist() + expected = numpy.tile(list(range(nnonglobals)), nmodels) + assert iparams == expected.tolist() + + @with_model(8) + def testChannelIndex(self): + # Test model index in concatenated + strides = [2, 3, 100, 1000, 1100, 1200, 3000] + for stride in strides: + x = self.fitmodel.xdata + x2 = x[::stride] + access_cnt = numpy.zeros(len(x2), dtype=int) + vstride = stride + if stride < 1000: + vstride = None + for idx in self.fitmodel._generate_idx_channels(len(x2), stride=vstride): + chunk = x2[idx] + access_cnt[idx] += 1 + assert all(numpy.diff(chunk) == stride) + assert all(access_cnt == 1) + + +def getSuite(auto=True): + testSuite = unittest.TestSuite() + if auto: + testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(testFitModel)) + else: + # use a predefined order + testSuite.addTest(testFitModel("testParameterIndex")) + testSuite.addTest(testFitModel("testChannelIndex")) + testSuite.addTest(testFitModel("testLinearFit")) + testSuite.addTest(testFitModel("testNonLinearFit")) + testSuite.addTest(testFitModel("testLinearFitConcat")) + testSuite.addTest(testFitModel("testNonLinearFitConcat")) + return testSuite + + +def test(auto=False): + unittest.TextTestRunner(verbosity=2).run(getSuite(auto=auto)) + + +if __name__ == "__main__": + test() diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py new file mode 100644 index 000000000..7c644b275 --- /dev/null +++ b/PyMca5/tests/SimpleModel.py @@ -0,0 +1,282 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + + +import numpy +from PyMca5.PyMcaMath.fitting import SpecfitFuns +from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel + + +class SimpleModel(Model): + """Model MCA data using a fixed list of peak positions and efficiencies""" + + def __init__(self): + self.config = { + "detector": {"zero": 0.0, "gain": 1.0, "wzero": 0.0, "wgain": 1.0}, + "matrix": {"positions": [], "concentrations": [], "efficiency": []}, + "fit": {"linear": False}, + "xmin": 0.0, + "xmax": 1.0, + } + self.xdata_raw = None + self.ydata_raw = None + self.ystd_raw = None + self.sigma_to_fwhm = 2 * numpy.sqrt(2 * numpy.log(2)) + super(SimpleModel, self).__init__() + + def __str__(self): + return "{}(npeaks={}, zero={}, gain={}, wzero={}, wgain={})".format( + self.__class__, self.npeaks, self.zero, self.gain, self.wzero, self.wgain + ) + + @property + def zero(self): + return self.config["detector"]["zero"] + + @zero.setter + def zero(self, value): + self.config["detector"]["zero"] = value + + @property + def gain(self): + return self.config["detector"]["gain"] + + @gain.setter + def gain(self, value): + self.config["detector"]["gain"] = value + + @property + def wzero(self): + return self.config["detector"]["wzero"] + + @wzero.setter + def wzero(self, value): + self.config["detector"]["wzero"] = value + + @property + def wgain(self): + return self.config["detector"]["wgain"] + + @wgain.setter + def wgain(self, value): + self.config["detector"]["wgain"] = value + + @property + def efficiency(self): + return self.config["matrix"]["efficiency"] + + @efficiency.setter + def efficiency(self, value): + arr = self.config["matrix"]["efficiency"] + self.config["matrix"]["efficiency"] = value + + @property + def positions(self): + return self.config["matrix"]["positions"] + + @positions.setter + def positions(self, value): + self.config["matrix"]["positions"] = value + + @property + def fwhms(self): + return self.zero + self.gain * self.positions + + @property + def areas(self): + return self.efficiency * self.concentrations + + @property + def concentrations(self): + return self.config["matrix"]["concentrations"] + + @concentrations.setter + def concentrations(self, value): + self.config["matrix"]["concentrations"] = value + + @property + def linear(self): + return self.config["fit"]["linear"] + + @linear.setter + def linear(self, value): + self.config["fit"]["linear"] = value + + @property + def idx_channels(self): + return slice(self.xmin, self.xmax) + + @property + def xdata(self): + if self.xdata_raw is None: + return None + else: + return self.xdata_raw[self.idx_channels] + + @xdata.setter + def xdata(self, values): + self.xdata_raw[self.idx_channels] = values + + @property + def xenergy(self): + return self.zero + self.gain * self.xdata + + @property + def ydata(self): + if self.ydata_raw is None: + return None + else: + return self.ydata_raw[self.idx_channels] + + @ydata.setter + def ydata(self, values): + self.ydata_raw[self.idx_channels] = values + + @property + def ystd(self): + if self.ystd_raw is None: + return None + else: + return self.ystd_raw[self.idx_channels] + + @ystd.setter + def ystd(self, values): + self.ystd_raw[self.idx_channels] = values + + @property + def nchannels(self): + return self.xmax - self.xmin + + @property + def npeaks(self): + return len(self.concentrations) + + @property + def _parameter_group_names(self): + return ["zero", "gain", "wzero", "wgain", "concentrations"] + + @property + def _linear_parameter_group_names(self): + return ["concentrations"] + + def _iter_parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + if linear_only: + names = self.linear_parameter_group_names + else: + names = self.parameter_group_names + for name in names: + if name == "zero": + yield name, 1 + elif name == "gain": + yield name, 1 + elif name == "wzero": + yield name, 1 + elif name == "wgain": + yield name, 1 + elif name == "concentrations": + yield name, self.npeaks + else: + raise ValueError(name) + + def evaluate(self, xdata=None): + """DEvaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + x = self.zero + self.gain * xdata + p = list(zip(self.areas, self.positions, self.fwhms)) + return SpecfitFuns.agauss(p, x) + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + x = self.zero + self.gain * xdata + it = zip(self.efficiency, self.positions, self.fwhms) + return numpy.array([SpecfitFuns.agauss([a, p, w], x) for a, p, w in it]) + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + x = self.zero + self.gain * xdata + name, i = self._parameter_name_from_index(param_idx) + if name == "concentrations": + p = self.positions[i] + a = self.efficiency[i] + w = self.wzero + self.wgain * p + y = SpecfitFuns.agauss([a, p, w], x) + else: + fwhms = self.fwhms + sigmas = fwhms / self.sigma_to_fwhm + y = x * 0.0 + for p, a, w, s in zip(self.positions, self.areas, fwhms, sigmas): + if name in ("zero", "gain"): + # Derivative to position + m = -(x - p) / s ** 2 + # Derivative to position param + if name == "gain": + m *= xdata + else: + # Derivative to FWHM + m = ((x - p) ** 2 / s ** 2 - 1) / (self.sigma_to_fwhm * s) + # Derivative to FWHM param + if name == "wgain": + m *= p + y += m * SpecfitFuns.agauss([a, p, w], x) + return y + + +class SimpleConcatModel(ConcatModel): + def __init__(self, ndetectors=1): + models = [SimpleModel() for i in range(ndetectors)] + shared_attributes = ["concentrations", "positions"] + super(SimpleConcatModel, self).__init__( + models, shared_attributes=shared_attributes + ) From dd4abb04e641469feb1d43fba02a52c9a0606d65 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 21 Jan 2021 22:37:04 +0100 Subject: [PATCH 04/74] New McaTheory (WIP) --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 981 +++++++++++++++++++ 1 file changed, 981 insertions(+) create mode 100644 PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py new file mode 100644 index 000000000..e091a2fe7 --- /dev/null +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -0,0 +1,981 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + +import os +import copy +import logging +import warnings +import numpy + +from PyMca5 import PyMcaDataDir +from PyMca5.PyMcaIO import ConfigDict +from PyMca5.PyMcaMath.fitting import SpecfitFuns +from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel + +from . import Elements +from . import ConcentrationsTool + +FISX = ConcentrationsTool.FISX +if FISX: + FisxHelper = ConcentrationsTool.FisxHelper + + +_logger = logging.getLogger(__name__) + + +def defaultConfigFilename(): + dirname = PyMcaDataDir.PYMCA_DATA_DIR + filename = os.path.join(dirname, "McaTheory.cfg") + if not os.path.exists(filename): + # Frozen version deals differently with the path + dirname = os.path.dirname(dirname) + filename = os.path.join(dirname, "McaTheory.cfg") + if not os.path.exists(filename): + if dirname.lower().endswith(".zip"): + dirname = os.path.dirname(dirname) + filename = os.path.join(dirname, "McaTheory.cfg") + if os.path.exists(filename): + return filename + else: + print("Cannot find file McaTheory.cfg") + raise IOError("File %s does not exist" % filename) + + +class McaTheoryConfigApi: + def __init__(self, initdict=None, filelist=None, **kw): + if initdict is None: + initdict = defaultConfigFilename() + if os.path.exists(initdict.split("::")[0]): + self.config = ConfigDict.ConfigDict(filelist=initdict) + else: + raise IOError("File %s does not exist" % initdict) + self._overwriteConfig(**kw) + self.attflag = kw.get("attenuatorsflag", 1) + + def _overwriteConfig(self, **kw): + if "config" in kw: + self.config.update(kw["config"]) + cfgfit = self.config["fit"] + cfgfit["sumflag"] = kw.get("sumflag", cfgfit["sumflag"]) + cfgfit["escapeflag"] = kw.get("escapeflag", cfgfit["escapeflag"]) + cfgfit["continuum"] = kw.get("continuum", cfgfit["continuum"]) + cfgfit["stripflag"] = kw.get("stripflag", cfgfit["stripflag"]) + cfgfit["maxiter"] = kw.get("maxiter", cfgfit["maxiter"]) + cfgfit["hypermetflag"] = kw.get("hypermetflag", cfgfit["hypermetflag"]) + + def _addMissingConfig(self): + """Add missing information to the configuration""" + cfgroot = self.config + + cfgroot["userattenuators"] = cfgroot.get("userattenuators", {}) + cfgroot["multilayer"] = cfgroot.get("multilayer", {}) + cfgroot["materials"] = cfgroot.get("materials", {}) + + cfgpeakshape = cfgroot["peakshape"] + cfgpeakshape["eta_factor"] = cfgpeakshape.get("eta_factor", 0.02) + cfgpeakshape["fixedeta_factor"] = cfgpeakshape.get("fixedeta_factor", 0) + cfgpeakshape["deltaeta_factor"] = cfgpeakshape.get( + "deltaeta_factor", cfgpeakshape["eta_factor"] + ) + + cfgfit = cfgroot["fit"] + cfgfit["fitfunction"] = cfgfit.get("fitfunction", None) + if cfgfit["fitfunction"] is None: + if cfgfit["hypermetflag"]: + cfgfit["fitfunction"] = 0 + else: + cfgfit["fitfunction"] = 1 + cfgfit["linearfitflag"] = cfgfit.get("linearfitflag", 0) + cfgfit["fitweight"] = cfgfit.get("fitweight", 1) + cfgfit["deltaonepeak"] = cfgfit.get("deltaonepeak", 0.010) + + cfgfit["energy"] = self._normalizeConfigListParam(cfgfit.get("energy", None)) + nenergies = len(cfgfit["energy"]) + cfgfit["energyweight"] = self._normalizeConfigListParam( + cfgfit.get("energyweight"), length=nenergies, default=1.0 + ) + cfgfit["energyflag"] = self._normalizeConfigListParam( + cfgfit.get("energyweight"), length=nenergies, default=1 + ) + cfgfit["energyscatter"] = self._normalizeConfigListParam( + cfgfit.get("energyweight"), length=nenergies, default=1 + ) + cfgfit["scatterflag"] = cfgfit.get("scatterflag", 0) + + cfgfit["stripalgorithm"] = cfgfit.get("stripalgorithm", 0) + cfgfit["snipwidth"] = cfgfit.get("snipwidth", 30) + cfgfit["linpolorder"] = cfgfit.get("linpolorder", 6) + cfgfit["exppolorder"] = cfgfit.get("exppolorder", 6) + cfgfit["stripconstant"] = cfgfit.get("stripconstant", 1.0) + cfgfit["stripwidth"] = int(cfgfit.get("stripwidth", 1)) + cfgfit["stripfilterwidth"] = int(cfgfit.get("stripfilterwidth", 5)) + cfgfit["stripiterations"] = int(cfgfit.get("stripiterations", 20000)) + cfgfit["stripanchorsflag"] = int(cfgfit.get("stripanchorsflag", 0)) + cfgfit["stripanchorslist"] = cfgfit.get("stripanchorslist", [0, 0, 0, 0]) + + cfgdetector = cfgroot["detector"] + cfgdetector["detene"] = cfgdetector.get("detene", 1.7420) + cfgdetector["ethreshold"] = cfgdetector.get("ethreshold", 0.020) + cfgdetector["nthreshold"] = cfgdetector.get("nthreshold", 4) + cfgdetector["ithreshold"] = cfgdetector.get("ithreshold", 1.0e-07) + + @staticmethod + def _normalizeConfigListParam(lst, length=None, default=0): + if isinstance(lst, (str, bytes)) or lst is None: + lst = [] + elif isinstance(lst, list): + pass + else: + lst = [lst] + if length is not None: + n = len(lst) + if n > length: + lst = lst[:length] + else: + lst += [default] * (length - n) + return lst + + def _sourceLines(self): + """ + :yields tuple: (energy, weight, scatter) + """ + cfg = self.config["fit"] + for energy, flag, weight, scatter in zip( + cfg["energy"], + cfg["energyflag"], + cfg["energyweight"], + cfg["energyscatter"], + ): + if energy and flag: + yield energy, weight, scatter + + @property + def _maxEnergy(self): + """ + :returns float or None: + """ + cfg = self.config["fit"] + energies = [energy for energy, _, _ in self._sourceLines()] + if energies: + return max(energies) + else: + return None + + @property + def _nSourceLines(self): + """ + :returns int: + """ + return len(list(self._sourceLines())) + + def _attenuators( + self, + matrix=False, + detector=False, + detectorfilters=False, + beamfilters=False, + detectorfunnyfilters=False, + yielddisabled=False, + ): + """Does not yield anything by default + + :yields list: + """ + cfg = self.config["attenuators"] + for name, alist in cfg.items(): + if not alist[0] and not yielddisabled: + continue + name = name.upper() + if name == "MATRIX": + # Sample description + # enable, formula, density, thickness, anglein, angleout, usescatteringangle, scatteringangle + if matrix: + if len(alist) == 6: + alist += [0, alist[4] + alist[5]] + yield alist[1:] + elif name == "DETECTOR": + # Detector description + # enable, formula, density, thickness + if detector: + yield alist[1:] + else: + # Filter + # enable, formula, density, thickness, funny + if len(alist) == 4: + alist.append(1.0) + if name.startswith("BEAMFILTER"): + if beamfilters: + yield alist[1:] + elif abs(alist[4] - 1.0) > 1.0e-10: + if detectorfunnyfilters: + yield alist[1:] + else: + if detectorfilters: + yield alist[1:] + + @property + def _matrix(self): + """ + :returns list or None: [formula, density, thickness, anglein, angleout, usescatteringangle, scatteringangle] + """ + if not self.attflag: + return + lst = list(self._attenuators(matrix=True)) + if lst: + return lst[0] + + @property + def _angleIn(self): + """Angle between sample surface and primary beam""" + lst = list(self._attenuators(matrix=True, yielddisabled=True)) + if lst: + return lst[0][3] + else: + _logger.warning("Sample incident angle set to 45 deg.") + return 45.0 + + @property + def _angleOut(self): + """Angle between sample surface and outgoing beam (emission or scattering)""" + lst = list(self._attenuators(matrix=True, yielddisabled=True)) + if lst: + return lst[0][4] + else: + _logger.warning("Sample incident angle set to 45 deg.") + return 45.0 + + @property + def _scatteringAngle(self): + """Angle between primary beam and outgoing beam (emission or scattering)""" + lst = list(self._attenuators(matrix=True, yielddisabled=True)) + if lst: + if lst[0][5]: + return lst[0][6] + else: + return self._angleIn + self._angleOut + else: + _logger.warning("Scattering angle set to 90 deg.") + return 90.0 + + @property + def _detector(self): + """ + :returns list or None: [formula, density, thickness] + """ + if not self.attflag: + return + lst = list(self._attenuators(detector=True)) + if lst: + return lst[0] + + def _multilayer(self): + """Yields the sample layers + + :yields list: [formula, density, thickness] + """ + if not self.attflag: + return + matrix = self._matrix + if matrix[0].upper() == "MULTILAYER": + cfg = self.config["multilayer"] + layerkeys = list(cfg.keys()) + layerkeys.sort() + for layer in layerkeys: + alist = cfg[layer] + if alist[0]: + yield alist[1:] + else: + yield matrix + + def _userAttenuators(self): + """ + :yields dict: {"energy": [], "transmission": []} + """ + if not self.attflag: + return + cfg = self.config["userattenuators"] + for tableDict in cfg.values(): + if tableDict["use"]: + yield tableDict + + def _emissionGroups(self): + """ + :yields list: [Z, symb, linegroupname] + """ + maxenergy = self._maxEnergy + cfg = self.config["peaks"] + for element, peaks in cfg.items(): + symb = element.capitalize() + Z = Elements.getz(symb) + if isinstance(peaks, list): + for peak in peaks: + yield [Z, symb, peak] + else: + yield [Z, symb, peaks] + + def _configureElementsModule(self): + """Configure the globals in the Elements module""" + for material, info in self.config["materials"].items(): + Elements.Material[material] = copy.deepcopy(info) + maxenergy = self._maxEnergy + for element in self.config["peaks"]: + symb = element.capitalize() + if maxenergy != Elements.Element[symb]["buildparameters"]["energy"]: + Elements.updateDict(energy=maxenergy) + break + + def _initializeConfig(self): + self._addMissingConfig() + self._configureElementsModule() + + @property + def _hypermet(self): + if self.config["fit"]["fitfunction"] == 0: + return self.config["fit"]["hypermetflag"] + else: + return 0 + + @property + def _hypermetShortTail(self): + return (self.config["fit"]["hypermetflag"] >> 1) & 1 + + @property + def _hypermetLongTail(self): + return (self.config["fit"]["hypermetflag"] >> 2) & 2 + + @property + def _hypermetStep(self): + return (self.config["fit"]["hypermetflag"] >> 3) & 3 + + +class McaTheoryLegacyApi: + def setdata(self, *args, **kw): + warnings.warn("McaTheory.setdata deprecated, please use setData", FutureWarning) + return self.setData(*args, **kw) + + def startfit(self, *args, **kw): + warnings.warn( + "McaTheory.startfit deprecated, please use startFit", FutureWarning + ) + return self.startFit(*args, **kw) + + +class McaTheory(McaTheoryConfigApi, McaTheoryLegacyApi, Model): + """Model for MCA data""" + + BAND_GAP = 0.00385 # For silicon + GAUSS_SIGMA_TO_FWHM = 2.3548 + MAX_ATTENUATION = 1.0e-300 + + def __init__(self, **kw): + super(McaTheory, self).__init__(**kw) + SpecfitFuns.fastagauss([1.0, 10.0, 1.0], numpy.arange(10.0)) + self.useFisxEscape(False) + + self.ydata0 = None + self.xdata0 = None + self.sigmay0 = None + self.strategyInstances = {} + + self.__toBeConfigured = False + self.__lastTime = None + self.lastxmin = None + self.lastxmax = None + self.laststrip = None + self.laststripconstant = None + self.laststripiterations = None + self.laststripalgorithm = None + self.lastsnipwidth = None + self.laststripwidth = None + self.laststripfilterwidth = None + self.laststripanchorsflag = None + self.laststripanchorslist = None + + self.__configure() + + def useFisxEscape(self, flag=None): + """Make sure the model uses fisx to calculate the escape peaks + when possible. + """ + if flag and FISX: + if ConcentrationsTool.FisxHelper.xcom is None: + FisxHelper.xcom = xcom = FisxHelper.getElementsInstance() + else: + xcom = ConcentrationsTool.FisxHelper.xcom + if hasattr(xcom, "setEscapeCacheEnabled"): + xcom.setEscapeCacheEnabled(1) + self._useFisxEscape = True + else: + self._useFisxEscape = False + else: + self._useFisxEscape = False + + def setConfiguration(self, ddict): + """ + The current fit configuration dictionary is updated, but not replaced, + by the input dictionary. + It returns a copy of the final fit configuration. + """ + return self.configure(ddict) + + def getConfiguration(self): + """ + returns a copy of the current fit configuration parameters + """ + return self.configure() + + def getStartingConfiguration(self): + # same output as calling configure but with the calling program + # knowing what is going on (no warning) + if self.__toBeConfigured: + return copy.deepcopy(self.__originalConfiguration) + else: + return self.configure() + + def configure(self, newdict=None): + if newdict in [None, {}]: + if self.__toBeConfigured: + _logger.debug( + "WARNING: This configuration is the one of last fit.\n" + "It does not correspond to the one of next fit." + ) + return copy.deepcopy(self.config) + self.config.update(newdict) + self.__toBeConfigured = False + self.__configure() + return copy.deepcopy(self.config) + + def __configure(self): + self._initializeConfig() + self._preCalculateParameterIndependent() + self._preCalculateParameterDependent() + + def _preCalculateParameterIndependent(self): + self._fluoRates = self._calcFluoRates() + self._calcFluoRateCorrections() + + # Line groups: nested lists + # line group + # -> emission/scattering line + # -> energy, rate, line name + self._lineGroups = list(self._getEmissionLines()) + self._lineGroups.extend(self._getScatterLines()) + + # Escape line groups: nested lists + # line group + # -> emission/scattering line + # -> escape line + # -> energy, rate, escape name + self._escapeLineGroups = [ + self._calcEscapePeaks([peak[0] for peak in peaks]) + for peaks in self._lineGroups + ] + + def _getEmissionLines(self): + """Yields a list of emission lines for each group with total + rate of 1 and sorted by energy. + + :yields list: [[energy, rate, "name"], + [energy, rate, "name"], + ...] + """ + for group in sorted(self._emissionGroups()): + yield self._getGroupEmissionLines(*group) + + def _getScatterLines(self): + """Yields a list for scattering lines for each source line. + + :yields list: [[energy, 1.0, "Peak"]] + """ + scatteringAngle = self._scatteringAngle * numpy.pi / 180.0 + angleFactor = 1.0 - numpy.cos(scatteringAngle) + for energy, _, scatter in self._sourceLines(): + if scatter: + energy /= 1.0 + (energy / 511.0) * angleFactor + yield [[energy, 1.0, "Peak"]] + + def _preCalculateParameterDependent(self): + pass + + def _calcFluoRates(self): + """Fluorescence rate for each emission line of each element. + Rate means fluorescence intensity divided by primary intensity. + + :returns None or dict: + """ + if self._matrix: + if self._maxEnergy: + multilayer = list(self._multilayer()) + if not multilayer: + text = "Your matrix is not properly defined.\n" + text += "If you used the graphical interface,\n" + text += "Please check the MATRIX tab" + raise ValueError(text) + + emissiongroups = sorted(self._emissionGroups()) + energylist, weightlist, scatterlist = zip(*self._sourceLines()) + detector = self._detector + attenuatorlist = list(self._attenuators(detectorfilters=True)) + userattenuatorlist = list(self._userAttenuators()) + funnyfilters = list(self._attenuators(detectorfunnyfilters=True)) + filterlist = list(self._attenuators(beamfilters=True)) + alphain = self._angleIn + alphaout = self._angleOut + return Elements.getMultilayerFluorescence( + multilayer, + energylist, + layerList=None, + weightList=weightlist, + fulloutput=1, + attenuators=attenuatorlist, + alphain=alphain, + alphaout=alphaout, + elementsList=emissiongroups, + cascade=True, + detector=detector, + funnyfilters=funnyfilters, + beamfilters=filterlist, + forcepresent=1, + userattenuators=userattenuatorlist, + ) + else: + text = "Invalid energy for matrix configuration.\n" + text += "Please check your BEAM parameters." + raise ValueError(text) + else: + if self._nSourceLines > 1: + raise ValueError("Multiple energies require a matrix definition") + else: + return None + + def _calcFluoRateCorrections(self): + """Higher-order interaction corrections on the fluorescence rates. + This will not be needed once fisx replaces the Elements module. + """ + self.config["fisx"] = {} + if not FISX or "concentrations" not in self.config: + return + secondary = self.config["concentrations"].get("usemultilayersecondary", False) + if secondary: + corrections = FisxHelper.getFisxCorrectionFactorsFromFitConfiguration( + self.config, elementsFromMatrix=False + ) + self.config["fisx"]["corrections"] = corrections + self.config["fisx"]["secondary"] = secondary + + def _getGroupEmissionLines(self, Z, symb, groupname): + """Return a list of emission lines with total rate of 1 and + sorted by energy. + + :param int Z: atomic number + :param str symb: for example "Fe" + :param str groupname: for example "K" + :returns list: [[energy, rate, "name"], + [energy, rate, "name"], + ...] + """ + if self._fluoRates is None: + groups = Elements.Element[symb] + else: + groups = self._fluoRates[0][symb] + + peaks = [] + lines = groups.get(groupname + " xrays", dict()) + if not lines: + return peaks + for line in lines: + lineinfo = groups[line] + if lineinfo["rate"] > 0.0: + peaks.append([lineinfo["energy"], lineinfo["rate"], line]) + + if self._fluoRates is None: + self._applyAttenuation(peaks, symb) + + totalrate = sum(peak[1] for peak in peaks) + if not totalrate: + text = "Intensity of %s %s is zero\n" % (symb, groupname) + text += "Too high attenuation?" + raise ZeroDivisionError(text) + for peak in peaks: + peak[1] /= totalrate + + ethreshold = self.config["fit"]["deltaonepeak"] + return Elements._filterPeaks( + peaks, + ethreshold=ethreshold, + ithreshold=0.0005, + nthreshold=None, + keeptotalrate=True, + ) + + def _applyAttenuation(self, peaks, symb): + """Apply attenuation of primary and secondary beams. + No high-order interactions are taken into account. + Only 1 primary beam energy can be used. + """ + self._applyMatrixAttenuation(peaks, symb) + self._applyBeamFilterAttenuation(peaks, symb) + self._applyDetectorFilterAttenuation(peaks, symb) + self._applyFunnyFilterAttenuation(peaks, symb) + self._applyDetectorAttenuation(peaks, symb) + for peak in peaks: + if peak[1] < self.MAX_ATTENUATION: + peak[1] = 0 + + def _iterLinearAttenuation(self, energies, **kw): + """Linear attenuation coefficients of matrix, detector, filters, ... + + :param list energies: + :param **kw: select the attenuator type to include + """ + for attenuator in self._attenuators(**kw): + formula, density, thickness, funnyfactor = attenuator + rhod = density * thickness + mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) + if len(energies) != 1 and len(mu["total"]) == 1: + mu = mu["total"] * len(energies) + else: + mu = mu["total"] + mulin = rhod * numpy.array(mu) + yield mulin, funnyfactor + + def _applyBeamFilterAttenuation(self, peaks, symb): + energies = Elements.Element[symb]["buildparameters"]["energy"] + if not energies: + raise ValueError("Invalid excitation energy") + + for mulin, _ in self._iterLinearAttenuation(energies, beamfilter=True): + transmission = numpy.exp(-mulin) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac + + def _applyDetectorFilterAttenuation(self, peaks, symb): + energies = [peak[0] for peak in peaks] + for mulin, _ in self._iterLinearAttenuation(energies, detectorfilter=True): + transmission = numpy.exp(-mulin) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac + + def _applyFunnyFilterAttenuation(self, peaks, symb): + firstfunnyfactor = None + energies = [peak[0] for peak in peaks] + for mulin, funnyfactor in self._iterLinearAttenuation( + energies, detectorfunnyfilter=True + ): + if (funnyfactor < 0.0) or (funnyfactor > 1.0): + text = ( + "Funny factor should be between 0.0 and 1.0., got %g" % funnyfactor + ) + raise ValueError(text) + transmission = numpy.exp(-mulin) + if firstfunnyfactor is None: + # only has to be multiplied once!!! + firstfunnyfactor = funnyfactor + transmission = funnyfactor * transmission + (1.0 - funnyfactor) + else: + if abs(firstfunnyfactor - funnyfactor) > 0.0001: + text = "All funny type attenuators must have same opening fraction" + raise ValueError(text) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac + + def _applyDetectorAttenuation(self, peaks, symb): + energies = [peak[0] for peak in peaks] + for mulin, _ in self._iterLinearAttenuation(energies, symb, detector=True): + attenuation = 1.0 - numpy.exp(-mulin) + for peak, frac in zip(peaks, attenuation): + peak[1] *= frac + + def _applyMatrixAttenuation(self, peaks, symb): + matrix = self._matrix + if not matrix: + return + maxenergy = Elements.Element[symb]["buildparameters"]["energy"] + if not maxenergy: + raise ValueError("Invalid excitation energy") + formula, density, thickness = matrix[:3] + alphaIn = self._angleIn + alphaOut = self._angleOut + + energies = [x[0] for x in peaks] + [maxenergy] + mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) + sinAlphaIn = numpy.sin(alphaIn * numpy.pi / 180.0) + sinAlphaOut = numpy.sin(alphaOut * numpy.pi / 180.0) + sinRatio = sinAlphaIn / sinAlphaOut + muSource = mu["total"][-1] + muFluo = numpy.array(mu["total"][:-1]) + + transmission = 1.0 / (muSource + muFluo * sinRatio) + rhod = density * thickness + if rhod > 0.0 and abs(sinAlphaIn) > 0.0: + expterm = -(muSource / sinAlphaIn + muFluo / sinAlphaOut) * rhod + transmission *= 1.0 - numpy.exp(expterm) + + for peak, frac in zip(peaks, transmission): + peak[1] *= frac + + def _applyUserAttenuators(self, peaks): + for userattenuator in self.config["userattenuators"]: + if self.config["userattenuators"][userattenuator]["use"]: + transmission = Elements.getTableTransmission( + self.config["userattenuators"][userattenuator], + [x[0] for x in peaks], + ) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac + + def _calcEscapePeaks(self, energies): + """For each energy a list of escape peaks with total rate of 1 + and sorted by energy. + + :param list energies: + :returns list: [[[energy, rate, "name"], + [energy, rate, "name"], + ...]] + """ + if not self.config["fit"]["escapeflag"]: + return [] + if self._useFisxEscape: + _logger.debug("Using fisx escape ratio's") + return self._calcFisxEscapeRatios(energies) + else: + return self._calcPymcaEscapeRatios(energies) + + def _calcFisxEscapeRatios(self, energies): + xcom = FisxHelper.xcom + detele = self.config["detector"]["detele"] + detector_composition = Elements.getMaterialMassFractions([detele], [1.0]) + ethreshold = self.config["detector"]["ethreshold"] + ithreshold = self.config["detector"]["ithreshold"] + nthreshold = self.config["detector"]["nthreshold"] + xcom.updateEscapeCache( + detector_composition, + energies, + energyThreshold=ethreshold, + intensityThreshold=ithreshold, + nThreshold=nthreshold, + ) + + escape_peaks = [] + for energy in energies: + epeaks = xcom.getEscape( + detector_composition, + energy, + energyThreshold=ethreshold, + intensityThreshold=ithreshold, + nThreshold=nthreshold, + ) + epeaks = [ + [epeakinfo["energy"], epeakinfo["rate"], name[:-3].replace("_", " ")] + for name, epeakinfo in epeaks.items() + ] + epeaks = Elements._filterPeaks( + epeaks, + ethreshold=ethreshold, + ithreshold=ithreshold, + nthreshold=nthreshold, + absoluteithreshold=True, + keeptotalrate=False, + ) + escape_peaks.append(epeaks) + return escape_peaks + + def _calcPymcaEscapeRatios(self, energies): + escape_peaks = [] + detele = self.config["detector"]["detele"] + for energy in energies: + peaks = Elements.getEscape( + [detele, 1.0, 1.0], + energy, + ethreshold=self.config["detector"]["ethreshold"], + ithreshold=self.config["detector"]["ithreshold"], + nthreshold=self.config["detector"]["nthreshold"], + ) + escape_peaks.append(peaks) + return escape_peaks + + @property + def zero(self): + return self.config["detector"]["zero"] + + @zero.setter + def zero(self, value): + self.config["detector"]["zero"] = value + + @property + def gain(self): + return self.config["detector"]["gain"] + + @gain.setter + def gain(self, value): + self.config["detector"]["gain"] = value + + @property + def fano(self): + return self.config["detector"]["fano"] + + @gain.setter + def fano(self, value): + self.config["detector"]["fano"] = value + + @property + def sum(self): + return self.config["detector"]["sum"] + + @sum.setter + def sum(self, value): + self.config["detector"]["sum"] = value + + @classmethod + def _calc_fwhm(cls, noise, fano, energy): + return numpy.sqrt( + noise * noise + + cls.BAND_GAP + * energy + * fano + * cls.GAUSS_SIGMA_TO_FWHM + * cls.GAUSS_SIGMA_TO_FWHM + ) + + @property + def eta_factor(self): + return self.config["peakshape"]["eta_factor"] + + @eta_factor.setter + def eta_factor(self, value): + self.config["peakshape"]["eta_factor"] = value + + @property + def step_heightratio(self): + return self.config["peakshape"]["step_heightratio"] + + @sum.setter + def step_heightratio(self, value): + self.config["peakshape"]["step_heightratio"] = value + + @property + def lt_sloperatio(self): + return self.config["peakshape"]["lt_sloperatio"] + + @lt_sloperatio.setter + def lt_sloperatio(self, value): + self.config["detector"]["lt_sloperatio"] = value + + @property + def lt_arearatio(self): + return self.config["peakshape"]["lt_arearatio"] + + @lt_arearatio.setter + def lt_arearatio(self, value): + self.config["peakshape"]["lt_arearatio"] = value + + @property + def st_sloperatio(self): + return self.config["peakshape"]["st_sloperatio"] + + @st_sloperatio.setter + def st_sloperatio(self, value): + self.config["peakshape"]["st_sloperatio"] = value + + @property + def st_arearatio(self): + return self.config["peakshape"]["st_arearatio"] + + @st_arearatio.setter + def st_arearatio(self, value): + self.config["peakshape"]["st_arearatio"] = value + + @property + def _parameter_group_names(self): + return [ + "zero", + "gain", + "noise", + "fano", + "sum", + "st_arearatio", + "st_sloperatio", + "lt_arearatio", + "lt_sloperatio", + "step_heightratio", + "eta_factor", + ] + + @property + def _linear_parameter_group_names(self): + raise NotImplementedError + + def _iter_parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + if linear_only: + names = self.linear_parameter_group_names + else: + names = self.parameter_group_names + hypermet = self._hypermet + for name in names: + if name == "zero": + yield name, 1 + elif name == "gain": + yield name, 1 + elif name == "noise": + yield name, 1 + elif name == "fano": + yield name, 1 + elif name == "sum": + yield name, 1 + elif name == "st_arearatio" and hypermet: + yield name, 1 + elif name == "st_sloperatio" and hypermet: + yield name, 1 + elif name == "lt_arearatio" and hypermet: + yield name, 1 + elif name == "lt_sloperatio" and hypermet: + yield name, 1 + elif name == "step_heightratio" and hypermet: + yield name, 1 + elif name == "eta_factor" and not hypermet: + yield name, 1 + else: + raise ValueError(name) + + +class MultiMcaTheory(ConcatModel): + def __init__(self, ndetectors=1): + models = [McaTheory() for i in range(ndetectors)] + shared_attributes = [] + super(MultiMcaTheory, self).__init__( + models, shared_attributes=shared_attributes + ) From d06a1051b7d86e8d735b7c85abe30a9ca66724c3 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 14:30:52 +0100 Subject: [PATCH 05/74] New McaTheory (WIP) --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 432 ++++++++++++++++--- PyMca5/tests/XrfTest.py | 229 +++++++--- 2 files changed, 555 insertions(+), 106 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index e091a2fe7..c106c5288 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -32,6 +32,7 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import os +import sys import copy import logging import warnings @@ -100,6 +101,7 @@ def _addMissingConfig(self): cfgroot["userattenuators"] = cfgroot.get("userattenuators", {}) cfgroot["multilayer"] = cfgroot.get("multilayer", {}) cfgroot["materials"] = cfgroot.get("materials", {}) + cfgroot["concentrations"] = cfgroot.get("concentrations", {}) cfgpeakshape = cfgroot["peakshape"] cfgpeakshape["eta_factor"] = cfgpeakshape.get("eta_factor", 0.02) @@ -119,16 +121,17 @@ def _addMissingConfig(self): cfgfit["fitweight"] = cfgfit.get("fitweight", 1) cfgfit["deltaonepeak"] = cfgfit.get("deltaonepeak", 0.010) - cfgfit["energy"] = self._normalizeConfigListParam(cfgfit.get("energy", None)) - nenergies = len(cfgfit["energy"]) - cfgfit["energyweight"] = self._normalizeConfigListParam( - cfgfit.get("energyweight"), length=nenergies, default=1.0 + cfgfit["energy"], idx = self._normalizeEnergyListParam( + cfgfit.get("energy", None) ) - cfgfit["energyflag"] = self._normalizeConfigListParam( - cfgfit.get("energyweight"), length=nenergies, default=1 + cfgfit["energyweight"], _ = self._normalizeEnergyListParam( + cfgfit.get("energyweight"), idx=idx, default=1.0 ) - cfgfit["energyscatter"] = self._normalizeConfigListParam( - cfgfit.get("energyweight"), length=nenergies, default=1 + cfgfit["energyflag"], _ = self._normalizeEnergyListParam( + cfgfit.get("energyweight"), idx=idx, default=1 + ) + cfgfit["energyscatter"], _ = self._normalizeEnergyListParam( + cfgfit.get("energyweight"), idx=idx, default=1 ) cfgfit["scatterflag"] = cfgfit.get("scatterflag", 0) @@ -149,27 +152,23 @@ def _addMissingConfig(self): cfgdetector["nthreshold"] = cfgdetector.get("nthreshold", 4) cfgdetector["ithreshold"] = cfgdetector.get("ithreshold", 1.0e-07) - @staticmethod - def _normalizeConfigListParam(lst, length=None, default=0): - if isinstance(lst, (str, bytes)) or lst is None: - lst = [] - elif isinstance(lst, list): - pass - else: + def _normalizeEnergyListParam(self, lst, idx=None, default=0): + if not isinstance(lst, list): lst = [lst] - if length is not None: - n = len(lst) - if n > length: - lst = lst[:length] - else: - lst += [default] * (length - n) - return lst + if idx is None: + idx = [ + i for i, v in enumerate(lst) if v and not isinstance(v, (str, bytes)) + ] + n = len(lst) + lst = [lst[i] if i < n else default for i in idx] + return lst, idx def _sourceLines(self): """ :yields tuple: (energy, weight, scatter) """ cfg = self.config["fit"] + scatterflag = cfg["scatterflag"] for energy, flag, weight, scatter in zip( cfg["energy"], cfg["energyflag"], @@ -177,14 +176,22 @@ def _sourceLines(self): cfg["energyscatter"], ): if energy and flag: - yield energy, weight, scatter + yield energy, weight, scatter and scatterflag + + def _scatterLines(self): + """Source lines that are included in the fir model + + :yields tuple: (energy, weight) + """ + for energy, weight, scatter in self._sourceLines(): + if scatter and energy > self.SCATTER_ENERGY_THRESHOLD: + yield energy, weight @property def _maxEnergy(self): """ :returns float or None: """ - cfg = self.config["fit"] energies = [energy for energy, _, _ in self._sourceLines()] if energies: return max(energies) @@ -198,6 +205,13 @@ def _nSourceLines(self): """ return len(list(self._sourceLines())) + @property + def _nRayleighLines(self): + """ + :returns int: + """ + return len(list(self._scatterLines())) + def _attenuators( self, matrix=False, @@ -332,7 +346,6 @@ def _emissionGroups(self): """ :yields list: [Z, symb, linegroupname] """ - maxenergy = self._maxEnergy cfg = self.config["peaks"] for element, peaks in cfg.items(): symb = element.capitalize() @@ -377,12 +390,34 @@ def _hypermetLongTail(self): def _hypermetStep(self): return (self.config["fit"]["hypermetflag"] >> 3) & 3 + def _anchorsIndices(self): + cfg = self.config["fit"] + if not cfg["stripanchorsflag"] or not cfg["stripanchorslist"]: + return + ravelled = self.xdata + for channel in cfg["stripanchorslist"]: + if channel <= ravelled[0]: + continue + index = numpy.nonzero(ravelled >= channel)[0] + if len(index): + index = min(index) + if index > 0: + yield index + class McaTheoryLegacyApi: def setdata(self, *args, **kw): warnings.warn("McaTheory.setdata deprecated, please use setData", FutureWarning) return self.setData(*args, **kw) + @property + def sigmay(self): + return self.ystd + + @property + def sigmay0(self): + return self.ystd0 + def startfit(self, *args, **kw): warnings.warn( "McaTheory.startfit deprecated, please use startFit", FutureWarning @@ -393,34 +428,35 @@ def startfit(self, *args, **kw): class McaTheory(McaTheoryConfigApi, McaTheoryLegacyApi, Model): """Model for MCA data""" - BAND_GAP = 0.00385 # For silicon + BAND_GAP = 0.00385 # keV, silicon GAUSS_SIGMA_TO_FWHM = 2.3548 MAX_ATTENUATION = 1.0e-300 + SCATTER_ENERGY_THRESHOLD = 0.2 # keV def __init__(self, **kw): super(McaTheory, self).__init__(**kw) + # TODO: done for some initialization of SpecfitFuns? SpecfitFuns.fastagauss([1.0, 10.0, 1.0], numpy.arange(10.0)) self.useFisxEscape(False) - self.ydata0 = None - self.xdata0 = None - self.sigmay0 = None + # XRF spectrum + self._ydata0 = None + self._xdata0 = None + self._std0 = None + self._ydata = None + self._xdata = None + self._std = None + self._lastXrange = None + + # XRF spectrum background + self._numBkg = None + self._lastNumBkgParams = None + + self._lastTime = None + self.strategyInstances = {} self.__toBeConfigured = False - self.__lastTime = None - self.lastxmin = None - self.lastxmax = None - self.laststrip = None - self.laststripconstant = None - self.laststripiterations = None - self.laststripalgorithm = None - self.lastsnipwidth = None - self.laststripwidth = None - self.laststripfilterwidth = None - self.laststripanchorsflag = None - self.laststripanchorslist = None - self.__configure() def useFisxEscape(self, flag=None): @@ -469,10 +505,10 @@ def configure(self, newdict=None): "WARNING: This configuration is the one of last fit.\n" "It does not correspond to the one of next fit." ) - return copy.deepcopy(self.config) - self.config.update(newdict) - self.__toBeConfigured = False - self.__configure() + else: + self.config.update(newdict) + self.__toBeConfigured = False + self.__configure() return copy.deepcopy(self.config) def __configure(self): @@ -515,14 +551,15 @@ def _getEmissionLines(self): def _getScatterLines(self): """Yields a list for scattering lines for each source line. - :yields list: [[energy, 1.0, "Peak"]] + :yields list: [[energy, 1.0, "Scatter %03d"]] """ scatteringAngle = self._scatteringAngle * numpy.pi / 180.0 angleFactor = 1.0 - numpy.cos(scatteringAngle) - for energy, _, scatter in self._sourceLines(): - if scatter: - energy /= 1.0 + (energy / 511.0) * angleFactor - yield [[energy, 1.0, "Peak"]] + for i, (en_elastic, _) in enumerate(self._scatterLines()): + en_inelastic = en_elastic / (1.0 + (en_elastic / 511.0) * angleFactor) + name = "Scatter %03d" % i + yield [[en_elastic, 1.0, name]] + yield [[en_inelastic, 1.0, name]] def _preCalculateParameterDependent(self): pass @@ -582,16 +619,16 @@ def _calcFluoRateCorrections(self): """Higher-order interaction corrections on the fluorescence rates. This will not be needed once fisx replaces the Elements module. """ - self.config["fisx"] = {} - if not FISX or "concentrations" not in self.config: + fisxcfg = self.config["fisx"] = {} + if not FISX: return secondary = self.config["concentrations"].get("usemultilayersecondary", False) if secondary: corrections = FisxHelper.getFisxCorrectionFactorsFromFitConfiguration( self.config, elementsFromMatrix=False ) - self.config["fisx"]["corrections"] = corrections - self.config["fisx"]["secondary"] = secondary + fisxcfg["corrections"] = corrections + fisxcfg["secondary"] = secondary def _getGroupEmissionLines(self, Z, symb, groupname): """Return a list of emission lines with total rate of 1 and @@ -824,6 +861,287 @@ def _calcPymcaEscapeRatios(self, energies): escape_peaks.append(peaks) return escape_peaks + @property + def xdata(self): + """Sorted and sliced view of xdata0""" + return self._xdata + + @property + def nchannels(self): + return len(self.xdata) + + @property + def ydata(self): + """Sorted and sliced view of ydata0""" + return self._ydata + + @property + def ystd(self): + """Sorted and sliced view of ystd0""" + return self._ystd + + @property + def xdata0(self): + return self._xdata0 + + @property + def ydata0(self): + return self._ydata0 + + @property + def ystd0(self): + return self._ystd0 + + @property + def ynumbkg(self): + """Get the numerical background (as opposed to the analytical background)""" + bkgparams = self._numBkgParams + if self._numBkg is not None and self._lastNumBkgParams == bkgparams: + return self._numBkg + if self.config["fit"]["stripflag"]: + signal = self._smooth(self.ydata) + anchorslist = list(self._anchorsIndices()) + if self.config["fit"]["stripalgorithm"] == 1: + self._numBkg = self._snip(signal, anchorslist) + else: + self._numBkg = self._strip(signal, anchorslist) + else: + self._numBkg = numpy.zeros_like(self.ydata) + self._lastNumBkgParams = bkgparams + return self._numBkg + + @property + def _numBkgParams(self): + cfg = self.config["fit"] + params = [ + "stripflag", + "stripalgorithm", + "stripfilterwidth", + "stripanchorsflag", + "stripanchorslist", + ] + if cfg["stripalgorithm"] == 1: + params += ["snipwidth"] + else: + params += ["stripwidth", "stripconstant", "stripiterations"] + return params + + def _snip(self, signal, anchorslist): + _logger.debug("CALCULATING SNIP") + n = len(signal) + if len(anchorslist): + anchorslist.sort() + else: + anchorslist = [0, n - 1] + + bkg = 0.0 * signal + lastAnchor = 0 + cfg = self.config["fit"] + width = cfg["snipwidth"] + for anchor in anchorslist: + if (anchor > lastAnchor) and (anchor < len(signal)): + bkg[lastAnchor:anchor] = SpecfitFuns.snip1d( + signal[lastAnchor:anchor], width, 0 + ) + lastAnchor = anchor + if lastAnchor < len(signal): + bkg[lastAnchor:] = SpecfitFuns.snip1d(signal[lastAnchor:], width, 0) + return bkg + + def _strip(self, signal, anchorslist): + cfg = self.config["fit"] + niter = cfg["stripiterations"] + if niter <= 0: + return numpy.zeros_like(signal) + min(signal) + + _logger.debug("CALCULATING STRIP") + if (niter > 1000) and (cfg["stripwidth"] == 1): + bkg = SpecfitFuns.subac( + signal, cfg["stripconstant"], niter / 20, 4, anchorslist + ) + bkg = SpecfitFuns.subac( + bkg, cfg["stripconstant"], niter / 4, cfg["stripwidth"], anchorslist + ) + else: + bkg = SpecfitFuns.subac( + signal, cfg["stripconstant"], niter, cfg["stripwidth"], anchorslist + ) + if niter > 1000: + # make sure to get something smooth + bkg = SpecfitFuns.subac(bkg, cfg["stripconstant"], 500, 1, anchorslist) + else: + # make sure to get something smooth but with less than + # 500 iterations + bkg = SpecfitFuns.subac( + bkg, + cfg["stripconstant"], + int(cfg["stripwidth"] * 2), + 1, + anchorslist, + ) + return bkg + + def _smooth(self, y): + try: + y = y.astype(dtype=numpy.float64) + w = self.config["fit"]["stripfilterwidth"] + ysmooth = SpecfitFuns.SavitskyGolay(y, w) + except Exception: + print("Unsuccessful Savitsky-Golay smoothing: %s" % sys.exc_info()) + raise + if ysmooth.size > 1: + fltr = [0.25, 0.5, 0.25] + ysmooth[1:-1] = numpy.convolve(ysmooth, fltr, mode=0) + ysmooth[0] = 0.5 * (ysmooth[0] + ysmooth[1]) + ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) + return ysmooth + + def setData(self, *var, **kw): + """ + Method to update the data to be fitted. + It accepts several combinations of input arguments, the simplest to + take into account is: + + setData(x, y, sigmay=None, xmin=None, xmax=None) + + x corresponds to the spectrum channels + y corresponds to the spectrum counts + sigmay is the uncertainty associated to the counts. If not given, + Poisson statistics will be assumed. If the fit configuration + is set to no weight, it will not be used. + xmin and xmax define the limits to be considered for performing the fit. + If the fit configuration flag self.config['fit']['use_limit'] is + set, they will be ignored. If xmin and xmax are not given, the + whole given spectrum will be considered for fitting. + time (seconds) is the factor associated to the flux, only used when using + a strategy based on concentrations + """ + if self.__toBeConfigured: + _logger.debug("setData RESTORE ORIGINAL CONFIGURATION") + self.configure(self.__originalConfiguration) + + if "y" in kw: + ydata0 = kw["y"] + elif len(var) > 1: + ydata0 = var[1] + elif len(var) == 1: + ydata0 = var[0] + else: + ydata0 = None + + if ydata0 is None: + return 1 + else: + ydata0 = numpy.ravel(ydata0) + + if "x" in kw: + xdata0 = kw["x"] + elif len(var) > 1: + xdata0 = var[0] + else: + xdata0 = None + + if xdata0 is None: + xdata0 = numpy.arange(len(ydata0)) + else: + xdata0 = numpy.ravel(xdata0) + + if "sigmay" in kw: + ystd0 = kw["sigmay"] + elif "stdy" in kw: + ystd0 = kw["stdy"] + elif len(var) > 2: + ystd0 = var[2] + else: + ystd0 = None + + if ystd0 is None: + # Poisson noise + valid = ydata0 > 0 + if valid.any(): + ystd0 = numpy.sqrt(abs(ydata0)) + ystd0[~valid] = ystd0[valid].min() + else: + ystd0 = numpy.ones_like(ystd0) + else: + ystd0 = numpy.ravel(ystd0) + + timeFactor = kw.get("time", None) + self._lastTime = timeFactor + if timeFactor is None: + cfgfit = self.config["fit"] + if self.config["concentrations"].get("useautotime", False): + if not self.config["concentrations"]["usematrix"]: + msg = "Requested to use time from data but not present!!" + raise ValueError(msg) + elif self.config["concentrations"].get("useautotime", False): + self.config["concentrations"]["time"] = timeFactor + + cfgfit = self.config["fit"] + + xmin = cfgfit["xmin"] + if not cfgfit["use_limit"]: + if "xmin" in kw: + xmin = kw["xmin"] + if xmin is not None: + cfgfit["xmin"] = xmin + else: + xmin = xdata0.min() + elif xdata0.size: + xmin = xdata0.min() + + xmax = cfgfit["xmax"] + if not cfgfit["use_limit"]: + if "xmax" in kw: + xmax = kw["xmax"] + if xmax is not None: + cfgfit["xmax"] = xmax + else: + xmax = xdata0.max() + elif xdata0.size: + xmax = xdata0.max() + + if self._cacheDataView(xdata0, ydata0, ystd0, xmin, xmax): + return 1 + self._xdata0 = xdata0 + self._ydata0 = ydata0 + self._ystd0 = ystd0 + self._lastXrange = xmin, xmax + + def _cacheDataView( + self, xdata0=None, ydata0=None, ystd0=None, xmin=None, xmax=None + ): + """Sorted and sliced view of original data""" + if xdata0 is None: + xdata0 = self.xdata0 + if xdata0 is None or not xdata0.size: + return 1 + if ydata0 is None: + ydata0 = self.ydata0 + if ystd0 is None: + ystd0 = self.ystd0 + if xmin is None: + xmin = self._lastXrange[0] + if xmax is None: + xmax = self._lastXrange[1] + + selection = numpy.isfinite(ydata0) + if xmin is not None: + selection &= xdata0 >= xmin + if xmax is not None: + selection &= xdata0 <= xmax + if not selection.any(): + return 1 + + idx = numpy.argsort(xdata0)[selection] + self._xdata = xdata0[idx] + self._ydata = ydata0[idx] + self._ystd = ystd0[idx] + self._numBkg = None + + def getLastTime(self): + return self._lastTime + @property def zero(self): return self.config["detector"]["zero"] @@ -844,7 +1162,7 @@ def gain(self, value): def fano(self): return self.config["detector"]["fano"] - @gain.setter + @fano.setter def fano(self, value): self.config["detector"]["fano"] = value @@ -879,7 +1197,7 @@ def eta_factor(self, value): def step_heightratio(self): return self.config["peakshape"]["step_heightratio"] - @sum.setter + @step_heightratio.setter def step_heightratio(self, value): self.config["peakshape"]["step_heightratio"] = value diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 46c3a67c8..232e0312b 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -33,6 +33,7 @@ import unittest import os import sys +import copy import numpy if sys.version_info < (3,): from StringIO import StringIO @@ -267,10 +268,8 @@ def testTrainingDataFilePresence(self): self.assertTrue(os.path.isfile(trainingDataFile), "File %s is not an actual file" % trainingDataFile) - def testTrainingDataFit(self): + def _readTrainingData(self): from PyMca5.PyMcaIO import specfilewrapper as specfile - from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory - from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool from PyMca5.PyMcaIO import ConfigDict trainingDataFile = os.path.join(self.dataDir, "XRFSpectrum.mca") self.assertTrue(os.path.isfile(trainingDataFile), @@ -285,18 +284,52 @@ def testTrainingDataFit(self): "Training data 1st scan should contain no MCAs") y = mcaData = sf[1].mca(1) sf = None + x = numpy.arange(y.size).astype(numpy.float64) # perform the actual XRF analysis configuration = ConfigDict.ConfigDict() configuration.readfp(StringIO(cfg)) + + return x, y, configuration + + def _readStainlessSteelData(self): + from PyMca5.PyMcaIO import specfilewrapper as specfile + from PyMca5.PyMcaIO import ConfigDict + + # read the data + dataFile = os.path.join(self.dataDir, "Steel.spe") + self.assertTrue(os.path.isfile(dataFile), + "File %s is not an actual file" % dataFile) + sf = specfile.Specfile(dataFile) + self.assertTrue(len(sf) == 1, "File %s cannot be read" % dataFile) + self.assertTrue(sf[0].nbmca() == 1, + "Spe file should contain MCA data") + y = counts = sf[0].mca(1) + x = channels = numpy.arange(y.size).astype(numpy.float64) + sf = None + + # read the fit configuration + configFile = os.path.join(self.dataDir, "Steel.cfg") + self.assertTrue(os.path.isfile(configFile), + "File %s is not an actual file" % configFile) + configuration = ConfigDict.ConfigDict() + configuration.read(configFile) + # configure the fit + # make sure no secondary excitations are used + configuration["concentrations"]["usemultilayersecondary"] = 0 + + return x, y, configuration + + def testTrainingDataFit(self): + from PyMca5.PyMcaIO import specfilewrapper as specfile + from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory + from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool + + x, y, configuration = self._readTrainingData() + + # perform the actual XRF analysis mcaFit = LegacyMcaTheory.LegacyMcaTheory() - configuration=mcaFit.configure(configuration) - x = numpy.arange(y.size).astype(numpy.float64) - mcaFit.setData(x, y, - xmin=configuration["fit"]["xmin"], - xmax=configuration["fit"]["xmax"]) - mcaFit.estimate() - fitResult, result = mcaFit.startFit(digest=1) + configuration, fitResult, result = self._configAndFit(x, y, configuration, mcaFit) # fit is already done, calculate the concentrations concentrationsConfiguration = configuration["concentrations"] @@ -350,39 +383,14 @@ def testTrainingDataFit(self): "Error for <%s> concentration %g != %g" % (key, internal, fp)) def testStainlessSteelDataFit(self): - from PyMca5.PyMcaIO import specfilewrapper as specfile from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool - from PyMca5.PyMcaIO import ConfigDict - # read the data - dataFile = os.path.join(self.dataDir, "Steel.spe") - self.assertTrue(os.path.isfile(dataFile), - "File %s is not an actual file" % dataFile) - sf = specfile.Specfile(dataFile) - self.assertTrue(len(sf) == 1, "File %s cannot be read" % dataFile) - self.assertTrue(sf[0].nbmca() == 1, - "Spe file should contain MCA data") - y = counts = sf[0].mca(1) - x = channels = numpy.arange(y.size).astype(numpy.float64) - sf = None + x, y, configuration = self._readStainlessSteelData() - # read the fit configuration - configFile = os.path.join(self.dataDir, "Steel.cfg") - self.assertTrue(os.path.isfile(configFile), - "File %s is not an actual file" % configFile) - configuration = ConfigDict.ConfigDict() - configuration.read(configFile) # configure the fit - # make sure no secondary excitations are used - configuration["concentrations"]["usemultilayersecondary"] = 0 mcaFit = LegacyMcaTheory.LegacyMcaTheory() - configuration=mcaFit.configure(configuration) - mcaFit.setData(x, y, - xmin=configuration["fit"]["xmin"], - xmax=configuration["fit"]["xmax"]) - mcaFit.estimate() - fitResult, result = mcaFit.startFit(digest=1) + configuration, fitResult, result = self._configAndFit(x, y, configuration, mcaFit) # concentrations # fit is already done, calculate the concentrations @@ -456,12 +464,7 @@ def testStainlessSteelDataFit(self): # in order to get the good fundamental parameters configuration["concentrations"]['usematrix'] = 1 configuration["concentrations"]["usemultilayersecondary"] = 2 - mcaFit.setConfiguration(configuration) - mcaFit.setData(x, y, - xmin=configuration["fit"]["xmin"], - xmax=configuration["fit"]["xmax"]) - mcaFit.estimate() - fitResult, result = mcaFit.startFit(digest=1) + configuration, fitResult, result = self._configAndFit(x, y, configuration, mcaFit) # concentrations # fit is already done, calculate the concentrations @@ -511,12 +514,7 @@ def testStainlessSteelDataFit(self): "Ni", "-", "-", "-","-","-"] mcaFit = LegacyMcaTheory.LegacyMcaTheory() - configuration=mcaFit.configure(configuration) - mcaFit.setData(x, y, - xmin=configuration["fit"]["xmin"], - xmax=configuration["fit"]["xmax"]) - mcaFit.estimate() - fitResult, result = mcaFit.startFit(digest=1) + configuration, fitResult, result = self._configAndFit(x, y, configuration, mcaFit) # concentrations # fit is already done, calculate the concentrations @@ -546,6 +544,139 @@ def testStainlessSteelDataFit(self): "Strategy: Element %s discrepancy too large %.1f %%" % \ (element.split()[0], delta)) + def testLegacyMcaTheory(self): + x, y, configuration = self._readTrainingData() + self._testLegacyMcaTheory(x, y, configuration) + + x, y, configuration = self._readStainlessSteelData() + + configuration["concentrations"]['usematrix'] = 0 + configuration["concentrations"]["usemultilayersecondary"] = 0 + self._testLegacyMcaTheory(x, y, configuration) + + configuration["concentrations"]['usematrix'] = 1 + configuration["concentrations"]["usemultilayersecondary"] = 2 + self._testLegacyMcaTheory(x, y, configuration) + + configuration["concentrations"]['usematrix'] = 0 + configuration["concentrations"]["usemultilayersecondary"] = 2 + configuration["attenuators"]["Matrix"] = [1, 'Fe', 1.0, 1.0, 45.0, 45.0] + configuration["fit"]["strategyflag"] = 1 + configuration["fit"]["strategy"] = "SingleLayerStrategy" + configuration["SingleLayerStrategy"] = {} + configuration["SingleLayerStrategy"]["layer"] = "Auto" + configuration["SingleLayerStrategy"]["iterations"] = 3 + configuration["SingleLayerStrategy"]["completer"] = "-" + configuration["SingleLayerStrategy"]["flags"] = [1, 1, 1, 1, 0, + 0, 0, 0, 0, 0] + configuration["SingleLayerStrategy"]["peaks"] = [ "Cr K", + "Mn K", "Fe Ka", + "Ni K", "-", "-", + "-","-","-","-"] + configuration["SingleLayerStrategy"]["materials"] = ["Cr", + "Mn", "Fe", + "Ni", "-", "-", + "-","-","-"] + self._testLegacyMcaTheory(x, y, configuration) + + def _testLegacyMcaTheory(self, x, y, configuration): + from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory + from PyMca5.PyMcaPhysics.xrf import NewClassMcaTheory + + mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() + _, fitResult1, result1 = self._configAndFit( + x, y, copy.deepcopy(configuration), mcaFitLegacy, tmpflag=True) + + mcaFit = NewClassMcaTheory.McaTheory() + _, fitResult2, result2 = self._configAndFit( + x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) + + # Compare data + numpy.testing.assert_array_equal(mcaFitLegacy.xdata.flat, mcaFit.xdata) + numpy.testing.assert_array_equal(mcaFitLegacy.ydata.flat, mcaFit.ydata) + numpy.testing.assert_array_equal(mcaFitLegacy.sigmay.flat, mcaFit.ystd) + + # Compare configuration + config1 = copy.deepcopy(mcaFitLegacy.config) + config2 = copy.deepcopy(mcaFit.config) + + # The new class removes invalid energies + n = len(config2["fit"]["energy"]) + for name in ["energy", "energyweight", "energyflag", "energyscatter"]: + if config1["fit"][name] is None: + config1["fit"][name] = [] + else: + config1["fit"][name] = config1["fit"][name][:n] + if len(config1["attenuators"]["Matrix"]) == 6: + lst = config1["attenuators"]["Matrix"] + lst.extend([0, lst[-1]+lst[-2]]) + + self._assertDeepEqual(config1, config2) + + # Compare fluo rate dictionaries + self.assertEqual(mcaFitLegacy._fluoRates, mcaFit._fluoRates) + + # Compare line groups + linegroups1 = mcaFitLegacy.PEAKS0 + linegroups2 = mcaFit._lineGroups + linegroups1 = [[[line[1], line[0], name] + for name, line in zip(names, lines)] + for names, lines in zip(mcaFitLegacy.PEAKS0NAMES, linegroups1)] + + self.assertEqual(len(linegroups1), len(linegroups2)) + for lg1, lg2 in zip(linegroups1, linegroups2): + self.assertEqual(len(lg1), len(lg2)) + for line1, line2 in zip(lg1, lg2): + self.assertEqual(line1[0], line2[0]) + numpy.testing.assert_allclose(line1[1], line2[1], rtol=1e-9) + self.assertEqual(line1[2], line2[2]) + + # Compare escape line groups + linegroups1 = mcaFitLegacy.PEAKS0ESCAPE + linegroups2 = mcaFit._escapeLineGroups + self.assertEqual(len(linegroups1), len(linegroups2)) + for lg1, lg2 in zip(linegroups1, linegroups2): + self.assertEqual(len(lg1), len(lg2)) + for elines1, elines2 in zip(lg1, lg2): + self.assertEqual(len(elines1), len(elines2)) + for line1, line2 in zip(elines1, elines2): + self.assertEqual(line1[0], line2[0]) + numpy.testing.assert_allclose(line1[1], line2[1], rtol=1e-9) + self.assertEqual(line1[2], line2[2]) + + # Compare fit results + self.assertEqual(fitResult1, fitResult2) + self.assertEqual(result1, result2) + + def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): + configuration = mcaFit.configure(configuration) + mcaFit.setData(x, y, + xmin=configuration["fit"]["xmin"], + xmax=configuration["fit"]["xmax"]) + if tmpflag: + return configuration, None, None + mcaFit.estimate() + fitResult1, result1 = mcaFit.startFit(digest=1) + return configuration, fitResult1, result1 + + def _assertDeepEqual(self, obj1, obj2): + """Better verbosity than assertEqual for deep structures + """ + if isinstance(obj1, dict): + self.assertEqual(set(obj1.keys()), set(obj2.keys())) + for k in obj1: + self._assertDeepEqual(obj1[k], obj2[k]) + elif isinstance(obj1, (list, tuple)): + if isinstance(obj1[0], (list, tuple, numpy.ndarray)): + self._assertDeepEqual(obj1, obj2) + else: + self.assertEqual(obj1, obj2) + elif isinstance(obj1, numpy.ndarray): + numpy.testing.assert_allclose(obj1, obj2, rtol=0) + else: + self.assertEqual(obj1, obj2) + + def getSuite(auto=True): testSuite = unittest.TestSuite() if auto: From bd72a596f4e20dd66e67a019f936e432aa73ae4a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 15:26:14 +0100 Subject: [PATCH 06/74] WIP --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 133 +++++++++++-------- PyMca5/tests/XrfTest.py | 1 + 2 files changed, 77 insertions(+), 57 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index c106c5288..0a3c5e40a 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -439,18 +439,22 @@ def __init__(self, **kw): SpecfitFuns.fastagauss([1.0, 10.0, 1.0], numpy.arange(10.0)) self.useFisxEscape(False) - # XRF spectrum + # Original XRF spectrum self._ydata0 = None self._xdata0 = None self._std0 = None + self._xmin0 = None + self._xmax0 = None + + # XRF spectrum to fit self._ydata = None self._xdata = None self._std = None - self._lastXrange = None + self._lastDataCacheParams = None # XRF spectrum background self._numBkg = None - self._lastNumBkgParams = None + self._lastNumBkgCacheParams = None self._lastTime = None @@ -864,6 +868,7 @@ def _calcPymcaEscapeRatios(self, energies): @property def xdata(self): """Sorted and sliced view of xdata0""" + self._refreshDataCache() return self._xdata @property @@ -873,13 +878,21 @@ def nchannels(self): @property def ydata(self): """Sorted and sliced view of ydata0""" + self._refreshDataCache() return self._ydata @property def ystd(self): """Sorted and sliced view of ystd0""" + self._refreshDataCache() return self._ystd + @property + def ynumbkg(self): + """Get the numerical background (as opposed to the analytical background)""" + self._refreshNumBkgCache() + return self._numBkg + @property def xdata0(self): return self._xdata0 @@ -893,25 +906,7 @@ def ystd0(self): return self._ystd0 @property - def ynumbkg(self): - """Get the numerical background (as opposed to the analytical background)""" - bkgparams = self._numBkgParams - if self._numBkg is not None and self._lastNumBkgParams == bkgparams: - return self._numBkg - if self.config["fit"]["stripflag"]: - signal = self._smooth(self.ydata) - anchorslist = list(self._anchorsIndices()) - if self.config["fit"]["stripalgorithm"] == 1: - self._numBkg = self._snip(signal, anchorslist) - else: - self._numBkg = self._strip(signal, anchorslist) - else: - self._numBkg = numpy.zeros_like(self.ydata) - self._lastNumBkgParams = bkgparams - return self._numBkg - - @property - def _numBkgParams(self): + def _numBkgCacheParams(self): cfg = self.config["fit"] params = [ "stripflag", @@ -926,6 +921,23 @@ def _numBkgParams(self): params += ["stripwidth", "stripconstant", "stripiterations"] return params + def _refreshNumBkgCache(self): + bkgparams = self._numBkgCacheParams + if self._lastNumBkgCacheParams == bkgparams: + return + elif self.ydata is None: + self._numBkg = None + elif self.config["fit"]["stripflag"]: + signal = self._smooth(self.ydata) + anchorslist = list(self._anchorsIndices()) + if self.config["fit"]["stripalgorithm"] == 1: + self._numBkg = self._snip(signal, anchorslist) + else: + self._numBkg = self._strip(signal, anchorslist) + else: + self._numBkg = numpy.zeros_like(self.ydata) + self._lastNumBkgCacheParams = bkgparams + def _snip(self, signal, anchorslist): _logger.debug("CALCULATING SNIP") n = len(signal) @@ -1062,7 +1074,7 @@ def setData(self, *var, **kw): ystd0 = numpy.sqrt(abs(ydata0)) ystd0[~valid] = ystd0[valid].min() else: - ystd0 = numpy.ones_like(ystd0) + ystd0 = numpy.ones_like(ydata0) else: ystd0 = numpy.ravel(ystd0) @@ -1077,57 +1089,61 @@ def setData(self, *var, **kw): elif self.config["concentrations"].get("useautotime", False): self.config["concentrations"]["time"] = timeFactor - cfgfit = self.config["fit"] - - xmin = cfgfit["xmin"] - if not cfgfit["use_limit"]: - if "xmin" in kw: - xmin = kw["xmin"] - if xmin is not None: - cfgfit["xmin"] = xmin - else: - xmin = xdata0.min() - elif xdata0.size: - xmin = xdata0.min() - - xmax = cfgfit["xmax"] - if not cfgfit["use_limit"]: - if "xmax" in kw: - xmax = kw["xmax"] - if xmax is not None: - cfgfit["xmax"] = xmax - else: - xmax = xdata0.max() - elif xdata0.size: - xmax = xdata0.max() - - if self._cacheDataView(xdata0, ydata0, ystd0, xmin, xmax): + self._xmin0 = kw.get("xmin", self.xmin) + self._xmax0 = kw.get("xmax", self.xmax) + if self._refreshDataCache(xdata0=xdata0, ydata0=ydata0, ystd0=ystd0): return 1 self._xdata0 = xdata0 self._ydata0 = ydata0 self._ystd0 = ystd0 - self._lastXrange = xmin, xmax - def _cacheDataView( - self, xdata0=None, ydata0=None, ystd0=None, xmin=None, xmax=None - ): + @property + def xmin(self): + """From config (if enabled) or the last `setData` call""" + cfgfit = self.config["fit"] + if cfgfit["use_limit"]: + return cfgfit["xmin"] + else: + return self._xmin0 + + @property + def xmax(self): + """From config (if enabled) or the last `setData` call""" + cfgfit = self.config["fit"] + if cfgfit["use_limit"]: + return cfgfit["xmax"] + else: + return self._xmax0 + + @property + def _dataCacheParams(self): + return self.xmin, self.xmax + + def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): """Sorted and sliced view of original data""" + params = self._dataCacheParams + if xdata0 is None and ydata0 is None and ystd0 is None: + if self._lastDataCacheParams == params: + return + if xdata0 is None: xdata0 = self.xdata0 if xdata0 is None or not xdata0.size: return 1 if ydata0 is None: ydata0 = self.ydata0 + if ydata0 is None or ydata0.size != xdata0.size: + return 1 if ystd0 is None: ystd0 = self.ystd0 - if xmin is None: - xmin = self._lastXrange[0] - if xmax is None: - xmax = self._lastXrange[1] + if ystd0 is None or ystd0.size != xdata0.size: + return 1 selection = numpy.isfinite(ydata0) + xmin = self.xmin if xmin is not None: selection &= xdata0 >= xmin + xmax = self.xmax if xmax is not None: selection &= xdata0 <= xmax if not selection.any(): @@ -1137,7 +1153,10 @@ def _cacheDataView( self._xdata = xdata0[idx] self._ydata = ydata0[idx] self._ystd = ystd0[idx] - self._numBkg = None + + # Fix data cache and invalidate background cache + self._lastDataCacheParams = params + self._lastNumBkgCacheParams = None def getLastTime(self): return self._lastTime diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 232e0312b..9d93aa822 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -595,6 +595,7 @@ def _testLegacyMcaTheory(self, x, y, configuration): numpy.testing.assert_array_equal(mcaFitLegacy.xdata.flat, mcaFit.xdata) numpy.testing.assert_array_equal(mcaFitLegacy.ydata.flat, mcaFit.ydata) numpy.testing.assert_array_equal(mcaFitLegacy.sigmay.flat, mcaFit.ystd) + numpy.testing.assert_array_equal(mcaFitLegacy.zz.flat, mcaFit.ynumbkg) # Compare configuration config1 = copy.deepcopy(mcaFitLegacy.config) From c18ab26b51c7b95c6206eb50ec510806a08e010a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 15:39:35 +0100 Subject: [PATCH 07/74] WIP --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 233 ++++++++++--------- 1 file changed, 119 insertions(+), 114 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 0a3c5e40a..b5ae10d3f 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -905,109 +905,6 @@ def ydata0(self): def ystd0(self): return self._ystd0 - @property - def _numBkgCacheParams(self): - cfg = self.config["fit"] - params = [ - "stripflag", - "stripalgorithm", - "stripfilterwidth", - "stripanchorsflag", - "stripanchorslist", - ] - if cfg["stripalgorithm"] == 1: - params += ["snipwidth"] - else: - params += ["stripwidth", "stripconstant", "stripiterations"] - return params - - def _refreshNumBkgCache(self): - bkgparams = self._numBkgCacheParams - if self._lastNumBkgCacheParams == bkgparams: - return - elif self.ydata is None: - self._numBkg = None - elif self.config["fit"]["stripflag"]: - signal = self._smooth(self.ydata) - anchorslist = list(self._anchorsIndices()) - if self.config["fit"]["stripalgorithm"] == 1: - self._numBkg = self._snip(signal, anchorslist) - else: - self._numBkg = self._strip(signal, anchorslist) - else: - self._numBkg = numpy.zeros_like(self.ydata) - self._lastNumBkgCacheParams = bkgparams - - def _snip(self, signal, anchorslist): - _logger.debug("CALCULATING SNIP") - n = len(signal) - if len(anchorslist): - anchorslist.sort() - else: - anchorslist = [0, n - 1] - - bkg = 0.0 * signal - lastAnchor = 0 - cfg = self.config["fit"] - width = cfg["snipwidth"] - for anchor in anchorslist: - if (anchor > lastAnchor) and (anchor < len(signal)): - bkg[lastAnchor:anchor] = SpecfitFuns.snip1d( - signal[lastAnchor:anchor], width, 0 - ) - lastAnchor = anchor - if lastAnchor < len(signal): - bkg[lastAnchor:] = SpecfitFuns.snip1d(signal[lastAnchor:], width, 0) - return bkg - - def _strip(self, signal, anchorslist): - cfg = self.config["fit"] - niter = cfg["stripiterations"] - if niter <= 0: - return numpy.zeros_like(signal) + min(signal) - - _logger.debug("CALCULATING STRIP") - if (niter > 1000) and (cfg["stripwidth"] == 1): - bkg = SpecfitFuns.subac( - signal, cfg["stripconstant"], niter / 20, 4, anchorslist - ) - bkg = SpecfitFuns.subac( - bkg, cfg["stripconstant"], niter / 4, cfg["stripwidth"], anchorslist - ) - else: - bkg = SpecfitFuns.subac( - signal, cfg["stripconstant"], niter, cfg["stripwidth"], anchorslist - ) - if niter > 1000: - # make sure to get something smooth - bkg = SpecfitFuns.subac(bkg, cfg["stripconstant"], 500, 1, anchorslist) - else: - # make sure to get something smooth but with less than - # 500 iterations - bkg = SpecfitFuns.subac( - bkg, - cfg["stripconstant"], - int(cfg["stripwidth"] * 2), - 1, - anchorslist, - ) - return bkg - - def _smooth(self, y): - try: - y = y.astype(dtype=numpy.float64) - w = self.config["fit"]["stripfilterwidth"] - ysmooth = SpecfitFuns.SavitskyGolay(y, w) - except Exception: - print("Unsuccessful Savitsky-Golay smoothing: %s" % sys.exc_info()) - raise - if ysmooth.size > 1: - fltr = [0.25, 0.5, 0.25] - ysmooth[1:-1] = numpy.convolve(ysmooth, fltr, mode=0) - ysmooth[0] = 0.5 * (ysmooth[0] + ysmooth[1]) - ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) - return ysmooth - def setData(self, *var, **kw): """ Method to update the data to be fitted. @@ -1091,11 +988,7 @@ def setData(self, *var, **kw): self._xmin0 = kw.get("xmin", self.xmin) self._xmax0 = kw.get("xmax", self.xmax) - if self._refreshDataCache(xdata0=xdata0, ydata0=ydata0, ystd0=ystd0): - return 1 - self._xdata0 = xdata0 - self._ydata0 = ydata0 - self._ystd0 = ystd0 + return self._refreshDataCache(xdata0=xdata0, ydata0=ydata0, ystd0=ystd0) @property def xmin(self): @@ -1115,17 +1008,22 @@ def xmax(self): else: return self._xmax0 + def getLastTime(self): + return self._lastTime + @property def _dataCacheParams(self): + """Any change in these parameter will invalidate the cache""" return self.xmin, self.xmax def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): - """Sorted and sliced view of original data""" + """Cache sorted and sliced view of the original XRF spectrum data""" params = self._dataCacheParams if xdata0 is None and ydata0 is None and ystd0 is None: if self._lastDataCacheParams == params: - return + return # the cached data is still valid + # Original XRF spectrum if xdata0 is None: xdata0 = self.xdata0 if xdata0 is None or not xdata0.size: @@ -1139,6 +1037,7 @@ def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): if ystd0 is None or ystd0.size != xdata0.size: return 1 + # XRF spectrum selection selection = numpy.isfinite(ydata0) xmin = self.xmin if xmin is not None: @@ -1149,17 +1048,123 @@ def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): if not selection.any(): return 1 + # Cache original and reformed XRF spectrum idx = numpy.argsort(xdata0)[selection] self._xdata = xdata0[idx] self._ydata = ydata0[idx] self._ystd = ystd0[idx] - - # Fix data cache and invalidate background cache + self._xdata0 = xdata0 + self._ydata0 = ydata0 + self._ystd0 = ystd0 self._lastDataCacheParams = params + + # Invalidate background cache self._lastNumBkgCacheParams = None - def getLastTime(self): - return self._lastTime + @property + def _numBkgCacheParams(self): + """Any change in these parameter will invalidate the cache""" + cfg = self.config["fit"] + params = [ + "stripflag", + "stripalgorithm", + "stripfilterwidth", + "stripanchorsflag", + "stripanchorslist", + ] + if cfg["stripalgorithm"] == 1: + params += ["snipwidth"] + else: + params += ["stripwidth", "stripconstant", "stripiterations"] + return params + + def _refreshNumBkgCache(self): + """Cache numerical background""" + bkgparams = self._numBkgCacheParams + if self._lastNumBkgCacheParams == bkgparams: + return # the cached data is still valid + elif self.ydata is None: + self._numBkg = None + elif self.config["fit"]["stripflag"]: + signal = self._smooth(self.ydata) + anchorslist = list(self._anchorsIndices()) + if self.config["fit"]["stripalgorithm"] == 1: + self._numBkg = self._snip(signal, anchorslist) + else: + self._numBkg = self._strip(signal, anchorslist) + else: + self._numBkg = numpy.zeros_like(self.ydata) + self._lastNumBkgCacheParams = bkgparams + + def _snip(self, signal, anchorslist): + _logger.debug("CALCULATING SNIP") + n = len(signal) + if len(anchorslist): + anchorslist.sort() + else: + anchorslist = [0, n - 1] + + bkg = 0.0 * signal + lastAnchor = 0 + cfg = self.config["fit"] + width = cfg["snipwidth"] + for anchor in anchorslist: + if (anchor > lastAnchor) and (anchor < len(signal)): + bkg[lastAnchor:anchor] = SpecfitFuns.snip1d( + signal[lastAnchor:anchor], width, 0 + ) + lastAnchor = anchor + if lastAnchor < len(signal): + bkg[lastAnchor:] = SpecfitFuns.snip1d(signal[lastAnchor:], width, 0) + return bkg + + def _strip(self, signal, anchorslist): + cfg = self.config["fit"] + niter = cfg["stripiterations"] + if niter <= 0: + return numpy.zeros_like(signal) + min(signal) + + _logger.debug("CALCULATING STRIP") + if (niter > 1000) and (cfg["stripwidth"] == 1): + bkg = SpecfitFuns.subac( + signal, cfg["stripconstant"], niter / 20, 4, anchorslist + ) + bkg = SpecfitFuns.subac( + bkg, cfg["stripconstant"], niter / 4, cfg["stripwidth"], anchorslist + ) + else: + bkg = SpecfitFuns.subac( + signal, cfg["stripconstant"], niter, cfg["stripwidth"], anchorslist + ) + if niter > 1000: + # make sure to get something smooth + bkg = SpecfitFuns.subac(bkg, cfg["stripconstant"], 500, 1, anchorslist) + else: + # make sure to get something smooth but with less than + # 500 iterations + bkg = SpecfitFuns.subac( + bkg, + cfg["stripconstant"], + int(cfg["stripwidth"] * 2), + 1, + anchorslist, + ) + return bkg + + def _smooth(self, y): + try: + y = y.astype(dtype=numpy.float64) + w = self.config["fit"]["stripfilterwidth"] + ysmooth = SpecfitFuns.SavitskyGolay(y, w) + except Exception: + print("Unsuccessful Savitsky-Golay smoothing: %s" % sys.exc_info()) + raise + if ysmooth.size > 1: + fltr = [0.25, 0.5, 0.25] + ysmooth[1:-1] = numpy.convolve(ysmooth, fltr, mode=0) + ysmooth[0] = 0.5 * (ysmooth[0] + ysmooth[1]) + ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) + return ysmooth @property def zero(self): From a43d5d61adbed31fbe2411eff20b268d70de0d74 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 15:51:37 +0100 Subject: [PATCH 08/74] WIP --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 9 ++++++--- PyMca5/tests/FitModelTest.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index b5ae10d3f..8e06515f1 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -1037,7 +1037,7 @@ def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): if ystd0 is None or ystd0.size != xdata0.size: return 1 - # XRF spectrum selection + # XRF spectrum view selection = numpy.isfinite(ydata0) xmin = self.xmin if xmin is not None: @@ -1048,7 +1048,7 @@ def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): if not selection.any(): return 1 - # Cache original and reformed XRF spectrum + # Cache the original XRF spectrum and its view idx = numpy.argsort(xdata0)[selection] self._xdata = xdata0[idx] self._ydata = ydata0[idx] @@ -1097,6 +1097,7 @@ def _refreshNumBkgCache(self): self._lastNumBkgCacheParams = bkgparams def _snip(self, signal, anchorslist): + """Apply SNIP filtering to a signal""" _logger.debug("CALCULATING SNIP") n = len(signal) if len(anchorslist): @@ -1119,10 +1120,11 @@ def _snip(self, signal, anchorslist): return bkg def _strip(self, signal, anchorslist): + """Apply STRIP filtering to a signal""" cfg = self.config["fit"] niter = cfg["stripiterations"] if niter <= 0: - return numpy.zeros_like(signal) + min(signal) + return numpy.full_like(signal, signal.min()) _logger.debug("CALCULATING STRIP") if (niter > 1000) and (cfg["stripwidth"] == 1): @@ -1152,6 +1154,7 @@ def _strip(self, signal, anchorslist): return bkg def _smooth(self, y): + """Smooth a signal""" try: y = y.astype(dtype=numpy.float64) w = self.config["fit"]["stripfilterwidth"] diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index 66bd7245e..78221b8c2 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -242,7 +242,7 @@ def _testNonLinearFit(self): # TODO: non-linear parameters not precise # numpy.testing.assert_allclose(self.fitmodel.parameters, expected1) numpy.testing.assert_allclose( - self.fitmodel.linear_parameters, expected2, rtol=1e-6 + self.fitmodel.linear_parameters, expected2, rtol=1e-5 ) def assert_result(self, result, expected): From f1f337457c3d5bd743f7f262fb2bccd0982d3f5f Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 17:13:36 +0100 Subject: [PATCH 09/74] Disable Travis --- .travis.yml => .travis.yml.bak | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .travis.yml => .travis.yml.bak (100%) diff --git a/.travis.yml b/.travis.yml.bak similarity index 100% rename from .travis.yml rename to .travis.yml.bak From 32b339c6522da201b5963e54391a14ae568aa19c Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Feb 2021 17:26:06 +0100 Subject: [PATCH 10/74] WIP --- PyMca5/PyMcaMath/fitting/Model.py | 59 ++++++++++++++++++-- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 20 ++++++- PyMca5/tests/SimpleModel.py | 2 +- PyMca5/tests/XrfTest.py | 10 ++++ 4 files changed, 84 insertions(+), 7 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index b66178f16..e1b912708 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -123,6 +123,10 @@ def parameters(self): def parameters(self, values): return self._set_parameters(values) + @property + def constraints(self): + return self._get_constraints() + @property def nparameters(self): return sum(tpl[1] for tpl in self._parameter_groups()) @@ -147,6 +151,10 @@ def linear_parameters(self): def linear_parameters(self, params): return self._set_parameters(params, linear_only=True) + @property + def linear_constraints(self): + return self._get_constraints(linear_only=True) + @property def nlinear_parameters(self): return sum(tpl[1] for tpl in self._parameter_groups(linear_only=True)) @@ -170,14 +178,38 @@ def _get_parameters(self, linear_only=False): """ i = 0 if linear_only: - params = numpy.zeros(self.nlinear_parameters) + nparams = self.nlinear_parameters else: - params = numpy.zeros(self.nparameters) + nparams = self.nparameters + params = numpy.zeros(nparams) for name, n in self._parameter_groups(linear_only=linear_only): params[i : i + n] = getattr(self, name) i += n return params + def _get_constraints(self, linear_only=False): + """ + :param bool linear_only: + :returns array: + """ + i = 0 + if linear_only: + nparams = self.nlinear_parameters + else: + nparams = self.nparameters + codes = numpy.zeros((nparams, 3), numpy.float64) + bspecified = False + for name, n in self._parameter_groups(linear_only=linear_only): + name += "_constraint" + if hasattr(self, name): + bspecified = True + codes[i : i + n] = getattr(self, name) + i += n + if bspecified: + return codes.T + else: + return None + def _set_parameters(self, params, linear_only=False): """ :returns values: @@ -348,20 +380,25 @@ def nonlinear_fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - initial = self.parameters + initial = parameters = self.parameters + constraints = self.constraints try: for i in range(max(self.niter_non_leastsquares, 1)): result = Gefit.LeastSquaresFit( self._evaluate, - initial, + parameters, model_deriv=self._derivative, xdata=self.xdata, ydata=self.ydata, sigmadata=self.ystd, + constrains=constraints, + maxiter=self.maxiter, + weightflag=self.weightflag, + deltachi=self.deltachi, fulloutput=full_output, ) if self.niter_non_leastsquares: - self.parameters = result[0] + parameters = self.parameters = result[0] self.non_leastsquares_increment() finally: self.parameters = initial @@ -376,6 +413,18 @@ def nonlinear_fit(self, full_output=False): ret["lastdeltachi"] = result[4] return ret + @property + def maxiter(self): + return 100 + + @property + def deltachi(self): + return None + + @property + def weightflag(self): + return 0 + def _evaluate(self, parameters, xdata): """Update parameters and evaluate model diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 8e06515f1..341f2e0cf 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -541,6 +541,10 @@ def _preCalculateParameterIndependent(self): for peaks in self._lineGroups ] + @property + def _nLineGroups(self): + return len(self._lineGroups) + def _getEmissionLines(self): """Yields a list of emission lines for each group with total rate of 1 and sorted by energy. @@ -1274,11 +1278,12 @@ def _parameter_group_names(self): "lt_sloperatio", "step_heightratio", "eta_factor", + "areas", ] @property def _linear_parameter_group_names(self): - raise NotImplementedError + return ["areas"] def _iter_parameter_groups(self, linear_only=False): """ @@ -1313,9 +1318,22 @@ def _iter_parameter_groups(self, linear_only=False): yield name, 1 elif name == "eta_factor" and not hypermet: yield name, 1 + elif name == "areas": + yield name, self._nLineGroups else: raise ValueError(name) + def evaluate(self, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + x = self.zero + self.gain * xdata + raise NotImplementedError + class MultiMcaTheory(ConcatModel): def __init__(self, ndetectors=1): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 7c644b275..cfe0795f0 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -213,7 +213,7 @@ def _iter_parameter_groups(self, linear_only=False): raise ValueError(name) def evaluate(self, xdata=None): - """DEvaluate model + """Evaluate model :param array xdata: length nxdata :returns array: nxdata diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 9d93aa822..4a7d563b4 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -583,14 +583,24 @@ def _testLegacyMcaTheory(self, x, y, configuration): from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import NewClassMcaTheory + import time + t0 = time.time() + mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() _, fitResult1, result1 = self._configAndFit( x, y, copy.deepcopy(configuration), mcaFitLegacy, tmpflag=True) + t1 = time.time() + mcaFit = NewClassMcaTheory.McaTheory() _, fitResult2, result2 = self._configAndFit( x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) + t2 = time.time() + + print("LEGACY TIME", t1-t0) + print("NEW TIME", t2-t1) + # Compare data numpy.testing.assert_array_equal(mcaFitLegacy.xdata.flat, mcaFit.xdata) numpy.testing.assert_array_equal(mcaFitLegacy.ydata.flat, mcaFit.ydata) From 638b0ba0cb8c2126d1032b377c5d15eeb9cf90aa Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 18 Feb 2021 14:01:40 +0100 Subject: [PATCH 11/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 341f2e0cf..86d06c37f 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -458,6 +458,10 @@ def __init__(self, **kw): self._lastTime = None + self._lineGroups = [] + self._fluoRates = [] + self._escapeLineGroups = [] + self.strategyInstances = {} self.__toBeConfigured = False @@ -528,6 +532,7 @@ def _preCalculateParameterIndependent(self): # line group # -> emission/scattering line # -> energy, rate, line name + # This is a filtered and normalized form of `_fluoRates` self._lineGroups = list(self._getEmissionLines()) self._lineGroups.extend(self._getScatterLines()) From 47ffe30ab3c34ebd615e57c3439daffe599c63ea Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 16:06:39 +0100 Subject: [PATCH 12/74] fixup --- PyMca5/tests/XrfTest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 4a7d563b4..33fb4b08c 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -588,17 +588,17 @@ def _testLegacyMcaTheory(self, x, y, configuration): mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() _, fitResult1, result1 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFitLegacy, tmpflag=True) + x, y, copy.deepcopy(configuration), mcaFitLegacy) t1 = time.time() mcaFit = NewClassMcaTheory.McaTheory() _, fitResult2, result2 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) + x, y, copy.deepcopy(configuration), mcaFit) t2 = time.time() - print("LEGACY TIME", t1-t0) + print("\nLEGACY TIME", t1-t0) print("NEW TIME", t2-t1) # Compare data @@ -659,13 +659,13 @@ def _testLegacyMcaTheory(self, x, y, configuration): self.assertEqual(fitResult1, fitResult2) self.assertEqual(result1, result2) - def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): + def _configAndFit(self, x, y, configuration, mcaFit): configuration = mcaFit.configure(configuration) mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], xmax=configuration["fit"]["xmax"]) - if tmpflag: - return configuration, None, None + return configuration, None, None + mcaFit.estimate() fitResult1, result1 = mcaFit.startFit(digest=1) return configuration, fitResult1, result1 From f723cca92d3f38ffd7e1c6c56d7cb6f7a770cc33 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 16:06:54 +0100 Subject: [PATCH 13/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 57 ++++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 86d06c37f..c8b28087b 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -433,6 +433,15 @@ class McaTheory(McaTheoryConfigApi, McaTheoryLegacyApi, Model): MAX_ATTENUATION = 1.0e-300 SCATTER_ENERGY_THRESHOLD = 0.2 # keV + CONTINUUM_LIST = [ + None, + "Constant", + "Linear", + "Parabolic", + "Linear Polynomial", + "Exp. Polynomial", + ] + def __init__(self, **kw): super(McaTheory, self).__init__(**kw) # TODO: done for some initialization of SpecfitFuns? @@ -566,7 +575,7 @@ def _getScatterLines(self): :yields list: [[energy, 1.0, "Scatter %03d"]] """ - scatteringAngle = self._scatteringAngle * numpy.pi / 180.0 + scatteringAngle = numpy.radians(self._scatteringAngle) angleFactor = 1.0 - numpy.cos(scatteringAngle) for i, (en_elastic, _) in enumerate(self._scatterLines()): en_inelastic = en_elastic / (1.0 + (en_elastic / 511.0) * angleFactor) @@ -779,8 +788,8 @@ def _applyMatrixAttenuation(self, peaks, symb): energies = [x[0] for x in peaks] + [maxenergy] mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) - sinAlphaIn = numpy.sin(alphaIn * numpy.pi / 180.0) - sinAlphaOut = numpy.sin(alphaOut * numpy.pi / 180.0) + sinAlphaIn = numpy.sin(numpy.radians(alphaIn)) + sinAlphaOut = numpy.sin(numpy.radians(alphaOut)) sinRatio = sinAlphaIn / sinAlphaOut muSource = mu["total"][-1] muFluo = numpy.array(mu["total"][:-1]) @@ -1204,12 +1213,51 @@ def fano(self, value): @property def sum(self): - return self.config["detector"]["sum"] + if self.config["fit"]["sumflag"]: + return self.config["detector"]["sum"] + else: + return 0.0 @sum.setter def sum(self, value): self.config["detector"]["sum"] = value + @property + def zero_constraint(self): + if self.config["detector"]["fixedzero"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.zero + delta = self.config["detector"]["deltazero"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def gain_constraint(self): + if self.config["detector"]["fixedgain"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.gain + delta = self.config["detector"]["deltagain"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def fano_constraint(self): + if self.config["detector"]["fixedfano"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.fano + delta = self.config["detector"]["deltafano"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def sum_constraint(self): + if self.config["detector"]["fixedsum"] or not self.config["fit"]["sumflag"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.sum + delta = self.config["detector"]["deltasum"] + return Gefit.CQUOTED, value + delta, value - delta + @classmethod def _calc_fwhm(cls, noise, fano, energy): return numpy.sqrt( @@ -1337,6 +1385,7 @@ def evaluate(self, xdata=None): if xdata is None: xdata = self.xdata x = self.zero + self.gain * xdata + fwhm = self._calc_fwhm(self.noise, self.fano, energy) raise NotImplementedError From 5ce8295d962921b7a2a3254281ea24a0a321948f Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 17:10:44 +0100 Subject: [PATCH 14/74] fixup Model --- PyMca5/PyMcaMath/fitting/Model.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index e1b912708..ef5581a47 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -219,7 +219,7 @@ def _set_parameters(self, params, linear_only=False): for name, n in self._parameter_groups(linear_only=linear_only): if n > 1: getattr(self, name)[:] = params[i : i + n] - else: + elif n == 1: setattr(self, name, params[i]) i += n @@ -361,10 +361,12 @@ def linear_fit(self, full_output=False): if self.niter_non_leastsquares: initial = self.linear_parameters try: + yshape = self.ydata.shape for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives().T # nchannels, npeaks + A = self.linear_derivatives().T # nchannels, nparams b = self.ydata # nchannels result = lstsq(A, b, digested_output=full_output) + b.shape = yshape if self.niter_non_leastsquares: self.linear_parameters = result[0] self.non_leastsquares_increment() From ad4ee1960744e063fb47da5d40de2b08247799ce Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 17:11:08 +0100 Subject: [PATCH 15/74] Polynomial models for McaTheory --- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 250 +++++++++++++++++++ PyMca5/tests/FitPolModelTest.py | 103 ++++++++ 2 files changed, 353 insertions(+) create mode 100644 PyMca5/PyMcaMath/fitting/PolynomialModels.py create mode 100644 PyMca5/tests/FitPolModelTest.py diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py new file mode 100644 index 000000000..ff729d9c0 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -0,0 +1,250 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + +import numpy +from PyMca5.PyMcaMath.fitting.Model import Model + + +class PolynomialModel(Model): + def __init__(self, degree=0): + self._xdata = None + self._ydata = None + self._mask = None + self._linear = None + self.degree = degree + super(PolynomialModel, self).__init__() + + @property + def degree(self): + return self.coefficients.size - 1 + + @degree.setter + def degree(self, n): + self.coefficients = numpy.zeros(n + 1) + + @property + def coefficients(self): + return self._coefficients + + @coefficients.setter + def coefficients(self, values): + self._coefficients = numpy.atleast_1d(values) + + @property + def xdata(self): + return self._xdata + + @xdata.setter + def xdata(self, values): + self._xdata = values + + @property + def ydata(self): + return self._ydata + + @ydata.setter + def ydata(self, values): + self._ydata = values + + @property + def ystd(self): + return None + + @property + def linear(self): + return self._linear + + @linear.setter + def linear(self, value): + self._linear = value + + +class LinearPolynomialModel(PolynomialModel): + """y = c0 + c1*x + c2*x^2 + ...""" + + @property + def _parameter_group_names(self): + return ["coefficients"] + + @property + def _linear_parameter_group_names(self): + return ["coefficients"] + + def _iter_parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + if linear_only: + names = self.linear_parameter_group_names + else: + names = self.parameter_group_names + for name in names: + if name == "coefficients": + yield name, self.degree + 1 + + def evaluate(self, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + coeff = self.coefficients + y = coeff[0] * numpy.ones_like(xdata) + for i in range(1, len(coeff)): + y += coeff[i] * (xdata ** i) + return y + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + return numpy.array( + [self.derivative(i, xdata=xdata) for i in range(self.degree + 1)] + ) + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + if param_idx == 0: + return numpy.ones_like(xdata) + else: + return xdata ** param_idx + + +class ExponentialPolynomialModel(PolynomialModel): + """y = c0 * exp[c1*x + c2*x^2 + ...]""" + + @property + def xdata(self): + return self._xdata + + @xdata.setter + def xdata(self, values): + self._xdata = values + + @property + def _parameter_group_names(self): + return ["factor", "expcoefficients"] + + @property + def _linear_parameter_group_names(self): + return ["factor"] + + @property + def factor(self): + return self.coefficients[0] + + @factor.setter + def factor(self, value): + self.coefficients[0] = value + + @property + def expcoefficients(self): + return self.coefficients[1:] + + @expcoefficients.setter + def expcoefficients(self, values): + self.coefficients[1:] = values + + def _iter_parameter_groups(self, linear_only=False): + """ + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group + """ + if linear_only: + names = self.linear_parameter_group_names + else: + names = self.parameter_group_names + for name in names: + if name == "factor": + yield name, 1 + elif name == "expcoefficients": + yield name, self.degree + + @staticmethod + def _exppol(coeff, x): + y = numpy.zeros_like(x) + for i in range(1, len(coeff)): + y += coeff[i] * (x ** i) + return coeff[0] * numpy.exp(y) + + def evaluate(self, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + return self._exppol(self.coefficients, xdata) + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + return self.derivative(0, xdata=xdata).reshape((1, xdata.size)) + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + coeff = self.coefficients + if param_idx == 0: + coeff = coeff.copy() + coeff[0] = 1.0 + y = self._exppol(coeff, xdata) + if param_idx: + y *= xdata ** param_idx + return y diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py new file mode 100644 index 000000000..8afd96cf8 --- /dev/null +++ b/PyMca5/tests/FitPolModelTest.py @@ -0,0 +1,103 @@ +# /*########################################################################## +# +# The PyMca X-Ray Fluorescence Toolkit +# +# Copyright (c) 2020 European Synchrotron Radiation Facility +# +# This file is part of the PyMca X-ray Fluorescence Toolkit developed at +# the ESRF by the Software group. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +#############################################################################*/ +__author__ = "Wout De Nolf" +__contact__ = "wout.de_nolf@esrf.eu" +__license__ = "MIT" +__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" + +import unittest +import numpy +from PyMca5.PyMcaMath.fitting import PolynomialModels + + +class testFitPolModel(unittest.TestCase): + def setUp(self): + self.random_state = numpy.random.RandomState(seed=0) + + def testLinearPol(self): + model = PolynomialModels.LinearPolynomialModel() + fitmodel = PolynomialModels.LinearPolynomialModel() + model.xdata = fitmodel.xdata = numpy.linspace(0, 100, 100) + + for degree in [0, 1, 5]: + ncoeff = degree + 1 + expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) + model.coefficients = expected + fitmodel.ydata = model.evaluate() + for linear in [True, False]: + with self.subTest(degree=degree, linear=linear): + fitmodel.linear = linear + fitmodel.coefficients = numpy.zeros_like(expected) + self.assertEqual(fitmodel.degree, degree) + result = fitmodel.fit()["parameters"] + numpy.testing.assert_allclose(result, expected, rtol=1e-4) + + def testExpPol(self): + model = PolynomialModels.ExponentialPolynomialModel() + fitmodel = PolynomialModels.ExponentialPolynomialModel() + model.xdata = fitmodel.xdata = numpy.linspace(-0.5, 0.5, 100) + + for degree in [0, 1, 5]: + ncoeff = degree + 1 + expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) + model.coefficients = expected + fitmodel.ydata = model.evaluate() + for linear in [True, False]: + with self.subTest(degree=degree, linear=linear): + fitmodel.linear = linear + if linear: + fitmodel.coefficients = expected.copy() + else: + fitmodel.coefficients = numpy.zeros_like(expected) + fitmodel.factor = 1 # Do not start from zero + self.assertEqual(fitmodel.degree, degree) + result = fitmodel.fit()["parameters"] + if linear: + numpy.testing.assert_allclose(result, expected[0], rtol=1e-4) + else: + numpy.testing.assert_allclose(result, expected, rtol=1e-4) + + +def getSuite(auto=True): + testSuite = unittest.TestSuite() + if auto: + testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(testFitModel)) + else: + # use a predefined order + testSuite.addTest(testFitPolModel("testLinearPol")) + testSuite.addTest(testFitPolModel("testExpPol")) + return testSuite + + +def test(auto=False): + unittest.TextTestRunner(verbosity=2).run(getSuite(auto=auto)) + + +if __name__ == "__main__": + test() From 77c23278afadf2ffab8c5ef1a379346de7b68902 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 17:40:57 +0100 Subject: [PATCH 16/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 151 +++++++++++++------ 1 file changed, 109 insertions(+), 42 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index c8b28087b..a3b096bb9 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -378,17 +378,21 @@ def _hypermet(self): else: return 0 + @property + def _hypermetGaussian(self): + return self._hypermet & 1 + @property def _hypermetShortTail(self): - return (self.config["fit"]["hypermetflag"] >> 1) & 1 + return (self._hypermet >> 1) & 1 @property def _hypermetLongTail(self): - return (self.config["fit"]["hypermetflag"] >> 2) & 2 + return (self._hypermet >> 2) & 2 @property def _hypermetStep(self): - return (self.config["fit"]["hypermetflag"] >> 3) & 3 + return (self._hypermet >> 3) & 3 def _anchorsIndices(self): cfg = self.config["fit"] @@ -1222,42 +1226,6 @@ def sum(self): def sum(self, value): self.config["detector"]["sum"] = value - @property - def zero_constraint(self): - if self.config["detector"]["fixedzero"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.zero - delta = self.config["detector"]["deltazero"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def gain_constraint(self): - if self.config["detector"]["fixedgain"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.gain - delta = self.config["detector"]["deltagain"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def fano_constraint(self): - if self.config["detector"]["fixedfano"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.fano - delta = self.config["detector"]["deltafano"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def sum_constraint(self): - if self.config["detector"]["fixedsum"] or not self.config["fit"]["sumflag"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.sum - delta = self.config["detector"]["deltasum"] - return Gefit.CQUOTED, value + delta, value - delta - @classmethod def _calc_fwhm(cls, noise, fano, energy): return numpy.sqrt( @@ -1279,7 +1247,10 @@ def eta_factor(self, value): @property def step_heightratio(self): - return self.config["peakshape"]["step_heightratio"] + if self._hypermetStep: + return self.config["peakshape"]["step_heightratio"] + else: + return 0 @step_heightratio.setter def step_heightratio(self, value): @@ -1295,7 +1266,10 @@ def lt_sloperatio(self, value): @property def lt_arearatio(self): - return self.config["peakshape"]["lt_arearatio"] + if self._hypermetLongTail: + return self.config["peakshape"]["lt_arearatio"] + else: + return 0 @lt_arearatio.setter def lt_arearatio(self, value): @@ -1311,12 +1285,105 @@ def st_sloperatio(self, value): @property def st_arearatio(self): - return self.config["peakshape"]["st_arearatio"] + if self._hypermetShortTail: + return self.config["peakshape"]["st_arearatio"] + else: + return 0 @st_arearatio.setter def st_arearatio(self, value): self.config["peakshape"]["st_arearatio"] = value + @property + def zero_constraint(self): + if self.config["detector"]["fixedzero"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.zero + delta = self.config["detector"]["deltazero"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def gain_constraint(self): + if self.config["detector"]["fixedgain"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.gain + delta = self.config["detector"]["deltagain"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def fano_constraint(self): + if self.config["detector"]["fixedfano"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.fano + delta = self.config["detector"]["deltafano"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def sum_constraint(self): + if self.config["detector"]["fixedsum"] or not self.config["fit"]["sumflag"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.sum + delta = self.config["detector"]["deltasum"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def eta_factor_constraint(self): + if self.config["detector"]["fixedeta_factor"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.eta_factor + delta = self.config["detector"]["deltaeta_factor"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def step_heightratio_constraint(self): + if self.config["detector"]["fixedstep_heightratio"] or not self._hypermetStep: + return Gefit.CFIXED, 0, 0 + else: + value = self.step_heightratio + delta = self.config["detector"]["deltastep_heightratio"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def lt_sloperatio_constraint(self): + if self.config["detector"]["fixedlt_sloperatio"] or not self._hypermetLongTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.lt_sloperatio + delta = self.config["detector"]["deltalt_sloperatio"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def lt_arearatio_constraint(self): + if self.config["detector"]["fixedlt_arearatio"] or not self._hypermetLongTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.lt_arearatio + delta = self.config["detector"]["deltalt_arearatio"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def st_sloperatio_constraint(self): + if self.config["detector"]["fixedst_sloperatio"] or not self._hypermetShortTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.st_sloperatio + delta = self.config["detector"]["deltast_sloperatio"] + return Gefit.CQUOTED, value + delta, value - delta + + @property + def st_arearatio_constraint(self): + if self.config["detector"]["fixedst_arearatio"] or not self._hypermetShortTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.st_arearatio + delta = self.config["detector"]["deltast_arearatio"] + return Gefit.CQUOTED, value + delta, value - delta + @property def _parameter_group_names(self): return [ From f3b8eeaca3adc1c95ea03f0dd58f907df1b7d182 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 18:18:22 +0100 Subject: [PATCH 17/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 71 ++++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index a3b096bb9..bfd79ebe6 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -42,6 +42,8 @@ from PyMca5.PyMcaIO import ConfigDict from PyMca5.PyMcaMath.fitting import SpecfitFuns from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel +from PyMca5.PyMcaMath.fitting.PolynomialModels import LinearPolynomialModel +from PyMca5.PyMcaMath.fitting.PolynomialModels import ExponentialPolynomialModel from . import Elements from . import ConcentrationsTool @@ -465,10 +467,15 @@ def __init__(self, **kw): self._std = None self._lastDataCacheParams = None - # XRF spectrum background + # XRF spectrum numerical background self._numBkg = None self._lastNumBkgCacheParams = None + # XRF spectrum analytical background + self._continuum = None + self._continuumModel = None + self._lastContinuumCacheParams = None + self._lastTime = None self._lineGroups = [] @@ -915,6 +922,12 @@ def ynumbkg(self): self._refreshNumBkgCache() return self._numBkg + @property + def ycontinuum(self): + """Get the analytical background (as opposed to the numerical background)""" + self._refreshContinuumCache() + return self._continuum + @property def xdata0(self): return self._xdata0 @@ -1080,14 +1093,12 @@ def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): self._ystd0 = ystd0 self._lastDataCacheParams = params - # Invalidate background cache - self._lastNumBkgCacheParams = None - @property def _numBkgCacheParams(self): """Any change in these parameter will invalidate the cache""" cfg = self.config["fit"] params = [ + id(self._lastDataCacheParams), "stripflag", "stripalgorithm", "stripfilterwidth", @@ -1118,6 +1129,52 @@ def _refreshNumBkgCache(self): self._numBkg = numpy.zeros_like(self.ydata) self._lastNumBkgCacheParams = bkgparams + @property + def _continuumCacheParams(self): + cfg = self.config["fit"] + params = [id(self._lastDataCacheParams), cfg["continuum"]] + if cfg["continuum"] == "Linear Polynomial": + params.append(cfg["linpolorder"]) + elif cfg["continuum"] == "Exp. Polynomial": + params.append(cfg["exppolorder"]) + return params + + def _refreshContinuumCache(self): + contparams = self._continuumCacheParams + if self._lastContinuumCacheParams == contparams: + return # the cached data is still valid + continuum = self.config["fit"]["continuum"] + if continuum is None: + model = None + elif continuum == "Constant": + model = LinearPolynomialModel(degree=0) + elif continuum == "Linear": + model = LinearPolynomialModel(degree=1) + elif continuum == "Parabolic": + model = LinearPolynomialModel(degree=2) + elif continuum == "Linear Polynomial": + model = LinearPolynomialModel(degree=self.config["fit"]["linpolorder"]) + elif continuum == "Exp. Polynomial": + model = ExponentialPolynomialModel(degree=self.config["fit"]["exppolorder"]) + else: + raise ValueError("Unknown continuum {}".format(continuum)) + self._continuumModel = model + if model is None: + if self.ydata is None: + self._continuum = None + else: + self._continuum = numpy.zeros_like(self.ydata) + else: + model.xdata = self.xdata + model.ydata = self.ynumbkg + self._continuum = model.evaluate() + self._lastContinuumCacheParams = contparams + + @property + def continuumModel(self): + self._refreshContinuumCache() + return self._continuumModel + def _snip(self, signal, anchorslist): """Apply SNIP filtering to a signal""" _logger.debug("CALCULATING SNIP") @@ -1399,11 +1456,13 @@ def _parameter_group_names(self): "step_heightratio", "eta_factor", "areas", + "continuum_linear", + "continuum_nonlinear", ] @property def _linear_parameter_group_names(self): - return ["areas"] + return ["areas", "continuum_linear"] def _iter_parameter_groups(self, linear_only=False): """ @@ -1440,6 +1499,8 @@ def _iter_parameter_groups(self, linear_only=False): yield name, 1 elif name == "areas": yield name, self._nLineGroups + elif name == "continuum_linear": + continuum else: raise ValueError(name) From 77ed00da1cc81676ba164e521e5a01420c721a41 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 9 Mar 2021 18:29:11 +0100 Subject: [PATCH 18/74] fixup --- PyMca5/tests/FitPolModelTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 8afd96cf8..9d40f9681 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -87,7 +87,7 @@ def testExpPol(self): def getSuite(auto=True): testSuite = unittest.TestSuite() if auto: - testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(testFitModel)) + testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(testFitPolModel)) else: # use a predefined order testSuite.addTest(testFitPolModel("testLinearPol")) From 16144b6ad89c56112ba7bd77ca0d007930fd042b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 10 Mar 2021 15:37:47 +0100 Subject: [PATCH 19/74] fixup polymodels --- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 27 +++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index ff729d9c0..adba1dede 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -36,12 +36,13 @@ class PolynomialModel(Model): - def __init__(self, degree=0): + def __init__(self, degree=0, maxiter=100): self._xdata = None self._ydata = None self._mask = None - self._linear = None + self._linear = True self.degree = degree + self.maxiter = maxiter super(PolynomialModel, self).__init__() @property @@ -88,10 +89,22 @@ def linear(self): def linear(self, value): self._linear = value + @property + def maxiter(self): + return self._maxiter + + @maxiter.setter + def maxiter(self, value): + self._maxiter = value + class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" + def __init__(self, **kw): + super(LinearPolynomialModel, self).__init__(**kw) + self._linear = True + @property def _parameter_group_names(self): return ["coefficients"] @@ -157,13 +170,9 @@ def derivative(self, param_idx, xdata=None): class ExponentialPolynomialModel(PolynomialModel): """y = c0 * exp[c1*x + c2*x^2 + ...]""" - @property - def xdata(self): - return self._xdata - - @xdata.setter - def xdata(self, values): - self._xdata = values + def __init__(self, **kw): + super(ExponentialPolynomialModel, self).__init__(**kw) + self._linear = False @property def _parameter_group_names(self): From ea93c0a02237207bc755feb4d51631b0d1fef643 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Mar 2021 11:27:18 +0100 Subject: [PATCH 20/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 270 +++++++++++++++---- PyMca5/tests/XrfTest.py | 9 +- 2 files changed, 228 insertions(+), 51 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index bfd79ebe6..035dd8a08 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -479,8 +479,11 @@ def __init__(self, **kw): self._lastTime = None self._lineGroups = [] + self.linegroup_areas = [] self._fluoRates = [] self._escapeLineGroups = [] + self._peakfunc = None + self._fastpeakfunc = None self.strategyInstances = {} @@ -551,24 +554,116 @@ def _preCalculateParameterIndependent(self): # Line groups: nested lists # line group # -> emission/scattering line - # -> energy, rate, line name + # [energy, rate, line name] # This is a filtered and normalized form of `_fluoRates` self._lineGroups = list(self._getEmissionLines()) self._lineGroups.extend(self._getScatterLines()) + self.linegroup_areas = numpy.zeros(len(self._lineGroups)) # Escape line groups: nested lists # line group # -> emission/scattering line # -> escape line - # -> energy, rate, escape name + # [energy, rate, escape name] self._escapeLineGroups = [ self._calcEscapePeaks([peak[0] for peak in peaks]) for peaks in self._lineGroups ] - @property - def _nLineGroups(self): - return len(self._lineGroups) + # Prepare peak shape calculations + self._prepare_peakfunc() + + def _prepare_peakfunc(self): + npeaks = sum(len(group) for group in self._lineGroups) + npeaks += sum( + len(escgroup) for group in self._escapeLineGroups for escgroup in group + ) + if self._hypermet: + # area, position, fwhm, ST AreaR, ST SlopeR, LT AreaR, LT SlopeR, STEP HeightR + htype = self._hypermet + + def peakfunc(params, x): + return SpecfitFuns.fastahypermet(params, x, htype) + + self._peakfunc = peakfunc + npeakparams = 8 + else: + # area, position, fwhm, eta + self._peakfunc = SpecfitFuns.apvoigt + npeakparams = 4 + self._peakfunc_params = numpy.zeros((npeaks, npeakparams)) + + def _fill_peakfunc_params(self): + """All parameters are in the energy domain (X-axis is energy, not channels)""" + parameters = self._peakfunc_params + + # Peak positions and areas + i = 0 + for group, escgroup, grouparea in zip( + self._lineGroups, self._escapeLineGroups, self.linegroup_areas + ): + if not escgroup: + escgroup = [[]] * len(group) + for (energy, rate, _), esclines in zip(group, escgroup): + peakarea = rate * grouparea + parameters[i, 0] = peakarea + parameters[i, 1] = energy + i += 1 + for escen, escrate, _ in esclines: + parameters[i, 0] = peakarea * escrate + parameters[i, 1] = escen + i += 1 + + # Area parameters from channel to energy domain + self._peakfunc_params[:, 0] *= self.gain + + # FWHM + parameters[:, 2] = self._peak_fwhm(parameters[:, 1]) + + # Other peak shape parameters + if self._hypermet: + shapeparams = [ + self.st_arearatio, + self.st_sloperatio, + self.lt_arearatio, + self.lt_sloperatio, + self.step_heightratio, + ] + else: + shapeparams = [self.eta_factor] + for i, param in enumerate(shapeparams, 3): + parameters[:, i] = param + + def _evaluate_peakprofiles(self, parameters, x, hypermet=None, fast=True): + """When providing parameters for more than one peak, the peak + profiles are added. + + :param array parameters: flat 1D array of parameters + :param array x: domain on which to evaluate the peak profiles + :param int or None hypermet: + :param bool fast: ??? + :returns array: same shape as x + """ + if hypermet is None: + hypermet = self._hypermet + if hypermet: + if fast: + return SpecfitFuns.fastahypermet(parameters, x, hypermet) + else: + return SpecfitFuns.ahypermet(parameters, x, hypermet) + else: + return SpecfitFuns.apvoigt(parameters, x) + + def _peak_fwhm(self, energy): + """Calculate the FWHM of a peak in the energy domain""" + return numpy.sqrt( + self.noise * self.noise + + self.BAND_GAP + * energy + * self.fano + * self.GAUSS_SIGMA_TO_FWHM + * self.GAUSS_SIGMA_TO_FWHM + ) def _getEmissionLines(self): """Yields a list of emission lines for each group with total @@ -900,6 +995,18 @@ def xdata(self): self._refreshDataCache() return self._xdata + @property + def xenergy(self): + return self_channels_to_energy(self.xdata) + + def _channels_to_energy(self, xchannels): + return self.zero + self.gain * xchannels + + @property + def xenergy_cen(self): + xchannels = self.xdata + return self_channels_to_energy(xchannels - numpy.mean(xchannels)) + @property def nchannels(self): return len(self.xdata) @@ -925,8 +1032,14 @@ def ynumbkg(self): @property def ycontinuum(self): """Get the analytical background (as opposed to the numerical background)""" - self._refreshContinuumCache() - return self._continuum + model = self.continuumModel + if model is None: + if self.ydata is None: + return None + else: + return numpy.zeros_like(self.ydata) + else: + return model.evaluate() @property def xdata0(self): @@ -1131,43 +1244,56 @@ def _refreshNumBkgCache(self): @property def _continuumCacheParams(self): - cfg = self.config["fit"] - params = [id(self._lastDataCacheParams), cfg["continuum"]] - if cfg["continuum"] == "Linear Polynomial": - params.append(cfg["linpolorder"]) - elif cfg["continuum"] == "Exp. Polynomial": - params.append(cfg["exppolorder"]) + cfgfit = self.config["fit"] + params = [id(self._lastDataCacheParams), cfgfit["continuum"]] + if cfgfit["continuum"] == "Linear Polynomial": + params.append(cfgfit["linpolorder"]) + elif cfgfit["continuum"] == "Exp. Polynomial": + params.append(cfgfit["exppolorder"]) return params def _refreshContinuumCache(self): contparams = self._continuumCacheParams if self._lastContinuumCacheParams == contparams: return # the cached data is still valid + + # Instantiate the model continuum = self.config["fit"]["continuum"] - if continuum is None: + if continuum is None or self.ynumbkg is None: model = None elif continuum == "Constant": - model = LinearPolynomialModel(degree=0) + model = LinearPolynomialModel(degree=0, maxiter=10) elif continuum == "Linear": - model = LinearPolynomialModel(degree=1) + model = LinearPolynomialModel(degree=1, maxiter=10) elif continuum == "Parabolic": - model = LinearPolynomialModel(degree=2) + model = LinearPolynomialModel(degree=2, maxiter=10) elif continuum == "Linear Polynomial": - model = LinearPolynomialModel(degree=self.config["fit"]["linpolorder"]) + model = LinearPolynomialModel( + degree=self.config["fit"]["linpolorder"], maxiter=10 + ) elif continuum == "Exp. Polynomial": model = ExponentialPolynomialModel(degree=self.config["fit"]["exppolorder"]) + estmodel = LinearPolynomialModel( + degree=self.config["fit"]["linpolorder"], maxiter=40 + ) else: raise ValueError("Unknown continuum {}".format(continuum)) self._continuumModel = model - if model is None: - if self.ydata is None: - self._continuum = None - else: - self._continuum = numpy.zeros_like(self.ydata) - else: - model.xdata = self.xdata + + # Estimate the polynomial coefficients by fitting the numerical background + if model is not None: + x = self.xdata + model.xdata = self.xenergy_cen model.ydata = self.ynumbkg - self._continuum = model.evaluate() + if continuum == "Exp. Polynomial": + estmodel.xdata = model.xdata + estmodel.ydata = numpy.log(model.ydata) + params = estmodel.fit()["parameters"] + params[0] = numpy.exp(params[0]) + model.parameters = params + else: + model.parameters = model.fit()["parameters"] + self._lastContinuumCacheParams = contparams @property @@ -1175,6 +1301,44 @@ def continuumModel(self): self._refreshContinuumCache() return self._continuumModel + @property + def continuum_coefficients(self): + model = self.continuumModel + if model is None: + return list() + else: + return model.parameters + + @continuum_coefficients.setter + def continuum_coefficients(self, values): + model = self.continuumModel + if model is not None: + model.parameters = values + + @property + def linpol_coefficients(self): + if isinstance(self.continuumModel, LinearPolynomialModel): + return self.continuum_coefficients + else: + return list() + + @linpol_coefficients.setter + def linpol_coefficients(self, values): + if isinstance(self.continuumModel, LinearPolynomialModel): + self.continuum_coefficients = values + + @property + def exppol_coefficients(self): + if isinstance(self.continuumModel, ExponentialPolynomialModel): + return self.continuum_coefficients + else: + return list() + + @exppol_coefficients.setter + def exppol_coefficients(self, values): + if isinstance(self.continuumModel, ExponentialPolynomialModel): + self.continuum_coefficients = values + def _snip(self, signal, anchorslist): """Apply SNIP filtering to a signal""" _logger.debug("CALCULATING SNIP") @@ -1283,17 +1447,6 @@ def sum(self): def sum(self, value): self.config["detector"]["sum"] = value - @classmethod - def _calc_fwhm(cls, noise, fano, energy): - return numpy.sqrt( - noise * noise - + cls.BAND_GAP - * energy - * fano - * cls.GAUSS_SIGMA_TO_FWHM - * cls.GAUSS_SIGMA_TO_FWHM - ) - @property def eta_factor(self): return self.config["peakshape"]["eta_factor"] @@ -1455,14 +1608,14 @@ def _parameter_group_names(self): "lt_sloperatio", "step_heightratio", "eta_factor", - "areas", - "continuum_linear", - "continuum_nonlinear", + "linegroup_areas", + "linpol_coefficients", + "exppol_coefficients", ] @property def _linear_parameter_group_names(self): - return ["areas", "continuum_linear"] + return ["linegroup_areas", "linpol_coefficients"] def _iter_parameter_groups(self, linear_only=False): """ @@ -1498,22 +1651,45 @@ def _iter_parameter_groups(self, linear_only=False): elif name == "eta_factor" and not hypermet: yield name, 1 elif name == "areas": - yield name, self._nLineGroups - elif name == "continuum_linear": - continuum + n = len(self.area) + if n: + yield name, n + elif name == "linpol": + n = len(self.linpol_coefficients) + if n: + yield name, n + elif name == "exppol": + n = len(self.exppol_coefficients) + if n: + yield name, n else: raise ValueError(name) def evaluate(self, xdata=None): - """Evaluate model + """Evaluate to MCA model + + y(xdata) = ybkg + ycont(C(xdata)) + A1*G1(E(xdata)) + A2*G2(E(xdata)) + ... + + xdata: MCA channels (positive integers) + + ybkg: numerical background derived from y + + ycont(x) = 0 # no analytical background + = c0 + c1*x + c2*x^2 + ... # linear polynomial + = c0 * exp[c1*x + c2*x^2 + ...] # exponential polynomial + + E(x) = zero + gain*x # energy + C(x) = E(x) - # centered energy + + Gi(x): several peaks with normalized total area :param array xdata: length nxdata :returns array: nxdata """ if xdata is None: xdata = self.xdata - x = self.zero + self.gain * xdata - fwhm = self._calc_fwhm(self.noise, self.fano, energy) + energy = self._channels_to_energy(xdata) + raise NotImplementedError diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 33fb4b08c..3af9a090e 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -588,13 +588,13 @@ def _testLegacyMcaTheory(self, x, y, configuration): mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() _, fitResult1, result1 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFitLegacy) + x, y, copy.deepcopy(configuration), mcaFitLegacy, tmpflag=True) t1 = time.time() mcaFit = NewClassMcaTheory.McaTheory() _, fitResult2, result2 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFit) + x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) t2 = time.time() @@ -659,12 +659,13 @@ def _testLegacyMcaTheory(self, x, y, configuration): self.assertEqual(fitResult1, fitResult2) self.assertEqual(result1, result2) - def _configAndFit(self, x, y, configuration, mcaFit): + def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): configuration = mcaFit.configure(configuration) mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], xmax=configuration["fit"]["xmax"]) - return configuration, None, None + if tmpflag: + return configuration, None, None mcaFit.estimate() fitResult1, result1 = mcaFit.startFit(digest=1) From e4d4220d85e87de4b255d6a68b63d171156007b3 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 11 Mar 2021 17:33:39 +0100 Subject: [PATCH 21/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 251 ++++++++++++++----- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 119 ++------- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 136 ++++++---- PyMca5/tests/FitModelTest.py | 33 ++- PyMca5/tests/FitPolModelTest.py | 29 ++- PyMca5/tests/SimpleModel.py | 16 +- 6 files changed, 352 insertions(+), 232 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index ef5581a47..b537378d4 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -108,20 +108,42 @@ def ystd(self): raise AttributeError from NotImplementedError @property - def ymodel(self): - return self.evaluate() + def yfitdata(self): + return self._ydata_to_fit(self.ydata) + + @property + def yfitstd(self): + return self._ystd_to_fit(self.ystd) + + @property + def yfullmodel(self): + """Model of ydata""" + return self.evaluate_fullmodel() + + @property + def yfitmodel(self): + """Model of yfitdata""" + return self.evaluate_fitmodel() @property def nchannels(self): raise AttributeError from NotImplementedError + @property + def fit_parameters(self): + return self._get_parameters(fitting=True) + + @fit_parameters.setter + def fit_parameters(self, values): + return self._set_parameters(values, fitting=True) + @property def parameters(self): - return self._get_parameters() + return self._get_parameters(fitting=False) @parameters.setter def parameters(self, values): - return self._set_parameters(values) + return self._set_parameters(values, fitting=False) @property def constraints(self): @@ -143,13 +165,21 @@ def parameter_group_names(self): def _parameter_group_names(self): raise AttributeError from NotImplementedError + @property + def linear_fit_parameters(self): + return self._get_parameters(linear_only=True, fitting=True) + + @linear_fit_parameters.setter + def linear_fit_parameters(self, params): + return self._set_parameters(params, linear_only=True, fitting=True) + @property def linear_parameters(self): - return self._get_parameters(linear_only=True) + return self._get_parameters(linear_only=True, fitting=False) @linear_parameters.setter def linear_parameters(self, params): - return self._set_parameters(params, linear_only=True) + return self._set_parameters(params, linear_only=True, fitting=False) @property def linear_constraints(self): @@ -171,9 +201,10 @@ def linear_parameter_group_names(self): def _linear_parameter_group_names(self): raise AttributeError from NotImplementedError - def _get_parameters(self, linear_only=False): + def _get_parameters(self, linear_only=False, fitting=True): """ :param bool linear_only: + :param bool fitting: :returns array: """ i = 0 @@ -185,7 +216,10 @@ def _get_parameters(self, linear_only=False): for name, n in self._parameter_groups(linear_only=linear_only): params[i : i + n] = getattr(self, name) i += n - return params + if fitting: + return self._parameters_to_fit(params) + else: + return params def _get_constraints(self, linear_only=False): """ @@ -210,11 +244,13 @@ def _get_constraints(self, linear_only=False): else: return None - def _set_parameters(self, params, linear_only=False): + def _set_parameters(self, params, linear_only=False, fitting=False): """ - :returns values: :param bool linear_only: + :param bool fitting: """ + if fitting: + params = self._fit_to_parameters(params) i = 0 for name, n in self._parameter_groups(linear_only=linear_only): if n > 1: @@ -232,34 +268,50 @@ def _filter_parameter_names(self, names): excluded = [] return [name for name in names if name in included and name not in excluded] - def evaluate(self, xdata=None): - """Evaluate model + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + return self._fit_to_ydata(self.evaluate_fitmodel(xdata=xdata)) + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + return self._fit_to_ydata(self.evaluate_linear_fitmodel(xdata=xdata)) + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. :param array xdata: length nxdata :returns array: nxdata """ raise NotImplementedError - def evaluate_linear(self, xdata=None): - """Derivate to a specific parameter + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. :param array xdata: length nxdata :returns array: n x nxdata """ - derivatives = self.linear_derivatives(xdata=xdata) + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) return self.linear_parameters.dot(derivatives) - def linear_decomposition(self, xdata=None): - """Linear decomposition + def linear_decomposition_fitmodel(self, xdata=None): + """Linear decomposition of the fit model. :param array xdata: length nxdata :returns array: nparams x nxdata """ - derivatives = self.linear_derivatives(xdata=xdata) + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) return self.linear_parameters[:, numpy.newaxis] * derivatives - def derivative(self, param_idx, xdata=None): - """Derivate to a specific parameter + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: length nxdata @@ -267,17 +319,19 @@ def derivative(self, param_idx, xdata=None): """ raise NotImplementedError - def derivatives(self, xdata=None): - """Derivates to all parameters + def derivatives_fitmodel(self, xdata=None): + """Derivates to all parameters of the fit model. :param array xdata: length nxdata :returns list(array): nparams x nxdata """ if xdata is None: xdata = self.xdata - return [self.derivative(i, xdata=xdata) for i in range(self.nparameters)] + return [ + self.derivative_fitmodel(i, xdata=xdata) for i in range(self.nparameters) + ] - def linear_derivatives(self, xdata=None): + def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: length nxdata @@ -359,22 +413,23 @@ def linear_fit(self, full_output=False): :returns dict: """ if self.niter_non_leastsquares: - initial = self.linear_parameters + keep = self.linear_parameters try: - yshape = self.ydata.shape + b = self.yfitdata # nchannels for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives().T # nchannels, nparams - b = self.ydata # nchannels - result = lstsq(A, b, digested_output=full_output) - b.shape = yshape + A = self.linear_derivatives_fitmodel().T # nchannels, nparams + result = lstsq(A, b.copy(), digested_output=full_output) if self.niter_non_leastsquares: - self.linear_parameters = result[0] + self.linear_fit_parameters = result[0] self.non_leastsquares_increment() finally: if self.niter_non_leastsquares: - self.linear_parameters = initial - result = {"linear": True, "parameters": result[0], "uncertainties": result[1]} - return result + self.linear_parameters = keep + return { + "linear": True, + "parameters": self._fit_to_parameters(result[0]), + "uncertainties": self._fit_to_uncertainties(result[1]), + } @enable_caching def nonlinear_fit(self, full_output=False): @@ -382,17 +437,20 @@ def nonlinear_fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - initial = parameters = self.parameters + keep = self.parameters constraints = self.constraints + xdata = self.xdata + ydata = self.yfitdata + ystd = self.yfitstd try: for i in range(max(self.niter_non_leastsquares, 1)): result = Gefit.LeastSquaresFit( - self._evaluate, - parameters, - model_deriv=self._derivative, - xdata=self.xdata, - ydata=self.ydata, - sigmadata=self.ystd, + self._evaluate_fitmodel, + self.fit_parameters, + model_deriv=self._derivative_fitmodel, + xdata=xdata, + ydata=ydata, + sigmadata=ystd, constrains=constraints, maxiter=self.maxiter, weightflag=self.weightflag, @@ -400,14 +458,14 @@ def nonlinear_fit(self, full_output=False): fulloutput=full_output, ) if self.niter_non_leastsquares: - parameters = self.parameters = result[0] + self.fit_parameters = parameters self.non_leastsquares_increment() finally: - self.parameters = initial + self.parameters = keep ret = { "linear": False, - "parameters": result[0], - "uncertainties": result[2], + "parameters": self._fit_to_parameters(result[0]), + "uncertainties": self._fit_to_uncertainties(result[2]), "chi2_red": result[1], } if full_output: @@ -415,6 +473,30 @@ def nonlinear_fit(self, full_output=False): ret["lastdeltachi"] = result[4] return ret + def _ydata_to_fit(self, ydata): + return ydata + + def _ystd_to_fit(self, ystd): + return ystd + + def _parameters_to_fit(self, params): + return params + + def _fit_to_ydata(self, yfit): + return yfit + + def _fit_to_parameters(self, params): + return params + + def _fit_to_uncertainties(self, uncertainties): + return uncertainties + + def _linear_parameters_to_fit(self, params): + return params + + def _fit_to_linear_parameters(self, params): + return params + @property def maxiter(self): return 100 @@ -427,17 +509,17 @@ def deltachi(self): def weightflag(self): return 0 - def _evaluate(self, parameters, xdata): + def _evaluate_fitmodel(self, parameters, xdata): """Update parameters and evaluate model :param array parameters: length nparams :param array xdata: length nxdata :returns array: nxdata """ - self.parameters = parameters - return self.evaluate(xdata=xdata) + self.fit_parameters = parameters + return self.evaluate_fitmodel(xdata=xdata) - def _derivative(self, parameters, param_idx, xdata): + def _derivative_fitmodel(self, parameters, param_idx, xdata): """Update parameters and return derivate to a specific parameter :param array parameters: length nparams @@ -445,8 +527,8 @@ def _derivative(self, parameters, param_idx, xdata): :param array xdata: length nxdata :returns array: nxdata """ - self.parameters = parameters - return self.derivative(param_idx, xdata=xdata) + self.fit_parameters = parameters + return self.derivative_fitmodel(param_idx, xdata=xdata) def use_fit_result(self, result): """ @@ -599,6 +681,14 @@ def ystd(self): def ystd(self, values): self._set_data("ystd", values) + @property + def yfitdata(self): + return self._get_data("yfitdata") + + @property + def yfitstd(self): + return self._get_data("yfitstd") + def _get_data(self, attr): """ :param str attr: @@ -664,21 +754,21 @@ def nshared_linear_parameters(self): with self._filter_parameter_context(shared=True): return self.model.nlinear_parameters - def _get_parameters(self, linear_only=False): + def _get_parameters(self, linear_only=False, fitting=False): """ :param bool linear_only: :returns array: """ return numpy.concatenate( [ - m._get_parameters(linear_only=linear_only) + m._get_parameters(linear_only=linear_only, fitting=fitting) for m in self._iter_parameter_models() ] ) - def _set_parameters(self, values, linear_only=False): + def _set_parameters(self, values, linear_only=False, fitting=False): """ - :returns values: + :paramm array values: :param bool linear_only: """ i = 0 @@ -688,7 +778,9 @@ def _set_parameters(self, values, linear_only=False): else: n = m.nparameters if n: - m._set_parameters(values[i : i + n], linear_only=linear_only) + m._set_parameters( + values[i : i + n], linear_only=linear_only, fitting=fitting + ) i += n def _iter_parameter_models(self): @@ -789,7 +881,7 @@ def shared_linear_parameters(self, values): with self._filter_parameter_context(shared=True): self.model.linear_parameters = values - def evaluate(self, xdata=None): + def _concatenate_evaluation(self, funcname, xdata=None): """Evaluate model :param array xdata: length nxdata @@ -799,11 +891,44 @@ def evaluate(self, xdata=None): xdata = self.xdata ret = xdata * 0.0 for idx, model in self._iter_models(xdata): - ret[idx] = model.evaluate(xdata=xdata[idx]) + func = getattr(model, funcname) + ret[idx] = func(xdata=xdata[idx]) return ret - def derivative(self, param_idx, xdata=None): - """Derivate to a specific parameter + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) + + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: length nxdata @@ -816,10 +941,10 @@ def derivative(self, param_idx, xdata=None): for model_idx, param_idx in self._parameter_model_index(param_idx): idx = idx_channels[model_idx] model = self._models[model_idx] - ret[idx] = model.derivative(param_idx, xdata=xdata[idx]) + ret[idx] = model.derivative_fitmodel(param_idx, xdata=xdata[idx]) return ret - def linear_derivatives(self, xdata=None): + def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: length nxdata @@ -829,7 +954,7 @@ def linear_derivatives(self, xdata=None): xdata = self.xdata ret = numpy.empty((self.nlinear_parameters, xdata.size)) for idx, model in self._iter_models(xdata): - ret[:, idx] = model.linear_derivatives(xdata=xdata[idx]) + ret[:, idx] = model.linear_derivatives_fitmodel(xdata=xdata[idx]) return ret def _iter_models(self, xdata): diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index adba1dede..5e06826c0 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -32,6 +32,7 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import numpy +from PyMca5.PyMcaMath.fitting import Gefit from PyMca5.PyMcaMath.fitting.Model import Model @@ -101,10 +102,6 @@ def maxiter(self, value): class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" - def __init__(self, **kw): - super(LinearPolynomialModel, self).__init__(**kw) - self._linear = True - @property def _parameter_group_names(self): return ["coefficients"] @@ -126,21 +123,21 @@ def _iter_parameter_groups(self, linear_only=False): if name == "coefficients": yield name, self.degree + 1 - def evaluate(self, xdata=None): - """Evaluate model + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model, not the full model. :param array xdata: length nxdata :returns array: nxdata """ if xdata is None: xdata = self.xdata - coeff = self.coefficients + coeff = self.fit_parameters y = coeff[0] * numpy.ones_like(xdata) for i in range(1, len(coeff)): y += coeff[i] * (xdata ** i) return y - def linear_derivatives(self, xdata=None): + def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: length nxdata @@ -149,10 +146,10 @@ def linear_derivatives(self, xdata=None): if xdata is None: xdata = self.xdata return numpy.array( - [self.derivative(i, xdata=xdata) for i in range(self.degree + 1)] + [self.derivative_fitmodel(i, xdata=xdata) for i in range(self.degree + 1)] ) - def derivative(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter :param int param_idx: @@ -167,93 +164,23 @@ def derivative(self, param_idx, xdata=None): return xdata ** param_idx -class ExponentialPolynomialModel(PolynomialModel): - """y = c0 * exp[c1*x + c2*x^2 + ...]""" +class ExponentialPolynomialModel(LinearPolynomialModel): + """y = c0 * exp[c1*x + c2*x^2 + ...] + yfit = log(y) = log(c1) + c1*x + c2*x^2 + ... + """ - def __init__(self, **kw): - super(ExponentialPolynomialModel, self).__init__(**kw) - self._linear = False - - @property - def _parameter_group_names(self): - return ["factor", "expcoefficients"] - - @property - def _linear_parameter_group_names(self): - return ["factor"] - - @property - def factor(self): - return self.coefficients[0] - - @factor.setter - def factor(self, value): - self.coefficients[0] = value - - @property - def expcoefficients(self): - return self.coefficients[1:] + def _ydata_to_fit(self, ydata): + return numpy.log(ydata) - @expcoefficients.setter - def expcoefficients(self, values): - self.coefficients[1:] = values - - def _iter_parameter_groups(self, linear_only=False): - """ - :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group - """ - if linear_only: - names = self.linear_parameter_group_names - else: - names = self.parameter_group_names - for name in names: - if name == "factor": - yield name, 1 - elif name == "expcoefficients": - yield name, self.degree - - @staticmethod - def _exppol(coeff, x): - y = numpy.zeros_like(x) - for i in range(1, len(coeff)): - y += coeff[i] * (x ** i) - return coeff[0] * numpy.exp(y) + def _fit_to_ydata(self, yfit): + return numpy.exp(yfit) - def evaluate(self, xdata=None): - """Evaluate model + def _parameters_to_fit(self, parameters): + parameters = parameters.copy() + parameters[0] = numpy.log(parameters[0]) + return parameters - :param array xdata: length nxdata - :returns array: nxdata - """ - if xdata is None: - xdata = self.xdata - return self._exppol(self.coefficients, xdata) - - def linear_derivatives(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - return self.derivative(0, xdata=xdata).reshape((1, xdata.size)) - - def derivative(self, param_idx, xdata=None): - """Derivate to a specific parameter - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - if xdata is None: - xdata = self.xdata - coeff = self.coefficients - if param_idx == 0: - coeff = coeff.copy() - coeff[0] = 1.0 - y = self._exppol(coeff, xdata) - if param_idx: - y *= xdata ** param_idx - return y + def _fit_to_parameters(self, parameters): + parameters = parameters.copy() + parameters[0] = numpy.exp(parameters[0]) + return parameters diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 035dd8a08..40453e02b 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -41,6 +41,7 @@ from PyMca5 import PyMcaDataDir from PyMca5.PyMcaIO import ConfigDict from PyMca5.PyMcaMath.fitting import SpecfitFuns +from PyMca5.PyMcaMath.fitting import Gefit from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel from PyMca5.PyMcaMath.fitting.PolynomialModels import LinearPolynomialModel from PyMca5.PyMcaMath.fitting.PolynomialModels import ExponentialPolynomialModel @@ -482,8 +483,6 @@ def __init__(self, **kw): self.linegroup_areas = [] self._fluoRates = [] self._escapeLineGroups = [] - self._peakfunc = None - self._fastpeakfunc = None self.strategyInstances = {} @@ -570,32 +569,19 @@ def _preCalculateParameterIndependent(self): for peaks in self._lineGroups ] - # Prepare peak shape calculations - self._prepare_peakfunc() - - def _prepare_peakfunc(self): + def _peak_profile_params(self): + """All parameters are in the energy domain (X-axis is energy, not channels)""" npeaks = sum(len(group) for group in self._lineGroups) npeaks += sum( len(escgroup) for group in self._escapeLineGroups for escgroup in group ) if self._hypermet: # area, position, fwhm, ST AreaR, ST SlopeR, LT AreaR, LT SlopeR, STEP HeightR - htype = self._hypermet - - def peakfunc(params, x): - return SpecfitFuns.fastahypermet(params, x, htype) - - self._peakfunc = peakfunc npeakparams = 8 else: # area, position, fwhm, eta - self._peakfunc = SpecfitFuns.apvoigt npeakparams = 4 - self._peakfunc_params = numpy.zeros((npeaks, npeakparams)) - - def _fill_peakfunc_params(self): - """All parameters are in the energy domain (X-axis is energy, not channels)""" - parameters = self._peakfunc_params + parameters = numpy.zeros((npeaks, npeakparams)) # Peak positions and areas i = 0 @@ -634,16 +620,20 @@ def _fill_peakfunc_params(self): for i, param in enumerate(shapeparams, 3): parameters[:, i] = param - def _evaluate_peakprofiles(self, parameters, x, hypermet=None, fast=True): + return parameters + + def _total_peakgroup_profile(self, parameters, x, hypermet=None, fast=True): """When providing parameters for more than one peak, the peak profiles are added. - :param array parameters: flat 1D array of parameters - :param array x: domain on which to evaluate the peak profiles + :param array parameters: 1D array of parameters + :param array x: 1D array :param int or None hypermet: :param bool fast: ??? :returns array: same shape as x """ + if parameters.size == 0: + return numpy.zeros_like(x) if hypermet is None: hypermet = self._hypermet if hypermet: @@ -997,15 +987,17 @@ def xdata(self): @property def xenergy(self): - return self_channels_to_energy(self.xdata) - - def _channels_to_energy(self, xchannels): - return self.zero + self.gain * xchannels + return self._channels_to_energy(self.xdata) @property - def xenergy_cen(self): - xchannels = self.xdata - return self_channels_to_energy(xchannels - numpy.mean(xchannels)) + def xpol(self): + return self._channels_to_xpol(self.xdata) + + def _channels_to_energy(self, x): + return self.zero + self.gain * x + + def _channels_to_xpol(self, x): + return self.zero + self.gain * (x - x.mean()) @property def nchannels(self): @@ -1272,9 +1264,8 @@ def _refreshContinuumCache(self): degree=self.config["fit"]["linpolorder"], maxiter=10 ) elif continuum == "Exp. Polynomial": - model = ExponentialPolynomialModel(degree=self.config["fit"]["exppolorder"]) - estmodel = LinearPolynomialModel( - degree=self.config["fit"]["linpolorder"], maxiter=40 + model = ExponentialPolynomialModel( + degree=self.config["fit"]["exppolorder"], maxiter=40 ) else: raise ValueError("Unknown continuum {}".format(continuum)) @@ -1283,16 +1274,10 @@ def _refreshContinuumCache(self): # Estimate the polynomial coefficients by fitting the numerical background if model is not None: x = self.xdata - model.xdata = self.xenergy_cen + model.xdata = self.xpol model.ydata = self.ynumbkg - if continuum == "Exp. Polynomial": - estmodel.xdata = model.xdata - estmodel.ydata = numpy.log(model.ydata) - params = estmodel.fit()["parameters"] - params[0] = numpy.exp(params[0]) - model.parameters = params - else: - model.parameters = model.fit()["parameters"] + result = model.fit() + model.use_fit_result(result) self._lastContinuumCacheParams = contparams @@ -1665,31 +1650,88 @@ def _iter_parameter_groups(self, linear_only=False): else: raise ValueError(name) + def _ydata_to_yfit(self, ydata): + return ydata - self.ynumbkg + def evaluate(self, xdata=None): - """Evaluate to MCA model + """Evaluate to MCA model (does not include the numerical background) - y(xdata) = ybkg + ycont(C(xdata)) + A1*G1(E(xdata)) + A2*G2(E(xdata)) + ... + y(xdata) = ybkg + ycont(P(xdata)) + A1*G1(E(xdata)) + A2*G2(E(xdata)) + ... xdata: MCA channels (positive integers) - ybkg: numerical background derived from y + ybkg = numerical background ycont(x) = 0 # no analytical background - = c0 + c1*x + c2*x^2 + ... # linear polynomial - = c0 * exp[c1*x + c2*x^2 + ...] # exponential polynomial + = c0 + c1*x + c2*x^2 + ... # linear polynomial + = c0 * exp[c1*x + c2*x^2 + ...] # exponential polynomial - E(x) = zero + gain*x # energy - C(x) = E(x) - # centered energy + E(x) = zero + gain*x + P(x) = E(x - ) Gi(x): several peaks with normalized total area :param array xdata: length nxdata :returns array: nxdata """ + # Evaluation domain if xdata is None: + binterp = True xdata = self.xdata + else: + binterp = False energy = self._channels_to_energy(xdata) + # Emission lines, scatter peaks and escape peaks + parameters = self._peak_profile_params() + y = self._total_peakgroup_profile(parameters, energy) + + # Analytical background + model = self.continuumModel + if model is not None: + xpol = self._channels_to_xpol(xdata) + y += model.evaluate(xdata=xpol) + + # Pile-up + pileupfactor = self.sum + if pileupfactor: + y *= pileupfactor * SpecfitFuns.pileup(y, min(xdata), zero, gain) + + # Numerical background + ybkg = self.ynumbkg + if ybkg is not None: + if binterp: + try: + binterp = numpy.allclose(xdata, self.xdata) + except ValueError: + binterp = True + if binterp: + ybkg = numpy.interp(xdata, self.xdata, ybkg) + y += ybkg + + return y + + def linear_derivatives(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + energy = self._channels_to_energy(xdata) + raise NotImplementedError + + def derivative(self, param_idx, xdata=None): + """Derivate to a specific parameter + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + energy = self._channels_to_energy(xdata) raise NotImplementedError diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index 78221b8c2..4a2043dc1 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -61,8 +61,11 @@ def create_model(self, nmodels): self.fitmodel = SimpleModel.SimpleConcatModel(ndetectors=nmodels) assert not self.fitmodel.linear self.init_random() - self.fitmodel.ydata = self.fitmodel.ymodel - numpy.testing.assert_array_equal(self.fitmodel.ydata, self.fitmodel.ymodel) + ydata = self.fitmodel.yfullmodel.copy() + self.fitmodel.ydata = ydata + numpy.testing.assert_array_equal(self.fitmodel.ydata, ydata) + numpy.testing.assert_array_equal(self.fitmodel.yfullmodel, ydata) + numpy.testing.assert_allclose(self.fitmodel.yfitmodel, ydata - 10) self.validate_model() def init_random(self, **kw): @@ -79,6 +82,7 @@ def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): self.nglobals = npeaks # concentrations model.xdata_raw = numpy.arange(nchannels) model.ydata_raw = numpy.full(nchannels, numpy.nan) + model.ybkg = 10 model.xmin = self.random_state.randint(low=0, high=10) model.xmax = self.random_state.randint(low=nchannels - 10, high=nchannels) model.zero = self.random_state.uniform(low=1, high=1.5) @@ -141,10 +145,17 @@ def _validate_model(self, model): assert model.nchannels == len(model.xdata) assert model.nparameters == len(model.parameters) assert model.nlinear_parameters == len(model.linear_parameters) - arr1 = model.evaluate() - arr2 = model.evaluate_linear() - arr3 = sum(model.linear_decomposition()) - arr4 = model.ymodel + + arr1 = model.evaluate_fullmodel() + arr2 = model.evaluate_linear_fullmodel() + arr3 = model.yfullmodel + numpy.testing.assert_allclose(arr1, arr2) + numpy.testing.assert_allclose(arr1, arr3) + + arr1 = model.evaluate_fitmodel() + arr2 = model.evaluate_linear_fitmodel() + arr3 = model.yfitmodel + arr4 = sum(model.linear_decomposition_fitmodel()) numpy.testing.assert_allclose(arr1, arr2) numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) @@ -181,7 +192,7 @@ def plot(self): names = m.parameter_names plt.figure() plt.plot(m.ydata, label="data") - plt.plot(m.ymodel, label="model") + plt.plot(m.yfitmodel, label="model") plt.legend() plt.figure() for y, name in zip(derivatives, names): @@ -205,11 +216,11 @@ def _testLinearFit(self): result = self.fitmodel.fit() self.assert_result(result, expected) - assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) assert not numpy.allclose(self.fitmodel.linear_parameters, expected) self.fitmodel.use_fit_result(result) - numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) numpy.testing.assert_allclose(self.fitmodel.linear_parameters, expected) @with_model(1) @@ -232,7 +243,7 @@ def _testNonLinearFit(self): # TODO: non-linear parameters not precise # self.assert_result(result, expected1) - assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.ymodel) + assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) assert not numpy.allclose(self.fitmodel.parameters, expected1) assert not numpy.allclose(self.fitmodel.linear_parameters, expected2) @@ -254,7 +265,7 @@ def assert_result(self, result, expected): def assert_ymodel(self): a = self.fitmodel.ydata - b = self.fitmodel.ymodel + b = self.fitmodel.yfullmodel mask = (a > 1) & (b > 1) assert mask.any() numpy.testing.assert_allclose(a[mask], b[mask], rtol=1e-3) diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 9d40f9681..a26d9d337 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -49,7 +49,13 @@ def testLinearPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected - fitmodel.ydata = model.evaluate() + fitmodel.ydata = model.yfullmodel + + numpy.testing.assert_array_equal(model.parameters, expected) + numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) + numpy.testing.assert_array_equal(fitmodel.yfitdata, model.yfitmodel) + numpy.testing.assert_array_equal(model.yfitmodel, model.yfullmodel) + for linear in [True, False]: with self.subTest(degree=degree, linear=linear): fitmodel.linear = linear @@ -67,21 +73,22 @@ def testExpPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected - fitmodel.ydata = model.evaluate() + fitmodel.ydata = model.yfullmodel + + numpy.testing.assert_array_equal(model.parameters, expected) + numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) + numpy.testing.assert_allclose(fitmodel.yfitdata, model.yfitmodel) + numpy.testing.assert_allclose(model.yfitmodel, numpy.log(model.yfullmodel)) + for linear in [True, False]: with self.subTest(degree=degree, linear=linear): fitmodel.linear = linear - if linear: - fitmodel.coefficients = expected.copy() - else: - fitmodel.coefficients = numpy.zeros_like(expected) - fitmodel.factor = 1 # Do not start from zero + fitmodel.coefficients = numpy.zeros_like(expected) + if not linear: + fitmodel.coefficients[0] = 0.1 self.assertEqual(fitmodel.degree, degree) result = fitmodel.fit()["parameters"] - if linear: - numpy.testing.assert_allclose(result, expected[0], rtol=1e-4) - else: - numpy.testing.assert_allclose(result, expected, rtol=1e-4) + numpy.testing.assert_allclose(result, expected) def getSuite(auto=True): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index cfe0795f0..915ec6d3f 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -51,6 +51,7 @@ def __init__(self): self.xdata_raw = None self.ydata_raw = None self.ystd_raw = None + self.ybkg = 0 self.sigma_to_fwhm = 2 * numpy.sqrt(2 * numpy.log(2)) super(SimpleModel, self).__init__() @@ -151,6 +152,12 @@ def xdata(self, values): def xenergy(self): return self.zero + self.gain * self.xdata + def _ydata_to_fit(self, ydata): + return ydata - self.ybkg + + def _fit_to_ydata(self, yfit): + return yfit + self.ybkg + @property def ydata(self): if self.ydata_raw is None: @@ -212,7 +219,7 @@ def _iter_parameter_groups(self, linear_only=False): else: raise ValueError(name) - def evaluate(self, xdata=None): + def evaluate_fitmodel(self, xdata=None): """Evaluate model :param array xdata: length nxdata @@ -222,9 +229,10 @@ def evaluate(self, xdata=None): xdata = self.xdata x = self.zero + self.gain * xdata p = list(zip(self.areas, self.positions, self.fwhms)) - return SpecfitFuns.agauss(p, x) + y = SpecfitFuns.agauss(p, x) + return y - def linear_derivatives(self, xdata=None): + def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: length nxdata @@ -236,7 +244,7 @@ def linear_derivatives(self, xdata=None): it = zip(self.efficiency, self.positions, self.fwhms) return numpy.array([SpecfitFuns.agauss([a, p, w], x) for a, p, w in it]) - def derivative(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter :param int param_idx: From 502b41ef2b5d96a974e8477e91281587734a3faa Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 6 May 2021 18:08:56 +0200 Subject: [PATCH 22/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 25 +- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 4 +- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 386 +++++++++++++------ PyMca5/tests/SimpleModel.py | 4 +- PyMca5/tests/XrfTest.py | 2 +- 5 files changed, 289 insertions(+), 132 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index b537378d4..9b6ff4f30 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -274,7 +274,8 @@ def evaluate_fullmodel(self, xdata=None): :param array xdata: length nxdata :returns array: nxdata """ - return self._fit_to_ydata(self.evaluate_fitmodel(xdata=xdata)) + y = self.evaluate_fitmodel(xdata=xdata) + return self._fit_to_ydata(y, xdata=xdata) def evaluate_linear_fullmodel(self, xdata=None): """Evaluate the full model. @@ -282,7 +283,8 @@ def evaluate_linear_fullmodel(self, xdata=None): :param array xdata: length nxdata :returns array: n x nxdata """ - return self._fit_to_ydata(self.evaluate_linear_fitmodel(xdata=xdata)) + y = self.evaluate_linear_fitmodel(xdata=xdata) + return self._fit_to_ydata(y, xdata=xdata) def evaluate_fitmodel(self, xdata=None): """Evaluate the fit model. @@ -356,7 +358,7 @@ def _parameter_groups(self, linear_only=False): :returns iterable(str, int): group name, nb. parameters in the group """ if self.caching_enabled: - cache = self._cache.setdefault("all_parameter_groups", {}) + cache = self._cache.setdefault("parameter_groups", {}) a = self.included_parameters b = self.excluded_parameters if a is not None: @@ -396,6 +398,17 @@ def _parameter_name_from_index(self, idx, linear_only=False): return name, idx - i i += n + def _parameter_indices_from_name(self, name, linear_only=False): + """Parameter group name to index range + + :returns int, int: start and end parameter index of the group + """ + i = 0 + for _name, n in self._parameter_groups(linear_only=linear_only): + if name == _name: + return i, i + n + i += n + def fit(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters @@ -473,16 +486,16 @@ def nonlinear_fit(self, full_output=False): ret["lastdeltachi"] = result[4] return ret - def _ydata_to_fit(self, ydata): + def _ydata_to_fit(self, ydata, xdata=None): return ydata - def _ystd_to_fit(self, ystd): + def _ystd_to_fit(self, ystd, xdata=None): return ystd def _parameters_to_fit(self, params): return params - def _fit_to_ydata(self, yfit): + def _fit_to_ydata(self, yfit, xdata=None): return yfit def _fit_to_parameters(self, params): diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index 5e06826c0..3fc719cc9 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -169,10 +169,10 @@ class ExponentialPolynomialModel(LinearPolynomialModel): yfit = log(y) = log(c1) + c1*x + c2*x^2 + ... """ - def _ydata_to_fit(self, ydata): + def _ydata_to_fit(self, ydata, xdata=None): return numpy.log(ydata) - def _fit_to_ydata(self, yfit): + def _fit_to_ydata(self, yfit, xdata=None): return numpy.exp(yfit) def _parameters_to_fit(self, parameters): diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 40453e02b..7f9a02a61 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -431,6 +431,26 @@ def startfit(self, *args, **kw): ) return self.startFit(*args, **kw) + def setConfiguration(self, ddict): + """ + The current fit configuration dictionary is updated, but not replaced, + by the input dictionary. + It returns a copy of the final fit configuration. + """ + return self.configure(ddict) + + def getConfiguration(self): + """ + returns a copy of the current fit configuration parameters + """ + return self.configure() + + def getStartingConfiguration(self): + """ + returns a copy of the current fit configuration parameters + """ + return self.configure() + class McaTheory(McaTheoryConfigApi, McaTheoryLegacyApi, Model): """Model for MCA data""" @@ -461,6 +481,7 @@ def __init__(self, **kw): self._std0 = None self._xmin0 = None self._xmax0 = None + self._expotime0 = None # XRF spectrum to fit self._ydata = None @@ -477,17 +498,14 @@ def __init__(self, **kw): self._continuumModel = None self._lastContinuumCacheParams = None - self._lastTime = None - + # XRF line groups self._lineGroups = [] - self.linegroup_areas = [] + self._linegroup_areas = [] self._fluoRates = [] self._escapeLineGroups = [] + self._lastAreasCacheParams = None - self.strategyInstances = {} - - self.__toBeConfigured = False - self.__configure() + self.configure() def useFisxEscape(self, flag=None): """Make sure the model uses fisx to calculate the escape peaks @@ -506,47 +524,21 @@ def useFisxEscape(self, flag=None): else: self._useFisxEscape = False - def setConfiguration(self, ddict): - """ - The current fit configuration dictionary is updated, but not replaced, - by the input dictionary. - It returns a copy of the final fit configuration. - """ - return self.configure(ddict) - - def getConfiguration(self): - """ - returns a copy of the current fit configuration parameters - """ - return self.configure() - - def getStartingConfiguration(self): - # same output as calling configure but with the calling program - # knowing what is going on (no warning) - if self.__toBeConfigured: - return copy.deepcopy(self.__originalConfiguration) - else: - return self.configure() - def configure(self, newdict=None): - if newdict in [None, {}]: - if self.__toBeConfigured: - _logger.debug( - "WARNING: This configuration is the one of last fit.\n" - "It does not correspond to the one of next fit." - ) - else: + if newdict: self.config.update(newdict) - self.__toBeConfigured = False - self.__configure() - return copy.deepcopy(self.config) - - def __configure(self): self._initializeConfig() self._preCalculateParameterIndependent() self._preCalculateParameterDependent() + return copy.deepcopy(self.config) + + def _preCalculateParameterDependent(self): + pass def _preCalculateParameterIndependent(self): + self._preCalculateLineGroups() + + def _preCalculateLineGroups(self): self._fluoRates = self._calcFluoRates() self._calcFluoRateCorrections() @@ -557,7 +549,7 @@ def _preCalculateParameterIndependent(self): # This is a filtered and normalized form of `_fluoRates` self._lineGroups = list(self._getEmissionLines()) self._lineGroups.extend(self._getScatterLines()) - self.linegroup_areas = numpy.zeros(len(self._lineGroups)) + self._linegroup_areas = numpy.ones(len(self._lineGroups)) # Escape line groups: nested lists # line group @@ -569,13 +561,27 @@ def _preCalculateParameterIndependent(self): for peaks in self._lineGroups ] - def _peak_profile_params(self): + def _peak_profile_params( + self, selected_groups=None, hypermet=None, normalize_peakgroups=False + ): """All parameters are in the energy domain (X-axis is energy, not channels)""" - npeaks = sum(len(group) for group in self._lineGroups) - npeaks += sum( - len(escgroup) for group in self._escapeLineGroups for escgroup in group - ) - if self._hypermet: + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self.linegroup_areas + if selected_groups is not None: + if not isinstance(selected_groups, (list, tuple)): + selected_groups = [selected_groups] + lineGroups = [lineGroups[i] for i in selected_groups] + escapeLineGroups = [escapeLineGroups[i] for i in selected_groups] + linegroup_areas = [linegroup_areas[i] for i in selected_groups] + if hypermet is None: + hypermet = self._hypermet + if normalize_peakgroups: + linegroup_areas = numpy.ones(len(linegroup_areas)) + + npeaks = sum(len(group) for group in lineGroups) + npeaks += sum(len(escgroup) for group in escapeLineGroups for escgroup in group) + if hypermet: # area, position, fwhm, ST AreaR, ST SlopeR, LT AreaR, LT SlopeR, STEP HeightR npeakparams = 8 else: @@ -586,7 +592,7 @@ def _peak_profile_params(self): # Peak positions and areas i = 0 for group, escgroup, grouparea in zip( - self._lineGroups, self._escapeLineGroups, self.linegroup_areas + lineGroups, escapeLineGroups, linegroup_areas ): if not escgroup: escgroup = [[]] * len(group) @@ -601,13 +607,13 @@ def _peak_profile_params(self): i += 1 # Area parameters from channel to energy domain - self._peakfunc_params[:, 0] *= self.gain + parameters[:, 0] *= self.gain # FWHM parameters[:, 2] = self._peak_fwhm(parameters[:, 1]) # Other peak shape parameters - if self._hypermet: + if hypermet: shapeparams = [ self.st_arearatio, self.st_sloperatio, @@ -622,11 +628,58 @@ def _peak_profile_params(self): return parameters + @property + def linegroup_areas(self): + self._refreshAreasCache() + return self._linegroup_areas + + def _refreshAreasCache(self): + params = self._areasCacheParams + if self._lastAreasCacheParams == params: + return # the cached data is still valid + self._estimateLineGroupAreas() + self._lastAreasCacheParams = params + + @property + def _areasCacheParams(self): + """Any change in these parameter will invalidate the cache""" + return id(self._dataCacheParams), id(self.linegroup_areas) + + def _estimateLineGroupAreas(self): + ydata = self._ydata_without_background() + xenergy = self.xenergy + emin = xenergy.min() + emax = xenergy.max() + factor = self.GAUSS_SIGMA_TO_FWHM * numpy.sqrt(2 * numpy.pi) + + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self._linegroup_areas + for i, (group, escgroup) in enumerate(zip(lineGroups, escapeLineGroups)): + if not escgroup: + escgroup = [[]] * len(group) + selected_energy = 0 + selected_rate = 0 + for (peakenergy, rate, _), esclines in zip(group, escgroup): + if peakenergy >= emin and peakenergy <= emax: + if rate > selected_rate: + selected_energy = peakenergy + for peakenergy, escrate, _ in esclines: + if peakenergy >= emin and peakenergy <= emax: + if rate * escrate > selected_rate: + selected_energy = peakenergy + if selected_energy: + height = ydata[(np.abs(xenergy - selected_energy)).argmin()] + fwhm = self._peak_fwhm(selected_energy) + linegroup_areas[i] = height * fwhm * factor # Gaussian + else: + linegroup_areas[i] = 0 # Fixed at zero + def _total_peakgroup_profile(self, parameters, x, hypermet=None, fast=True): """When providing parameters for more than one peak, the peak profiles are added. - :param array parameters: 1D array of parameters + :param array parameters: npeaks x nparams :param array x: 1D array :param int or None hypermet: :param bool fast: ??? @@ -679,9 +732,6 @@ def _getScatterLines(self): yield [[en_elastic, 1.0, name]] yield [[en_inelastic, 1.0, name]] - def _preCalculateParameterDependent(self): - pass - def _calcFluoRates(self): """Fluorescence rate for each emission line of each element. Rate means fluorescence intensity divided by primary intensity. @@ -1015,23 +1065,49 @@ def ystd(self): self._refreshDataCache() return self._ystd - @property - def ynumbkg(self): - """Get the numerical background (as opposed to the analytical background)""" - self._refreshNumBkgCache() - return self._numBkg + def ynumbkg(self, xdata=None): + ybkg = self._ynumbkg + if ybkg is None: + return ybkg + if xdata is not None: + try: + binterp = numpy.allclose(xdata, self.xdata) + except ValueError: + binterp = True + if binterp: + ybkg = numpy.interp(xdata, self.xdata, ybkg) + return ybkg - @property - def ycontinuum(self): + def ycontinuum(self, xdata=None): """Get the analytical background (as opposed to the numerical background)""" + if xdata is None: + xdata = self.xdata model = self.continuumModel if model is None: - if self.ydata is None: + if xdata is None: return None else: - return numpy.zeros_like(self.ydata) + return numpy.zeros(len(xdata)) + else: + return model.evaluate_fullmodel(xdata=xdata) + + def ybackground(self, xdata=None): + contbkg = self.ycontinuum(xdata=xdata) + numbkg = self.ynumbkg(xdata=xdata) + if numbkg is None: + return contbkg else: - return model.evaluate() + return contbkg + numbkg + + @property + def _ynumbkg(self): + self._refreshNumBkgCache() + return self._numBkg + + @property + def continuumModel(self): + self._refreshContinuumCache() + return self._continuumModel @property def xdata0(self): @@ -1065,10 +1141,6 @@ def setData(self, *var, **kw): time (seconds) is the factor associated to the flux, only used when using a strategy based on concentrations """ - if self.__toBeConfigured: - _logger.debug("setData RESTORE ORIGINAL CONFIGURATION") - self.configure(self.__originalConfiguration) - if "y" in kw: ydata0 = kw["y"] elif len(var) > 1: @@ -1116,7 +1188,7 @@ def setData(self, *var, **kw): ystd0 = numpy.ravel(ystd0) timeFactor = kw.get("time", None) - self._lastTime = timeFactor + self._expotime0 = timeFactor if timeFactor is None: cfgfit = self.config["fit"] if self.config["concentrations"].get("useautotime", False): @@ -1148,8 +1220,16 @@ def xmax(self): else: return self._xmax0 + @property + def emin(self): + return self._channels_to_energy(self.xmin) + + @property + def emax(self): + return self._channels_to_energy(self.xmax) + def getLastTime(self): - return self._lastTime + return self._expotime0 @property def _dataCacheParams(self): @@ -1203,7 +1283,6 @@ def _numBkgCacheParams(self): """Any change in these parameter will invalidate the cache""" cfg = self.config["fit"] params = [ - id(self._lastDataCacheParams), "stripflag", "stripalgorithm", "stripfilterwidth", @@ -1214,6 +1293,8 @@ def _numBkgCacheParams(self): params += ["snipwidth"] else: params += ["stripwidth", "stripconstant", "stripiterations"] + params = [cfg[p] for p in params] + params.append(id(self._lastDataCacheParams)) return params def _refreshNumBkgCache(self): @@ -1251,7 +1332,7 @@ def _refreshContinuumCache(self): # Instantiate the model continuum = self.config["fit"]["continuum"] - if continuum is None or self.ynumbkg is None: + if continuum is None or self._ynumbkg is None: model = None elif continuum == "Constant": model = LinearPolynomialModel(degree=0, maxiter=10) @@ -1275,17 +1356,12 @@ def _refreshContinuumCache(self): if model is not None: x = self.xdata model.xdata = self.xpol - model.ydata = self.ynumbkg + model.ydata = self.ynumbkg() result = model.fit() model.use_fit_result(result) self._lastContinuumCacheParams = contparams - @property - def continuumModel(self): - self._refreshContinuumCache() - return self._continuumModel - @property def continuum_coefficients(self): model = self.continuumModel @@ -1579,6 +1655,12 @@ def st_arearatio_constraint(self): delta = self.config["detector"]["deltast_arearatio"] return Gefit.CQUOTED, value + delta, value - delta + @property + def linegroup_areas_constraint(self): + fixed = Gefit.CFIXED, 0, 0 + positive = Gefit.CPOSITIVE, 0, 0 + return [positive if area else fixed for area in self.linegroup_areas] + @property def _parameter_group_names(self): return [ @@ -1635,8 +1717,8 @@ def _iter_parameter_groups(self, linear_only=False): yield name, 1 elif name == "eta_factor" and not hypermet: yield name, 1 - elif name == "areas": - n = len(self.area) + elif name == "linegroup_areas": + n = len(self.linegroup_areas) if n: yield name, n elif name == "linpol": @@ -1650,17 +1732,12 @@ def _iter_parameter_groups(self, linear_only=False): else: raise ValueError(name) - def _ydata_to_yfit(self, ydata): - return ydata - self.ynumbkg - - def evaluate(self, xdata=None): + def evaluate_fitmodel(self, xdata=None): """Evaluate to MCA model (does not include the numerical background) - y(xdata) = ybkg + ycont(P(xdata)) + A1*G1(E(xdata)) + A2*G2(E(xdata)) + ... + y(x) = ycont(P(x)) + A1*G1(E(x)) + A2*G2(E(x)) + ... - xdata: MCA channels (positive integers) - - ybkg = numerical background + x: MCA channels (positive integers) ycont(x) = 0 # no analytical background = c0 + c1*x + c2*x^2 + ... # linear polynomial @@ -1671,47 +1748,62 @@ def evaluate(self, xdata=None): Gi(x): several peaks with normalized total area - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: + :returns array: """ + parameters = self._peak_profile_params() + return self.mcatheory(parameters, xdata=xdata) + + def mcatheory( + self, parameters, xdata=None, hypermet=None, continuum=None, summing=None + ): # Evaluation domain if xdata is None: - binterp = True xdata = self.xdata - else: - binterp = False energy = self._channels_to_energy(xdata) # Emission lines, scatter peaks and escape peaks - parameters = self._peak_profile_params() - y = self._total_peakgroup_profile(parameters, energy) + y = self._total_peakgroup_profile(parameters, energy, hypermet=hypermet) # Analytical background - model = self.continuumModel - if model is not None: - xpol = self._channels_to_xpol(xdata) - y += model.evaluate(xdata=xpol) + if continuum or continuum is None: + model = self.continuumModel + if model is not None: + xpol = self._channels_to_xpol(xdata) + y += model.evaluate_fullmodel(xdata=xpol) # Pile-up - pileupfactor = self.sum - if pileupfactor: - y *= pileupfactor * SpecfitFuns.pileup(y, min(xdata), zero, gain) - - # Numerical background - ybkg = self.ynumbkg - if ybkg is not None: - if binterp: - try: - binterp = numpy.allclose(xdata, self.xdata) - except ValueError: - binterp = True - if binterp: - ybkg = numpy.interp(xdata, self.xdata, ybkg) - y += ybkg + if summing or summing is None: + pileupfactor = self.sum + if pileupfactor: + y *= pileupfactor * SpecfitFuns.pileup(y, min(xdata), zero, gain) return y - def linear_derivatives(self, xdata=None): + def _ydata_to_fit(self, ydata, xdata=None): + """The fitting is done after subtracting the numerical background""" + ybkg = self.ynumbkg(xdata=xdata) + if ybkg is None: + return ydata + else: + return ydata - ybkg + + def _ydata_without_background(self, ydata, xdata=None): + ybkg = self.ybackground(xdata=xdata) + if ybkg is None: + return ydata + else: + return ydata - ybkg + + def _fit_to_ydata(self, yfit, xdata=None): + """The numerical background is not included in the fit model""" + ybkg = self.ynumbkg(xdata=xdata) + if ybkg is None: + return ydata + else: + return ydata + ybkg + + def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: length nxdata @@ -1722,17 +1814,69 @@ def linear_derivatives(self, xdata=None): energy = self._channels_to_energy(xdata) raise NotImplementedError - def derivative(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter :param int param_idx: :param array xdata: length nxdata :returns array: nxdata """ - if xdata is None: - xdata = self.xdata - energy = self._channels_to_energy(xdata) - raise NotImplementedError + name, pgroupi = self._parameter_name_from_index(param_idx) + if name == "zero": + raise NotImplementedError + elif name == "gain": + raise NotImplementedError + elif name == "noise": + raise NotImplementedError + elif name == "fano": + raise NotImplementedError + elif name == "sum": + raise NotImplementedError + elif name == "st_arearatio" and hypermet: + raise NotImplementedError + elif name == "st_sloperatio" and hypermet: + raise NotImplementedError + elif name == "lt_arearatio" and hypermet: + raise NotImplementedError + elif name == "lt_sloperatio" and hypermet: + raise NotImplementedError + elif name == "step_heightratio" and hypermet: + raise NotImplementedError + elif name == "eta_factor" and not hypermet: + raise NotImplementedError + elif name == "linegroup_areas": + parameters = self._peak_profile_params( + selected_groups=[pgroupi], normalize_peakgroups=True + ) + return self.mcatheory(parameters, xdata=xdata) + elif name == "linpol": + return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) + elif name == "exppol": + return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) + else: + raise ValueError(name) + + def _numerical_derivative(self, parameters, index, xdata=None): + parameters = parameters.copy() + p0 = parameters[index] + delta = (p0[index] + p0[index]) * 0.00001 + parameters[index] = p0 + delta + f1 = self.mcatheory(parameters, xdata=xdata) + parameters[index] = p0 - delta + f2 = self.mcatheory(parameters, xdata=xdata) + return (f1 - f2) / (2.0 * delta) + + @property + def maxiter(self): + return self.config["fit"]["maxiter"] + + @property + def deltachi(self): + return self.config["fit"]["deltachi"] + + @property + def weightflag(self): + return self.config["fit"]["fitweight"] class MultiMcaTheory(ConcatModel): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 915ec6d3f..4ebfae85d 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -152,10 +152,10 @@ def xdata(self, values): def xenergy(self): return self.zero + self.gain * self.xdata - def _ydata_to_fit(self, ydata): + def _ydata_to_fit(self, ydata, xdata=None): return ydata - self.ybkg - def _fit_to_ydata(self, yfit): + def _fit_to_ydata(self, yfit, xdata=None): return yfit + self.ybkg @property diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 3af9a090e..628027ce4 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -605,7 +605,7 @@ def _testLegacyMcaTheory(self, x, y, configuration): numpy.testing.assert_array_equal(mcaFitLegacy.xdata.flat, mcaFit.xdata) numpy.testing.assert_array_equal(mcaFitLegacy.ydata.flat, mcaFit.ydata) numpy.testing.assert_array_equal(mcaFitLegacy.sigmay.flat, mcaFit.ystd) - numpy.testing.assert_array_equal(mcaFitLegacy.zz.flat, mcaFit.ynumbkg) + numpy.testing.assert_array_equal(mcaFitLegacy.zz.flat, mcaFit.ynumbkg()) # Compare configuration config1 = copy.deepcopy(mcaFitLegacy.config) From f86ee74bb6a88c2089cfc096026306ccc543f276 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 7 May 2021 12:30:03 +0200 Subject: [PATCH 23/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 2 +- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 1615 +++++++++--------- 2 files changed, 820 insertions(+), 797 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 9b6ff4f30..cc11f6220 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -471,7 +471,7 @@ def nonlinear_fit(self, full_output=False): fulloutput=full_output, ) if self.niter_non_leastsquares: - self.fit_parameters = parameters + self.fit_parameters = result[0] self.non_leastsquares_increment() finally: self.parameters = keep diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 7f9a02a61..7ce55314a 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -76,6 +76,8 @@ def defaultConfigFilename(): class McaTheoryConfigApi: + """API on top of an MCA configuration""" + def __init__(self, initdict=None, filelist=None, **kw): if initdict is None: initdict = defaultConfigFilename() @@ -85,6 +87,7 @@ def __init__(self, initdict=None, filelist=None, **kw): raise IOError("File %s does not exist" % initdict) self._overwriteConfig(**kw) self.attflag = kw.get("attenuatorsflag", 1) + self.configure() def _overwriteConfig(self, **kw): if "config" in kw: @@ -370,6 +373,12 @@ def _configureElementsModule(self): Elements.updateDict(energy=maxenergy) break + def configure(self, newdict=None): + if newdict: + self.config.update(newdict) + self._initializeConfig() + return copy.deepcopy(self.config) + def _initializeConfig(self): self._addMissingConfig() self._configureElementsModule() @@ -452,29 +461,10 @@ def getStartingConfiguration(self): return self.configure() -class McaTheory(McaTheoryConfigApi, McaTheoryLegacyApi, Model): - """Model for MCA data""" - - BAND_GAP = 0.00385 # keV, silicon - GAUSS_SIGMA_TO_FWHM = 2.3548 - MAX_ATTENUATION = 1.0e-300 - SCATTER_ENERGY_THRESHOLD = 0.2 # keV - - CONTINUUM_LIST = [ - None, - "Constant", - "Linear", - "Parabolic", - "Linear Polynomial", - "Exp. Polynomial", - ] +class McaTheoryDataApi(McaTheoryConfigApi): + """Add API for handling a single XRF spectrum (MCA data)""" def __init__(self, **kw): - super(McaTheory, self).__init__(**kw) - # TODO: done for some initialization of SpecfitFuns? - SpecfitFuns.fastagauss([1.0, 10.0, 1.0], numpy.arange(10.0)) - self.useFisxEscape(False) - # Original XRF spectrum self._ydata0 = None self._xdata0 = None @@ -489,892 +479,921 @@ def __init__(self, **kw): self._std = None self._lastDataCacheParams = None - # XRF spectrum numerical background - self._numBkg = None - self._lastNumBkgCacheParams = None - - # XRF spectrum analytical background - self._continuum = None - self._continuumModel = None - self._lastContinuumCacheParams = None + super(McaTheoryDataApi, self).__init__(**kw) - # XRF line groups - self._lineGroups = [] - self._linegroup_areas = [] - self._fluoRates = [] - self._escapeLineGroups = [] - self._lastAreasCacheParams = None + @property + def xdata(self): + """Sorted and sliced view of xdata0""" + self._refreshDataCache() + return self._xdata - self.configure() + @property + def nchannels(self): + return len(self.xdata) - def useFisxEscape(self, flag=None): - """Make sure the model uses fisx to calculate the escape peaks - when possible. - """ - if flag and FISX: - if ConcentrationsTool.FisxHelper.xcom is None: - FisxHelper.xcom = xcom = FisxHelper.getElementsInstance() - else: - xcom = ConcentrationsTool.FisxHelper.xcom - if hasattr(xcom, "setEscapeCacheEnabled"): - xcom.setEscapeCacheEnabled(1) - self._useFisxEscape = True - else: - self._useFisxEscape = False - else: - self._useFisxEscape = False + @property + def ydata(self): + """Sorted and sliced view of ydata0""" + self._refreshDataCache() + return self._ydata - def configure(self, newdict=None): - if newdict: - self.config.update(newdict) - self._initializeConfig() - self._preCalculateParameterIndependent() - self._preCalculateParameterDependent() - return copy.deepcopy(self.config) + @property + def ystd(self): + """Sorted and sliced view of ystd0""" + self._refreshDataCache() + return self._ystd - def _preCalculateParameterDependent(self): - pass + @property + def xdata0(self): + return self._xdata0 - def _preCalculateParameterIndependent(self): - self._preCalculateLineGroups() + @property + def ydata0(self): + return self._ydata0 - def _preCalculateLineGroups(self): - self._fluoRates = self._calcFluoRates() - self._calcFluoRateCorrections() + @property + def ystd0(self): + return self._ystd0 - # Line groups: nested lists - # line group - # -> emission/scattering line - # [energy, rate, line name] - # This is a filtered and normalized form of `_fluoRates` - self._lineGroups = list(self._getEmissionLines()) - self._lineGroups.extend(self._getScatterLines()) - self._linegroup_areas = numpy.ones(len(self._lineGroups)) + def setData(self, *var, **kw): + """ + Method to update the data to be fitted. + It accepts several combinations of input arguments, the simplest to + take into account is: - # Escape line groups: nested lists - # line group - # -> emission/scattering line - # -> escape line - # [energy, rate, escape name] - self._escapeLineGroups = [ - self._calcEscapePeaks([peak[0] for peak in peaks]) - for peaks in self._lineGroups - ] + setData(x, y, sigmay=None, xmin=None, xmax=None) - def _peak_profile_params( - self, selected_groups=None, hypermet=None, normalize_peakgroups=False - ): - """All parameters are in the energy domain (X-axis is energy, not channels)""" - lineGroups = self._lineGroups - escapeLineGroups = self._escapeLineGroups - linegroup_areas = self.linegroup_areas - if selected_groups is not None: - if not isinstance(selected_groups, (list, tuple)): - selected_groups = [selected_groups] - lineGroups = [lineGroups[i] for i in selected_groups] - escapeLineGroups = [escapeLineGroups[i] for i in selected_groups] - linegroup_areas = [linegroup_areas[i] for i in selected_groups] - if hypermet is None: - hypermet = self._hypermet - if normalize_peakgroups: - linegroup_areas = numpy.ones(len(linegroup_areas)) + x corresponds to the spectrum channels + y corresponds to the spectrum counts + sigmay is the uncertainty associated to the counts. If not given, + Poisson statistics will be assumed. If the fit configuration + is set to no weight, it will not be used. + xmin and xmax define the limits to be considered for performing the fit. + If the fit configuration flag self.config['fit']['use_limit'] is + set, they will be ignored. If xmin and xmax are not given, the + whole given spectrum will be considered for fitting. + time (seconds) is the factor associated to the flux, only used when using + a strategy based on concentrations + """ + if "y" in kw: + ydata0 = kw["y"] + elif len(var) > 1: + ydata0 = var[1] + elif len(var) == 1: + ydata0 = var[0] + else: + ydata0 = None - npeaks = sum(len(group) for group in lineGroups) - npeaks += sum(len(escgroup) for group in escapeLineGroups for escgroup in group) - if hypermet: - # area, position, fwhm, ST AreaR, ST SlopeR, LT AreaR, LT SlopeR, STEP HeightR - npeakparams = 8 + if ydata0 is None: + return 1 else: - # area, position, fwhm, eta - npeakparams = 4 - parameters = numpy.zeros((npeaks, npeakparams)) + ydata0 = numpy.ravel(ydata0) - # Peak positions and areas - i = 0 - for group, escgroup, grouparea in zip( - lineGroups, escapeLineGroups, linegroup_areas - ): - if not escgroup: - escgroup = [[]] * len(group) - for (energy, rate, _), esclines in zip(group, escgroup): - peakarea = rate * grouparea - parameters[i, 0] = peakarea - parameters[i, 1] = energy - i += 1 - for escen, escrate, _ in esclines: - parameters[i, 0] = peakarea * escrate - parameters[i, 1] = escen - i += 1 + if "x" in kw: + xdata0 = kw["x"] + elif len(var) > 1: + xdata0 = var[0] + else: + xdata0 = None - # Area parameters from channel to energy domain - parameters[:, 0] *= self.gain + if xdata0 is None: + xdata0 = numpy.arange(len(ydata0)) + else: + xdata0 = numpy.ravel(xdata0) - # FWHM - parameters[:, 2] = self._peak_fwhm(parameters[:, 1]) + if "sigmay" in kw: + ystd0 = kw["sigmay"] + elif "stdy" in kw: + ystd0 = kw["stdy"] + elif len(var) > 2: + ystd0 = var[2] + else: + ystd0 = None - # Other peak shape parameters - if hypermet: - shapeparams = [ - self.st_arearatio, - self.st_sloperatio, - self.lt_arearatio, - self.lt_sloperatio, - self.step_heightratio, - ] + if ystd0 is None: + # Poisson noise + valid = ydata0 > 0 + if valid.any(): + ystd0 = numpy.sqrt(abs(ydata0)) + ystd0[~valid] = ystd0[valid].min() + else: + ystd0 = numpy.ones_like(ydata0) else: - shapeparams = [self.eta_factor] - for i, param in enumerate(shapeparams, 3): - parameters[:, i] = param + ystd0 = numpy.ravel(ystd0) - return parameters + timeFactor = kw.get("time", None) + self._expotime0 = timeFactor + if timeFactor is None: + if self.config["concentrations"].get("useautotime", False): + if not self.config["concentrations"]["usematrix"]: + msg = "Requested to use time from data but not present!!" + raise ValueError(msg) + elif self.config["concentrations"].get("useautotime", False): + self.config["concentrations"]["time"] = timeFactor - @property - def linegroup_areas(self): - self._refreshAreasCache() - return self._linegroup_areas + self._xmin0 = kw.get("xmin", self.xmin) + self._xmax0 = kw.get("xmax", self.xmax) + return self._refreshDataCache(xdata0=xdata0, ydata0=ydata0, ystd0=ystd0) - def _refreshAreasCache(self): - params = self._areasCacheParams - if self._lastAreasCacheParams == params: - return # the cached data is still valid - self._estimateLineGroupAreas() - self._lastAreasCacheParams = params + @property + def xmin(self): + """From config (if enabled) or the last `setData` call""" + cfgfit = self.config["fit"] + if cfgfit["use_limit"]: + return cfgfit["xmin"] + else: + return self._xmin0 @property - def _areasCacheParams(self): - """Any change in these parameter will invalidate the cache""" - return id(self._dataCacheParams), id(self.linegroup_areas) + def xmax(self): + """From config (if enabled) or the last `setData` call""" + cfgfit = self.config["fit"] + if cfgfit["use_limit"]: + return cfgfit["xmax"] + else: + return self._xmax0 - def _estimateLineGroupAreas(self): - ydata = self._ydata_without_background() - xenergy = self.xenergy - emin = xenergy.min() - emax = xenergy.max() - factor = self.GAUSS_SIGMA_TO_FWHM * numpy.sqrt(2 * numpy.pi) + def getLastTime(self): + return self._expotime0 - lineGroups = self._lineGroups - escapeLineGroups = self._escapeLineGroups - linegroup_areas = self._linegroup_areas - for i, (group, escgroup) in enumerate(zip(lineGroups, escapeLineGroups)): - if not escgroup: - escgroup = [[]] * len(group) - selected_energy = 0 - selected_rate = 0 - for (peakenergy, rate, _), esclines in zip(group, escgroup): - if peakenergy >= emin and peakenergy <= emax: - if rate > selected_rate: - selected_energy = peakenergy - for peakenergy, escrate, _ in esclines: - if peakenergy >= emin and peakenergy <= emax: - if rate * escrate > selected_rate: - selected_energy = peakenergy - if selected_energy: - height = ydata[(np.abs(xenergy - selected_energy)).argmin()] - fwhm = self._peak_fwhm(selected_energy) - linegroup_areas[i] = height * fwhm * factor # Gaussian - else: - linegroup_areas[i] = 0 # Fixed at zero + @property + def _dataCacheParams(self): + """Any change in these parameter will invalidate the cache""" + return self.xmin, self.xmax - def _total_peakgroup_profile(self, parameters, x, hypermet=None, fast=True): - """When providing parameters for more than one peak, the peak - profiles are added. + def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): + """Cache sorted and sliced view of the original XRF spectrum data""" + params = self._dataCacheParams + if xdata0 is None and ydata0 is None and ystd0 is None: + if self._lastDataCacheParams == params: + return # the cached data is still valid - :param array parameters: npeaks x nparams - :param array x: 1D array - :param int or None hypermet: - :param bool fast: ??? - :returns array: same shape as x - """ - if parameters.size == 0: - return numpy.zeros_like(x) - if hypermet is None: - hypermet = self._hypermet - if hypermet: - if fast: - return SpecfitFuns.fastahypermet(parameters, x, hypermet) - else: - return SpecfitFuns.ahypermet(parameters, x, hypermet) - else: - return SpecfitFuns.apvoigt(parameters, x) + # Original XRF spectrum + if xdata0 is None: + xdata0 = self.xdata0 + if xdata0 is None or not xdata0.size: + return 1 + if ydata0 is None: + ydata0 = self.ydata0 + if ydata0 is None or ydata0.size != xdata0.size: + return 1 + if ystd0 is None: + ystd0 = self.ystd0 + if ystd0 is None or ystd0.size != xdata0.size: + return 1 - def _peak_fwhm(self, energy): - """Calculate the FWHM of a peak in the energy domain""" - return numpy.sqrt( - self.noise * self.noise - + self.BAND_GAP - * energy - * self.fano - * self.GAUSS_SIGMA_TO_FWHM - * self.GAUSS_SIGMA_TO_FWHM - ) + # XRF spectrum view + selection = numpy.isfinite(ydata0) + xmin = self.xmin + if xmin is not None: + selection &= xdata0 >= xmin + xmax = self.xmax + if xmax is not None: + selection &= xdata0 <= xmax + if not selection.any(): + return 1 - def _getEmissionLines(self): - """Yields a list of emission lines for each group with total - rate of 1 and sorted by energy. + # Cache the original XRF spectrum and its view + idx = numpy.argsort(xdata0)[selection] + self._xdata = xdata0[idx] + self._ydata = ydata0[idx] + self._ystd = ystd0[idx] + self._xdata0 = xdata0 + self._ydata0 = ydata0 + self._ystd0 = ystd0 + self._lastDataCacheParams = params - :yields list: [[energy, rate, "name"], - [energy, rate, "name"], - ...] - """ - for group in sorted(self._emissionGroups()): - yield self._getGroupEmissionLines(*group) - def _getScatterLines(self): - """Yields a list for scattering lines for each source line. +class McaTheoryBackground(McaTheoryDataApi): + """Model for the background of an XRF spectrum""" - :yields list: [[energy, 1.0, "Scatter %03d"]] - """ - scatteringAngle = numpy.radians(self._scatteringAngle) - angleFactor = 1.0 - numpy.cos(scatteringAngle) - for i, (en_elastic, _) in enumerate(self._scatterLines()): - en_inelastic = en_elastic / (1.0 + (en_elastic / 511.0) * angleFactor) - name = "Scatter %03d" % i - yield [[en_elastic, 1.0, name]] - yield [[en_inelastic, 1.0, name]] + CONTINUUM_LIST = [ + None, + "Constant", + "Linear", + "Parabolic", + "Linear Polynomial", + "Exp. Polynomial", + ] - def _calcFluoRates(self): - """Fluorescence rate for each emission line of each element. - Rate means fluorescence intensity divided by primary intensity. + def __init__(self, **kw): + # Numerical background + self._numBkg = None + self._lastNumBkgCacheParams = None - :returns None or dict: - """ - if self._matrix: - if self._maxEnergy: - multilayer = list(self._multilayer()) - if not multilayer: - text = "Your matrix is not properly defined.\n" - text += "If you used the graphical interface,\n" - text += "Please check the MATRIX tab" - raise ValueError(text) + # Analytical background + self._continuum = None + self._continuumModel = None + self._lastContinuumCacheParams = None - emissiongroups = sorted(self._emissionGroups()) - energylist, weightlist, scatterlist = zip(*self._sourceLines()) - detector = self._detector - attenuatorlist = list(self._attenuators(detectorfilters=True)) - userattenuatorlist = list(self._userAttenuators()) - funnyfilters = list(self._attenuators(detectorfunnyfilters=True)) - filterlist = list(self._attenuators(beamfilters=True)) - alphain = self._angleIn - alphaout = self._angleOut - return Elements.getMultilayerFluorescence( - multilayer, - energylist, - layerList=None, - weightList=weightlist, - fulloutput=1, - attenuators=attenuatorlist, - alphain=alphain, - alphaout=alphaout, - elementsList=emissiongroups, - cascade=True, - detector=detector, - funnyfilters=funnyfilters, - beamfilters=filterlist, - forcepresent=1, - userattenuators=userattenuatorlist, - ) + super(McaTheoryBackground, self).__init__(**kw) + + def ynumbkg(self, xdata=None): + """Get the numerical background (as opposed to the analytical background)""" + ybkg = self._ynumbkg + if ybkg is None: + return ybkg + if xdata is not None: + try: + binterp = numpy.allclose(xdata, self.xdata) + except ValueError: + binterp = True + if binterp: + ybkg = numpy.interp(xdata, self.xdata, ybkg) + return ybkg + + def ycontinuum(self, xdata=None): + """Get the analytical background (as opposed to the numerical background)""" + if xdata is None: + xdata = self.xdata + model = self.continuumModel + if model is None: + if xdata is None: + return None else: - text = "Invalid energy for matrix configuration.\n" - text += "Please check your BEAM parameters." - raise ValueError(text) + return numpy.zeros(len(xdata)) else: - if self._nSourceLines > 1: - raise ValueError("Multiple energies require a matrix definition") + return model.evaluate_fullmodel(xdata=xdata) + + def ybackground(self, xdata=None): + """Get the total background""" + contbkg = self.ycontinuum(xdata=xdata) + numbkg = self.ynumbkg(xdata=xdata) + if numbkg is None: + return contbkg + else: + return contbkg + numbkg + + @property + def _ynumbkg(self): + self._refreshNumBkgCache() + return self._numBkg + + @property + def continuumModel(self): + self._refreshContinuumCache() + return self._continuumModel + + @property + def _numBkgCacheParams(self): + """Any change in these parameter will invalidate the cache""" + cfg = self.config["fit"] + params = [ + "stripflag", + "stripalgorithm", + "stripfilterwidth", + "stripanchorsflag", + "stripanchorslist", + ] + if cfg["stripalgorithm"] == 1: + params += ["snipwidth"] + else: + params += ["stripwidth", "stripconstant", "stripiterations"] + params = [cfg[p] for p in params] + params.append(id(self._lastDataCacheParams)) + return params + + def _refreshNumBkgCache(self): + """Cache numerical background""" + bkgparams = self._numBkgCacheParams + if self._lastNumBkgCacheParams == bkgparams: + return # the cached data is still valid + elif self.ydata is None: + self._numBkg = None + elif self.config["fit"]["stripflag"]: + signal = self._smooth(self.ydata) + anchorslist = list(self._anchorsIndices()) + if self.config["fit"]["stripalgorithm"] == 1: + self._numBkg = self._snip(signal, anchorslist) else: - return None + self._numBkg = self._strip(signal, anchorslist) + else: + self._numBkg = numpy.zeros_like(self.ydata) + self._lastNumBkgCacheParams = bkgparams - def _calcFluoRateCorrections(self): - """Higher-order interaction corrections on the fluorescence rates. - This will not be needed once fisx replaces the Elements module. - """ - fisxcfg = self.config["fisx"] = {} - if not FISX: - return - secondary = self.config["concentrations"].get("usemultilayersecondary", False) - if secondary: - corrections = FisxHelper.getFisxCorrectionFactorsFromFitConfiguration( - self.config, elementsFromMatrix=False - ) - fisxcfg["corrections"] = corrections - fisxcfg["secondary"] = secondary + @property + def _continuumCacheParams(self): + cfgfit = self.config["fit"] + params = [id(self._lastDataCacheParams), cfgfit["continuum"]] + if cfgfit["continuum"] == "Linear Polynomial": + params.append(cfgfit["linpolorder"]) + elif cfgfit["continuum"] == "Exp. Polynomial": + params.append(cfgfit["exppolorder"]) + return params - def _getGroupEmissionLines(self, Z, symb, groupname): - """Return a list of emission lines with total rate of 1 and - sorted by energy. + def _refreshContinuumCache(self): + contparams = self._continuumCacheParams + if self._lastContinuumCacheParams == contparams: + return # the cached data is still valid - :param int Z: atomic number - :param str symb: for example "Fe" - :param str groupname: for example "K" - :returns list: [[energy, rate, "name"], - [energy, rate, "name"], - ...] - """ - if self._fluoRates is None: - groups = Elements.Element[symb] + # Instantiate the model + continuum = self.config["fit"]["continuum"] + if continuum is None or self._ynumbkg is None: + model = None + elif continuum == "Constant": + model = LinearPolynomialModel(degree=0, maxiter=10) + elif continuum == "Linear": + model = LinearPolynomialModel(degree=1, maxiter=10) + elif continuum == "Parabolic": + model = LinearPolynomialModel(degree=2, maxiter=10) + elif continuum == "Linear Polynomial": + model = LinearPolynomialModel( + degree=self.config["fit"]["linpolorder"], maxiter=10 + ) + elif continuum == "Exp. Polynomial": + model = ExponentialPolynomialModel( + degree=self.config["fit"]["exppolorder"], maxiter=40 + ) else: - groups = self._fluoRates[0][symb] + raise ValueError("Unknown continuum {}".format(continuum)) + self._continuumModel = model - peaks = [] - lines = groups.get(groupname + " xrays", dict()) - if not lines: - return peaks - for line in lines: - lineinfo = groups[line] - if lineinfo["rate"] > 0.0: - peaks.append([lineinfo["energy"], lineinfo["rate"], line]) + # Estimate the polynomial coefficients by fitting the numerical background + if model is not None: + model.xdata = self.xpol + model.ydata = self.ynumbkg() + result = model.fit() + model.use_fit_result(result) - if self._fluoRates is None: - self._applyAttenuation(peaks, symb) + self._lastContinuumCacheParams = contparams - totalrate = sum(peak[1] for peak in peaks) - if not totalrate: - text = "Intensity of %s %s is zero\n" % (symb, groupname) - text += "Too high attenuation?" - raise ZeroDivisionError(text) - for peak in peaks: - peak[1] /= totalrate + @property + def xpol(self): + return self._channelsToXpol(self.xdata) - ethreshold = self.config["fit"]["deltaonepeak"] - return Elements._filterPeaks( - peaks, - ethreshold=ethreshold, - ithreshold=0.0005, - nthreshold=None, - keeptotalrate=True, - ) + def _channelsToXpol(self, x): + raise NotImplementedError - def _applyAttenuation(self, peaks, symb): - """Apply attenuation of primary and secondary beams. - No high-order interactions are taken into account. - Only 1 primary beam energy can be used. - """ - self._applyMatrixAttenuation(peaks, symb) - self._applyBeamFilterAttenuation(peaks, symb) - self._applyDetectorFilterAttenuation(peaks, symb) - self._applyFunnyFilterAttenuation(peaks, symb) - self._applyDetectorAttenuation(peaks, symb) - for peak in peaks: - if peak[1] < self.MAX_ATTENUATION: - peak[1] = 0 + @property + def continuum_coefficients(self): + model = self.continuumModel + if model is None: + return list() + else: + return model.parameters - def _iterLinearAttenuation(self, energies, **kw): - """Linear attenuation coefficients of matrix, detector, filters, ... + @continuum_coefficients.setter + def continuum_coefficients(self, values): + model = self.continuumModel + if model is not None: + model.parameters = values - :param list energies: - :param **kw: select the attenuator type to include - """ - for attenuator in self._attenuators(**kw): - formula, density, thickness, funnyfactor = attenuator - rhod = density * thickness - mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) - if len(energies) != 1 and len(mu["total"]) == 1: - mu = mu["total"] * len(energies) - else: - mu = mu["total"] - mulin = rhod * numpy.array(mu) - yield mulin, funnyfactor - def _applyBeamFilterAttenuation(self, peaks, symb): - energies = Elements.Element[symb]["buildparameters"]["energy"] - if not energies: - raise ValueError("Invalid excitation energy") +class McaTheory(McaTheoryBackground, McaTheoryLegacyApi, Model): + """Model for MCA data""" - for mulin, _ in self._iterLinearAttenuation(energies, beamfilter=True): - transmission = numpy.exp(-mulin) - for peak, frac in zip(peaks, transmission): - peak[1] *= frac + BAND_GAP = 0.00385 # keV, silicon + GAUSS_SIGMA_TO_FWHM = 2 * numpy.sqrt(2 * numpy.log(2)) # 2.3548 + FULL_ATTENUATION = 1.0e-300 # intensity assumed to be zero + SCATTER_ENERGY_THRESHOLD = 0.2 # keV - def _applyDetectorFilterAttenuation(self, peaks, symb): - energies = [peak[0] for peak in peaks] - for mulin, _ in self._iterLinearAttenuation(energies, detectorfilter=True): - transmission = numpy.exp(-mulin) - for peak, frac in zip(peaks, transmission): - peak[1] *= frac + def __init__(self, **kw): + # TODO: done for some initialization of SpecfitFuns? + SpecfitFuns.fastagauss([1.0, 10.0, 1.0], numpy.arange(10.0)) + self.useFisxEscape(flag=False, apply=False) - def _applyFunnyFilterAttenuation(self, peaks, symb): - firstfunnyfactor = None - energies = [peak[0] for peak in peaks] - for mulin, funnyfactor in self._iterLinearAttenuation( - energies, detectorfunnyfilter=True - ): - if (funnyfactor < 0.0) or (funnyfactor > 1.0): - text = ( - "Funny factor should be between 0.0 and 1.0., got %g" % funnyfactor - ) - raise ValueError(text) - transmission = numpy.exp(-mulin) - if firstfunnyfactor is None: - # only has to be multiplied once!!! - firstfunnyfactor = funnyfactor - transmission = funnyfactor * transmission + (1.0 - funnyfactor) + # XRF line groups + self._lineGroups = [] + self._linegroup_areas = [] + self._fluoRates = [] + self._escapeLineGroups = [] + self._lastAreasCacheParams = None + + super(McaTheory, self).__init__(**kw) + + def useFisxEscape(self, flag=None, apply=True): + """Make sure the model uses fisx to calculate the escape peaks + when possible. + """ + if flag and FISX: + if ConcentrationsTool.FisxHelper.xcom is None: + FisxHelper.xcom = xcom = FisxHelper.getElementsInstance() else: - if abs(firstfunnyfactor - funnyfactor) > 0.0001: - text = "All funny type attenuators must have same opening fraction" - raise ValueError(text) - for peak, frac in zip(peaks, transmission): - peak[1] *= frac + xcom = ConcentrationsTool.FisxHelper.xcom + if hasattr(xcom, "setEscapeCacheEnabled"): + xcom.setEscapeCacheEnabled(1) + self._useFisxEscape = True + else: + self._useFisxEscape = False + else: + self._useFisxEscape = False + if apply: + self.configure() - def _applyDetectorAttenuation(self, peaks, symb): - energies = [peak[0] for peak in peaks] - for mulin, _ in self._iterLinearAttenuation(energies, symb, detector=True): - attenuation = 1.0 - numpy.exp(-mulin) - for peak, frac in zip(peaks, attenuation): - peak[1] *= frac + def _initializeConfig(self): + super(McaTheory, self)._initializeConfig() + self._preCalculateParameterIndependent() + self._preCalculateParameterDependent() - def _applyMatrixAttenuation(self, peaks, symb): - matrix = self._matrix - if not matrix: - return - maxenergy = Elements.Element[symb]["buildparameters"]["energy"] - if not maxenergy: - raise ValueError("Invalid excitation energy") - formula, density, thickness = matrix[:3] - alphaIn = self._angleIn - alphaOut = self._angleOut + def _preCalculateParameterDependent(self): + """Pre-calculate things that depend on the fit parameters""" + pass - energies = [x[0] for x in peaks] + [maxenergy] - mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) - sinAlphaIn = numpy.sin(numpy.radians(alphaIn)) - sinAlphaOut = numpy.sin(numpy.radians(alphaOut)) - sinRatio = sinAlphaIn / sinAlphaOut - muSource = mu["total"][-1] - muFluo = numpy.array(mu["total"][:-1]) + def _preCalculateParameterIndependent(self): + """Pre-calculate things that do not depend on the fit parameters""" + self._preCalculateLineGroups() - transmission = 1.0 / (muSource + muFluo * sinRatio) - rhod = density * thickness - if rhod > 0.0 and abs(sinAlphaIn) > 0.0: - expterm = -(muSource / sinAlphaIn + muFluo / sinAlphaOut) * rhod - transmission *= 1.0 - numpy.exp(expterm) + def _preCalculateLineGroups(self): + """Calculate fluorescence and escape rates for emission and scatter line groups""" + self._fluoRates = self._calcFluoRates() + self._calcFluoRateCorrections() - for peak, frac in zip(peaks, transmission): - peak[1] *= frac + # Line groups: nested lists + # line group + # -> emission/scattering line + # [energy, rate, line name] + # This is a filtered and normalized form of `_fluoRates` + self._lineGroups = list(self._getEmissionLines()) + self._lineGroups.extend(self._getScatterLines()) + self._linegroup_areas = numpy.ones(len(self._lineGroups)) - def _applyUserAttenuators(self, peaks): - for userattenuator in self.config["userattenuators"]: - if self.config["userattenuators"][userattenuator]["use"]: - transmission = Elements.getTableTransmission( - self.config["userattenuators"][userattenuator], - [x[0] for x in peaks], - ) - for peak, frac in zip(peaks, transmission): - peak[1] *= frac + # Escape line groups: nested lists + # line group + # -> emission/scattering line + # -> escape line + # [energy, rate, escape name] + self._escapeLineGroups = [ + self._calcEscapePeaks([peak[0] for peak in peaks]) + for peaks in self._lineGroups + ] - def _calcEscapePeaks(self, energies): - """For each energy a list of escape peaks with total rate of 1 - and sorted by energy. + def _peak_profile_params( + self, selected_groups=None, hypermet=None, normalize_peakgroups=False + ): + """All parameters are in the energy domain (X-axis is energy, not channels)""" + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self.linegroup_areas + if selected_groups is not None: + if not isinstance(selected_groups, (list, tuple)): + selected_groups = [selected_groups] + lineGroups = [lineGroups[i] for i in selected_groups] + escapeLineGroups = [escapeLineGroups[i] for i in selected_groups] + linegroup_areas = [linegroup_areas[i] for i in selected_groups] + if hypermet is None: + hypermet = self._hypermet + if normalize_peakgroups: + linegroup_areas = numpy.ones(len(linegroup_areas)) - :param list energies: - :returns list: [[[energy, rate, "name"], - [energy, rate, "name"], - ...]] - """ - if not self.config["fit"]["escapeflag"]: - return [] - if self._useFisxEscape: - _logger.debug("Using fisx escape ratio's") - return self._calcFisxEscapeRatios(energies) + npeaks = sum(len(group) for group in lineGroups) + npeaks += sum(len(escgroup) for group in escapeLineGroups for escgroup in group) + if hypermet: + # area, position, fwhm, ST AreaR, ST SlopeR, LT AreaR, LT SlopeR, STEP HeightR + npeakparams = 8 else: - return self._calcPymcaEscapeRatios(energies) + # area, position, fwhm, eta + npeakparams = 4 + parameters = numpy.zeros((npeaks, npeakparams)) - def _calcFisxEscapeRatios(self, energies): - xcom = FisxHelper.xcom - detele = self.config["detector"]["detele"] - detector_composition = Elements.getMaterialMassFractions([detele], [1.0]) - ethreshold = self.config["detector"]["ethreshold"] - ithreshold = self.config["detector"]["ithreshold"] - nthreshold = self.config["detector"]["nthreshold"] - xcom.updateEscapeCache( - detector_composition, - energies, - energyThreshold=ethreshold, - intensityThreshold=ithreshold, - nThreshold=nthreshold, - ) + # Peak positions and areas + i = 0 + for group, escgroup, grouparea in zip( + lineGroups, escapeLineGroups, linegroup_areas + ): + if not escgroup: + escgroup = [[]] * len(group) + for (energy, rate, _), esclines in zip(group, escgroup): + peakarea = rate * grouparea + parameters[i, 0] = peakarea + parameters[i, 1] = energy + i += 1 + for escen, escrate, _ in esclines: + parameters[i, 0] = peakarea * escrate + parameters[i, 1] = escen + i += 1 - escape_peaks = [] - for energy in energies: - epeaks = xcom.getEscape( - detector_composition, - energy, - energyThreshold=ethreshold, - intensityThreshold=ithreshold, - nThreshold=nthreshold, - ) - epeaks = [ - [epeakinfo["energy"], epeakinfo["rate"], name[:-3].replace("_", " ")] - for name, epeakinfo in epeaks.items() - ] - epeaks = Elements._filterPeaks( - epeaks, - ethreshold=ethreshold, - ithreshold=ithreshold, - nthreshold=nthreshold, - absoluteithreshold=True, - keeptotalrate=False, - ) - escape_peaks.append(epeaks) - return escape_peaks + # Area parameters from channel to energy domain + parameters[:, 0] *= self.gain - def _calcPymcaEscapeRatios(self, energies): - escape_peaks = [] - detele = self.config["detector"]["detele"] - for energy in energies: - peaks = Elements.getEscape( - [detele, 1.0, 1.0], - energy, - ethreshold=self.config["detector"]["ethreshold"], - ithreshold=self.config["detector"]["ithreshold"], - nthreshold=self.config["detector"]["nthreshold"], - ) - escape_peaks.append(peaks) - return escape_peaks + # FWHM + parameters[:, 2] = self._peakFWHM(parameters[:, 1]) - @property - def xdata(self): - """Sorted and sliced view of xdata0""" - self._refreshDataCache() - return self._xdata + # Other peak shape parameters + if hypermet: + shapeparams = [ + self.st_arearatio, + self.st_sloperatio, + self.lt_arearatio, + self.lt_sloperatio, + self.step_heightratio, + ] + else: + shapeparams = [self.eta_factor] + for i, param in enumerate(shapeparams, 3): + parameters[:, i] = param - @property - def xenergy(self): - return self._channels_to_energy(self.xdata) + return parameters @property - def xpol(self): - return self._channels_to_xpol(self.xdata) - - def _channels_to_energy(self, x): - return self.zero + self.gain * x + def linegroup_areas(self): + self._refreshAreasCache() + return self._linegroup_areas - def _channels_to_xpol(self, x): - return self.zero + self.gain * (x - x.mean()) + def _refreshAreasCache(self): + params = self._areasCacheParams + if self._lastAreasCacheParams == params: + return # the cached data is still valid + self._estimateLineGroupAreas() + self._lastAreasCacheParams = params @property - def nchannels(self): - return len(self.xdata) + def _areasCacheParams(self): + """Any change in these parameter will invalidate the cache""" + return id(self._dataCacheParams), id(self.linegroup_areas) - @property - def ydata(self): - """Sorted and sliced view of ydata0""" - self._refreshDataCache() - return self._ydata + def _estimateLineGroupAreas(self): + ydata = self._ydata_without_background() + xenergy = self.xenergy + emin = xenergy.min() + emax = xenergy.max() + factor = self.GAUSS_SIGMA_TO_FWHM * numpy.sqrt(2 * numpy.pi) - @property - def ystd(self): - """Sorted and sliced view of ystd0""" - self._refreshDataCache() - return self._ystd + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self._linegroup_areas + for i, (group, escgroup) in enumerate(zip(lineGroups, escapeLineGroups)): + if not escgroup: + escgroup = [[]] * len(group) + selected_energy = 0 + selected_rate = 0 + for (peakenergy, rate, _), esclines in zip(group, escgroup): + if peakenergy >= emin and peakenergy <= emax: + if rate > selected_rate: + selected_energy = peakenergy + for peakenergy, escrate, _ in esclines: + if peakenergy >= emin and peakenergy <= emax: + if rate * escrate > selected_rate: + selected_energy = peakenergy + if selected_energy: + height = ydata[(numpy.abs(xenergy - selected_energy)).argmin()] + fwhm = self._peakFWHM(selected_energy) + linegroup_areas[i] = height * fwhm * factor # Gaussian + else: + linegroup_areas[i] = 0 # Fixed at zero - def ynumbkg(self, xdata=None): - ybkg = self._ynumbkg - if ybkg is None: - return ybkg - if xdata is not None: - try: - binterp = numpy.allclose(xdata, self.xdata) - except ValueError: - binterp = True - if binterp: - ybkg = numpy.interp(xdata, self.xdata, ybkg) - return ybkg + def _totalPeakGroupProfile(self, parameters, x, hypermet=None, fast=True): + """When providing parameters for more than one peak, the peak + profiles are added. - def ycontinuum(self, xdata=None): - """Get the analytical background (as opposed to the numerical background)""" - if xdata is None: - xdata = self.xdata - model = self.continuumModel - if model is None: - if xdata is None: - return None + :param array parameters: npeaks x nparams + :param array x: 1D array + :param int or None hypermet: + :param bool fast: ??? + :returns array: same shape as x + """ + if parameters.size == 0: + return numpy.zeros_like(x) + if hypermet is None: + hypermet = self._hypermet + if hypermet: + if fast: + return SpecfitFuns.fastahypermet(parameters, x, hypermet) else: - return numpy.zeros(len(xdata)) + return SpecfitFuns.ahypermet(parameters, x, hypermet) else: - return model.evaluate_fullmodel(xdata=xdata) + return SpecfitFuns.apvoigt(parameters, x) - def ybackground(self, xdata=None): - contbkg = self.ycontinuum(xdata=xdata) - numbkg = self.ynumbkg(xdata=xdata) - if numbkg is None: - return contbkg - else: - return contbkg + numbkg + def _peakFWHM(self, energy): + """Calculate the FWHM of a peak in the energy domain""" + return numpy.sqrt( + self.noise * self.noise + + self.BAND_GAP + * energy + * self.fano + * self.GAUSS_SIGMA_TO_FWHM + * self.GAUSS_SIGMA_TO_FWHM + ) + + def _getEmissionLines(self): + """Yields a list of emission lines for each group with total + rate of 1 and sorted by energy. + + :yields list: [[energy, rate, "name"], + [energy, rate, "name"], + ...] + """ + for group in sorted(self._emissionGroups()): + yield self._getGroupEmissionLines(*group) - @property - def _ynumbkg(self): - self._refreshNumBkgCache() - return self._numBkg + def _getScatterLines(self): + """Yields a list for scattering lines for each source line. - @property - def continuumModel(self): - self._refreshContinuumCache() - return self._continuumModel + :yields list: [[energy, 1.0, "Scatter %03d"]] + """ + scatteringAngle = numpy.radians(self._scatteringAngle) + angleFactor = 1.0 - numpy.cos(scatteringAngle) + for i, (en_elastic, _) in enumerate(self._scatterLines()): + en_inelastic = en_elastic / (1.0 + (en_elastic / 511.0) * angleFactor) + name = "Scatter %03d" % i + yield [[en_elastic, 1.0, name]] + yield [[en_inelastic, 1.0, name]] - @property - def xdata0(self): - return self._xdata0 + def _calcFluoRates(self): + """Fluorescence rate for each emission line of each element. + Rate means fluorescence intensity divided by primary intensity. - @property - def ydata0(self): - return self._ydata0 + :returns None or dict: + """ + if self._matrix: + if self._maxEnergy: + multilayer = list(self._multilayer()) + if not multilayer: + text = "Your matrix is not properly defined.\n" + text += "If you used the graphical interface,\n" + text += "Please check the MATRIX tab" + raise ValueError(text) - @property - def ystd0(self): - return self._ystd0 + emissiongroups = sorted(self._emissionGroups()) + energylist, weightlist, scatterlist = zip(*self._sourceLines()) + detector = self._detector + attenuatorlist = list(self._attenuators(detectorfilters=True)) + userattenuatorlist = list(self._userAttenuators()) + funnyfilters = list(self._attenuators(detectorfunnyfilters=True)) + filterlist = list(self._attenuators(beamfilters=True)) + alphain = self._angleIn + alphaout = self._angleOut + return Elements.getMultilayerFluorescence( + multilayer, + energylist, + layerList=None, + weightList=weightlist, + fulloutput=1, + attenuators=attenuatorlist, + alphain=alphain, + alphaout=alphaout, + elementsList=emissiongroups, + cascade=True, + detector=detector, + funnyfilters=funnyfilters, + beamfilters=filterlist, + forcepresent=1, + userattenuators=userattenuatorlist, + ) + else: + text = "Invalid energy for matrix configuration.\n" + text += "Please check your BEAM parameters." + raise ValueError(text) + else: + if self._nSourceLines > 1: + raise ValueError("Multiple energies require a matrix definition") + else: + return None - def setData(self, *var, **kw): + def _calcFluoRateCorrections(self): + """Higher-order interaction corrections on the fluorescence rates. + This will not be needed once fisx replaces the Elements module. """ - Method to update the data to be fitted. - It accepts several combinations of input arguments, the simplest to - take into account is: + fisxcfg = self.config["fisx"] = {} + if not FISX: + return + secondary = self.config["concentrations"].get("usemultilayersecondary", False) + if secondary: + corrections = FisxHelper.getFisxCorrectionFactorsFromFitConfiguration( + self.config, elementsFromMatrix=False + ) + fisxcfg["corrections"] = corrections + fisxcfg["secondary"] = secondary - setData(x, y, sigmay=None, xmin=None, xmax=None) + def _getGroupEmissionLines(self, Z, symb, groupname): + """Return a list of emission lines with total rate of 1 and + sorted by energy. - x corresponds to the spectrum channels - y corresponds to the spectrum counts - sigmay is the uncertainty associated to the counts. If not given, - Poisson statistics will be assumed. If the fit configuration - is set to no weight, it will not be used. - xmin and xmax define the limits to be considered for performing the fit. - If the fit configuration flag self.config['fit']['use_limit'] is - set, they will be ignored. If xmin and xmax are not given, the - whole given spectrum will be considered for fitting. - time (seconds) is the factor associated to the flux, only used when using - a strategy based on concentrations + :param int Z: atomic number + :param str symb: for example "Fe" + :param str groupname: for example "K" + :returns list: [[energy, rate, "name"], + [energy, rate, "name"], + ...] """ - if "y" in kw: - ydata0 = kw["y"] - elif len(var) > 1: - ydata0 = var[1] - elif len(var) == 1: - ydata0 = var[0] + if self._fluoRates is None: + groups = Elements.Element[symb] else: - ydata0 = None + groups = self._fluoRates[0][symb] - if ydata0 is None: - return 1 - else: - ydata0 = numpy.ravel(ydata0) + peaks = [] + lines = groups.get(groupname + " xrays", dict()) + if not lines: + return peaks + for line in lines: + lineinfo = groups[line] + if lineinfo["rate"] > 0.0: + peaks.append([lineinfo["energy"], lineinfo["rate"], line]) - if "x" in kw: - xdata0 = kw["x"] - elif len(var) > 1: - xdata0 = var[0] - else: - xdata0 = None + if self._fluoRates is None: + self._applyAttenuation(peaks, symb) - if xdata0 is None: - xdata0 = numpy.arange(len(ydata0)) - else: - xdata0 = numpy.ravel(xdata0) + totalrate = sum(peak[1] for peak in peaks) + if not totalrate: + text = "Intensity of %s %s is zero\n" % (symb, groupname) + text += "Too high attenuation?" + raise ZeroDivisionError(text) + for peak in peaks: + peak[1] /= totalrate - if "sigmay" in kw: - ystd0 = kw["sigmay"] - elif "stdy" in kw: - ystd0 = kw["stdy"] - elif len(var) > 2: - ystd0 = var[2] - else: - ystd0 = None + ethreshold = self.config["fit"]["deltaonepeak"] + return Elements._filterPeaks( + peaks, + ethreshold=ethreshold, + ithreshold=0.0005, + nthreshold=None, + keeptotalrate=True, + ) - if ystd0 is None: - # Poisson noise - valid = ydata0 > 0 - if valid.any(): - ystd0 = numpy.sqrt(abs(ydata0)) - ystd0[~valid] = ystd0[valid].min() - else: - ystd0 = numpy.ones_like(ydata0) - else: - ystd0 = numpy.ravel(ystd0) + def _applyAttenuation(self, peaks, symb): + """Apply attenuation of primary and secondary beams. + No high-order interactions are taken into account. + Only 1 primary beam energy can be used. + """ + self._applyMatrixAttenuation(peaks, symb) + self._applyBeamFilterAttenuation(peaks, symb) + self._applyDetectorFilterAttenuation(peaks, symb) + self._applyFunnyFilterAttenuation(peaks, symb) + self._applyDetectorAttenuation(peaks, symb) + for peak in peaks: + if peak[1] < self.FULL_ATTENUATION: + peak[1] = 0 - timeFactor = kw.get("time", None) - self._expotime0 = timeFactor - if timeFactor is None: - cfgfit = self.config["fit"] - if self.config["concentrations"].get("useautotime", False): - if not self.config["concentrations"]["usematrix"]: - msg = "Requested to use time from data but not present!!" - raise ValueError(msg) - elif self.config["concentrations"].get("useautotime", False): - self.config["concentrations"]["time"] = timeFactor + def _iterLinearAttenuation(self, energies, **kw): + """Linear attenuation coefficients of matrix, detector, filters, ... - self._xmin0 = kw.get("xmin", self.xmin) - self._xmax0 = kw.get("xmax", self.xmax) - return self._refreshDataCache(xdata0=xdata0, ydata0=ydata0, ystd0=ystd0) + :param list energies: + :param **kw: select the attenuator type to include + """ + for attenuator in self._attenuators(**kw): + formula, density, thickness, funnyfactor = attenuator + rhod = density * thickness + mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) + if len(energies) != 1 and len(mu["total"]) == 1: + mu = mu["total"] * len(energies) + else: + mu = mu["total"] + mulin = rhod * numpy.array(mu) + yield mulin, funnyfactor - @property - def xmin(self): - """From config (if enabled) or the last `setData` call""" - cfgfit = self.config["fit"] - if cfgfit["use_limit"]: - return cfgfit["xmin"] - else: - return self._xmin0 + def _applyBeamFilterAttenuation(self, peaks, symb): + energies = Elements.Element[symb]["buildparameters"]["energy"] + if not energies: + raise ValueError("Invalid excitation energy") - @property - def xmax(self): - """From config (if enabled) or the last `setData` call""" - cfgfit = self.config["fit"] - if cfgfit["use_limit"]: - return cfgfit["xmax"] - else: - return self._xmax0 + for mulin, _ in self._iterLinearAttenuation(energies, beamfilter=True): + transmission = numpy.exp(-mulin) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac - @property - def emin(self): - return self._channels_to_energy(self.xmin) + def _applyDetectorFilterAttenuation(self, peaks, symb): + energies = [peak[0] for peak in peaks] + for mulin, _ in self._iterLinearAttenuation(energies, detectorfilter=True): + transmission = numpy.exp(-mulin) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac - @property - def emax(self): - return self._channels_to_energy(self.xmax) + def _applyFunnyFilterAttenuation(self, peaks, symb): + firstfunnyfactor = None + energies = [peak[0] for peak in peaks] + for mulin, funnyfactor in self._iterLinearAttenuation( + energies, detectorfunnyfilter=True + ): + if (funnyfactor < 0.0) or (funnyfactor > 1.0): + text = ( + "Funny factor should be between 0.0 and 1.0., got %g" % funnyfactor + ) + raise ValueError(text) + transmission = numpy.exp(-mulin) + if firstfunnyfactor is None: + # only has to be multiplied once!!! + firstfunnyfactor = funnyfactor + transmission = funnyfactor * transmission + (1.0 - funnyfactor) + else: + if abs(firstfunnyfactor - funnyfactor) > 0.0001: + text = "All funny type attenuators must have same opening fraction" + raise ValueError(text) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac - def getLastTime(self): - return self._expotime0 + def _applyDetectorAttenuation(self, peaks, symb): + energies = [peak[0] for peak in peaks] + for mulin, _ in self._iterLinearAttenuation(energies, symb, detector=True): + attenuation = 1.0 - numpy.exp(-mulin) + for peak, frac in zip(peaks, attenuation): + peak[1] *= frac - @property - def _dataCacheParams(self): - """Any change in these parameter will invalidate the cache""" - return self.xmin, self.xmax + def _applyMatrixAttenuation(self, peaks, symb): + matrix = self._matrix + if not matrix: + return + maxenergy = Elements.Element[symb]["buildparameters"]["energy"] + if not maxenergy: + raise ValueError("Invalid excitation energy") + formula, density, thickness = matrix[:3] + alphaIn = self._angleIn + alphaOut = self._angleOut - def _refreshDataCache(self, xdata0=None, ydata0=None, ystd0=None): - """Cache sorted and sliced view of the original XRF spectrum data""" - params = self._dataCacheParams - if xdata0 is None and ydata0 is None and ystd0 is None: - if self._lastDataCacheParams == params: - return # the cached data is still valid + energies = [x[0] for x in peaks] + [maxenergy] + mu = Elements.getMaterialMassAttenuationCoefficients(formula, 1.0, energies) + sinAlphaIn = numpy.sin(numpy.radians(alphaIn)) + sinAlphaOut = numpy.sin(numpy.radians(alphaOut)) + sinRatio = sinAlphaIn / sinAlphaOut + muSource = mu["total"][-1] + muFluo = numpy.array(mu["total"][:-1]) - # Original XRF spectrum - if xdata0 is None: - xdata0 = self.xdata0 - if xdata0 is None or not xdata0.size: - return 1 - if ydata0 is None: - ydata0 = self.ydata0 - if ydata0 is None or ydata0.size != xdata0.size: - return 1 - if ystd0 is None: - ystd0 = self.ystd0 - if ystd0 is None or ystd0.size != xdata0.size: - return 1 + transmission = 1.0 / (muSource + muFluo * sinRatio) + rhod = density * thickness + if rhod > 0.0 and abs(sinAlphaIn) > 0.0: + expterm = -(muSource / sinAlphaIn + muFluo / sinAlphaOut) * rhod + transmission *= 1.0 - numpy.exp(expterm) - # XRF spectrum view - selection = numpy.isfinite(ydata0) - xmin = self.xmin - if xmin is not None: - selection &= xdata0 >= xmin - xmax = self.xmax - if xmax is not None: - selection &= xdata0 <= xmax - if not selection.any(): - return 1 + for peak, frac in zip(peaks, transmission): + peak[1] *= frac - # Cache the original XRF spectrum and its view - idx = numpy.argsort(xdata0)[selection] - self._xdata = xdata0[idx] - self._ydata = ydata0[idx] - self._ystd = ystd0[idx] - self._xdata0 = xdata0 - self._ydata0 = ydata0 - self._ystd0 = ystd0 - self._lastDataCacheParams = params + def _applyUserAttenuators(self, peaks): + for userattenuator in self.config["userattenuators"]: + if self.config["userattenuators"][userattenuator]["use"]: + transmission = Elements.getTableTransmission( + self.config["userattenuators"][userattenuator], + [x[0] for x in peaks], + ) + for peak, frac in zip(peaks, transmission): + peak[1] *= frac - @property - def _numBkgCacheParams(self): - """Any change in these parameter will invalidate the cache""" - cfg = self.config["fit"] - params = [ - "stripflag", - "stripalgorithm", - "stripfilterwidth", - "stripanchorsflag", - "stripanchorslist", - ] - if cfg["stripalgorithm"] == 1: - params += ["snipwidth"] - else: - params += ["stripwidth", "stripconstant", "stripiterations"] - params = [cfg[p] for p in params] - params.append(id(self._lastDataCacheParams)) - return params + def _calcEscapePeaks(self, energies): + """For each energy a list of escape peaks with total rate of 1 + and sorted by energy. - def _refreshNumBkgCache(self): - """Cache numerical background""" - bkgparams = self._numBkgCacheParams - if self._lastNumBkgCacheParams == bkgparams: - return # the cached data is still valid - elif self.ydata is None: - self._numBkg = None - elif self.config["fit"]["stripflag"]: - signal = self._smooth(self.ydata) - anchorslist = list(self._anchorsIndices()) - if self.config["fit"]["stripalgorithm"] == 1: - self._numBkg = self._snip(signal, anchorslist) - else: - self._numBkg = self._strip(signal, anchorslist) + :param list energies: + :returns list: [[[energy, rate, "name"], + [energy, rate, "name"], + ...]] + """ + if not self.config["fit"]["escapeflag"]: + return [] + if self._useFisxEscape: + _logger.debug("Using fisx escape ratio's") + return self._calcFisxEscapeRatios(energies) else: - self._numBkg = numpy.zeros_like(self.ydata) - self._lastNumBkgCacheParams = bkgparams - - @property - def _continuumCacheParams(self): - cfgfit = self.config["fit"] - params = [id(self._lastDataCacheParams), cfgfit["continuum"]] - if cfgfit["continuum"] == "Linear Polynomial": - params.append(cfgfit["linpolorder"]) - elif cfgfit["continuum"] == "Exp. Polynomial": - params.append(cfgfit["exppolorder"]) - return params + return self._calcPymcaEscapeRatios(energies) - def _refreshContinuumCache(self): - contparams = self._continuumCacheParams - if self._lastContinuumCacheParams == contparams: - return # the cached data is still valid + def _calcFisxEscapeRatios(self, energies): + xcom = FisxHelper.xcom + detele = self.config["detector"]["detele"] + detector_composition = Elements.getMaterialMassFractions([detele], [1.0]) + ethreshold = self.config["detector"]["ethreshold"] + ithreshold = self.config["detector"]["ithreshold"] + nthreshold = self.config["detector"]["nthreshold"] + xcom.updateEscapeCache( + detector_composition, + energies, + energyThreshold=ethreshold, + intensityThreshold=ithreshold, + nThreshold=nthreshold, + ) - # Instantiate the model - continuum = self.config["fit"]["continuum"] - if continuum is None or self._ynumbkg is None: - model = None - elif continuum == "Constant": - model = LinearPolynomialModel(degree=0, maxiter=10) - elif continuum == "Linear": - model = LinearPolynomialModel(degree=1, maxiter=10) - elif continuum == "Parabolic": - model = LinearPolynomialModel(degree=2, maxiter=10) - elif continuum == "Linear Polynomial": - model = LinearPolynomialModel( - degree=self.config["fit"]["linpolorder"], maxiter=10 + escape_peaks = [] + for energy in energies: + epeaks = xcom.getEscape( + detector_composition, + energy, + energyThreshold=ethreshold, + intensityThreshold=ithreshold, + nThreshold=nthreshold, ) - elif continuum == "Exp. Polynomial": - model = ExponentialPolynomialModel( - degree=self.config["fit"]["exppolorder"], maxiter=40 + epeaks = [ + [epeakinfo["energy"], epeakinfo["rate"], name[:-3].replace("_", " ")] + for name, epeakinfo in epeaks.items() + ] + epeaks = Elements._filterPeaks( + epeaks, + ethreshold=ethreshold, + ithreshold=ithreshold, + nthreshold=nthreshold, + absoluteithreshold=True, + keeptotalrate=False, ) - else: - raise ValueError("Unknown continuum {}".format(continuum)) - self._continuumModel = model - - # Estimate the polynomial coefficients by fitting the numerical background - if model is not None: - x = self.xdata - model.xdata = self.xpol - model.ydata = self.ynumbkg() - result = model.fit() - model.use_fit_result(result) + escape_peaks.append(epeaks) + return escape_peaks - self._lastContinuumCacheParams = contparams + def _calcPymcaEscapeRatios(self, energies): + escape_peaks = [] + detele = self.config["detector"]["detele"] + for energy in energies: + peaks = Elements.getEscape( + [detele, 1.0, 1.0], + energy, + ethreshold=self.config["detector"]["ethreshold"], + ithreshold=self.config["detector"]["ithreshold"], + nthreshold=self.config["detector"]["nthreshold"], + ) + escape_peaks.append(peaks) + return escape_peaks @property - def continuum_coefficients(self): - model = self.continuumModel - if model is None: - return list() - else: - return model.parameters + def xenergy(self): + return self._channelsToEnergy(self.xdata) - @continuum_coefficients.setter - def continuum_coefficients(self, values): - model = self.continuumModel - if model is not None: - model.parameters = values + def _channelsToEnergy(self, x): + return self.zero + self.gain * x + + def _channelsToXpol(self, x): + return self.zero + self.gain * (x - x.mean()) @property def linpol_coefficients(self): @@ -1757,26 +1776,29 @@ def evaluate_fitmodel(self, xdata=None): def mcatheory( self, parameters, xdata=None, hypermet=None, continuum=None, summing=None ): + """The parameters are the raw peak parameters, not the fit parameters""" # Evaluation domain if xdata is None: xdata = self.xdata - energy = self._channels_to_energy(xdata) + energy = self._channelsToEnergy(xdata) # Emission lines, scatter peaks and escape peaks - y = self._total_peakgroup_profile(parameters, energy, hypermet=hypermet) + y = self._totalPeakGroupProfile(parameters, energy, hypermet=hypermet) # Analytical background if continuum or continuum is None: model = self.continuumModel if model is not None: - xpol = self._channels_to_xpol(xdata) + xpol = self._channelsToXpol(xdata) y += model.evaluate_fullmodel(xdata=xpol) # Pile-up if summing or summing is None: pileupfactor = self.sum if pileupfactor: - y *= pileupfactor * SpecfitFuns.pileup(y, min(xdata), zero, gain) + y *= pileupfactor * SpecfitFuns.pileup( + y, min(xdata), self.zero, self.gain + ) return y @@ -1799,9 +1821,9 @@ def _fit_to_ydata(self, yfit, xdata=None): """The numerical background is not included in the fit model""" ybkg = self.ynumbkg(xdata=xdata) if ybkg is None: - return ydata + return yfit else: - return ydata + ybkg + return yfit + ybkg def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters @@ -1811,7 +1833,7 @@ def linear_derivatives_fitmodel(self, xdata=None): """ if xdata is None: xdata = self.xdata - energy = self._channels_to_energy(xdata) + energy = self._channelsToEnergy(xdata) raise NotImplementedError def derivative_fitmodel(self, param_idx, xdata=None): @@ -1822,6 +1844,7 @@ def derivative_fitmodel(self, param_idx, xdata=None): :returns array: nxdata """ name, pgroupi = self._parameter_name_from_index(param_idx) + hypermet = self.hypermet if name == "zero": raise NotImplementedError elif name == "gain": From 9b77af6e0c6da3162b82162fe8c9318a4b92ef8e Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 7 May 2021 15:22:54 +0200 Subject: [PATCH 24/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 20 ++++++++++---------- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 4 ---- PyMca5/tests/FitModelTest.py | 2 +- PyMca5/tests/SimpleModel.py | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index cc11f6220..9070c7c6a 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -126,8 +126,8 @@ def yfitmodel(self): return self.evaluate_fitmodel() @property - def nchannels(self): - raise AttributeError from NotImplementedError + def ndata(self): + return len(self.xdata) @property def fit_parameters(self): @@ -428,9 +428,9 @@ def linear_fit(self, full_output=False): if self.niter_non_leastsquares: keep = self.linear_parameters try: - b = self.yfitdata # nchannels + b = self.yfitdata # ndata for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives_fitmodel().T # nchannels, nparams + A = self.linear_derivatives_fitmodel().T # ndata, nparams result = lstsq(A, b.copy(), digested_output=full_output) if self.niter_non_leastsquares: self.linear_fit_parameters = result[0] @@ -663,12 +663,12 @@ def nmodels(self): return len(self._models) @property - def nchannels(self): + def ndata(self): nmodels = self.nmodels if nmodels == 0: return 0 else: - return sum([m.nchannels for m in self._models]) + return sum([m.ndata for m in self._models]) @property def xdata(self): @@ -722,7 +722,7 @@ def _set_data(self, attr, values): :param str attr: :param array values: """ - if len(values) != self.nchannels: + if len(values) != self.ndata: raise ValueError("Not the expected number of channels") for idx, model in self._iter_models(values): setattr(model, attr, values[idx]) @@ -998,14 +998,14 @@ def _generate_idx_channels(self, nconcat, stride=None): """Yield slice of the concatenated data for each model. The concatenated data could be sliced as `xdata[::stride]`. """ - nchannels = [m.nchannels for m in self._models] + ndata = [m.ndata for m in self._models] if not stride: - stride, remain = divmod(sum(nchannels), nconcat) + stride, remain = divmod(sum(ndata), nconcat) stride += remain > 0 start = 0 offset = 0 i = 0 - for n in nchannels: + for n in ndata: # Index of model in concatenated xdata due to slicing stop = start + n lst = list(range(start + offset, stop, stride)) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 7ce55314a..d1b60604a 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -487,10 +487,6 @@ def xdata(self): self._refreshDataCache() return self._xdata - @property - def nchannels(self): - return len(self.xdata) - @property def ydata(self): """Sorted and sliced view of ydata0""" diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index 4a2043dc1..b63e24449 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -142,7 +142,7 @@ def _validate_model(self, model): assert not model.excluded_parameters assert not model.included_parameters - assert model.nchannels == len(model.xdata) + assert model.ndata == len(model.xdata) assert model.nparameters == len(model.parameters) assert model.nlinear_parameters == len(model.linear_parameters) diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 4ebfae85d..902a44dd0 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -181,7 +181,7 @@ def ystd(self, values): self.ystd_raw[self.idx_channels] = values @property - def nchannels(self): + def ndata(self): return self.xmax - self.xmin @property From 75b3c4be1bb68e3c49d6271fba3ad5585c4a1dc2 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 7 May 2021 15:37:20 +0200 Subject: [PATCH 25/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 104 ++++++++++++++---------------- 1 file changed, 49 insertions(+), 55 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 9070c7c6a..5e053f4ac 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -40,54 +40,48 @@ from PyMca5.PyMcaMath.fitting import Gefit -def enable_caching(method): - @functools.wraps(method) - def cache_wrapper(self, *args, **kw): - with self.caching_context(): - return method(self, *args, **kw) - - return cache_wrapper - - -def memoize(method): - @functools.wraps(method) - def cache_wrapper(self): - if self.caching_enabled: - name = method.__qualname__ - if name in self._cache: - return self._cache[name] - else: - r = self._cache[name] = method(self) - return r - - return cache_wrapper - - -def memoize_property(method): - return property(memoize(method)) - - -class Cashed(object): +class Cached(object): def __init__(self): - self._cache = None + self._cache = dict() @contextmanager - def caching_context(self): - reset = self._cache is None + def cachingContext(self, field): + reset = not self.cachingEnabled(field) if reset: - self._cache = {} + self._cache[field] = dict() try: yield finally: if reset: - self._cache = None + del self._cache[field] - @property - def caching_enabled(self): - return self._cache is not None + def cachingEnabled(self, field): + return field in self._cache + def getCache(self, field, *subfields): + if field in self._cache: + ret = self._cache[field] + for field in subfields: + if field in ret: + ret = ret[field] + else: + ret = ret[field] = dict() + return ret + else: + return None -class Model(Cashed): + @staticmethod + def enableCaching(field): + def decorator(method): + @functools.wraps(method) + def cache_wrapper(self, *args, **kw): + with self.cachingContext(field): + return method(self, *args, **kw) + return cache_wrapper + return decorator + + +class Model(Cached): """Evaluation and derivatives of a model to be used in least-squares fitting.""" def __init__(self): @@ -357,8 +351,10 @@ def _parameter_groups(self, linear_only=False): :param bool linear_only: :returns iterable(str, int): group name, nb. parameters in the group """ - if self.caching_enabled: - cache = self._cache.setdefault("parameter_groups", {}) + cache = self.getCache("fit", "parameter_groups") + if cache is None: + it = self._iter_parameter_groups(linear_only=linear_only) + else: a = self.included_parameters b = self.excluded_parameters if a is not None: @@ -371,8 +367,6 @@ def _parameter_groups(self, linear_only=False): it = cache[key] = list( self._iter_parameter_groups(linear_only=linear_only) ) - else: - it = self._iter_parameter_groups(linear_only=linear_only) return it def _iter_parameter_names(self, linear_only=False): @@ -419,7 +413,7 @@ def fit(self, full_output=False): else: return self.nonlinear_fit(full_output=full_output) - @enable_caching + @Cached.enableCaching("fit") def linear_fit(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters @@ -444,7 +438,7 @@ def linear_fit(self, full_output=False): "uncertainties": self._fit_to_uncertainties(result[1]), } - @enable_caching + @Cached.enableCaching("fit") def nonlinear_fit(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters @@ -828,15 +822,15 @@ def _parameter_model_index(self, idx, linear_only=False): :param int idx: :returns iterable(tuple): model index, parameter index in this model """ - if self.caching_enabled: - cache = self._cache.setdefault("parameter_model_index", {}) + cache = self.getCache("fit", "parameter_model_index") + if cache is None: + it = self._iter_parameter_index(idx, linear_only=linear_only) + else: it = cache.get(idx) if it is None: it = cache[idx] = list( self._iter_parameter_index(idx, linear_only=linear_only) - ) - else: - it = self._iter_parameter_index(idx, linear_only=linear_only) + ) return it def _iter_parameter_index(self, idx, linear_only=False): @@ -985,14 +979,14 @@ def _idx_channels(self, nconcat): :param int nconcat: :returns list(slice): """ - if self.caching_enabled: - cache = self._cache.setdefault("idx_channels", {}) + cache = self.getCache("fit", "idx_channels") + if cache is None: + return list(self._generate_idx_channels(nconcat)) + else: if nconcat != cache.get("nconcat"): cache["idx"] = list(self._generate_idx_channels(nconcat)) cache["nconcat"] = nconcat return cache["idx"] - else: - return list(self._generate_idx_channels(nconcat)) def _generate_idx_channels(self, nconcat, stride=None): """Yield slice of the concatenated data for each model. @@ -1022,10 +1016,10 @@ def _generate_idx_channels(self, nconcat, stride=None): yield idx @contextmanager - def caching_context(self): + def cachingContext(self, field): with ExitStack() as stack: - ctx = super(ConcatModel, self).caching_context() + ctx = super(ConcatModel, self).cachingContext(field) stack.enter_context(ctx) for m in self._models: - stack.enter_context(m.caching_context()) + stack.enter_context(m.cachingContext(field)) yield From 43afc3bfed6ff58615efab5bdc1ca09c00e867bb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 7 May 2021 15:39:06 +0200 Subject: [PATCH 26/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 40 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 5e053f4ac..3cceb7671 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -45,39 +45,41 @@ def __init__(self): self._cache = dict() @contextmanager - def cachingContext(self, field): - reset = not self.cachingEnabled(field) + def cachingContext(self, cachename): + reset = not self.cachingEnabled(cachename) if reset: - self._cache[field] = dict() + self._cache[cachename] = dict() try: yield finally: if reset: - del self._cache[field] + del self._cache[cachename] - def cachingEnabled(self, field): - return field in self._cache + def cachingEnabled(self, cachename): + return cachename in self._cache - def getCache(self, field, *subfields): - if field in self._cache: - ret = self._cache[field] - for field in subfields: - if field in ret: - ret = ret[field] + def getCache(self, cachename, *subnames): + if cachename in self._cache: + ret = self._cache[cachename] + for cachename in subnames: + if cachename in ret: + ret = ret[cachename] else: - ret = ret[field] = dict() + ret = ret[cachename] = dict() return ret else: return None @staticmethod - def enableCaching(field): + def enableCaching(cachename): def decorator(method): @functools.wraps(method) def cache_wrapper(self, *args, **kw): - with self.cachingContext(field): + with self.cachingContext(cachename): return method(self, *args, **kw) + return cache_wrapper + return decorator @@ -830,7 +832,7 @@ def _parameter_model_index(self, idx, linear_only=False): if it is None: it = cache[idx] = list( self._iter_parameter_index(idx, linear_only=linear_only) - ) + ) return it def _iter_parameter_index(self, idx, linear_only=False): @@ -1016,10 +1018,10 @@ def _generate_idx_channels(self, nconcat, stride=None): yield idx @contextmanager - def cachingContext(self, field): + def cachingContext(self, cachename): with ExitStack() as stack: - ctx = super(ConcatModel, self).cachingContext(field) + ctx = super(ConcatModel, self).cachingContext(cachename) stack.enter_context(ctx) for m in self._models: - stack.enter_context(m.cachingContext(field)) + stack.enter_context(m.cachingContext(cachename)) yield From fec57491f129d7eef0f5efba96e97a65c92b12bf Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 7 May 2021 19:10:11 +0200 Subject: [PATCH 27/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 51 ++- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 315 +++++++++++++------ PyMca5/tests/XrfTest.py | 7 +- 3 files changed, 267 insertions(+), 106 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 3cceb7671..f6e99f1d7 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -421,37 +421,50 @@ def linear_fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - if self.niter_non_leastsquares: - keep = self.linear_parameters - try: + with self._linear_fit_context(): b = self.yfitdata # ndata for i in range(max(self.niter_non_leastsquares, 1)): A = self.linear_derivatives_fitmodel().T # ndata, nparams - result = lstsq(A, b.copy(), digested_output=full_output) + result = lstsq( + A, + b.copy(), + uncertainties=True, + covariances=False, + digested_output=False, + ) if self.niter_non_leastsquares: - self.linear_fit_parameters = result[0] + if full_output: + self.linear_fit_parameters = result[0] + else: + self.linear_fit_parameters = result["parameters"] self.non_leastsquares_increment() - finally: - if self.niter_non_leastsquares: - self.linear_parameters = keep return { "linear": True, "parameters": self._fit_to_parameters(result[0]), "uncertainties": self._fit_to_uncertainties(result[1]), } + @contextmanager + def _linear_fit_context(self): + if self.niter_non_leastsquares: + keep = self.linear_parameters + try: + yield + finally: + if self.niter_non_leastsquares: + self.linear_parameters = keep + @Cached.enableCaching("fit") def nonlinear_fit(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters :returns dict: """ - keep = self.parameters - constraints = self.constraints - xdata = self.xdata - ydata = self.yfitdata - ystd = self.yfitstd - try: + with self._nonlinear_fit_context(): + constraints = self.constraints + xdata = self.xdata + ydata = self.yfitdata + ystd = self.yfitstd for i in range(max(self.niter_non_leastsquares, 1)): result = Gefit.LeastSquaresFit( self._evaluate_fitmodel, @@ -469,8 +482,6 @@ def nonlinear_fit(self, full_output=False): if self.niter_non_leastsquares: self.fit_parameters = result[0] self.non_leastsquares_increment() - finally: - self.parameters = keep ret = { "linear": False, "parameters": self._fit_to_parameters(result[0]), @@ -482,6 +493,14 @@ def nonlinear_fit(self, full_output=False): ret["lastdeltachi"] = result[4] return ret + @contextmanager + def _nonlinear_fit_context(self): + keep = self.parameters + try: + yield + finally: + self.parameters = keep + def _ydata_to_fit(self, ydata, xdata=None): return ydata diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index d1b60604a..3a9542fcc 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -36,6 +36,7 @@ import copy import logging import warnings +from contextlib import contextmanager import numpy from PyMca5 import PyMcaDataDir @@ -88,6 +89,7 @@ def __init__(self, initdict=None, filelist=None, **kw): self._overwriteConfig(**kw) self.attflag = kw.get("attenuatorsflag", 1) self.configure() + super(McaTheoryConfigApi, self).__init__() def _overwriteConfig(self, **kw): if "config" in kw: @@ -460,6 +462,14 @@ def getStartingConfiguration(self): """ return self.configure() + def estimate(self): + warnings.warn("McaTheory.estimate is deprecated (does nothing)", FutureWarning) + + def specfitestimate(self, x, y, z, xscaling=1.0, yscaling=1.0): + warnings.warn( + "McaTheory.specfitestimate is deprecated (does nothing)", FutureWarning + ) + class McaTheoryDataApi(McaTheoryConfigApi): """Add API for handling a single XRF spectrum (MCA data)""" @@ -684,12 +694,21 @@ def __init__(self, **kw): super(McaTheoryBackground, self).__init__(**kw) + @property + def hasNumBkg(self): + return self._ynumbkg is not None + def ynumbkg(self, xdata=None): """Get the numerical background (as opposed to the analytical background)""" ybkg = self._ynumbkg if ybkg is None: - return ybkg + if xdata is None: + return numpy.zeros(self.ndata) + else: + return numpy.zeros(len(xdata)) if xdata is not None: + # The numerical background is calculated on self.xdata + # so we need to interpolate on xdata try: binterp = numpy.allclose(xdata, self.xdata) except ValueError: @@ -698,27 +717,27 @@ def ynumbkg(self, xdata=None): ybkg = numpy.interp(xdata, self.xdata, ybkg) return ybkg + @property + def hasContinuum(self): + return self.continuumModel is not None + def ycontinuum(self, xdata=None): """Get the analytical background (as opposed to the numerical background)""" - if xdata is None: - xdata = self.xdata model = self.continuumModel if model is None: if xdata is None: - return None + return numpy.zeros(self.ndata) else: return numpy.zeros(len(xdata)) - else: - return model.evaluate_fullmodel(xdata=xdata) + return model.evaluate_fullmodel(xdata=xdata) + + @property + def hasBackground(self): + return self.hasNumBkg or self.hasContinuum def ybackground(self, xdata=None): """Get the total background""" - contbkg = self.ycontinuum(xdata=xdata) - numbkg = self.ynumbkg(xdata=xdata) - if numbkg is None: - return contbkg - else: - return contbkg + numbkg + return self.ycontinuum(xdata=xdata) + self.ynumbkg(xdata=xdata) @property def _ynumbkg(self): @@ -767,13 +786,39 @@ def _refreshNumBkgCache(self): self._numBkg = numpy.zeros_like(self.ydata) self._lastNumBkgCacheParams = bkgparams + @property + def continuum(self): + return self.config["fit"]["continuum"] + + @continuum.setter + def continuum(self, value): + self.config["fit"]["continuum_name"] = self.CONTINUUM_LIST[value] + self.config["fit"]["continuum"] = value + + @property + def continuum_name(self): + try: + return self.config["fit"]["continuum_name"] + except AttributeError: + return self.CONTINUUM_LIST[value] + + @continuum_name.setter + def continuum_name(self, value): + self.config["fit"]["continuum"] = self.CONTINUUM_LIST.index(name) + self.config["fit"]["continuum_name"] = name + + def _initializeConfig(self): + super(McaTheoryBackground, self)._initializeConfig() + self.continuum = self.continuum # verify continuum name and index + @property def _continuumCacheParams(self): cfgfit = self.config["fit"] - params = [id(self._lastDataCacheParams), cfgfit["continuum"]] - if cfgfit["continuum"] == "Linear Polynomial": + continuum = self.continuum_name + params = [id(self._lastDataCacheParams), continuum] + if continuum == "Linear Polynomial": params.append(cfgfit["linpolorder"]) - elif cfgfit["continuum"] == "Exp. Polynomial": + elif continuum == "Exp. Polynomial": params.append(cfgfit["exppolorder"]) return params @@ -783,8 +828,8 @@ def _refreshContinuumCache(self): return # the cached data is still valid # Instantiate the model - continuum = self.config["fit"]["continuum"] - if continuum is None or self._ynumbkg is None: + continuum = self.continuum_name + if continuum is None: model = None elif continuum == "Constant": model = LinearPolynomialModel(degree=0, maxiter=10) @@ -849,8 +894,9 @@ def __init__(self, **kw): self.useFisxEscape(flag=False, apply=False) # XRF line groups + self._nLineGroups = 0 self._lineGroups = [] - self._linegroup_areas = [] + self._lineGroupAreas = [] self._fluoRates = [] self._escapeLineGroups = [] self._lastAreasCacheParams = None @@ -901,7 +947,8 @@ def _preCalculateLineGroups(self): # This is a filtered and normalized form of `_fluoRates` self._lineGroups = list(self._getEmissionLines()) self._lineGroups.extend(self._getScatterLines()) - self._linegroup_areas = numpy.ones(len(self._lineGroups)) + self._nLineGroups = len(self._lineGroups) + self._lineGroupAreas = numpy.ones(self._nLineGroups) # Escape line groups: nested lists # line group @@ -913,10 +960,12 @@ def _preCalculateLineGroups(self): for peaks in self._lineGroups ] - def _peak_profile_params( - self, selected_groups=None, hypermet=None, normalize_peakgroups=False + def _peakProfileParams( + self, hypermet=None, selected_groups=None, normalized_peakgroups=False ): - """All parameters are in the energy domain (X-axis is energy, not channels)""" + """Raw parameters of emission/scatter/escape peaks. All parameters are + defined in the energy domain (X-axis is energy, not channels). + """ lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups linegroup_areas = self.linegroup_areas @@ -928,7 +977,7 @@ def _peak_profile_params( linegroup_areas = [linegroup_areas[i] for i in selected_groups] if hypermet is None: hypermet = self._hypermet - if normalize_peakgroups: + if normalized_peakgroups: linegroup_areas = numpy.ones(len(linegroup_areas)) npeaks = sum(len(group) for group in lineGroups) @@ -940,6 +989,8 @@ def _peak_profile_params( # area, position, fwhm, eta npeakparams = 4 parameters = numpy.zeros((npeaks, npeakparams)) + if not parameters.size: + return parameters # Peak positions and areas i = 0 @@ -983,7 +1034,7 @@ def _peak_profile_params( @property def linegroup_areas(self): self._refreshAreasCache() - return self._linegroup_areas + return self._lineGroupAreas def _refreshAreasCache(self): params = self._areasCacheParams @@ -995,10 +1046,13 @@ def _refreshAreasCache(self): @property def _areasCacheParams(self): """Any change in these parameter will invalidate the cache""" - return id(self._dataCacheParams), id(self.linegroup_areas) + return id(self._dataCacheParams), id(self._lineGroupAreas) def _estimateLineGroupAreas(self): - ydata = self._ydata_without_background() + if self.hasBackground: + ydata = self.ydata - self.ybackground() + else: + ydata = self.ydata xenergy = self.xenergy emin = xenergy.min() emax = xenergy.max() @@ -1006,7 +1060,7 @@ def _estimateLineGroupAreas(self): lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups - linegroup_areas = self._linegroup_areas + linegroup_areas = self._lineGroupAreas for i, (group, escgroup) in enumerate(zip(lineGroups, escapeLineGroups)): if not escgroup: escgroup = [[]] * len(group) @@ -1027,27 +1081,45 @@ def _estimateLineGroupAreas(self): else: linegroup_areas[i] = 0 # Fixed at zero - def _totalPeakGroupProfile(self, parameters, x, hypermet=None, fast=True): - """When providing parameters for more than one peak, the peak - profiles are added. + def _evaluatePeakProfiles( + self, + xdata=None, + hypermet=None, + fast=True, + selected_groups=None, + normalized_peakgroups=False, + ): + """Summed peak profiles of emission/scatter/escape peaks - :param array parameters: npeaks x nparams - :param array x: 1D array + :param array xdata: 1D array :param int or None hypermet: :param bool fast: ??? + :param bool selected_groups: + :param bool normalized_peakgroups: :returns array: same shape as x """ + parameters = self._peakProfileParams( + hypermet=hypermet, + selected_groups=selected_groups, + normalized_peakgroups=normalized_peakgroups, + ) if parameters.size == 0: - return numpy.zeros_like(x) + if xdata is None: + return numpy.zeros(self.ndata) + else: + return numpy.zeros(len(xdata)) + if xdata is None: + xdata = self.xdata + energy = self._channelsToEnergy(xdata) if hypermet is None: hypermet = self._hypermet if hypermet: if fast: - return SpecfitFuns.fastahypermet(parameters, x, hypermet) + return SpecfitFuns.fastahypermet(parameters, energy, hypermet) else: - return SpecfitFuns.ahypermet(parameters, x, hypermet) + return SpecfitFuns.ahypermet(parameters, energy, hypermet) else: - return SpecfitFuns.apvoigt(parameters, x) + return SpecfitFuns.apvoigt(parameters, energy) def _peakFWHM(self, energy): """Calculate the FWHM of a peak in the energy domain""" @@ -1504,6 +1576,14 @@ def gain(self): def gain(self, value): self.config["detector"]["gain"] = value + @property + def noise(self): + return self.config["detector"]["noise"] + + @noise.setter + def noise(self, value): + self.config["detector"]["noise"] = value + @property def fano(self): return self.config["detector"]["fano"] @@ -1748,6 +1828,17 @@ def _iter_parameter_groups(self, linear_only=False): raise ValueError(name) def evaluate_fitmodel(self, xdata=None): + return self.mcatheory(xdata=xdata) + + def mcatheory( + self, + xdata=None, + hypermet=None, + continuum=None, + summing=None, + selected_groups=None, + normalized_peakgroups=False, + ): """Evaluate to MCA model (does not include the numerical background) y(x) = ycont(P(x)) + A1*G1(E(x)) + A2*G2(E(x)) + ... @@ -1764,62 +1855,65 @@ def evaluate_fitmodel(self, xdata=None): Gi(x): several peaks with normalized total area :param array xdata: + :param int hypermet: + :param bool continuum: + :param bool summing: + :param list selected_groups: + :param bool normalized_peakgroups: :returns array: """ - parameters = self._peak_profile_params() - return self.mcatheory(parameters, xdata=xdata) - - def mcatheory( - self, parameters, xdata=None, hypermet=None, continuum=None, summing=None - ): - """The parameters are the raw peak parameters, not the fit parameters""" - # Evaluation domain - if xdata is None: - xdata = self.xdata - energy = self._channelsToEnergy(xdata) - # Emission lines, scatter peaks and escape peaks - y = self._totalPeakGroupProfile(parameters, energy, hypermet=hypermet) + y = self._evaluatePeakProfiles( + xdata=xdata, + hypermet=hypermet, + selected_groups=selected_groups, + normalized_peakgroups=normalized_peakgroups, + ) # Analytical background if continuum or continuum is None: - model = self.continuumModel - if model is not None: - xpol = self._channelsToXpol(xdata) - y += model.evaluate_fullmodel(xdata=xpol) + y += self.ycontinuum(xdata=xdata) # Pile-up if summing or summing is None: - pileupfactor = self.sum - if pileupfactor: - y *= pileupfactor * SpecfitFuns.pileup( - y, min(xdata), self.zero, self.gain - ) + y += self.ypileup(y, xdata=xdata) return y + def ypileup(self, ymodel, xdata=None): + """The model contains the peaks and the continuum""" + pileupfactor = self.sum + if not pileupfactor: + if xdata is None: + return numpy.zeros(self.ndata) + else: + return numpy.zeros(len(xdata)) + if xdata is None: + xdata = self.xdata + return pileupfactor * SpecfitFuns.pileup( + ymodel, min(xdata), self.zero, self.gain + ) + def _ydata_to_fit(self, ydata, xdata=None): """The fitting is done after subtracting the numerical background""" - ybkg = self.ynumbkg(xdata=xdata) - if ybkg is None: - return ydata - else: - return ydata - ybkg + if self.hasNumBkg: + ydata = ydata - self.ynumbkg(xdata=xdata) + if self.linear: + if self.hasPileUp: + ymodel = self.mcatheory(xdata=xdata, summing=False) + ydata = ydata - self.ypileup(ymodel, xdata=xdata) + return ydata - def _ydata_without_background(self, ydata, xdata=None): - ybkg = self.ybackground(xdata=xdata) - if ybkg is None: - return ydata - else: - return ydata - ybkg + @property + def hasPileUp(self): + return bool(self.sum) def _fit_to_ydata(self, yfit, xdata=None): """The numerical background is not included in the fit model""" - ybkg = self.ynumbkg(xdata=xdata) - if ybkg is None: - return yfit + if self.hasNumBkg: + return yfit + self.ynumbkg(xdata=xdata) else: - return yfit + ybkg + return yfit def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters @@ -1827,10 +1921,23 @@ def linear_derivatives_fitmodel(self, xdata=None): :param array xdata: length nxdata :returns array: nparams x nxdata """ - if xdata is None: - xdata = self.xdata - energy = self._channelsToEnergy(xdata) - raise NotImplementedError + derivatives = [] + # Derivatives to peak group areas + for pgroupi in range(self._nLineGroups): + derivatives.append( + self.mcatheory( + xdata=xdata, + selected_groups=[pgroupi], + normalized_peakgroups=True, + continuum=False, + summing=False, + ) + ) + # Derivatives to polynomial coefficients + if isinstance(self.continuumModel, LinearPolynomialModel): + arr = self.continuumModel.linear_derivatives_fitmodel(xdata=xdata) + derivatives += arr.tolist() + return numpy.array(derivatives) def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter @@ -1864,10 +1971,12 @@ def derivative_fitmodel(self, param_idx, xdata=None): elif name == "eta_factor" and not hypermet: raise NotImplementedError elif name == "linegroup_areas": - parameters = self._peak_profile_params( - selected_groups=[pgroupi], normalize_peakgroups=True + return self.mcatheory( + selected_groups=[pgroupi], + normalized_peakgroups=True, + xdata=xdata, + continuum=False, ) - return self.mcatheory(parameters, xdata=xdata) elif name == "linpol": return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) elif name == "exppol": @@ -1875,14 +1984,19 @@ def derivative_fitmodel(self, param_idx, xdata=None): else: raise ValueError(name) - def _numerical_derivative(self, parameters, index, xdata=None): - parameters = parameters.copy() - p0 = parameters[index] - delta = (p0[index] + p0[index]) * 0.00001 - parameters[index] = p0 + delta - f1 = self.mcatheory(parameters, xdata=xdata) - parameters[index] = p0 - delta - f2 = self.mcatheory(parameters, xdata=xdata) + def _numerical_derivative(self, index, xdata=None): + keep = self.parameters + p0 = self.parameters[index] + try: + delta = (p0[index] + p0[index]) * 0.00001 + parameters[index] = p0 + delta + self.parameters = parameters + f1 = self.evaluate_fitmodel(xdata=xdata) + parameters[index] = p0 - delta + self.parameters = parameters + f2 = self.evaluate_fitmodel(xdata=xdata) + finally: + parameters = keep return (f1 - f2) / (2.0 * delta) @property @@ -1897,6 +2011,31 @@ def deltachi(self): def weightflag(self): return self.config["fit"]["fitweight"] + @property + def linear(self): + return self.config["fit"].get("linearfitflag", 0) + + @linear.setter + def linear(self, value): + self.config["fit"]["linearfitflag"] = value + + @contextmanager + def _nonlinear_fit_context(self): + with super(McaTheory, self)._nonlinear_fit_context(): + if abs(self.zero) < 1.0e-10: + self.zero = 0.0 + yield + + def startFit(self, digest=0, linear=None): + if linear is not None: + keep = self.linear + self.linear = linear + result = super(McaTheory, self).fit(full_output=digest) + if linear is not None: + self.linear = linear + # TODO digest result + return None, None + class MultiMcaTheory(ConcatModel): def __init__(self, ndetectors=1): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 628027ce4..e7282bab2 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -611,7 +611,7 @@ def _testLegacyMcaTheory(self, x, y, configuration): config1 = copy.deepcopy(mcaFitLegacy.config) config2 = copy.deepcopy(mcaFit.config) - # The new class removes invalid energies + # Remove expected differences n = len(config2["fit"]["energy"]) for name in ["energy", "energyweight", "energyflag", "energyscatter"]: if config1["fit"][name] is None: @@ -621,6 +621,7 @@ def _testLegacyMcaTheory(self, x, y, configuration): if len(config1["attenuators"]["Matrix"]) == 6: lst = config1["attenuators"]["Matrix"] lst.extend([0, lst[-1]+lst[-2]]) + config1["fit"]["continuum_name"] = config2["fit"]["continuum_name"] self._assertDeepEqual(config1, config2) @@ -664,10 +665,12 @@ def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], xmax=configuration["fit"]["xmax"]) + + mcaFit.estimate() + if tmpflag: return configuration, None, None - mcaFit.estimate() fitResult1, result1 = mcaFit.startFit(digest=1) return configuration, fitResult1, result1 From caf4a38c5054854eb1650a0a3adf6970ad96eed0 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 26 May 2021 14:54:52 +0200 Subject: [PATCH 28/74] fixup --- PyMca5/PyMcaMath/fitting/Model.py | 150 +++++++++++------ PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 159 +++++++++++++++---- PyMca5/tests/XrfTest.py | 7 +- 3 files changed, 235 insertions(+), 81 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index f6e99f1d7..32b2f922f 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -84,11 +84,57 @@ def cache_wrapper(self, *args, **kw): class Model(Cached): - """Evaluation and derivatives of a model to be used in least-squares fitting.""" + """Evaluation and derivatives of a model to be used in least-squares fitting. + + Each group of fit parameters is exposed as a property. Associated fit contraints + are optional properties. + + There is a "fit model" and a "full model". The full model describes the data, + the fit model describes the pre-processed data (for example smoothed, + numerical back subtracted, ...). By default the full model and the fit model + are identical. + + Fitting is done with linear least-squares optimization (no iterations) or + non-linear least-squares optimization (iterative). In addition a + non-least-squares optimization loop can be added on top (iterative). + + The abstract class provides the following abstract methods/properties: + * Data to fit: + * xdata: + * ydata: + * ystd: + * Fit Model: + * linear: linear or non-linear fitting + * evaluate_fitmodel: + * derivative_fitmodel: derivatives of all parameters + * linear_derivatives_fitmodel: derivatives of the linear parameters + * non_leastsquares_increment: non-least-squares optimization step + * Model parameters: + * _parameter_group_names: names of all parameter groups + * _linear_parameter_group_names: names of the linear parameter groups + * _iter_parameter_groups: provides parameter group names and number of parameters + + Example: + + ```python + model = MyModel() + model.xdata = xdata + model.ydata = ydata + model.ystd = ydata**0.5 + + plt.plot(model.xdata, model.ydata, label="data") + plt.plot(model.xdata, model.ymodel, label="initial") + + result = model.fit() + model.use_fit_result(result) + + plt.plot(model.xdata, model.ymodel, label="fit") + ``` + """ def __init__(self): - self.included_parameters = None - self.excluded_parameters = None + self._included_parameters = None # for model concatenation + self._excluded_parameters = None # for model concatenation super(Model, self).__init__() @property @@ -125,14 +171,6 @@ def yfitmodel(self): def ndata(self): return len(self.xdata) - @property - def fit_parameters(self): - return self._get_parameters(fitting=True) - - @fit_parameters.setter - def fit_parameters(self, values): - return self._set_parameters(values, fitting=True) - @property def parameters(self): return self._get_parameters(fitting=False) @@ -141,6 +179,14 @@ def parameters(self): def parameters(self, values): return self._set_parameters(values, fitting=False) + @property + def fit_parameters(self): + return self._get_parameters(fitting=True) + + @fit_parameters.setter + def fit_parameters(self, values): + return self._set_parameters(values, fitting=True) + @property def constraints(self): return self._get_constraints() @@ -161,14 +207,6 @@ def parameter_group_names(self): def _parameter_group_names(self): raise AttributeError from NotImplementedError - @property - def linear_fit_parameters(self): - return self._get_parameters(linear_only=True, fitting=True) - - @linear_fit_parameters.setter - def linear_fit_parameters(self, params): - return self._set_parameters(params, linear_only=True, fitting=True) - @property def linear_parameters(self): return self._get_parameters(linear_only=True, fitting=False) @@ -177,6 +215,14 @@ def linear_parameters(self): def linear_parameters(self, params): return self._set_parameters(params, linear_only=True, fitting=False) + @property + def linear_fit_parameters(self): + return self._get_parameters(linear_only=True, fitting=True) + + @linear_fit_parameters.setter + def linear_fit_parameters(self, params): + return self._set_parameters(params, linear_only=True, fitting=True) + @property def linear_constraints(self): return self._get_constraints(linear_only=True) @@ -256,8 +302,8 @@ def _set_parameters(self, params, linear_only=False, fitting=False): i += n def _filter_parameter_names(self, names): - included = self.included_parameters - excluded = self.excluded_parameters + included = self._included_parameters + excluded = self._excluded_parameters if included is None: included = names if excluded is None: @@ -357,8 +403,8 @@ def _parameter_groups(self, linear_only=False): if cache is None: it = self._iter_parameter_groups(linear_only=linear_only) else: - a = self.included_parameters - b = self.excluded_parameters + a = self._included_parameters + b = self._excluded_parameters if a is not None: a = tuple(sorted(a)) if b is not None: @@ -430,19 +476,16 @@ def linear_fit(self, full_output=False): b.copy(), uncertainties=True, covariances=False, - digested_output=False, + digested_output=True, ) if self.niter_non_leastsquares: - if full_output: - self.linear_fit_parameters = result[0] - else: - self.linear_fit_parameters = result["parameters"] + self.linear_fit_parameters = result["parameters"] self.non_leastsquares_increment() - return { - "linear": True, - "parameters": self._fit_to_parameters(result[0]), - "uncertainties": self._fit_to_uncertainties(result[1]), - } + result["linear"] = True + result["parameters"] = self._fit_to_parameters(result["parameters"]) + result["uncertainties"] = self._fit_to_uncertainties(result["uncertainties"]) + result.pop("svd") + return result @contextmanager def _linear_fit_context(self): @@ -567,6 +610,21 @@ def use_fit_result(self, result): else: self.parameters = result["parameters"] + @contextmanager + def use_fit_result_context(self, result): + if result["linear"]: + keep = self.linear_parameters + else: + keep = self.parameters + self.use_fit_result(result) + try: + yield + finally: + if result["linear"]: + self.linear_parameters = keep + else: + self.parameters = keep + @property def niter_non_leastsquares(self): return 0 @@ -587,8 +645,8 @@ def __init__(self, models, shared_attributes=None): self._models = models self.__fixed_shared_attributes = { "linear", - "included_parameters", - "excluded_parameters", + "_included_parameters", + "_excluded_parameters", } self.shared_attributes = shared_attributes super(ConcatModel, self).__init__() @@ -683,7 +741,7 @@ def ndata(self): if nmodels == 0: return 0 else: - return sum([m.ndata for m in self._models]) + return sum(m.ndata for m in self._models) @property def xdata(self): @@ -744,25 +802,25 @@ def _set_data(self, attr, values): @contextmanager def _filter_parameter_context(self, shared=True): - keepex = self.excluded_parameters - keepinc = self.included_parameters + keepex = self._excluded_parameters + keepin = self._included_parameters try: if shared: - if keepinc: - self.included_parameters = list( - set(keepinc) - set(self.shared_attributes) + if keepin: + self._included_parameters = list( + set(keepin) - set(self.shared_attributes) ) else: - self.included_parameters = self.shared_attributes + self._included_parameters = self.shared_attributes else: if keepex: - self.excluded_parameters.extend(self.shared_attributes) + self._excluded_parameters.extend(self.shared_attributes) else: - self.excluded_parameters = self.shared_attributes + self._excluded_parameters = self.shared_attributes yield finally: - self.excluded_parameters = keepex - self.included_parameters = keepinc + self._excluded_parameters = keepex + self._included_parameters = keepin @property def nparameters(self): diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 3a9542fcc..85c0b1783 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -364,6 +364,10 @@ def _emissionGroups(self): else: yield [Z, symb, peaks] + @property + def emissionGroupNames(self): + return [symb + " " + line for _, symb, line in sorted(self._emissionGroups())] + def _configureElementsModule(self): """Configure the globals in the Elements module""" for material, info in self.config["materials"].items(): @@ -800,10 +804,10 @@ def continuum_name(self): try: return self.config["fit"]["continuum_name"] except AttributeError: - return self.CONTINUUM_LIST[value] + return self.CONTINUUM_LIST[self.continuum] @continuum_name.setter - def continuum_name(self, value): + def continuum_name(self, name): self.config["fit"]["continuum"] = self.CONTINUUM_LIST.index(name) self.config["fit"]["continuum_name"] = name @@ -827,7 +831,7 @@ def _refreshContinuumCache(self): if self._lastContinuumCacheParams == contparams: return # the cached data is still valid - # Instantiate the model + # Instantiate the continuum model continuum = self.continuum_name if continuum is None: model = None @@ -1800,33 +1804,87 @@ def _iter_parameter_groups(self, linear_only=False): yield name, 1 elif name == "sum": yield name, 1 - elif name == "st_arearatio" and hypermet: - yield name, 1 - elif name == "st_sloperatio" and hypermet: - yield name, 1 - elif name == "lt_arearatio" and hypermet: - yield name, 1 - elif name == "lt_sloperatio" and hypermet: - yield name, 1 - elif name == "step_heightratio" and hypermet: - yield name, 1 - elif name == "eta_factor" and not hypermet: - yield name, 1 + elif name == "st_arearatio": + if hypermet: + yield name, 1 + elif name == "st_sloperatio": + if hypermet: + yield name, 1 + elif name == "lt_arearatio": + if hypermet: + yield name, 1 + elif name == "lt_sloperatio": + if hypermet: + yield name, 1 + elif name == "step_heightratio": + if hypermet: + yield name, 1 + elif name == "eta_factor": + if not hypermet: + yield name, 1 elif name == "linegroup_areas": n = len(self.linegroup_areas) if n: yield name, n - elif name == "linpol": + elif name == "linpol_coefficients": n = len(self.linpol_coefficients) if n: yield name, n - elif name == "exppol": + elif name == "exppol_coefficients": n = len(self.exppol_coefficients) if n: yield name, n else: raise ValueError(name) + def _convert_parameter_names(self, names): + linegroup_names = None + for name in names: + if "linegroup_areas" in name: + if not linegroup_names: + linegroup_names = self.emissionGroupNames + idx = int(name.replace("linegroup_areas", "")) + yield linegroup_names[idx] + elif "linpol_coefficients" in name: + if self.continuum < self.CONTINUUM_LIST.index("Linear Polynomial"): + idx = int(name.replace("linpol_coefficients", "")) + if idx == 0: + yield "Constant" + elif idx == 1: + yield "1st Order" + elif idx == 2: + yield "2nd Order" + else: + yield name.replace("linpol_coefficients", "A") + elif "exppol_coefficients" in name: + yield name.replace("linpol_coefficients", "A") + elif name == "st_arearatio": + yield "ST AreaR" + elif name == "st_sloperatio": + yield "ST SlopeR" + elif name == "lt_arearatio": + yield "LT AreaR" + elif name == "lt_sloperatio": + yield "LT SlopeR" + elif name == "step_heightratio": + yield "STEP HeightR" + elif name == "eta_factor": + yield "Eta Factor" + else: + yield name.capitalize() + + @property + def parameter_names(self): + return list( + self._convert_parameter_names(super(McaTheory, self).parameter_names) + ) + + @property + def linear_parameter_names(self): + return list( + self._convert_parameter_names(super(McaTheory, self).linear_parameter_names) + ) + def evaluate_fitmodel(self, xdata=None): return self.mcatheory(xdata=xdata) @@ -1834,8 +1892,8 @@ def mcatheory( self, xdata=None, hypermet=None, - continuum=None, - summing=None, + continuum=True, + summing=True, selected_groups=None, normalized_peakgroups=False, ): @@ -1871,11 +1929,11 @@ def mcatheory( ) # Analytical background - if continuum or continuum is None: + if continuum and self.continuumModel is not None: y += self.ycontinuum(xdata=xdata) # Pile-up - if summing or summing is None: + if summing and self.sum: y += self.ypileup(y, xdata=xdata) return y @@ -1947,7 +2005,6 @@ def derivative_fitmodel(self, param_idx, xdata=None): :returns array: nxdata """ name, pgroupi = self._parameter_name_from_index(param_idx) - hypermet = self.hypermet if name == "zero": raise NotImplementedError elif name == "gain": @@ -1958,17 +2015,17 @@ def derivative_fitmodel(self, param_idx, xdata=None): raise NotImplementedError elif name == "sum": raise NotImplementedError - elif name == "st_arearatio" and hypermet: + elif name == "st_arearatio": raise NotImplementedError - elif name == "st_sloperatio" and hypermet: + elif name == "st_sloperatio": raise NotImplementedError - elif name == "lt_arearatio" and hypermet: + elif name == "lt_arearatio": raise NotImplementedError - elif name == "lt_sloperatio" and hypermet: + elif name == "lt_sloperatio": raise NotImplementedError - elif name == "step_heightratio" and hypermet: + elif name == "step_heightratio": raise NotImplementedError - elif name == "eta_factor" and not hypermet: + elif name == "eta_factor": raise NotImplementedError elif name == "linegroup_areas": return self.mcatheory( @@ -1978,15 +2035,18 @@ def derivative_fitmodel(self, param_idx, xdata=None): continuum=False, ) elif name == "linpol": + # TODO: subtract param return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) elif name == "exppol": + # TODO: subtract param return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) else: raise ValueError(name) def _numerical_derivative(self, index, xdata=None): keep = self.parameters - p0 = self.parameters[index] + parameters = keep.copy() + p0 = parameters[index] try: delta = (p0[index] + p0[index]) * 0.00001 parameters[index] = p0 + delta @@ -1996,7 +2056,7 @@ def _numerical_derivative(self, index, xdata=None): self.parameters = parameters f2 = self.evaluate_fitmodel(xdata=xdata) finally: - parameters = keep + self.parameters = keep return (f1 - f2) / (2.0 * delta) @property @@ -2027,14 +2087,47 @@ def _nonlinear_fit_context(self): yield def startFit(self, digest=0, linear=None): + """Fit with legacy output""" if linear is not None: keep = self.linear self.linear = linear - result = super(McaTheory, self).fit(full_output=digest) + result = super(McaTheory, self).fit(full_output=True) + if linear is not None: self.linear = linear - # TODO digest result - return None, None + + self._last_fit_result = result + if digest: + return self._legacyresult(result), self.digestresult() + else: + return self._legacyresult(result) + + @staticmethod + def _legacyresult(result): + if result["linear"]: + return ( + result["parameters"].tolist(), + numpy.nan, + result["uncertainties"].tolist(), + 1, + numpy.nan, + ) + else: + return ( + result["parameters"].tolist(), + result["chi2_red"], + result["uncertainties"].tolist(), + result["niter"], + result["lastdeltachi"], + ) + + def digestresult(self): + with self.use_fit_result_context(self._last_fit_result): + return dict() + + def imagingDigestResult(self): + with self.use_fit_result_context(self._last_fit_result): + return dict() class MultiMcaTheory(ConcatModel): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index e7282bab2..a661ff992 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -656,9 +656,12 @@ def _testLegacyMcaTheory(self, x, y, configuration): numpy.testing.assert_allclose(line1[1], line2[1], rtol=1e-9) self.assertEqual(line1[2], line2[2]) + # Compares parameter names + #self.assertEqual(mcaFitLegacy.PARAMETERS, mcaFit.parameter_names) + # Compare fit results - self.assertEqual(fitResult1, fitResult2) - self.assertEqual(result1, result2) + #self.assertEqual(fitResult1[0], fitResult2[0]) + #self.assertEqual(result1, result2) def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): configuration = mcaFit.configure(configuration) From 69c9b923715616adfe6a907ca903f0782fd445aa Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 1 Jun 2021 20:43:36 +0200 Subject: [PATCH 29/74] parameter decorator --- PyMca5/PyMcaMath/fitting/Model.py | 261 +++++++--- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 33 +- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 519 ++++++++++--------- PyMca5/tests/FitModelTest.py | 173 ++++--- PyMca5/tests/FitPolModelTest.py | 84 +-- PyMca5/tests/SimpleModel.py | 46 +- PyMca5/tests/XrfTest.py | 63 ++- 7 files changed, 672 insertions(+), 507 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 32b2f922f..39810fd7e 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -62,10 +62,11 @@ def getCache(self, cachename, *subnames): if cachename in self._cache: ret = self._cache[cachename] for cachename in subnames: - if cachename in ret: + try: + ret = ret[cachename] + except KeyError: + ret[cachename] = dict() ret = ret[cachename] - else: - ret = ret[cachename] = dict() return ret else: return None @@ -83,11 +84,87 @@ def cache_wrapper(self, *args, **kw): return decorator +class parameter(property): + """Usages: + + .. highlight:: python + .. code-block:: python + + class MyClass(Model): + + def __init__(self): + self._myparam = 0. + + @parameter + def myparam(self): + return self._myparam + + @myparam.setter # optional + def myparam(self, value): + self._myparam = value + + @myparam.counter # optional + def myparam(self): + return 1 + + @myparam.constraints # optional + def myparam(self): + return 1, 0, 0 + """ + + def __init__(self, fget=None, fset=None, fdel=None, doc=""): + if fget is not None: + fget = self._param_getter(fget) + if fset is not None: + fset = self._param_setter(fset) + super().__init__(fget=fget, fset=fset, fdel=fdel, doc=doc) + self.fcount = lambda _: None + self.fconstraints = lambda _: None + + def getter(self, fget): + if fget is not None: + fget = self._param_getter(fget) + return super().getter(fget) + + def setter(self, fset): + if fset is not None: + fset = self._param_setter(fset) + return super().setter(fset) + + def counter(self, fcount): + self.fcount = fcount + return self + + def constraints(self, fconstraints): + self.fconstraints = fconstraints + return self + + @classmethod + def _param_getter(cls, fget): + @functools.wraps(fget) + def wrapper(self): + return fget(self) + + return wrapper + + @classmethod + def _param_setter(cls, fset): + @functools.wraps(fset) + def wrapper(self, value): + return fset(self, value) + + return wrapper + + +class linear_parameter(parameter): + pass + + class Model(Cached): """Evaluation and derivatives of a model to be used in least-squares fitting. - Each group of fit parameters is exposed as a property. Associated fit contraints - are optional properties. + The `parameter` and `linear_parameter` work like python's `property` decorator. + Associated fit contraints are optional properties. There is a "fit model" and a "full model". The full model describes the data, the fit model describes the pre-processed data (for example smoothed, @@ -109,10 +186,6 @@ class Model(Cached): * derivative_fitmodel: derivatives of all parameters * linear_derivatives_fitmodel: derivatives of the linear parameters * non_leastsquares_increment: non-least-squares optimization step - * Model parameters: - * _parameter_group_names: names of all parameter groups - * _linear_parameter_group_names: names of the linear parameter groups - * _iter_parameter_groups: provides parameter group names and number of parameters Example: @@ -137,6 +210,17 @@ def __init__(self): self._excluded_parameters = None # for model concatenation super(Model, self).__init__() + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + allp = cls._PARAMETER_GROUP_NAMES = list() + linp = cls._LINEAR_PARAMETER_GROUP_NAMES = list() + for name in sorted(dir(cls)): + attr = getattr(cls, name) + if isinstance(attr, parameter): + allp.append(name) + if isinstance(attr, linear_parameter): + linp.append(name) + @property def xdata(self): raise AttributeError from NotImplementedError @@ -173,6 +257,7 @@ def ndata(self): @property def parameters(self): + """Only enabled parameters""" return self._get_parameters(fitting=False) @parameters.setter @@ -181,6 +266,7 @@ def parameters(self, values): @property def fit_parameters(self): + """Only enabled parameters""" return self._get_parameters(fitting=True) @fit_parameters.setter @@ -189,26 +275,27 @@ def fit_parameters(self, values): @property def constraints(self): + """Only enabled parameters""" return self._get_constraints() @property def nparameters(self): + """Only enabled parameters""" return sum(tpl[1] for tpl in self._parameter_groups()) @property def parameter_names(self): + """Only enabled parameters""" return list(self._iter_parameter_names()) @property - def parameter_group_names(self): - return self._filter_parameter_names(self._parameter_group_names) - - @property - def _parameter_group_names(self): - raise AttributeError from NotImplementedError + def all_parameter_group_names(self): + """All parameters, not only the enabled ones""" + return list(self._filter_parameter_names(self._PARAMETER_GROUP_NAMES)) @property def linear_parameters(self): + """Only enabled parameters""" return self._get_parameters(linear_only=True, fitting=False) @linear_parameters.setter @@ -217,6 +304,7 @@ def linear_parameters(self, params): @property def linear_fit_parameters(self): + """Only enabled parameters""" return self._get_parameters(linear_only=True, fitting=True) @linear_fit_parameters.setter @@ -225,23 +313,23 @@ def linear_fit_parameters(self, params): @property def linear_constraints(self): + """Only enabled parameters""" return self._get_constraints(linear_only=True) @property def nlinear_parameters(self): + """Only enabled parameters""" return sum(tpl[1] for tpl in self._parameter_groups(linear_only=True)) @property def linear_parameter_names(self): + """Only enabled parameters""" return list(self._iter_parameter_names(linear_only=True)) @property - def linear_parameter_group_names(self): - return self._filter_parameter_names(self._linear_parameter_group_names) - - @property - def _linear_parameter_group_names(self): - raise AttributeError from NotImplementedError + def all_linear_parameter_group_names(self): + """All parameters, not only the enabled ones""" + return list(self._filter_parameter_names(self._LINEAR_PARAMETER_GROUP_NAMES)) def _get_parameters(self, linear_only=False, fitting=True): """ @@ -249,12 +337,12 @@ def _get_parameters(self, linear_only=False, fitting=True): :param bool fitting: :returns array: """ - i = 0 if linear_only: nparams = self.nlinear_parameters else: nparams = self.nparameters params = numpy.zeros(nparams) + i = 0 for name, n in self._parameter_groups(linear_only=linear_only): params[i : i + n] = getattr(self, name) i += n @@ -266,25 +354,20 @@ def _get_parameters(self, linear_only=False, fitting=True): def _get_constraints(self, linear_only=False): """ :param bool linear_only: - :returns array: + :returns array: nparams x 3 """ - i = 0 if linear_only: nparams = self.nlinear_parameters else: nparams = self.nparameters codes = numpy.zeros((nparams, 3), numpy.float64) - bspecified = False + i = 0 for name, n in self._parameter_groups(linear_only=linear_only): - name += "_constraint" - if hasattr(self, name): - bspecified = True - codes[i : i + n] = getattr(self, name) + constraints = getattr(self.__class__, name).fconstraints(self) + if constraints is not None: + codes[i : i + n] = constraints i += n - if bspecified: - return codes.T - else: - return None + return codes def _set_parameters(self, params, linear_only=False, fitting=False): """ @@ -304,11 +387,12 @@ def _set_parameters(self, params, linear_only=False, fitting=False): def _filter_parameter_names(self, names): included = self._included_parameters excluded = self._excluded_parameters - if included is None: - included = names - if excluded is None: - excluded = [] - return [name for name in names if name in included and name not in excluded] + for name in names: + if included is not None and name not in included: + continue + if excluded is not None and name in excluded: + continue + yield name def evaluate_fullmodel(self, xdata=None): """Evaluate the full model. @@ -387,15 +471,9 @@ def linear_derivatives_fitmodel(self, xdata=None): def linear(self): raise AttributeError from NotImplementedError - def _iter_parameter_groups(self, linear_only=False): - """ - :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group - """ - raise NotImplementedError - def _parameter_groups(self, linear_only=False): - """ + """Returns an iterator over name and count of enabled parameters + :param bool linear_only: :returns iterable(str, int): group name, nb. parameters in the group """ @@ -417,8 +495,31 @@ def _parameter_groups(self, linear_only=False): ) return it - def _iter_parameter_names(self, linear_only=False): + def _iter_parameter_groups(self, linear_only=False): + """Helper for `_parameter_groups`. + + :param bool linear_only: + :yields (str, int): group name, nb. parameters in the group """ + if linear_only: + names = self.all_linear_parameter_group_names + else: + names = self.all_parameter_group_names + for name in names: + param = getattr(self.__class__, name) + n = param.fcount(self) + if n is None: + value = getattr(self, name) + try: + n = len(value) + except TypeError: + n = 1 + if n: + yield name, n + + def _iter_parameter_names(self, linear_only=False): + """Only enabled parameters + :param bool linear_only: :yields str: """ @@ -504,7 +605,7 @@ def nonlinear_fit(self, full_output=False): :returns dict: """ with self._nonlinear_fit_context(): - constraints = self.constraints + constraints = self.constraints.T xdata = self.xdata ydata = self.yfitdata ystd = self.yfitstd @@ -642,6 +743,8 @@ def __init__(self, models, shared_attributes=None): for model in models: if not isinstance(model, Model): raise ValueError("'models' must be a list of type 'Model'") + if len(set(type(m) for m in models)) > 1: + raise ValueError("Multiple model types are currently not supported") self._models = models self.__fixed_shared_attributes = { "linear", @@ -651,6 +754,19 @@ def __init__(self, models, shared_attributes=None): self.shared_attributes = shared_attributes super(ConcatModel, self).__init__() + @contextmanager + def cachingContext(self, cachename): + with ExitStack() as stack: + ctx = super(ConcatModel, self).cachingContext(cachename) + stack.enter_context(ctx) + for m in self._models: + stack.enter_context(m.cachingContext(cachename)) + yield + + @property + def nmodels(self): + return len(self._models) + @property def model(self): """Model used to get/set shared attributes""" @@ -663,7 +779,7 @@ def __getattr__(self, name): raise AttributeError(name) def __setattr__(self, name, value): - """Set shared attribute""" + """Set the attributed of the models when shared""" if ( name != "_models" and self.nmodels @@ -731,10 +847,6 @@ def share_attributes(self, shared_attributes=None): for name, value in adict.items(): setattr(model, name, value) - @property - def nmodels(self): - return len(self._models) - @property def ndata(self): nmodels = self.nmodels @@ -822,6 +934,21 @@ def _filter_parameter_context(self, shared=True): self._excluded_parameters = keepex self._included_parameters = keepin + def _iter_parameter_models(self): + with self._filter_parameter_context(shared=True): + yield self.model + with self._filter_parameter_context(shared=False): + for m in self._models: + yield m + + def _iter_models_types(self): + modeltypes = set() + for model in self._models: + modeltype = type(model) + if modeltype not in modeltypes: + modeltypes.add(modeltype) + yield model + @property def nparameters(self): return sum(m.nparameters for m in self._iter_parameter_models()) @@ -869,16 +996,17 @@ def _set_parameters(self, values, linear_only=False, fitting=False): ) i += n - def _iter_parameter_models(self): - """Iterate over models which are temporarily configured so that - after iterations, all parameters provided. - :yields Model: + def _get_constraints(self, linear_only=False): """ - with self._filter_parameter_context(shared=True): - yield self.model - with self._filter_parameter_context(shared=False): - for m in self._models: - yield m + :param bool linear_only: + :returns array: nparams x 3 + """ + return numpy.concatenate( + [ + m._get_constraints(linear_only=linear_only) + for m in self._iter_parameter_models() + ] + ) def _parameter_groups(self, linear_only=False): """ @@ -1093,12 +1221,3 @@ def _generate_idx_channels(self, nconcat, stride=None): offset -= n start = stop yield idx - - @contextmanager - def cachingContext(self, cachename): - with ExitStack() as stack: - ctx = super(ConcatModel, self).cachingContext(cachename) - stack.enter_context(ctx) - for m in self._models: - stack.enter_context(m.cachingContext(cachename)) - yield diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index 3fc719cc9..4b8e82c01 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -32,8 +32,8 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import numpy -from PyMca5.PyMcaMath.fitting import Gefit from PyMca5.PyMcaMath.fitting.Model import Model +from PyMca5.PyMcaMath.fitting.Model import linear_parameter class PolynomialModel(Model): @@ -48,19 +48,21 @@ def __init__(self, degree=0, maxiter=100): @property def degree(self): - return self.coefficients.size - 1 + return self._coefficients.size - 1 @degree.setter def degree(self, n): - self.coefficients = numpy.zeros(n + 1) + if n < 0: + raise ValueError("degree must be a positive integer") + self._coefficients = numpy.zeros(n + 1) - @property + @linear_parameter def coefficients(self): return self._coefficients @coefficients.setter def coefficients(self, values): - self._coefficients = numpy.atleast_1d(values) + self._coefficients[:] = values @property def xdata(self): @@ -102,27 +104,6 @@ def maxiter(self, value): class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" - @property - def _parameter_group_names(self): - return ["coefficients"] - - @property - def _linear_parameter_group_names(self): - return ["coefficients"] - - def _iter_parameter_groups(self, linear_only=False): - """ - :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group - """ - if linear_only: - names = self.linear_parameter_group_names - else: - names = self.parameter_group_names - for name in names: - if name == "coefficients": - yield name, self.degree + 1 - def evaluate_fitmodel(self, xdata=None): """Evaluate the fit model, not the full model. diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 85c0b1783..65d196fe0 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -43,10 +43,14 @@ from PyMca5.PyMcaIO import ConfigDict from PyMca5.PyMcaMath.fitting import SpecfitFuns from PyMca5.PyMcaMath.fitting import Gefit -from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel +from PyMca5.PyMcaMath.fitting.Model import Model +from PyMca5.PyMcaMath.fitting.Model import ConcatModel +from PyMca5.PyMcaMath.fitting.Model import parameter +from PyMca5.PyMcaMath.fitting.Model import linear_parameter from PyMca5.PyMcaMath.fitting.PolynomialModels import LinearPolynomialModel from PyMca5.PyMcaMath.fitting.PolynomialModels import ExponentialPolynomialModel + from . import Elements from . import ConcentrationsTool @@ -969,6 +973,12 @@ def _peakProfileParams( ): """Raw parameters of emission/scatter/escape peaks. All parameters are defined in the energy domain (X-axis is energy, not channels). + + :param int hypermet: + :param list or None selected_groups: all groups when `None` + no groups when empty list (npeaks == 0) + :param bool normalized_peakgroups: + :returns array: npeaks x npeakparams """ lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups @@ -1035,11 +1045,25 @@ def _peakProfileParams( return parameters - @property + @linear_parameter def linegroup_areas(self): self._refreshAreasCache() return self._lineGroupAreas + @linegroup_areas.setter + def linegroup_areas(self, values): + self._lineGroupAreas[:] = values + + @linegroup_areas.counter + def linegroup_areas(self): + return self._nLineGroups + + @linegroup_areas.constraints + def linegroup_areas(self): + fixed = Gefit.CFIXED, 0, 0 + positive = Gefit.CPOSITIVE, 0, 0 + return [positive if area else fixed for area in self.linegroup_areas] + def _refreshAreasCache(self): params = self._areasCacheParams if self._lastAreasCacheParams == params: @@ -1060,7 +1084,10 @@ def _estimateLineGroupAreas(self): xenergy = self.xenergy emin = xenergy.min() emax = xenergy.max() - factor = self.GAUSS_SIGMA_TO_FWHM * numpy.sqrt(2 * numpy.pi) + factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM + gain = self.gain + + keep = list() lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups @@ -1079,12 +1106,24 @@ def _estimateLineGroupAreas(self): if rate * escrate > selected_rate: selected_energy = peakenergy if selected_energy: - height = ydata[(numpy.abs(xenergy - selected_energy)).argmin()] + chan = (numpy.abs(xenergy - selected_energy)).argmin() + height = ydata[chan] + keep.append((chan, height)) fwhm = self._peakFWHM(selected_energy) linegroup_areas[i] = height * fwhm * factor # Gaussian else: linegroup_areas[i] = 0 # Fixed at zero + if False: + print(linegroup_areas) + import matplotlib.pyplot as plt + + plt.figure() + plt.plot(ydata) + x, y = zip(*keep) + plt.plot(x, y, "o") + plt.show() + def _evaluatePeakProfiles( self, xdata=None, @@ -1097,7 +1136,7 @@ def _evaluatePeakProfiles( :param array xdata: 1D array :param int or None hypermet: - :param bool fast: ??? + :param bool fast: use lookup table for calculating exponentials :param bool selected_groups: :param bool normalized_peakgroups: :returns array: same shape as x @@ -1467,7 +1506,7 @@ def _channelsToEnergy(self, x): def _channelsToXpol(self, x): return self.zero + self.gain * (x - x.mean()) - @property + @linear_parameter def linpol_coefficients(self): if isinstance(self.continuumModel, LinearPolynomialModel): return self.continuum_coefficients @@ -1479,7 +1518,7 @@ def linpol_coefficients(self, values): if isinstance(self.continuumModel, LinearPolynomialModel): self.continuum_coefficients = values - @property + @parameter def exppol_coefficients(self): if isinstance(self.continuumModel, ExponentialPolynomialModel): return self.continuum_coefficients @@ -1564,7 +1603,7 @@ def _smooth(self, y): ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) return ysmooth - @property + @parameter def zero(self): return self.config["detector"]["zero"] @@ -1572,7 +1611,16 @@ def zero(self): def zero(self, value): self.config["detector"]["zero"] = value - @property + @zero.constraints + def zero(self): + if self.config["detector"]["fixedzero"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.zero + delta = self.config["detector"]["deltazero"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def gain(self): return self.config["detector"]["gain"] @@ -1580,7 +1628,16 @@ def gain(self): def gain(self, value): self.config["detector"]["gain"] = value - @property + @gain.constraints + def gain(self): + if self.config["detector"]["fixedgain"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.gain + delta = self.config["detector"]["deltagain"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def noise(self): return self.config["detector"]["noise"] @@ -1588,7 +1645,7 @@ def noise(self): def noise(self, value): self.config["detector"]["noise"] = value - @property + @parameter def fano(self): return self.config["detector"]["fano"] @@ -1596,7 +1653,16 @@ def fano(self): def fano(self, value): self.config["detector"]["fano"] = value - @property + @fano.constraints + def fano(self): + if self.config["detector"]["fixedfano"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.fano + delta = self.config["detector"]["deltafano"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def sum(self): if self.config["fit"]["sumflag"]: return self.config["detector"]["sum"] @@ -1607,7 +1673,16 @@ def sum(self): def sum(self, value): self.config["detector"]["sum"] = value - @property + @sum.constraints + def sum(self): + if self.config["detector"]["fixedsum"] or not self.config["fit"]["sumflag"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.sum + delta = self.config["detector"]["deltasum"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def eta_factor(self): return self.config["peakshape"]["eta_factor"] @@ -1615,7 +1690,23 @@ def eta_factor(self): def eta_factor(self, value): self.config["peakshape"]["eta_factor"] = value - @property + @eta_factor.counter + def eta_factor(self): + if self._hypermet: + return 0 + else: + return 1 + + @eta_factor.constraints + def eta_factor(self): + if self.config["peakshape"]["fixedeta_factor"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.eta_factor + delta = self.config["peakshape"]["deltaeta_factor"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def step_heightratio(self): if self._hypermetStep: return self.config["peakshape"]["step_heightratio"] @@ -1626,15 +1717,47 @@ def step_heightratio(self): def step_heightratio(self, value): self.config["peakshape"]["step_heightratio"] = value - @property + @step_heightratio.counter + def step_heightratio(self): + if self._hypermet: + return 1 + else: + return 0 + + @step_heightratio.constraints + def step_heightratio(self): + if self.config["peakshape"]["fixedstep_heightratio"] or not self._hypermetStep: + return Gefit.CFIXED, 0, 0 + else: + value = self.step_heightratio + delta = self.config["peakshape"]["deltastep_heightratio"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def lt_sloperatio(self): return self.config["peakshape"]["lt_sloperatio"] @lt_sloperatio.setter def lt_sloperatio(self, value): - self.config["detector"]["lt_sloperatio"] = value + self.config["peakshape"]["lt_sloperatio"] = value - @property + @lt_sloperatio.counter + def lt_sloperatio(self): + if self._hypermet: + return 1 + else: + return 0 + + @lt_sloperatio.constraints + def lt_sloperatio(self): + if self.config["peakshape"]["fixedlt_sloperatio"] or not self._hypermetLongTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.lt_sloperatio + delta = self.config["peakshape"]["deltalt_sloperatio"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def lt_arearatio(self): if self._hypermetLongTail: return self.config["peakshape"]["lt_arearatio"] @@ -1645,7 +1768,23 @@ def lt_arearatio(self): def lt_arearatio(self, value): self.config["peakshape"]["lt_arearatio"] = value - @property + @lt_arearatio.counter + def lt_arearatio(self): + if self._hypermet: + return 1 + else: + return 0 + + @lt_arearatio.constraints + def lt_arearatio(self): + if self.config["peakshape"]["fixedlt_arearatio"] or not self._hypermetLongTail: + return Gefit.CFIXED, 0, 0 + else: + value = self.lt_arearatio + delta = self.config["peakshape"]["deltalt_arearatio"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def st_sloperatio(self): return self.config["peakshape"]["st_sloperatio"] @@ -1653,7 +1792,26 @@ def st_sloperatio(self): def st_sloperatio(self, value): self.config["peakshape"]["st_sloperatio"] = value - @property + @st_sloperatio.counter + def st_sloperatio(self): + if self._hypermet: + return 1 + else: + return 0 + + @st_sloperatio.constraints + def st_sloperatio(self): + if ( + self.config["peakshape"]["fixedst_sloperatio"] + or not self._hypermetShortTail + ): + return Gefit.CFIXED, 0, 0 + else: + value = self.st_sloperatio + delta = self.config["peakshape"]["deltast_sloperatio"] + return Gefit.CQUOTED, value - delta, value + delta + + @parameter def st_arearatio(self): if self._hypermetShortTail: return self.config["peakshape"]["st_arearatio"] @@ -1664,178 +1822,21 @@ def st_arearatio(self): def st_arearatio(self, value): self.config["peakshape"]["st_arearatio"] = value - @property - def zero_constraint(self): - if self.config["detector"]["fixedzero"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.zero - delta = self.config["detector"]["deltazero"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def gain_constraint(self): - if self.config["detector"]["fixedgain"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.gain - delta = self.config["detector"]["deltagain"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def fano_constraint(self): - if self.config["detector"]["fixedfano"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.fano - delta = self.config["detector"]["deltafano"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def sum_constraint(self): - if self.config["detector"]["fixedsum"] or not self.config["fit"]["sumflag"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.sum - delta = self.config["detector"]["deltasum"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def eta_factor_constraint(self): - if self.config["detector"]["fixedeta_factor"]: - return Gefit.CFIXED, 0, 0 - else: - value = self.eta_factor - delta = self.config["detector"]["deltaeta_factor"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def step_heightratio_constraint(self): - if self.config["detector"]["fixedstep_heightratio"] or not self._hypermetStep: - return Gefit.CFIXED, 0, 0 - else: - value = self.step_heightratio - delta = self.config["detector"]["deltastep_heightratio"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def lt_sloperatio_constraint(self): - if self.config["detector"]["fixedlt_sloperatio"] or not self._hypermetLongTail: - return Gefit.CFIXED, 0, 0 - else: - value = self.lt_sloperatio - delta = self.config["detector"]["deltalt_sloperatio"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def lt_arearatio_constraint(self): - if self.config["detector"]["fixedlt_arearatio"] or not self._hypermetLongTail: - return Gefit.CFIXED, 0, 0 - else: - value = self.lt_arearatio - delta = self.config["detector"]["deltalt_arearatio"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def st_sloperatio_constraint(self): - if self.config["detector"]["fixedst_sloperatio"] or not self._hypermetShortTail: - return Gefit.CFIXED, 0, 0 + @st_arearatio.counter + def st_arearatio(self): + if self._hypermet: + return 1 else: - value = self.st_sloperatio - delta = self.config["detector"]["deltast_sloperatio"] - return Gefit.CQUOTED, value + delta, value - delta + return 0 - @property - def st_arearatio_constraint(self): - if self.config["detector"]["fixedst_arearatio"] or not self._hypermetShortTail: + @st_arearatio.constraints + def st_arearatio(self): + if self.config["peakshape"]["fixedst_arearatio"] or not self._hypermetShortTail: return Gefit.CFIXED, 0, 0 else: value = self.st_arearatio - delta = self.config["detector"]["deltast_arearatio"] - return Gefit.CQUOTED, value + delta, value - delta - - @property - def linegroup_areas_constraint(self): - fixed = Gefit.CFIXED, 0, 0 - positive = Gefit.CPOSITIVE, 0, 0 - return [positive if area else fixed for area in self.linegroup_areas] - - @property - def _parameter_group_names(self): - return [ - "zero", - "gain", - "noise", - "fano", - "sum", - "st_arearatio", - "st_sloperatio", - "lt_arearatio", - "lt_sloperatio", - "step_heightratio", - "eta_factor", - "linegroup_areas", - "linpol_coefficients", - "exppol_coefficients", - ] - - @property - def _linear_parameter_group_names(self): - return ["linegroup_areas", "linpol_coefficients"] - - def _iter_parameter_groups(self, linear_only=False): - """ - :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group - """ - if linear_only: - names = self.linear_parameter_group_names - else: - names = self.parameter_group_names - hypermet = self._hypermet - for name in names: - if name == "zero": - yield name, 1 - elif name == "gain": - yield name, 1 - elif name == "noise": - yield name, 1 - elif name == "fano": - yield name, 1 - elif name == "sum": - yield name, 1 - elif name == "st_arearatio": - if hypermet: - yield name, 1 - elif name == "st_sloperatio": - if hypermet: - yield name, 1 - elif name == "lt_arearatio": - if hypermet: - yield name, 1 - elif name == "lt_sloperatio": - if hypermet: - yield name, 1 - elif name == "step_heightratio": - if hypermet: - yield name, 1 - elif name == "eta_factor": - if not hypermet: - yield name, 1 - elif name == "linegroup_areas": - n = len(self.linegroup_areas) - if n: - yield name, n - elif name == "linpol_coefficients": - n = len(self.linpol_coefficients) - if n: - yield name, n - elif name == "exppol_coefficients": - n = len(self.exppol_coefficients) - if n: - yield name, n - else: - raise ValueError(name) + delta = self.config["peakshape"]["deltast_arearatio"] + return Gefit.CQUOTED, value - delta, value + delta def _convert_parameter_names(self, names): linegroup_names = None @@ -1899,7 +1900,7 @@ def mcatheory( ): """Evaluate to MCA model (does not include the numerical background) - y(x) = ycont(P(x)) + A1*G1(E(x)) + A2*G2(E(x)) + ... + y(x) = ycont(P(x)) + A1*F1(E(x)) + A2*F2(E(x)) + ... x: MCA channels (positive integers) @@ -1910,7 +1911,26 @@ def mcatheory( E(x) = zero + gain*x P(x) = E(x - ) - Gi(x): several peaks with normalized total area + Fi(x): emission, scatter or escape peak + + Hypermet: + + F(x) = A * Gnorm(x, u, s) + + st_arearatio*A * Tnorm(x, u, s, st_sloperatio) + + lt_arearatio*A * Tnorm(x, u, s, lt_sloperatio) + + step_heightratio*A * u/(sqrt(2*pi)*s) * Snorm(x, u, s) + + A: the area of the gaussian part, not the entire peak + + Gaussian is normalized in ]-inf, inf[ + Gnorm(x, u, s) = exp[-(x-u)^2/(2*s^2)] / (sqrt(2*pi)*s) + + Step is normalized in [0, inf[ + Snorm(x, u, s) = erfc[(x-u)/(sqrt(2)*s)] / (2 * u) + + Tail is normalized in ]-inf, inf[ + Tnorm(x, u, s, r) = erfc[(x-u)/(sqrt(2)*s) + s/(sqrt(2)*r)] + * exp[(x-u)/r + (s/r)^2/2] / (2 * r) :param array xdata: :param int hypermet: @@ -1939,7 +1959,9 @@ def mcatheory( return y def ypileup(self, ymodel, xdata=None): - """The model contains the peaks and the continuum""" + """The ymodel contains the peaks and the continuum. + Pileup is + """ pileupfactor = self.sum if not pileupfactor: if xdata is None: @@ -1949,17 +1971,16 @@ def ypileup(self, ymodel, xdata=None): if xdata is None: xdata = self.xdata return pileupfactor * SpecfitFuns.pileup( - ymodel, min(xdata), self.zero, self.gain + ymodel, min(xdata), self.zero, self.gain, 0 ) def _ydata_to_fit(self, ydata, xdata=None): """The fitting is done after subtracting the numerical background""" if self.hasNumBkg: ydata = ydata - self.ynumbkg(xdata=xdata) - if self.linear: - if self.hasPileUp: - ymodel = self.mcatheory(xdata=xdata, summing=False) - ydata = ydata - self.ypileup(ymodel, xdata=xdata) + if self.linear and self.hasPileUp: + ymodel = self.mcatheory(xdata=xdata, summing=False) + ydata = ydata - self.ypileup(ymodel, xdata=xdata) return ydata @property @@ -1997,6 +2018,15 @@ def linear_derivatives_fitmodel(self, xdata=None): derivatives += arr.tolist() return numpy.array(derivatives) + @contextmanager + def _keep_parameters(self): + # TODO: wait for parameters refactoring + keep = self.parameters + try: + yield + finally: + self.parameters = keep + def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter @@ -2004,60 +2034,65 @@ def derivative_fitmodel(self, param_idx, xdata=None): :param array xdata: length nxdata :returns array: nxdata """ - name, pgroupi = self._parameter_name_from_index(param_idx) - if name == "zero": - raise NotImplementedError - elif name == "gain": - raise NotImplementedError - elif name == "noise": - raise NotImplementedError - elif name == "fano": - raise NotImplementedError - elif name == "sum": - raise NotImplementedError - elif name == "st_arearatio": - raise NotImplementedError - elif name == "st_sloperatio": - raise NotImplementedError - elif name == "lt_arearatio": - raise NotImplementedError - elif name == "lt_sloperatio": - raise NotImplementedError - elif name == "step_heightratio": - raise NotImplementedError - elif name == "eta_factor": - raise NotImplementedError - elif name == "linegroup_areas": - return self.mcatheory( - selected_groups=[pgroupi], - normalized_peakgroups=True, - xdata=xdata, - continuum=False, + with self._keep_parameters(): + name, pgroupi = self._parameter_name_from_index(param_idx) + print( + name, + "index=", + param_idx, + "index in group=", + pgroupi, + "value=", + self.parameters[param_idx], ) - elif name == "linpol": - # TODO: subtract param - return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) - elif name == "exppol": - # TODO: subtract param - return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) - else: - raise ValueError(name) + if name == "st_arearatio": + self.parameters[param_idx] = 1 + return self.mcatheory(xdata=xdata, hypermet=2, continuum=False) + elif name == "lt_arearatio": + self.parameters[param_idx] = 1 + return self.mcatheory(xdata=xdata, hypermet=4, continuum=False) + elif name == "step_heightratio": + self.parameters[param_idx] = 1 + return self.mcatheory(xdata=xdata, hypermet=8, continuum=False) + elif name == "linegroup_areas": + self.parameters[param_idx] = 1 + return self.mcatheory( + selected_groups=[pgroupi], + xdata=xdata, + continuum=False, + ) + elif name == "linpol": + # TODO: subtract param + return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) + elif name == "exppol": + # TODO: subtract param + return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) + else: + return self._numerical_derivative(param_idx, xdata=xdata) + + def _numerical_derivative(self, param_idx, xdata=None): + with self._keep_parameters(): + parameters = self.parameters + p0 = parameters[param_idx] + + # Choose delta to be a small fraction of the + # parameter value but not too small, otherwise + # the derivative is zero. + delta = p0 * 1e-5 + if delta < 0: + delta = min(delta, -1e-12) + else: + delta = max(delta, 1e-12) - def _numerical_derivative(self, index, xdata=None): - keep = self.parameters - parameters = keep.copy() - p0 = parameters[index] - try: - delta = (p0[index] + p0[index]) * 0.00001 - parameters[index] = p0 + delta + parameters[param_idx] = p0 + delta self.parameters = parameters f1 = self.evaluate_fitmodel(xdata=xdata) - parameters[index] = p0 - delta + + parameters[param_idx] = p0 - delta self.parameters = parameters f2 = self.evaluate_fitmodel(xdata=xdata) - finally: - self.parameters = keep - return (f1 - f2) / (2.0 * delta) + + return (f1 - f2) / (2.0 * delta) @property def maxiter(self): diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index b63e24449..ec68efca8 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -52,14 +52,16 @@ def inner2(self, *args, **kw): class testFitModel(unittest.TestCase): def setUp(self): - self.random_state = numpy.random.RandomState(seed=0) + self.random_state = numpy.random.RandomState(seed=100) def create_model(self, nmodels): + self.nmodels = nmodels + self.is_concat = nmodels != 1 if nmodels == 1: self.fitmodel = SimpleModel.SimpleModel() else: self.fitmodel = SimpleModel.SimpleConcatModel(ndetectors=nmodels) - assert not self.fitmodel.linear + self.assertTrue(not self.fitmodel.linear) self.init_random() ydata = self.fitmodel.yfullmodel.copy() self.fitmodel.ydata = ydata @@ -69,7 +71,7 @@ def create_model(self, nmodels): self.validate_model() def init_random(self, **kw): - if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): + if self.is_concat: for model in self.fitmodel._models: self._init_random(model, **kw) self.fitmodel.shared_attributes = self.fitmodel.shared_attributes @@ -78,8 +80,8 @@ def init_random(self, **kw): def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): """Peaks close to the border will cause the nlls to fail""" - self.nnonglobals = 4 # zero, gain, wzero, wgain - self.nglobals = npeaks # concentrations + self.npeaks = npeaks # concentrations + self.nshapeparams = 4 # zero, gain, wzero, wgain model.xdata_raw = numpy.arange(nchannels) model.ydata_raw = numpy.full(nchannels, numpy.nan) model.ybkg = 10 @@ -99,52 +101,55 @@ def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): model.efficiency = self.random_state.uniform(low=5000, high=6000, size=npeaks) def modify_random(self, only_linear=False): - if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): - self._modify_random_concat(only_linear=only_linear) - else: - self._modify_random(only_linear=only_linear) + self._modify_random(only_linear=only_linear) self.validate_model() # self.plot() def _modify_random(self, only_linear=False): + porg = self.fitmodel.parameters.copy() + plinorg = self.fitmodel.linear_parameters.copy() if only_linear: - p = self.fitmodel.parameters plin = self.fitmodel.linear_parameters - plin *= numpy.random.uniform(0.5, 0.8, len(plin)) + plin *= self.random_state.uniform(0.5, 0.8, len(plin)) + self.fitmodel.linear_parameters = plin + numpy.testing.assert_array_equal(self.fitmodel.linear_parameters, plin) + p = self.fitmodel.parameters else: p = self.fitmodel.parameters - plin = self.fitmodel.linear_parameters - p *= numpy.random.uniform(0.95, 1, len(p)) - plin *= numpy.random.uniform(0.5, 0.8, len(plin)) + p *= self.random_state.uniform(0.95, 1, len(p)) self.fitmodel.parameters = p - assert numpy.array_equal(self.fitmodel.parameters, p) - self.fitmodel.linear_parameters = plin - assert numpy.array_equal(self.fitmodel.linear_parameters, plin) - return p + numpy.testing.assert_array_equal(self.fitmodel.parameters, p) + plin = self.fitmodel.linear_parameters - def _modify_random_concat(self, only_linear=False): - p = self._modify_random(only_linear=only_linear) - assert not numpy.array_equal( - self.fitmodel.parameters[: self.nglobals], p[: self.nglobals] - ) - assert numpy.array_equal( - self.fitmodel.parameters[self.nglobals :], p[self.nglobals :] - ) + for param_idx, param_name in enumerate(self.fitmodel.parameter_names): + if "concentration" in param_name or not only_linear: + self.assertNotEqual(p[param_idx], porg[param_idx], msg=param_name) + else: + self.assertEqual(p[param_idx], porg[param_idx], msg=param_name) - def validate_model(self): - self._validate_model(self.fitmodel) - if isinstance(self.fitmodel, SimpleModel.SimpleConcatModel): - for model in self.fitmodel._models: - self._validate_model(model) + for param_idx, param_name in enumerate(self.fitmodel.linear_parameter_names): + self.assertNotEqual(plin[param_idx], plinorg[param_idx], msg=param_name) - def _validate_model(self, model): - is_concat = isinstance(model, SimpleModel.SimpleConcatModel) + return p - assert not model.excluded_parameters - assert not model.included_parameters - assert model.ndata == len(model.xdata) - assert model.nparameters == len(model.parameters) - assert model.nlinear_parameters == len(model.linear_parameters) + def validate_model(self): + self._validate_model(self.fitmodel, self.is_concat) + if self.is_concat: + for model in self.fitmodel._models: + self._validate_model(model, False) + + def _validate_model(self, model, is_concat): + if not is_concat: + # Alphabetic order + expected = ["concentrations", "gain", "wgain", "wzero", "zero"] + self.assertEqual(model.all_parameter_group_names, expected) + expected = ["concentrations"] + self.assertEqual(model.all_linear_parameter_group_names, expected) + self.assertTrue(not model._excluded_parameters) + self.assertTrue(not model._included_parameters) + self.assertEqual(model.ndata, len(model.xdata)) + self.assertEqual(model.nparameters, len(model.parameters)) + self.assertEqual(model.nlinear_parameters, len(model.linear_parameters)) arr1 = model.evaluate_fullmodel() arr2 = model.evaluate_linear_fullmodel() @@ -160,29 +165,28 @@ def _validate_model(self, model): numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) - nonlin_names = ["zero", "gain", "wzero", "wgain"] - lin_names = ["concentrations" + str(i) for i in range(self.nglobals)] - names = nonlin_names + lin_names + # Alphabetic order + nonlin_names = ["gain", "wgain", "wzero", "zero"] + lin_names = ["concentrations" + str(i) for i in range(self.npeaks)] + names = lin_names + nonlin_names if is_concat: model.validate_shared_attributes() - assert model.nshared_parameters == self.nglobals - assert model.nshared_linear_parameters == self.nglobals + self.assertEqual(model.nshared_parameters, self.npeaks) + self.assertEqual(model.nshared_linear_parameters, self.npeaks) nmodels = model.nmodels - nonglobal_names = [ + names = lin_names + [ name + str(i) for i in range(nmodels) for name in nonlin_names ] - global_names = lin_names - names = global_names + nonglobal_names - n = self.nglobals + self.nnonglobals * nmodels - assert model.nparameters == n - assert model.nlinear_parameters == self.nglobals - assert model.parameter_names == names - assert model.linear_parameter_names == lin_names + n = self.npeaks + self.nshapeparams * nmodels + self.assertEqual(model.nparameters, n) + self.assertEqual(model.nlinear_parameters, self.npeaks) + self.assertEqual(model.parameter_names, names) + self.assertEqual(model.linear_parameter_names, lin_names) else: - assert model.nparameters == self.nglobals + self.nnonglobals - assert model.nlinear_parameters == self.nglobals - assert model.parameter_names == names - assert model.linear_parameter_names == lin_names + self.assertEqual(model.nparameters, self.npeaks + self.nshapeparams) + self.assertEqual(model.nlinear_parameters, self.npeaks) + self.assertEqual(model.parameter_names, names) + self.assertEqual(model.linear_parameter_names, lin_names) def plot(self): import matplotlib.pyplot as plt @@ -216,8 +220,10 @@ def _testLinearFit(self): result = self.fitmodel.fit() self.assert_result(result, expected) - assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - assert not numpy.allclose(self.fitmodel.linear_parameters, expected) + self.assertTrue( + not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + ) + self.assertTrue(not numpy.allclose(self.fitmodel.linear_parameters, expected)) self.fitmodel.use_fit_result(result) numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) @@ -238,14 +244,17 @@ def _testNonLinearFit(self): self.modify_random(only_linear=False) # from PyMca5.PyMcaMisc.ProfilingUtils import profile - # with profile(memory=False, filename="testNonLinearFit.pyprof"): + # filename = "testNonLinearFit{}.pyprof".format(self.nmodels) + # with profile(memory=False, filename=filename): result = self.fitmodel.fit(full_output=True) # TODO: non-linear parameters not precise # self.assert_result(result, expected1) - assert not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - assert not numpy.allclose(self.fitmodel.parameters, expected1) - assert not numpy.allclose(self.fitmodel.linear_parameters, expected2) + self.assertTrue( + not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + ) + self.assertTrue(not numpy.allclose(self.fitmodel.parameters, expected1)) + self.assertTrue(not numpy.allclose(self.fitmodel.linear_parameters, expected2)) self.fitmodel.use_fit_result(result) # self.plot() @@ -253,7 +262,7 @@ def _testNonLinearFit(self): # TODO: non-linear parameters not precise # numpy.testing.assert_allclose(self.fitmodel.parameters, expected1) numpy.testing.assert_allclose( - self.fitmodel.linear_parameters, expected2, rtol=1e-5 + self.fitmodel.linear_parameters, expected2, rtol=1e-3 ) def assert_result(self, result, expected): @@ -261,47 +270,53 @@ def assert_result(self, result, expected): pstd = numpy.asarray(result["uncertainties"]) ll = p - 3 * pstd ul = p + 3 * pstd - assert all((expected >= ll) & (expected <= ul)) + self.assertTrue(all((expected >= ll) & (expected <= ul))) def assert_ymodel(self): a = self.fitmodel.ydata b = self.fitmodel.yfullmodel mask = (a > 1) & (b > 1) - assert mask.any() + self.assertTrue(mask.any()) numpy.testing.assert_allclose(a[mask], b[mask], rtol=1e-3) @with_model(8) def testParameterIndex(self): # Test parameter index conversion from concatenated model to single model nmodels = self.fitmodel.nmodels - nglobals = self.nglobals + npeaks = self.npeaks for linear in [False, True]: self.fitmodel.linear = linear if linear: - nnonglobals = 0 + nshapeparams = 0 else: - nnonglobals = self.nnonglobals + nshapeparams = self.nshapeparams imodels = [] iparams = [] - for param_idx in range(self.fitmodel.nparameters): + for param_idx, param_name in enumerate(self.fitmodel.parameter_names): lst = list( self.fitmodel._parameter_model_index(param_idx, linear_only=linear) ) + # imodel: model indices + # iparam: index of the parameter in the corresponing models if lst: imodel, iparam = list(zip(*lst)) else: imodel, iparam = tuple(), tuple() - if param_idx < nglobals: - assert imodel == tuple(range(nmodels)) - assert iparam == tuple([nnonglobals + param_idx] * nmodels) + if "concentrations" in param_name: + offset = 0 + # Shared parameters + self.assertEqual(imodel, tuple(range(nmodels))) + self.assertEqual(iparam, (param_idx - offset,) * nmodels) else: + # Non-shared peak shape parameters + offset = npeaks imodels.extend(imodel) - iparams.extend(iparam) - assert len(imodels) == nnonglobals * nmodels - expected = numpy.repeat(list(range(nmodels)), nnonglobals) - assert imodels == expected.tolist() - expected = numpy.tile(list(range(nnonglobals)), nmodels) - assert iparams == expected.tolist() + iparams.extend(i - offset for i in iparam) + self.assertEqual(len(imodels), nshapeparams * nmodels) + expected = numpy.repeat(list(range(nmodels)), nshapeparams) + self.assertEqual(imodels, expected.tolist()) + expected = numpy.tile(list(range(nshapeparams)), nmodels) + self.assertEqual(iparams, expected.tolist()) @with_model(8) def testChannelIndex(self): @@ -317,8 +332,8 @@ def testChannelIndex(self): for idx in self.fitmodel._generate_idx_channels(len(x2), stride=vstride): chunk = x2[idx] access_cnt[idx] += 1 - assert all(numpy.diff(chunk) == stride) - assert all(access_cnt == 1) + self.assertTrue(all(numpy.diff(chunk) == stride)) + self.assertTrue(all(access_cnt == 1)) def getSuite(auto=True): diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index a26d9d337..365718631 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -46,23 +46,30 @@ def testLinearPol(self): model.xdata = fitmodel.xdata = numpy.linspace(0, 100, 100) for degree in [0, 1, 5]: - ncoeff = degree + 1 - expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) - model.coefficients = expected - fitmodel.ydata = model.yfullmodel - - numpy.testing.assert_array_equal(model.parameters, expected) - numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) - numpy.testing.assert_array_equal(fitmodel.yfitdata, model.yfitmodel) - numpy.testing.assert_array_equal(model.yfitmodel, model.yfullmodel) - - for linear in [True, False]: - with self.subTest(degree=degree, linear=linear): - fitmodel.linear = linear - fitmodel.coefficients = numpy.zeros_like(expected) - self.assertEqual(fitmodel.degree, degree) - result = fitmodel.fit()["parameters"] - numpy.testing.assert_allclose(result, expected, rtol=1e-4) + with self.subTest(degree=degree): + model.degree = degree + fitmodel.degree = degree + ncoeff = degree + 1 + expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) + model.coefficients = expected + self.assertEqual(model.all_parameter_group_names, ["coefficients"]) + self.assertEqual( + model.all_linear_parameter_group_names, ["coefficients"] + ) + numpy.testing.assert_array_equal(model.parameters, expected) + + fitmodel.ydata = model.yfullmodel + numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) + numpy.testing.assert_array_equal(fitmodel.yfitdata, model.yfitmodel) + numpy.testing.assert_array_equal(model.yfitmodel, model.yfullmodel) + + for linear in [True, False]: + with self.subTest(degree=degree, linear=linear): + fitmodel.linear = linear + fitmodel.coefficients = numpy.zeros_like(expected) + self.assertEqual(fitmodel.degree, degree) + result = fitmodel.fit()["parameters"] + numpy.testing.assert_allclose(result, expected, rtol=1e-4) def testExpPol(self): model = PolynomialModels.ExponentialPolynomialModel() @@ -70,25 +77,30 @@ def testExpPol(self): model.xdata = fitmodel.xdata = numpy.linspace(-0.5, 0.5, 100) for degree in [0, 1, 5]: - ncoeff = degree + 1 - expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) - model.coefficients = expected - fitmodel.ydata = model.yfullmodel - - numpy.testing.assert_array_equal(model.parameters, expected) - numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) - numpy.testing.assert_allclose(fitmodel.yfitdata, model.yfitmodel) - numpy.testing.assert_allclose(model.yfitmodel, numpy.log(model.yfullmodel)) - - for linear in [True, False]: - with self.subTest(degree=degree, linear=linear): - fitmodel.linear = linear - fitmodel.coefficients = numpy.zeros_like(expected) - if not linear: - fitmodel.coefficients[0] = 0.1 - self.assertEqual(fitmodel.degree, degree) - result = fitmodel.fit()["parameters"] - numpy.testing.assert_allclose(result, expected) + with self.subTest(degree=degree): + model.degree = degree + fitmodel.degree = degree + ncoeff = degree + 1 + expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) + model.coefficients = expected + numpy.testing.assert_array_equal(model.parameters, expected) + + fitmodel.ydata = model.yfullmodel + numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) + numpy.testing.assert_allclose(fitmodel.yfitdata, model.yfitmodel) + numpy.testing.assert_allclose( + model.yfitmodel, numpy.log(model.yfullmodel) + ) + + for linear in [True, False]: + with self.subTest(degree=degree, linear=linear): + fitmodel.linear = linear + fitmodel.coefficients = numpy.zeros_like(expected) + if not linear: + fitmodel.coefficients[0] = 0.1 + self.assertEqual(fitmodel.degree, degree) + result = fitmodel.fit()["parameters"] + numpy.testing.assert_allclose(result, expected) def getSuite(auto=True): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 902a44dd0..f48adcafa 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -34,7 +34,10 @@ import numpy from PyMca5.PyMcaMath.fitting import SpecfitFuns -from PyMca5.PyMcaMath.fitting.Model import Model, ConcatModel +from PyMca5.PyMcaMath.fitting.Model import Model +from PyMca5.PyMcaMath.fitting.Model import ConcatModel +from PyMca5.PyMcaMath.fitting.Model import parameter +from PyMca5.PyMcaMath.fitting.Model import linear_parameter class SimpleModel(Model): @@ -60,7 +63,7 @@ def __str__(self): self.__class__, self.npeaks, self.zero, self.gain, self.wzero, self.wgain ) - @property + @parameter def zero(self): return self.config["detector"]["zero"] @@ -68,7 +71,7 @@ def zero(self): def zero(self, value): self.config["detector"]["zero"] = value - @property + @parameter def gain(self): return self.config["detector"]["gain"] @@ -76,7 +79,7 @@ def gain(self): def gain(self, value): self.config["detector"]["gain"] = value - @property + @parameter def wzero(self): return self.config["detector"]["wzero"] @@ -84,7 +87,7 @@ def wzero(self): def wzero(self, value): self.config["detector"]["wzero"] = value - @property + @parameter def wgain(self): return self.config["detector"]["wgain"] @@ -117,7 +120,7 @@ def fwhms(self): def areas(self): return self.efficiency * self.concentrations - @property + @linear_parameter def concentrations(self): return self.config["matrix"]["concentrations"] @@ -188,37 +191,6 @@ def ndata(self): def npeaks(self): return len(self.concentrations) - @property - def _parameter_group_names(self): - return ["zero", "gain", "wzero", "wgain", "concentrations"] - - @property - def _linear_parameter_group_names(self): - return ["concentrations"] - - def _iter_parameter_groups(self, linear_only=False): - """ - :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group - """ - if linear_only: - names = self.linear_parameter_group_names - else: - names = self.parameter_group_names - for name in names: - if name == "zero": - yield name, 1 - elif name == "gain": - yield name, 1 - elif name == "wzero": - yield name, 1 - elif name == "wgain": - yield name, 1 - elif name == "concentrations": - yield name, self.npeaks - else: - raise ValueError(name) - def evaluate_fitmodel(self, xdata=None): """Evaluate model diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index a661ff992..e0190d7ee 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -425,7 +425,7 @@ def testStainlessSteelDataFit(self): self.assertTrue( testValue > 0.30, "Expected Cr concentration above 0.30 got %.3f" % testValue) - # chek the sum of concentration of main components is above 1 + # check the sum of concentration of main components is above 1 # because of neglecting higher order excitations elements = ["Cr K", "V K", "Mn K", "Fe Ka", "Ni K"] total = 0.0 @@ -448,7 +448,7 @@ def testStainlessSteelDataFit(self): abs(concentrationsResult["mass fraction"]["Fe Ka"] - 0.65) < 0.03, "Invalid Fe Concentration Using Tertiary Excitation") - # chek the sum of concentration of main components is above 1 + # check the sum of concentration of main components is above 1 elements = ["Cr K", "Mn K", "Fe Ka", "Ni K"] total = 0.0 for element in elements: @@ -529,7 +529,7 @@ def testStainlessSteelDataFit(self): fluorates = mcaFit._fluoRates, addinfo=True) - # chek the sum of concentration of main components is above 1 + # check the sum of concentration of main components is above 1 elements = ["Cr K", "Mn K", "Fe Ka", "Ni K"] total = 0.0 for element in elements: @@ -544,19 +544,19 @@ def testStainlessSteelDataFit(self): "Strategy: Element %s discrepancy too large %.1f %%" % \ (element.split()[0], delta)) - def testLegacyMcaTheory(self): + def testCompareLegacyMcaTheory(self): x, y, configuration = self._readTrainingData() - self._testLegacyMcaTheory(x, y, configuration) + self._testCompareLegacyMcaTheory(x, y, configuration) x, y, configuration = self._readStainlessSteelData() configuration["concentrations"]['usematrix'] = 0 configuration["concentrations"]["usemultilayersecondary"] = 0 - self._testLegacyMcaTheory(x, y, configuration) + self._testCompareLegacyMcaTheory(x, y, configuration) configuration["concentrations"]['usematrix'] = 1 configuration["concentrations"]["usemultilayersecondary"] = 2 - self._testLegacyMcaTheory(x, y, configuration) + self._testCompareLegacyMcaTheory(x, y, configuration) configuration["concentrations"]['usematrix'] = 0 configuration["concentrations"]["usemultilayersecondary"] = 2 @@ -577,9 +577,9 @@ def testLegacyMcaTheory(self): "Mn", "Fe", "Ni", "-", "-", "-","-","-"] - self._testLegacyMcaTheory(x, y, configuration) + self._testCompareLegacyMcaTheory(x, y, configuration) - def _testLegacyMcaTheory(self, x, y, configuration): + def _testCompareLegacyMcaTheory(self, x, y, configuration): from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import NewClassMcaTheory @@ -596,6 +596,35 @@ def _testLegacyMcaTheory(self, x, y, configuration): _, fitResult2, result2 = self._configAndFit( x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) + import matplotlib.pyplot as plt + from pprint import pprint + + if False: + mcaFit.linear = False + print(mcaFit.linegroup_areas) + result = mcaFit.fit(full_output=True) + pprint(result["niter"]) + mcaFit.use_fit_result(result) + print(mcaFit.linegroup_areas) + + if False: + plt.plot(mcaFit.ydata, label="data") + plt.plot(mcaFit.ynumbkg(), label="ynumbkg") + plt.plot(mcaFit.yfullmodel, label="model") + plt.yscale("log") + plt.legend() + plt.show() + + if False: + for i, name in enumerate(mcaFit.parameter_names): + yd = mcaFit.derivative_fitmodel(i) + yd_num = mcaFit._numerical_derivative(i) + plt.plot(yd, label="yd") + plt.plot(yd_num, label="yd_num") + plt.legend() + plt.title(name) + plt.show() + t2 = time.time() print("\nLEGACY TIME", t1-t0) @@ -677,22 +706,23 @@ def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): fitResult1, result1 = mcaFit.startFit(digest=1) return configuration, fitResult1, result1 - def _assertDeepEqual(self, obj1, obj2): + def _assertDeepEqual(self, obj1, obj2, _nodes=tuple()): """Better verbosity than assertEqual for deep structures """ + err_msg = "->".join(_nodes) if isinstance(obj1, dict): - self.assertEqual(set(obj1.keys()), set(obj2.keys())) + self.assertEqual(set(obj1.keys()), set(obj2.keys()), err_msg) for k in obj1: - self._assertDeepEqual(obj1[k], obj2[k]) + self._assertDeepEqual(obj1[k], obj2[k], _nodes=_nodes+(k,)) elif isinstance(obj1, (list, tuple)): if isinstance(obj1[0], (list, tuple, numpy.ndarray)): - self._assertDeepEqual(obj1, obj2) + self._assertDeepEqual(obj1, obj2, _nodes=_nodes+(len(_nodes),)) else: - self.assertEqual(obj1, obj2) + self.assertEqual(obj1, obj2, err_msg) elif isinstance(obj1, numpy.ndarray): - numpy.testing.assert_allclose(obj1, obj2, rtol=0) + numpy.testing.assert_allclose(obj1, obj2, rtol=0, err_msg=err_msg) else: - self.assertEqual(obj1, obj2) + self.assertEqual(obj1, obj2, err_msg) def getSuite(auto=True): @@ -705,6 +735,7 @@ def getSuite(auto=True): testSuite.addTest(testXrf("testTrainingDataFilePresence")) testSuite.addTest(testXrf("testTrainingDataFit")) testSuite.addTest(testXrf("testStainlessSteelDataFit")) + testSuite.addTest(testXrf("testCompareLegacyMcaTheory")) return testSuite def test(auto=False): From df22bf3d6f8c9ee9e724880308d4817a77c61392 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 2 Jun 2021 20:02:50 +0200 Subject: [PATCH 30/74] fixup parameter caching --- PyMca5/PyMcaMath/fitting/Model.py | 1115 ++++++++++-------- PyMca5/PyMcaMath/fitting/PolynomialModels.py | 6 + PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 2 +- PyMca5/tests/FitModelTest.py | 15 +- PyMca5/tests/FitPolModelTest.py | 6 +- 5 files changed, 654 insertions(+), 490 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 39810fd7e..34b329778 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -40,7 +40,73 @@ from PyMca5.PyMcaMath.fitting import Gefit -class Cached(object): +class ModelUserInterface: + """The user part of the interface for all fit models. Need to be + implemented by all classes derived from `Model`. + """ + + @property + def xdata(self): + raise AttributeError from NotImplementedError + + @xdata.setter + def xdata(self, value): + raise AttributeError from NotImplementedError + + @property + def ydata(self): + raise AttributeError from NotImplementedError + + @ydata.setter + def ydata(self, value): + raise AttributeError from NotImplementedError + + @property + def ystd(self): + raise AttributeError from NotImplementedError + + @ystd.setter + def ystd(self, value): + raise AttributeError from NotImplementedError + + @property + def linear(self): + raise AttributeError from NotImplementedError + + @linear.setter + def linear(self, value): + raise AttributeError from NotImplementedError + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + raise NotImplementedError + + def non_leastsquares_increment(self): + raise NotImplementedError + + +class Cached: def __init__(self): self._cache = dict() @@ -85,7 +151,7 @@ def cache_wrapper(self, *args, **kw): class parameter(property): - """Usages: + """Usage: .. highlight:: python .. code-block:: python @@ -118,8 +184,8 @@ def __init__(self, fget=None, fset=None, fdel=None, doc=""): if fset is not None: fset = self._param_setter(fset) super().__init__(fget=fget, fset=fset, fdel=fdel, doc=doc) - self.fcount = lambda _: None - self.fconstraints = lambda _: None + self.fcount = self._fcount_default() + self.fconstraints = self._fconstraints_default() def getter(self, fget): if fget is not None: @@ -143,7 +209,7 @@ def constraints(self, fconstraints): def _param_getter(cls, fget): @functools.wraps(fget) def wrapper(self): - return fget(self) + return self._get_parameter(fget) return wrapper @@ -151,95 +217,99 @@ def wrapper(self): def _param_setter(cls, fset): @functools.wraps(fset) def wrapper(self, value): - return fset(self, value) + return self._set_parameter(fset, value) return wrapper + def _fcount_default(self): + def fcount(wself): + try: + return len(self.fget(wself)) + except TypeError: + return 1 + + return fcount + + def _fconstraints_default(self): + def fconstraints(wself): + return numpy.zeros((self.fcount(wself), 3)) + + return fconstraints + class linear_parameter(parameter): pass -class Model(Cached): - """Evaluation and derivatives of a model to be used in least-squares fitting. +class ModelInterface(Cached, ModelUserInterface): + """Interface for all fit models (Model and ConcatModel derived classes).""" - The `parameter` and `linear_parameter` work like python's `property` decorator. - Associated fit contraints are optional properties. + @property + def parameter_group_names(self): + raise AttributeError from NotImplementedError - There is a "fit model" and a "full model". The full model describes the data, - the fit model describes the pre-processed data (for example smoothed, - numerical back subtracted, ...). By default the full model and the fit model - are identical. + @property + def linear_parameter_group_names(self): + raise AttributeError from NotImplementedError - Fitting is done with linear least-squares optimization (no iterations) or - non-linear least-squares optimization (iterative). In addition a - non-least-squares optimization loop can be added on top (iterative). - - The abstract class provides the following abstract methods/properties: - * Data to fit: - * xdata: - * ydata: - * ystd: - * Fit Model: - * linear: linear or non-linear fitting - * evaluate_fitmodel: - * derivative_fitmodel: derivatives of all parameters - * linear_derivatives_fitmodel: derivatives of the linear parameters - * non_leastsquares_increment: non-least-squares optimization step + def _parameter_groups(self, linear_only=False): + """Yield name and count of enabled parameter groups - Example: + :param bool linear_only: + :yields str, int: group name, nb. parameters in the group + """ + raise NotImplementedError - ```python - model = MyModel() - model.xdata = xdata - model.ydata = ydata - model.ystd = ydata**0.5 + def _get_parameters(self, linear_only=False, fitting=True): + """ + :param bool linear_only: + :param bool fitting: + :returns array: + """ + raise NotImplementedError - plt.plot(model.xdata, model.ydata, label="data") - plt.plot(model.xdata, model.ymodel, label="initial") + def _get_constraints(self, linear_only=False): + """ + :param bool linear_only: + :returns array: nparams x 3 + """ + raise NotImplementedError - result = model.fit() - model.use_fit_result(result) + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. - plt.plot(model.xdata, model.ymodel, label="fit") - ``` - """ + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError - def __init__(self): - self._included_parameters = None # for model concatenation - self._excluded_parameters = None # for model concatenation - super(Model, self).__init__() + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - allp = cls._PARAMETER_GROUP_NAMES = list() - linp = cls._LINEAR_PARAMETER_GROUP_NAMES = list() - for name in sorted(dir(cls)): - attr = getattr(cls, name) - if isinstance(attr, parameter): - allp.append(name) - if isinstance(attr, linear_parameter): - linp.append(name) + :param array xdata: length nxdata + :returns array: n x nxdata + """ + raise NotImplementedError - @property - def xdata(self): - raise AttributeError from NotImplementedError + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. - @property - def ydata(self): - raise AttributeError from NotImplementedError + :param array xdata: length nxdata + :returns array: n x nxdata + """ + raise NotImplementedError @property - def ystd(self): + def ndata(self): raise AttributeError from NotImplementedError @property def yfitdata(self): - return self._ydata_to_fit(self.ydata) + raise AttributeError from NotImplementedError @property def yfitstd(self): - return self._ystd_to_fit(self.ystd) + raise AttributeError from NotImplementedError @property def yfullmodel(self): @@ -251,183 +321,209 @@ def yfitmodel(self): """Model of yfitdata""" return self.evaluate_fitmodel() - @property - def ndata(self): - return len(self.xdata) - @property def parameters(self): - """Only enabled parameters""" return self._get_parameters(fitting=False) - @parameters.setter - def parameters(self, values): - return self._set_parameters(values, fitting=False) + @property + def linear_parameters(self): + return self._get_parameters(linear_only=True, fitting=False) @property def fit_parameters(self): - """Only enabled parameters""" return self._get_parameters(fitting=True) - @fit_parameters.setter - def fit_parameters(self, values): - return self._set_parameters(values, fitting=True) + @property + def linear_fit_parameters(self): + return self._get_parameters(linear_only=True, fitting=True) @property def constraints(self): - """Only enabled parameters""" return self._get_constraints() @property - def nparameters(self): - """Only enabled parameters""" - return sum(tpl[1] for tpl in self._parameter_groups()) - - @property - def parameter_names(self): - """Only enabled parameters""" - return list(self._iter_parameter_names()) + def linear_constraints(self): + return self._get_constraints(linear_only=True) - @property - def all_parameter_group_names(self): - """All parameters, not only the enabled ones""" - return list(self._filter_parameter_names(self._PARAMETER_GROUP_NAMES)) + @parameters.setter + def parameters(self, values): + return self._set_parameters(values, fitting=False) - @property - def linear_parameters(self): - """Only enabled parameters""" - return self._get_parameters(linear_only=True, fitting=False) + @fit_parameters.setter + def fit_parameters(self, values): + return self._set_parameters(values, fitting=True) @linear_parameters.setter - def linear_parameters(self, params): - return self._set_parameters(params, linear_only=True, fitting=False) - - @property - def linear_fit_parameters(self): - """Only enabled parameters""" - return self._get_parameters(linear_only=True, fitting=True) + def linear_parameters(self, values): + return self._set_parameters(values, linear_only=True, fitting=False) @linear_fit_parameters.setter - def linear_fit_parameters(self, params): - return self._set_parameters(params, linear_only=True, fitting=True) + def linear_fit_parameters(self, values): + return self._set_parameters(values, linear_only=True, fitting=True) @property - def linear_constraints(self): - """Only enabled parameters""" - return self._get_constraints(linear_only=True) + def nparameters(self): + return sum(n for _, n in self._parameter_groups()) @property def nlinear_parameters(self): - """Only enabled parameters""" - return sum(tpl[1] for tpl in self._parameter_groups(linear_only=True)) + return sum(n for _, n in self._parameter_groups(linear_only=True)) + + @property + def parameter_names(self): + return list(self._iter_parameter_names()) @property def linear_parameter_names(self): - """Only enabled parameters""" return list(self._iter_parameter_names(linear_only=True)) - @property - def all_linear_parameter_group_names(self): - """All parameters, not only the enabled ones""" - return list(self._filter_parameter_names(self._LINEAR_PARAMETER_GROUP_NAMES)) + def _iter_parameter_names(self, linear_only=False): + for name, n in self._parameter_groups(linear_only=linear_only): + if n > 1: + for i in range(n): + yield name + str(i) + else: + yield name - def _get_parameters(self, linear_only=False, fitting=True): + def fit(self, full_output=False): """ - :param bool linear_only: - :param bool fitting: - :returns array: + :param bool full_output: add statistics to fitted parameters + :returns dict: """ - if linear_only: - nparams = self.nlinear_parameters - else: - nparams = self.nparameters - params = numpy.zeros(nparams) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - params[i : i + n] = getattr(self, name) - i += n - if fitting: - return self._parameters_to_fit(params) + if self.linear: + return self.linear_fit(full_output=full_output) else: - return params + return self.nonlinear_fit(full_output=full_output) - def _get_constraints(self, linear_only=False): + def linear_fit(self, full_output=False): """ - :param bool linear_only: - :returns array: nparams x 3 + :param bool full_output: add statistics to fitted parameters + :returns dict: """ - if linear_only: - nparams = self.nlinear_parameters - else: - nparams = self.nparameters - codes = numpy.zeros((nparams, 3), numpy.float64) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - constraints = getattr(self.__class__, name).fconstraints(self) - if constraints is not None: - codes[i : i + n] = constraints - i += n - return codes + with self._linear_fit_context(): + b = self.yfitdata # ndata + for i in range(max(self.niter_non_leastsquares, 1)): + A = self.linear_derivatives_fitmodel().T # ndata, nparams + result = lstsq( + A, + b.copy(), + uncertainties=True, + covariances=False, + digested_output=True, + ) + if self.niter_non_leastsquares: + self.linear_fit_parameters = result["parameters"] + self.non_leastsquares_increment() + result["linear"] = True + result["parameters"] = self._fit_to_linear_parameters(result["parameters"]) + result["uncertainties"] = self._fit_to_linear_uncertainties( + result["uncertainties"] + ) + result.pop("svd") + return result - def _set_parameters(self, params, linear_only=False, fitting=False): + def nonlinear_fit(self, full_output=False): """ - :param bool linear_only: - :param bool fitting: + :param bool full_output: add statistics to fitted parameters + :returns dict: """ - if fitting: - params = self._fit_to_parameters(params) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - if n > 1: - getattr(self, name)[:] = params[i : i + n] - elif n == 1: - setattr(self, name, params[i]) - i += n + with self._nonlinear_fit_context(): + constraints = self.constraints.T + xdata = self.xdata + ydata = self.yfitdata + ystd = self.yfitstd + for i in range(max(self.niter_non_leastsquares, 1)): + result = Gefit.LeastSquaresFit( + self._gefit_evaluate_fitmodel, + self.fit_parameters, + model_deriv=self._gefit_derivative_fitmodel, + xdata=xdata, + ydata=ydata, + sigmadata=ystd, + constrains=constraints, + maxiter=self.maxiter, + weightflag=self.weightflag, + deltachi=self.deltachi, + fulloutput=full_output, + ) + if self.niter_non_leastsquares: + self.fit_parameters = result[0] + self.non_leastsquares_increment() + ret = { + "linear": False, + "parameters": self._fit_to_parameters(result[0]), + "uncertainties": self._fit_to_uncertainties(result[2]), + "chi2_red": result[1], + } + if full_output: + ret["niter"] = result[3] + ret["lastdeltachi"] = result[4] + return ret - def _filter_parameter_names(self, names): - included = self._included_parameters - excluded = self._excluded_parameters - for name in names: - if included is not None and name not in included: - continue - if excluded is not None and name in excluded: - continue - yield name + @property + def maxiter(self): + return 100 - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. + @property + def deltachi(self): + return None - :param array xdata: length nxdata - :returns array: nxdata - """ - y = self.evaluate_fitmodel(xdata=xdata) - return self._fit_to_ydata(y, xdata=xdata) + @property + def weightflag(self): + return 0 - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. + @property + def niter_non_leastsquares(self): + return 0 - :param array xdata: length nxdata - :returns array: n x nxdata - """ - y = self.evaluate_linear_fitmodel(xdata=xdata) - return self._fit_to_ydata(y, xdata=xdata) + @contextmanager + def _linear_fit_context(self): + with self.cachingContext("fit"): + with self._linear_context(True): + with self._cache_parameters_context(): + yield - def evaluate_fitmodel(self, xdata=None): - """Evaluate the fit model. + @contextmanager + def _nonlinear_fit_context(self): + with self.cachingContext("fit"): + with self._linear_context(False): + with self._cache_parameters_context(): + yield + + @contextmanager + def _cache_parameters_context(self): + with self.cachingContext("parameters"): + yield + + @contextmanager + def _linear_context(self, linear): + keep = self.linear + self.linear = linear + try: + yield + finally: + self.linear = keep + def _gefit_evaluate_fitmodel(self, parameters, xdata): + """Update parameters and evaluate model + + :param array parameters: length nparams :param array xdata: length nxdata :returns array: nxdata """ - raise NotImplementedError + self.fit_parameters = parameters + return self.evaluate_fitmodel(xdata=xdata) - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. + def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): + """Update parameters and return derivate to a specific parameter + :param array parameters: length nparams + :param int param_idx: :param array xdata: length nxdata - :returns array: n x nxdata + :returns array: nxdata """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - return self.linear_parameters.dot(derivatives) + self.fit_parameters = parameters + return self.derivative_fitmodel(param_idx, xdata=xdata) def linear_decomposition_fitmodel(self, xdata=None): """Linear decomposition of the fit model. @@ -435,101 +531,286 @@ def linear_decomposition_fitmodel(self, xdata=None): :param array xdata: length nxdata :returns array: nparams x nxdata """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - return self.linear_parameters[:, numpy.newaxis] * derivatives - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + return self.linear_parameters[:, numpy.newaxis] * derivatives + + def derivatives_fitmodel(self, xdata=None): + """Derivates to all parameters of the fit model. + + :param array xdata: length nxdata + :returns list(array): nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + return [ + self.derivative_fitmodel(i, xdata=xdata) for i in range(self.nparameters) + ] + + def use_fit_result(self, result): + """ + :param dict result: + """ + if result["linear"]: + self.linear_parameters = result["parameters"] + else: + self.parameters = result["parameters"] + + @contextmanager + def use_fit_result_context(self, result): + with self._linear_context(result["linear"]): + with self._cache_parameters_context(): + self.use_fit_result(result) + yield + + def _parameters_to_fit(self, params): + return params + + def _linear_parameters_to_fit(self, params): + return params + + def _fit_to_parameters(self, params): + return params + + def _fit_to_linear_parameters(self, params): + return params + + def _fit_to_linear_uncertainties(self, uncertainties): + return uncertainties + + def _fit_to_uncertainties(self, uncertainties): + return uncertainties + + +class Model(ModelInterface): + """Evaluation and derivatives of a model to be used in least-squares fitting. + + Derived classes: + + * implement the ModelUserInterface. + * add parameter like a python property by using the `parameter` or + `linear_parameter` decorators instead of the `property` decortor. + + There is a "fit model" and a "full model". The full model describes the data, + the fit model describes the pre-processed data (for example smoothed, + numerical back subtracted, ...). By default the full model and the fit model + are identical. + + Fitting is done with linear least-squares optimization (not iterative) + or non-linear least-squares optimization (iterative). An outer loop of + non-least-squares optimization can be enabled (iterative). + + Example: + + .. code-block:: python + + model = MyModel() + model.xdata = xdata + model.ydata = ydata + model.ystd = ydata**0.5 + + plt.plot(model.xdata, model.ydata, label="data") + plt.plot(model.xdata, model.ymodel, label="initial") + + result = model.fit() + model.use_fit_result(result) + + plt.plot(model.xdata, model.ymodel, label="fit") + """ + + def __init__(self): + self._included_parameters = None # for ConcatModel + self._excluded_parameters = None # for ConcatModel + super(Model, self).__init__() + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + allp = cls._PARAMETER_GROUP_NAMES = list() + linp = cls._LINEAR_PARAMETER_GROUP_NAMES = list() + for name in sorted(dir(cls)): + attr = getattr(cls, name) + if isinstance(attr, parameter): + allp.append(name) + if isinstance(attr, linear_parameter): + linp.append(name) + + @property + def parameter_group_names(self): + return list(self._filter_parameter_names(self._PARAMETER_GROUP_NAMES)) + + @property + def linear_parameter_group_names(self): + return list(self._filter_parameter_names(self._LINEAR_PARAMETER_GROUP_NAMES)) + + def _filter_parameter_names(self, names): + included = self._included_parameters + excluded = self._excluded_parameters + for name in names: + if included is not None and name not in included: + continue + if excluded is not None and name in excluded: + continue + yield name + + def _get_parameters(self, linear_only=False, fitting=True): + """ + :param bool linear_only: + :param bool fitting: + :returns array: + """ + cache = self.getCache("parameters") + if cache is None: + return self._get_parameters_notcached( + linear_only=linear_only, fitting=fitting + ) + + key = self._parameters_cache_key() + parameters = cache.get(key, None) + if parameters is None: + parameters = cache[key] = self._get_parameters_notcached( + linear_only=linear_only, fitting=fitting + ) + return parameters + + def _get_parameters_notcached(self, linear_only=False, fitting=True): + """Helper for `_get_parameters`""" + if linear_only: + nparams = self.nlinear_parameters + else: + nparams = self.nparameters + params = numpy.zeros(nparams) + i = 0 + for name, n in self._parameter_groups(linear_only=linear_only): + params[i : i + n] = getattr(self, name) + i += n + if fitting: + if linear_only: + return self._linear_parameters_to_fit(params) + else: + return self._parameters_to_fit(params) + else: + return params + + def _get_parameter(self, fget): + """Helper for parameter getters.""" + parameters = self.getCache("parameters") + if parameters is None: + return fget(self) + + key = self._parameters_cache_key() + parameters = parameters.get(key, None) + if parameters is None: + return fget(self) + + idx = self._parameter_slice_from_groupname(fget.__name__) + return parameters[idx] + + def _get_constraints(self, linear_only=False): + """ + :param bool linear_only: + :returns array: nparams x 3 + """ + if linear_only: + nparams = self.nlinear_parameters + else: + nparams = self.nparameters + codes = numpy.zeros((nparams, 3), numpy.float64) + i = 0 + for name, n in self._parameter_groups(linear_only=linear_only): + codes[i : i + n] = getattr(self.__class__, name).fconstraints(self) + i += n + return codes - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata + def _set_parameters(self, params, linear_only=False, fitting=False): """ - raise NotImplementedError + :param bool linear_only: + :param bool fitting: + """ + cache = self.getCache("parameters") + if cache is None: + self._set_parameters_notcached( + params, linear_only=linear_only, fitting=fitting + ) + else: + key = self._parameters_cache_key() + cache[key] = params - def derivatives_fitmodel(self, xdata=None): - """Derivates to all parameters of the fit model. + def _set_parameters_notcached(self, params, linear_only=False, fitting=False): + """Helper of `_set_parameters` - :param array xdata: length nxdata - :returns list(array): nparams x nxdata + :param bool linear_only: + :param bool fitting: """ - if xdata is None: - xdata = self.xdata - return [ - self.derivative_fitmodel(i, xdata=xdata) for i in range(self.nparameters) - ] + if fitting: + if linear_only: + params = self._fit_to_linear_parameters(params) + else: + params = self._fit_to_parameters(params) + i = 0 + for name, n in self._parameter_groups(linear_only=linear_only): + if n > 1: + getattr(self, name)[:] = params[i : i + n] + elif n == 1: + setattr(self, name, params[i]) + i += n - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters + def _set_parameter(self, fset, value): + """Helper for parameter setters""" + parameters = self.getCache("parameters") + if parameters is None: + return fset(self, value) - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - raise NotImplementedError + key = self._parameters_cache_key() + parameters = parameters.get(key, None) + if parameters is None: + return fset(self, value) - @property - def linear(self): - raise AttributeError from NotImplementedError + idx = self._parameter_slice_from_groupname(fset.__name__) + parameters[idx] = value def _parameter_groups(self, linear_only=False): - """Returns an iterator over name and count of enabled parameters + """Yield name and count of enabled parameter groups :param bool linear_only: - :returns iterable(str, int): group name, nb. parameters in the group + :yields str, int: group name, nb. parameters in the group """ cache = self.getCache("fit", "parameter_groups") if cache is None: - it = self._iter_parameter_groups(linear_only=linear_only) - else: - a = self._included_parameters - b = self._excluded_parameters - if a is not None: - a = tuple(sorted(a)) - if b is not None: - b = tuple(sorted(b)) - key = a, b - it = cache.get(key) - if it is None: - it = cache[key] = list( - self._iter_parameter_groups(linear_only=linear_only) - ) - return it + yield from self._parameter_groups_notcached(linear_only=linear_only) + return - def _iter_parameter_groups(self, linear_only=False): + key = self._parameters_cache_key() + it = cache.get(key) + if it is None: + it = cache[key] = list( + self._parameter_groups_notcached(linear_only=linear_only) + ) + yield from it + + def _parameters_cache_key(self): + a = self._included_parameters + b = self._excluded_parameters + if a is not None: + a = tuple(sorted(a)) + if b is not None: + b = tuple(sorted(b)) + return a, b + + def _parameter_groups_notcached(self, linear_only=False): """Helper for `_parameter_groups`. :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group + :yields str, int: group name, nb. parameters in the group """ if linear_only: - names = self.all_linear_parameter_group_names + names = self.linear_parameter_group_names else: - names = self.all_parameter_group_names + names = self.parameter_group_names for name in names: param = getattr(self.__class__, name) n = param.fcount(self) - if n is None: - value = getattr(self, name) - try: - n = len(value) - except TypeError: - n = 1 if n: yield name, n - def _iter_parameter_names(self, linear_only=False): - """Only enabled parameters - - :param bool linear_only: - :yields str: - """ - for name, n in self._parameter_groups(linear_only=linear_only): - if n > 1: - for i in range(n): - yield name + str(i) - else: - yield name - def _parameter_name_from_index(self, idx, linear_only=False): """Parameter index to group name and group index @@ -541,109 +822,31 @@ def _parameter_name_from_index(self, idx, linear_only=False): return name, idx - i i += n - def _parameter_indices_from_name(self, name, linear_only=False): + def _parameter_slice_from_groupname(self, name, linear_only=False): """Parameter group name to index range - :returns int, int: start and end parameter index of the group + :returns int or slice: slice of parameter group in all parameters """ i = 0 for _name, n in self._parameter_groups(linear_only=linear_only): if name == _name: - return i, i + n + if n == 1: + return i + else: + return slice(i, i + n) i += n - def fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - if self.linear: - return self.linear_fit(full_output=full_output) - else: - return self.nonlinear_fit(full_output=full_output) - - @Cached.enableCaching("fit") - def linear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self._linear_fit_context(): - b = self.yfitdata # ndata - for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives_fitmodel().T # ndata, nparams - result = lstsq( - A, - b.copy(), - uncertainties=True, - covariances=False, - digested_output=True, - ) - if self.niter_non_leastsquares: - self.linear_fit_parameters = result["parameters"] - self.non_leastsquares_increment() - result["linear"] = True - result["parameters"] = self._fit_to_parameters(result["parameters"]) - result["uncertainties"] = self._fit_to_uncertainties(result["uncertainties"]) - result.pop("svd") - return result - - @contextmanager - def _linear_fit_context(self): - if self.niter_non_leastsquares: - keep = self.linear_parameters - try: - yield - finally: - if self.niter_non_leastsquares: - self.linear_parameters = keep + @property + def ndata(self): + return len(self.xdata) - @Cached.enableCaching("fit") - def nonlinear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self._nonlinear_fit_context(): - constraints = self.constraints.T - xdata = self.xdata - ydata = self.yfitdata - ystd = self.yfitstd - for i in range(max(self.niter_non_leastsquares, 1)): - result = Gefit.LeastSquaresFit( - self._evaluate_fitmodel, - self.fit_parameters, - model_deriv=self._derivative_fitmodel, - xdata=xdata, - ydata=ydata, - sigmadata=ystd, - constrains=constraints, - maxiter=self.maxiter, - weightflag=self.weightflag, - deltachi=self.deltachi, - fulloutput=full_output, - ) - if self.niter_non_leastsquares: - self.fit_parameters = result[0] - self.non_leastsquares_increment() - ret = { - "linear": False, - "parameters": self._fit_to_parameters(result[0]), - "uncertainties": self._fit_to_uncertainties(result[2]), - "chi2_red": result[1], - } - if full_output: - ret["niter"] = result[3] - ret["lastdeltachi"] = result[4] - return ret + @property + def yfitdata(self): + return self._ydata_to_fit(self.ydata) - @contextmanager - def _nonlinear_fit_context(self): - keep = self.parameters - try: - yield - finally: - self.parameters = keep + @property + def yfitstd(self): + return self._ystd_to_fit(self.ystd) def _ydata_to_fit(self, ydata, xdata=None): return ydata @@ -651,90 +854,38 @@ def _ydata_to_fit(self, ydata, xdata=None): def _ystd_to_fit(self, ystd, xdata=None): return ystd - def _parameters_to_fit(self, params): - return params - def _fit_to_ydata(self, yfit, xdata=None): return yfit - def _fit_to_parameters(self, params): - return params - - def _fit_to_uncertainties(self, uncertainties): - return uncertainties - - def _linear_parameters_to_fit(self, params): - return params - - def _fit_to_linear_parameters(self, params): - return params - - @property - def maxiter(self): - return 100 - - @property - def deltachi(self): - return None - - @property - def weightflag(self): - return 0 - - def _evaluate_fitmodel(self, parameters, xdata): - """Update parameters and evaluate model + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. - :param array parameters: length nparams :param array xdata: length nxdata :returns array: nxdata """ - self.fit_parameters = parameters - return self.evaluate_fitmodel(xdata=xdata) + y = self.evaluate_fitmodel(xdata=xdata) + return self._fit_to_ydata(y, xdata=xdata) - def _derivative_fitmodel(self, parameters, param_idx, xdata): - """Update parameters and return derivate to a specific parameter + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. - :param array parameters: length nparams - :param int param_idx: :param array xdata: length nxdata - :returns array: nxdata - """ - self.fit_parameters = parameters - return self.derivative_fitmodel(param_idx, xdata=xdata) - - def use_fit_result(self, result): - """ - :param dict result: + :returns array: n x nxdata """ - if result["linear"]: - self.linear_parameters = result["parameters"] - else: - self.parameters = result["parameters"] - - @contextmanager - def use_fit_result_context(self, result): - if result["linear"]: - keep = self.linear_parameters - else: - keep = self.parameters - self.use_fit_result(result) - try: - yield - finally: - if result["linear"]: - self.linear_parameters = keep - else: - self.parameters = keep + y = self.evaluate_linear_fitmodel(xdata=xdata) + return self._fit_to_ydata(y, xdata=xdata) - @property - def niter_non_leastsquares(self): - return 0 + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. - def non_leastsquares_increment(self): - raise NotImplementedError + :param array xdata: length nxdata + :returns array: n x nxdata + """ + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + return self.linear_parameters.dot(derivatives) -class ConcatModel(Model): +class ConcatModel(ModelInterface): """Concatenated model with shared parameters""" def __init__(self, models, shared_attributes=None): @@ -748,16 +899,18 @@ def __init__(self, models, shared_attributes=None): self._models = models self.__fixed_shared_attributes = { "linear", + "niter_non_leastsquares", "_included_parameters", "_excluded_parameters", } self.shared_attributes = shared_attributes - super(ConcatModel, self).__init__() + super().__init__() @contextmanager def cachingContext(self, cachename): + """Enter the same caching context for all models""" with ExitStack() as stack: - ctx = super(ConcatModel, self).cachingContext(cachename) + ctx = super().cachingContext(cachename) stack.enter_context(ctx) for m in self._models: stack.enter_context(m.cachingContext(cachename)) @@ -768,28 +921,33 @@ def nmodels(self): return len(self._models) @property - def model(self): + def shared_model(self): """Model used to get/set shared attributes""" return self._models[0] + @property + def _all_other_models(self): + """All models except for `shared_model`""" + return self._models[1:] + def __getattr__(self, name): """Get shared attribute""" if self.nmodels and name in self.shared_attributes: - return getattr(self.model, name) + return getattr(self.shared_model, name) raise AttributeError(name) def __setattr__(self, name, value): - """Set the attributed of the models when shared""" + """Set the attributes of all models when shared""" if ( name != "_models" and self.nmodels - and hasattr(self.model, name) + and hasattr(self.shared_model, name) and name in self.shared_attributes ): for m in self._models: setattr(m, name, value) else: - super(ConcatModel, self).__setattr__(name, value) + super().__setattr__(name, value) @property def shared_attributes(self): @@ -824,12 +982,12 @@ def validate_shared_attributes(self, shared_attributes=None): if shared_attributes is None: shared_attributes = self._shared_attributes for name in shared_attributes: - value = getattr(self.model, name) + value = getattr(self.shared_model, name) if isinstance(value, (Sequence, MutableMapping, numpy.ndarray)): - for m in self._models[1:]: + for m in self._all_other_models: assert id(value) == id(getattr(m, name)), name else: - for m in self._models[1:]: + for m in self._all_other_models: assert value == getattr(m, name), name def share_attributes(self, shared_attributes=None): @@ -841,11 +999,14 @@ def share_attributes(self, shared_attributes=None): return if shared_attributes is None: shared_attributes = self._shared_attributes - model = self.model + model = self.shared_model adict = {name: getattr(model, name) for name in shared_attributes} - for model in self._models[1:]: + for model in self._all_other_models: for name, value in adict.items(): - setattr(model, name, value) + try: + setattr(model, name, value) + except AttributeError: + pass # no setter @property def ndata(self): @@ -896,8 +1057,8 @@ def _get_data(self, attr): if nmodels == 0: return None elif nmodels == 1: - return getattr(self.model, attr) - elif getattr(self.model, attr) is None: + return getattr(self.shared_model, attr) + elif getattr(self.shared_model, attr) is None: return None else: return numpy.concatenate([getattr(m, attr) for m in self._models]) @@ -909,13 +1070,13 @@ def _set_data(self, attr, values): """ if len(values) != self.ndata: raise ValueError("Not the expected number of channels") - for idx, model in self._iter_models(values): + for idx, model in self._iter_model_data_slices(values): setattr(model, attr, values[idx]) @contextmanager def _filter_parameter_context(self, shared=True): - keepex = self._excluded_parameters - keepin = self._included_parameters + keepex = self._excluded_parameters # shared between all models + keepin = self._included_parameters # shared between all models try: if shared: if keepin: @@ -935,13 +1096,16 @@ def _filter_parameter_context(self, shared=True): self._included_parameters = keepin def _iter_parameter_models(self): + """Yields models which are in such a state that they have + either shared or non-shared parameters enabled. + """ with self._filter_parameter_context(shared=True): - yield self.model + yield self.shared_model with self._filter_parameter_context(shared=False): for m in self._models: yield m - def _iter_models_types(self): + def _iter_model_data_slices_types(self): modeltypes = set() for model in self._models: modeltype = type(model) @@ -949,25 +1113,17 @@ def _iter_models_types(self): modeltypes.add(modeltype) yield model - @property - def nparameters(self): - return sum(m.nparameters for m in self._iter_parameter_models()) - - @property - def nlinear_parameters(self): - return sum(m.nlinear_parameters for m in self._iter_parameter_models()) - @property def nshared_parameters(self): with self._filter_parameter_context(shared=True): - return self.model.nparameters + return self.shared_model.nparameters @property def nshared_linear_parameters(self): with self._filter_parameter_context(shared=True): - return self.model.nlinear_parameters + return self.shared_model.nlinear_parameters - def _get_parameters(self, linear_only=False, fitting=False): + def _get_parameters(self, linear_only=False, fitting=True): """ :param bool linear_only: :returns array: @@ -1009,16 +1165,19 @@ def _get_constraints(self, linear_only=False): ) def _parameter_groups(self, linear_only=False): - """ + """Yield name and count of enabled parameter groups + :param bool linear_only: - :yields (str, int): group name, nb. parameters in the group + :yields str, int: group name, nb. parameters in the group """ with self._filter_parameter_context(shared=True): - for item in self.model._parameter_groups(linear_only=linear_only): + for item in self.shared_model._parameter_groups(linear_only=linear_only): yield item with self._filter_parameter_context(shared=False): for i, m in enumerate(self._models): - for name, n in self.model._parameter_groups(linear_only=linear_only): + for name, n in self.shared_model._parameter_groups( + linear_only=linear_only + ): yield name + str(i), n def _parameter_model_index(self, idx, linear_only=False): @@ -1027,18 +1186,19 @@ def _parameter_model_index(self, idx, linear_only=False): :param bool linear_only: :param int idx: - :returns iterable(tuple): model index, parameter index in this model + :yields (int, int): model index, parameter index in this model """ cache = self.getCache("fit", "parameter_model_index") if cache is None: - it = self._iter_parameter_index(idx, linear_only=linear_only) - else: - it = cache.get(idx) - if it is None: - it = cache[idx] = list( - self._iter_parameter_index(idx, linear_only=linear_only) - ) - return it + yield from self._iter_parameter_index(idx, linear_only=linear_only) + return + + it = cache.get(idx) + if it is None: + it = cache[idx] = list( + self._iter_parameter_index(idx, linear_only=linear_only) + ) + yield from it def _iter_parameter_index(self, idx, linear_only=False): """Convert parameter index of ConcatModel to a parameter indices @@ -1078,22 +1238,22 @@ def _iter_parameter_index(self, idx, linear_only=False): @property def shared_parameters(self): with self._filter_parameter_context(shared=True): - return self.model.parameters + return self.shared_model.parameters @shared_parameters.setter def shared_parameters(self, values): with self._filter_parameter_context(shared=True): - self.model.parameters = values + self.shared_model.parameters = values @property def shared_linear_parameters(self): with self._filter_parameter_context(shared=True): - return self.model.linear_parameters + return self.shared_model.linear_parameters @shared_linear_parameters.setter def shared_linear_parameters(self, values): with self._filter_parameter_context(shared=True): - self.model.linear_parameters = values + self.shared_model.linear_parameters = values def _concatenate_evaluation(self, funcname, xdata=None): """Evaluate model @@ -1104,7 +1264,7 @@ def _concatenate_evaluation(self, funcname, xdata=None): if xdata is None: xdata = self.xdata ret = xdata * 0.0 - for idx, model in self._iter_models(xdata): + for idx, model in self._iter_model_data_slices(xdata): func = getattr(model, funcname) ret[idx] = func(xdata=xdata[idx]) return ret @@ -1151,9 +1311,9 @@ def derivative_fitmodel(self, param_idx, xdata=None): if xdata is None: xdata = self.xdata ret = xdata * 0.0 - idx_channels = self._idx_channels(len(xdata)) + model_data_slices = self._model_data_slices(len(xdata)) for model_idx, param_idx in self._parameter_model_index(param_idx): - idx = idx_channels[model_idx] + idx = model_data_slices[model_idx] model = self._models[model_idx] ret[idx] = model.derivative_fitmodel(param_idx, xdata=xdata[idx]) return ret @@ -1167,35 +1327,34 @@ def linear_derivatives_fitmodel(self, xdata=None): if xdata is None: xdata = self.xdata ret = numpy.empty((self.nlinear_parameters, xdata.size)) - for idx, model in self._iter_models(xdata): + for idx, model in self._iter_model_data_slices(xdata): ret[:, idx] = model.linear_derivatives_fitmodel(xdata=xdata[idx]) return ret - def _iter_models(self, xdata): - """Loop over the models and yield xdata slice - + def _iter_model_data_slices(self, xdata): + """ :param array xdata: :yields (slice, Model): """ - for item in zip(self._idx_channels(len(xdata)), self._models): + for item in zip(self._model_data_slices(len(xdata)), self._models): yield item - def _idx_channels(self, nconcat): - """Index of each model in the concatenated data + def _model_data_slices(self, nconcat): + """Slice of each model in the concatenated data :param int nconcat: :returns list(slice): """ - cache = self.getCache("fit", "idx_channels") + cache = self.getCache("fit", "model_data_slices") if cache is None: - return list(self._generate_idx_channels(nconcat)) + return list(self._generate_model_data_slices(nconcat)) else: if nconcat != cache.get("nconcat"): - cache["idx"] = list(self._generate_idx_channels(nconcat)) + cache["idx"] = list(self._generate_model_data_slices(nconcat)) cache["nconcat"] = nconcat return cache["idx"] - def _generate_idx_channels(self, nconcat, stride=None): + def _generate_model_data_slices(self, nconcat, stride=None): """Yield slice of the concatenated data for each model. The concatenated data could be sliced as `xdata[::stride]`. """ diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index 4b8e82c01..eacba31eb 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -165,3 +165,9 @@ def _fit_to_parameters(self, parameters): parameters = parameters.copy() parameters[0] = numpy.exp(parameters[0]) return parameters + + def _linear_parameters_to_fit(self, parameters): + return self._parameters_to_fit(parameters) + + def _fit_to_linear_parameters(self, parameters): + return self._fit_to_parameters(parameters) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 65d196fe0..3ae3389af 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -2168,7 +2168,7 @@ def imagingDigestResult(self): class MultiMcaTheory(ConcatModel): def __init__(self, ndetectors=1): models = [McaTheory() for i in range(ndetectors)] - shared_attributes = [] + shared_attributes = [] # nothing shared yet super(MultiMcaTheory, self).__init__( models, shared_attributes=shared_attributes ) diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index ec68efca8..f29aab279 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -40,10 +40,9 @@ def with_model(nmodels): def inner1(method): def inner2(self, *args, **kw): self.create_model(nmodels=nmodels) - try: - return method(self, *args, **kw) - finally: - self.validate_model() + result = method(self, *args, **kw) + self.validate_model() + return result return inner2 @@ -142,9 +141,9 @@ def _validate_model(self, model, is_concat): if not is_concat: # Alphabetic order expected = ["concentrations", "gain", "wgain", "wzero", "zero"] - self.assertEqual(model.all_parameter_group_names, expected) + self.assertEqual(model.parameter_group_names, expected) expected = ["concentrations"] - self.assertEqual(model.all_linear_parameter_group_names, expected) + self.assertEqual(model.linear_parameter_group_names, expected) self.assertTrue(not model._excluded_parameters) self.assertTrue(not model._included_parameters) self.assertEqual(model.ndata, len(model.xdata)) @@ -329,7 +328,9 @@ def testChannelIndex(self): vstride = stride if stride < 1000: vstride = None - for idx in self.fitmodel._generate_idx_channels(len(x2), stride=vstride): + for idx in self.fitmodel._generate_model_data_slices( + len(x2), stride=vstride + ): chunk = x2[idx] access_cnt[idx] += 1 self.assertTrue(all(numpy.diff(chunk) == stride)) diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 365718631..62232890f 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -52,10 +52,8 @@ def testLinearPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected - self.assertEqual(model.all_parameter_group_names, ["coefficients"]) - self.assertEqual( - model.all_linear_parameter_group_names, ["coefficients"] - ) + self.assertEqual(model.parameter_group_names, ["coefficients"]) + self.assertEqual(model.linear_parameter_group_names, ["coefficients"]) numpy.testing.assert_array_equal(model.parameters, expected) fitmodel.ydata = model.yfullmodel From 6632b1f9ca67f5fc3c7a4656b11651361ff763b9 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 22 Jun 2021 19:00:02 +0200 Subject: [PATCH 31/74] fixup --- PyMca5/PyMcaMath/fitting/CachedInterface.py | 164 ++ PyMca5/PyMcaMath/fitting/ConcatModel.py | 501 +++++++ PyMca5/PyMcaMath/fitting/LinkedInterface.py | 176 +++ PyMca5/PyMcaMath/fitting/Model.py | 1313 ++--------------- PyMca5/PyMcaMath/fitting/ModelInterface.py | 426 ++++++ .../fitting/ModelParameterInterface.py | 353 +++++ PyMca5/PyMcaMath/fitting/PolynomialModels.py | 65 +- PyMca5/PyMcaMath/fitting/PropertyUtils.py | 57 + PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 14 +- PyMca5/tests/CachedInterfaceTest.py | 127 ++ PyMca5/tests/FitModelTest.py | 78 +- PyMca5/tests/FitPolModelTest.py | 7 +- PyMca5/tests/LinkedInterfaceTest.py | 211 +++ PyMca5/tests/ModelParameterInterfaceTest.py | 92 ++ PyMca5/tests/SimpleModel.py | 75 +- PyMca5/tests/XrfTest.py | 5 +- 16 files changed, 2394 insertions(+), 1270 deletions(-) create mode 100644 PyMca5/PyMcaMath/fitting/CachedInterface.py create mode 100644 PyMca5/PyMcaMath/fitting/ConcatModel.py create mode 100644 PyMca5/PyMcaMath/fitting/LinkedInterface.py create mode 100644 PyMca5/PyMcaMath/fitting/ModelInterface.py create mode 100644 PyMca5/PyMcaMath/fitting/ModelParameterInterface.py create mode 100644 PyMca5/PyMcaMath/fitting/PropertyUtils.py create mode 100644 PyMca5/tests/CachedInterfaceTest.py create mode 100644 PyMca5/tests/LinkedInterfaceTest.py create mode 100644 PyMca5/tests/ModelParameterInterfaceTest.py diff --git a/PyMca5/PyMcaMath/fitting/CachedInterface.py b/PyMca5/PyMcaMath/fitting/CachedInterface.py new file mode 100644 index 000000000..d7ff7cb28 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/CachedInterface.py @@ -0,0 +1,164 @@ +import functools +from contextlib import contextmanager +from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property + + +class cached_property(wrapped_property): + def _wrap_getter(self, fget): + fget = super()._wrap_getter(fget) + + @functools.wraps(fget) + def wrapper(oself): + return oself._cached_property_fget(fget) + + return wrapper + + def _wrap_setter(self, fset): + fset = super()._wrap_setter(fset) + + @functools.wraps(fset) + def wrapper(oself, value): + return oself._cached_property_fset(fset, value) + + return wrapper + + +class CachedInterface: + _CACHED_PROPERTIES = tuple() + + def __init_subclass__(subcls, **kwargs): + super().__init_subclass__(**kwargs) + allp = list() + for name, attr in vars(subcls).items(): + if isinstance(attr, cached_property): + allp.append(name) + subcls._CACHED_PROPERTIES = subcls._CACHED_PROPERTIES + tuple(allp) + + @classmethod + def _cached_properties(self): + return self._CACHED_PROPERTIES + + def __init__(self): + self._cache_object = None + self._cache_object._cache_root = dict() + super().__init__() + + @property + def _cache_object(self): + if self.__external_cache_object is None: + return self + else: + return self.__external_cache_object + + @_cache_object.setter + def _cache_object(self, obj): + if obj is not None and not isinstance(obj, CachedInterface): + raise TypeError(obj, type(obj)) + self.__external_cache_object = obj + + @contextmanager + def cachingContext(self, cachename): + cache_root = self._cache_object._cache_root + new_context_entry = cachename not in cache_root + if new_context_entry: + cache_root[cachename] = dict() + try: + yield cache_root[cachename] + finally: + if new_context_entry: + del cache_root[cachename] + + def cachingEnabled(self, cachename): + return cachename in self._cache_object._cache_root + + def getCache(self, cachename, *subnames): + cache_root = self._cache_object._cache_root + if cachename not in cache_root: + return None + ret = cache_root[cachename] + for cachename in subnames: + try: + ret = ret[cachename] + except KeyError: + ret[cachename] = dict() + ret = ret[cachename] + return ret + + @contextmanager + def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): + values_cache = self._get_property_values_cache(**cacheoptions) + if values_cache is not None: + # Re-entering this context should not affect anything + yield values_cache + return + + if start_cache is None: + # Fill and empty cache with property values + cache_object = self._cache_object + key = cache_object._property_cache_key(**cacheoptions) + values_cache = cache_object._create_empty_cache(key, **cacheoptions) + nameindexmap = dict() + for name in self._cached_properties(): + index = cache_object._property_cache_index(name) + nameindexmap[name] = index + values_cache[index] = getattr(self, name) + else: + values_cache = start_cache + nameindexmap = None + + with self.cachingContext("_cached_properties"): + # Initialize the property values cache + self._set_property_values_cache(values_cache, **cacheoptions) + yield values_cache + + if persist: + # Set property values to the cached values + if nameindexmap is None: + cache_object = self._cache_object + for name in self._cached_properties(): + index = cache_object._property_cache_index(name) + setattr(self, name, values_cache[index]) + else: + for name, index in nameindexmap.items(): + setattr(self, name, values_cache[index]) + + def _get_property_values_cache(self, **cacheoptions): + caches = self.getCache("_cached_properties") + if caches is None: + return None + key = self._cache_object._property_cache_key(**cacheoptions) + return caches.get(key, None) + + def _set_property_values_cache(self, values_cache, **cacheoptions): + caches = self.getCache("_cached_properties") + if caches is None: + return + cache_object = self._cache_object + key = cache_object._property_cache_key(**cacheoptions) + caches[key] = values_cache + + def _cached_property_fget(self, fget): + values_cache = self._get_property_values_cache() + if values_cache is None: + return fget(self) + index = self._cache_object._property_cache_index(fget.__name__) + return values_cache[index] + + def _cached_property_fset(self, fset, value): + values_cache = self._get_property_values_cache() + if values_cache is None: + return fset(self, value) + index = self._cache_object._property_cache_index(fset.__name__) + values_cache[index] = value + + def _create_empty_cache(self, key, **cacheoptions): + # By default the property cache is a dictionary + return dict() + + def _property_cache_index(self, name): + # By default the property cache index is its name + return name + + def _property_cache_key(self, **cacheoptions): + # By default we only manage 1 cache (None) + return None diff --git a/PyMca5/PyMcaMath/fitting/ConcatModel.py b/PyMca5/PyMcaMath/fitting/ConcatModel.py new file mode 100644 index 000000000..4b1dd49c6 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/ConcatModel.py @@ -0,0 +1,501 @@ +import numpy +from collections.abc import Sequence, MutableMapping +from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface +from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface +from PyMca5.PyMcaMath.fitting.Model import Model + + +class ConcatModel(LinkedContainerInterface, ModelInterface): + """Concatenated model with shared parameters""" + + def __init__(self, models, shared_attributes=None): + if not isinstance(models, Sequence): + models = [models] + for model in models: + if not isinstance(model, Model): + raise ValueError("'models' must be a list of type 'Model'") + + super().__init__(models) + + self.__fixed_shared_attributes = { + "linear", + "niter_non_leastsquares", + "_included_parameters", + "_excluded_parameters", + } + self.shared_attributes = shared_attributes + + def _iter_context_managers(self, context_name): + ctxmgr = getattr(super(), context_name, None) + if ctxmgr is not None: + yield ctxmgr + for model in self._models: + yield getattr(model, context_name) + + @property + def nmodels(self): + return len(self._models) + + @property + def shared_model(self): + """Model used to get/set shared attributes""" + return self._models[0] + + @property + def _all_other_models(self): + """All models except for `shared_model`""" + return self._models[1:] + + def __getattr__(self, name): + """Get shared attribute""" + if self.nmodels and name in self.shared_attributes: + return getattr(self.shared_model, name) + raise AttributeError(name) + + def __setattr__(self, name, value): + """Set the attributes of all models when shared""" + if ( + name != "_models" + and self.nmodels + and hasattr(self.shared_model, name) + and name in self.shared_attributes + ): + for model in self._models: + setattr(model, name, value) + else: + super().__setattr__(name, value) + + @property + def shared_attributes(self): + """Attributes shared between the fit models (they should have the same value)""" + return self._shared_attributes + + @shared_attributes.setter + def shared_attributes(self, shared_attributes): + """ + :param Sequence(str) shared_attributes: + """ + if shared_attributes is None: + shared_attributes = set() + else: + shared_attributes = set(shared_attributes) + shared_attributes |= self.__fixed_shared_attributes + if self.nmodels <= 1: + self._shared_attributes = shared_attributes + return + self.share_attributes(shared_attributes) + self.validate_shared_attributes(shared_attributes) + self._shared_attributes = shared_attributes + + def validate_shared_attributes(self, shared_attributes=None): + """Check whether attributes are shared + + :param Sequence(str) shared_attributes: + :raises AssertionError: + """ + if self.nmodels <= 1: + return + if shared_attributes is None: + shared_attributes = self._shared_attributes + for name in shared_attributes: + value = getattr(self.shared_model, name) + if isinstance(value, (Sequence, MutableMapping, numpy.ndarray)): + for model in self._all_other_models: + assert id(value) == id(getattr(model, name)), name + else: + for model in self._all_other_models: + assert value == getattr(model, name), name + + def share_attributes(self, shared_attributes=None): + """Ensure attributes are shared + + :param Sequence(str) shared_attributes: + """ + if self.nmodels <= 1: + return + if shared_attributes is None: + shared_attributes = self._shared_attributes + model = self.shared_model + adict = {name: getattr(model, name) for name in shared_attributes} + for model in self._all_other_models: + for name, value in adict.items(): + try: + setattr(model, name, value) + except AttributeError: + pass # no setter + + @property + def ndata(self): + nmodels = self.nmodels + if nmodels == 0: + return 0 + else: + return sum(model.ndata for model in self._models) + + @property + def xdata(self): + return self._get_data("xdata") + + @xdata.setter + def xdata(self, values): + self._set_data("xdata", values) + + @property + def ydata(self): + return self._get_data("ydata") + + @ydata.setter + def ydata(self, values): + self._set_data("ydata", values) + + @property + def ystd(self): + return self._get_data("ystd") + + @ystd.setter + def ystd(self, values): + self._set_data("ystd", values) + + @property + def yfitdata(self): + return self._get_data("yfitdata") + + @property + def yfitstd(self): + return self._get_data("yfitstd") + + def _get_data(self, attr): + """ + :param str attr: + :returns array: + """ + nmodels = self.nmodels + if nmodels == 0: + return None + elif nmodels == 1: + return getattr(self.shared_model, attr) + elif getattr(self.shared_model, attr) is None: + return None + else: + return numpy.concatenate([getattr(model, attr) for model in self._models]) + + def _set_data(self, attr, values): + """ + :param str attr: + :param array values: + """ + if len(values) != self.ndata: + raise ValueError("Not the expected number of channels") + for idx, model in self._iter_model_data_slices(values): + setattr(model, attr, values[idx]) + + @contextmanager + def _filter_parameter_context(self, shared=True): + keepex = self._excluded_parameters # shared between all models + keepin = self._included_parameters # shared between all models + try: + if shared: + if keepin: + self._included_parameters = list( + set(keepin) - set(self.shared_attributes) + ) + else: + self._included_parameters = self.shared_attributes + else: + if keepex: + self._excluded_parameters.extend(self.shared_attributes) + else: + self._excluded_parameters = self.shared_attributes + yield + finally: + self._excluded_parameters = keepex + self._included_parameters = keepin + + def _iter_parameter_models(self): + """Yields models which are in such a state that they have + either shared or non-shared parameters enabled. + """ + with self._filter_parameter_context(shared=True): + yield self.shared_model + with self._filter_parameter_context(shared=False): + for model in self._models: + yield model + + def _iter_model_data_slices_types(self): + modeltypes = set() + for model in self._models: + modeltype = type(model) + if modeltype not in modeltypes: + modeltypes.add(modeltype) + yield model + + @property + def nshared_parameters(self): + with self._filter_parameter_context(shared=True): + return self.shared_model.nparameters + + @property + def nshared_linear_parameters(self): + with self._filter_parameter_context(shared=True): + return self.shared_model.nlinear_parameters + + def _get_parameters(self, linear_only=None): + """ + :param bool linear_only: + :returns array: + """ + return numpy.concatenate( + [ + model._get_parameters(linear_only=linear_only) + for model in self._iter_parameter_models() + ] + ) + + def _set_parameters(self, values, linear_only=None): + """ + :paramm array values: + :param bool linear_only: + """ + if linear_only is None: + linear_only = self.linear + i = 0 + for model in self._iter_parameter_models(): + if linear_only: + n = model.nlinear_parameters + else: + n = model.nparameters + if n: + model._set_parameters(values[i : i + n], linear_only=linear_only) + i += n + self.share_attributes() # TODO: find a better way to share parameters + + def _get_constraints(self, linear_only=None): + """ + :param bool linear_only: + :returns array: nparams x 3 + """ + return numpy.concatenate( + [ + model._get_constraints(linear_only=linear_only) + for model in self._iter_parameter_models() + ] + ) + + def _parameter_groups(self, linear_only=None): + """Yield name and count of enabled parameter groups + + :param bool linear_only: + :yields str, int: group name, nb. parameters in the group + """ + with self._filter_parameter_context(shared=True): + for item in self.shared_model._parameter_groups(linear_only=linear_only): + yield item + with self._filter_parameter_context(shared=False): + for i, model in enumerate(self._models): + for name, n in self.shared_model._parameter_groups( + linear_only=linear_only + ): + yield name + str(i), n + + def _parameter_model_index(self, idx, linear_only=None): + """Convert parameter index of ConcatModel to a parameter indices + of the underlying models (only one when parameter is not shared). + + :param bool linear_only: + :param int idx: + :yields (int, int): model index, parameter index in this model + """ + cache = self.getCache("fit", "parameter_model_index") + if cache is None: + yield from self._iter_parameter_index(idx, linear_only=linear_only) + return + + it = cache.get(idx) + if it is None: + it = cache[idx] = list( + self._iter_parameter_index(idx, linear_only=linear_only) + ) + yield from it + + def _iter_parameter_index(self, idx, linear_only=None): + """Convert parameter index of ConcatModel to a parameter indices + of the underlying models (only one when parameter is not shared). + + :param bool linear_only: + :param int idx: + :yields (int, int): model index, parameter index in this model + """ + if linear_only is None: + linear_only = self.linear + if linear_only: + nshared = self.nshared_linear_parameters + else: + nshared = self.nshared_parameters + shared_attributes = self.shared_attributes + if idx < nshared: + for i, model in enumerate(self._models): + iglobal = 0 + imodel = 0 + for name, n in model._parameter_groups(linear_only=linear_only): + if name in shared_attributes: + if idx >= iglobal and idx < (iglobal + n): + yield i, imodel + idx - iglobal + iglobal += n + imodel += n + else: + iglobal = nshared + for i, model in enumerate(self._models): + imodel = 0 + for name, n in model._parameter_groups(linear_only=linear_only): + if name not in shared_attributes: + if idx >= iglobal and idx < (iglobal + n): + yield i, imodel + idx - iglobal + return + iglobal += n + imodel += n + + @property + def shared_parameters(self): + with self._filter_parameter_context(shared=True): + return self.shared_model.parameters + + @shared_parameters.setter + def shared_parameters(self, values): + with self._filter_parameter_context(shared=True): + self.shared_model.parameters = values + + @property + def shared_linear_parameters(self): + with self._filter_parameter_context(shared=True): + return self.shared_model.linear_parameters + + @shared_linear_parameters.setter + def shared_linear_parameters(self, values): + with self._filter_parameter_context(shared=True): + self.shared_model.linear_parameters = values + + def _concatenate_evaluation(self, funcname, xdata=None): + """Evaluate model + + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + ret = numpy.empty(len(xdata)) + for idx, model in self._iter_model_data_slices(xdata): + func = getattr(model, funcname) + ret[idx] = func(xdata=xdata[idx]) + return ret + + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) + + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + if xdata is None: + xdata = self.xdata + ret = numpy.empty(len(xdata)) + model_data_slices = self._model_data_slices(len(xdata)) + for model_idx, param_idx in self._parameter_model_index(param_idx): + idx = model_data_slices[model_idx] + model = self._models[model_idx] + ret[idx] = model.derivative_fitmodel(param_idx, xdata=xdata[idx]) + return ret + + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + if xdata is None: + xdata = self.xdata + ret = numpy.empty((self.nlinear_parameters, len(xdata))) + for idx, model in self._iter_model_data_slices(xdata): + ret[:, idx] = model.linear_derivatives_fitmodel(xdata=xdata[idx]) + return ret + + def _iter_model_data_slices(self, xdata): + """ + :param array xdata: + :yields (slice, Model): + """ + for item in zip(self._model_data_slices(len(xdata)), self._models): + yield item + + def _model_data_slices(self, nconcat): + """Slice of each model in the concatenated data + + :param int nconcat: + :returns list(slice): + """ + cache = self.getCache("fit", "model_data_slices") + if cache is None: + return list(self._generate_model_data_slices(nconcat)) + else: + if nconcat != cache.get("nconcat"): + cache["idx"] = list(self._generate_model_data_slices(nconcat)) + cache["nconcat"] = nconcat + return cache["idx"] + + def _generate_model_data_slices(self, nconcat, stride=None): + """Yield slice of the concatenated data for each model. + The concatenated data could be sliced as `xdata[::stride]`. + """ + ndata = [model.ndata for model in self._models] + if not stride: + stride, remain = divmod(sum(ndata), nconcat) + stride += remain > 0 + start = 0 + offset = 0 + i = 0 + for n in ndata: + # Index of model in concatenated xdata due to slicing + stop = start + n + lst = list(range(start + offset, stop, stride)) + nlst = len(lst) + # Index of model in concatenated xdata after slicing + idx = slice(i, i + nlst) + i += nlst + # Prepare for next model + if lst: + offset = lst[-1] + stride - stop + else: + offset -= n + start = stop + yield idx diff --git a/PyMca5/PyMcaMath/fitting/LinkedInterface.py b/PyMca5/PyMcaMath/fitting/LinkedInterface.py new file mode 100644 index 000000000..5db33444b --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/LinkedInterface.py @@ -0,0 +1,176 @@ +import functools +from contextlib import ExitStack, contextmanager +from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property + + +class linked_property(wrapped_property): + """Setting a linked property of one object + will set that property for all linked objects + """ + def _wrap_setter(self, fset): + propname = fset.__name__ + fset = super()._wrap_setter(fset) + + @functools.wraps(fset) + def wrapper(oself, value): + ret = fset(oself, value) + if oself.propagation_is_enabled(propname): + for instance in oself._filter_class_has_linked_property( + oself._non_propagating_instances, propname + ): + setattr(instance, propname, value) + return ret + + return wrapper + + +def linked_contextmanager(method): + """Entering the context manager of one object + will enter the context manager of linked objects + """ + context_name = method.__name__ + ctxmethod = contextmanager(method) + + @functools.wraps(method) + def wrapper(self, *args, **kw): + with ExitStack() as stack: + ctx = ctxmethod(self, *args, **kw) + stack.enter_context(ctx) + for instance in self._non_propagating_instances: + ctxmgr = getattr(instance, context_name) + if ctxmgr is not None: + ctx = ctxmgr(*args, **kw) + stack.enter_context(ctx) + yield + + return contextmanager(wrapper) + + +class LinkedInterface: + """Every class that uses the link decorators needs + to derived from this class. + """ + def __init__(self): + self.__enabled_linked_properties = dict() + self.__linked_instances = list() + self.__propagate = True + super().__init__() + + def propagation_is_enabled(self, name): + if not self.__linked_instances: + return False + return self.property_is_linked(name) + + def property_is_linked(self, name): + return self.__enabled_linked_properties.get(name, False) + + def disable_property_link(self, *names): + for name in names: + if self.has_linked_property(name): + self.__enabled_linked_properties[name] = False + + def enable_property_link(self, *names): + for name in names: + if self.has_linked_property(name): + self.__enabled_linked_properties[name] = True + + @property + def linked_instances(self): + return self.__linked_instances + + @linked_instances.setter + def linked_instances(self, instances): + self._propagated_linked_instances_setter(instances) + + def _propagated_linked_instances_setter(self, instances): + others = list() + for instance in instances: + if instance is self: + continue + if not isinstance(instance, LinkedInterface): + raise TypeError(type(instance), "can only link objects of the 'LinkedInterface' type") + others.append(instance) + self.__linked_instances = others + for instance in others: + instance._unpropagated_linked_instances_setter(instances) + + def _unpropagated_linked_instances_setter(self, instances): + self.__linked_instances = [i for i in instances if i is not self] + + @property + def _non_propagating_instances(self): + if not self.__propagate: + return + for instance in self.linked_instances: + with instance._disable_propagation(): + yield instance + + @contextmanager + def _disable_propagation(self): + keep = self.__propagate + self.__propagate = False + try: + yield + finally: + self.__propagate = keep + + @classmethod + def has_linked_property(cls, prop_name): + prop = getattr(cls, prop_name, None) + return isinstance(prop, linked_property) + + @staticmethod + def _filter_class_has_linked_property(instances, prop_name): + for instance in instances: + if instance.has_linked_property(prop_name): + yield instance + + +class LinkedContainerInterface: + """Classes that manage LinkedInterface objects should + derive from this class. + """ + def __init__(self, linked_instances): + self.linked_instances = linked_instances + super().__init__() + + @property + def linked_instances(self): + return self.__linked_instances + + @linked_instances.setter + def linked_instances(self, linked_instances): + linked_instances[0].linked_instances = linked_instances + self.__linked_instances = linked_instances + + def instances_with_linked_property(self, prop_name): + yield from LinkedInterface._filter_class_has_linked_property(self.linked_instances, prop_name) + + def instance_with_linked_property(self, prop_name): + for instance in self.instances_with_linked_property(prop_name): + return instance + return None + + def get_linked_property(self, prop_name): + instance = self.instance_with_linked_property(prop_name) + if instance is None: + raise ValueError(f"No instance has linked property {repr(prop_name)}") + return getattr(instance, prop_name) + + def set_linked_property(self, prop_name, value): + instance = self.instance_with_linked_property(prop_name) + if instance is None: + raise ValueError(f"No instance has linked property {repr(prop_name)}") + setattr(instance, prop_name, value) + + def disable_property_link(self, *names): + for name in names: + for instance in self.instances_with_linked_property(name): + instance.disable_property_link(name) + + def enable_property_link(self, *names): + for name in names: + value = self.get_linked_property(name) + for i, instance in enumerate(self.instances_with_linked_property(name)): + instance.enable_property_link(name) + self.set_linked_property(name, value) diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 34b329778..2e4264f82 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -1,587 +1,9 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - - -import functools -from collections.abc import Sequence, MutableMapping import numpy -from contextlib import contextmanager, ExitStack -from PyMca5.PyMcaMath.linalg import lstsq -from PyMca5.PyMcaMath.fitting import Gefit +from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface -class ModelUserInterface: - """The user part of the interface for all fit models. Need to be - implemented by all classes derived from `Model`. - """ - - @property - def xdata(self): - raise AttributeError from NotImplementedError - - @xdata.setter - def xdata(self, value): - raise AttributeError from NotImplementedError - - @property - def ydata(self): - raise AttributeError from NotImplementedError - - @ydata.setter - def ydata(self, value): - raise AttributeError from NotImplementedError - - @property - def ystd(self): - raise AttributeError from NotImplementedError - - @ystd.setter - def ystd(self, value): - raise AttributeError from NotImplementedError - - @property - def linear(self): - raise AttributeError from NotImplementedError - - @linear.setter - def linear(self, value): - raise AttributeError from NotImplementedError - - def evaluate_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - raise NotImplementedError - - def non_leastsquares_increment(self): - raise NotImplementedError - - -class Cached: - def __init__(self): - self._cache = dict() - - @contextmanager - def cachingContext(self, cachename): - reset = not self.cachingEnabled(cachename) - if reset: - self._cache[cachename] = dict() - try: - yield - finally: - if reset: - del self._cache[cachename] - - def cachingEnabled(self, cachename): - return cachename in self._cache - - def getCache(self, cachename, *subnames): - if cachename in self._cache: - ret = self._cache[cachename] - for cachename in subnames: - try: - ret = ret[cachename] - except KeyError: - ret[cachename] = dict() - ret = ret[cachename] - return ret - else: - return None - - @staticmethod - def enableCaching(cachename): - def decorator(method): - @functools.wraps(method) - def cache_wrapper(self, *args, **kw): - with self.cachingContext(cachename): - return method(self, *args, **kw) - - return cache_wrapper - - return decorator - - -class parameter(property): - """Usage: - - .. highlight:: python - .. code-block:: python - - class MyClass(Model): - - def __init__(self): - self._myparam = 0. - - @parameter - def myparam(self): - return self._myparam - - @myparam.setter # optional - def myparam(self, value): - self._myparam = value - - @myparam.counter # optional - def myparam(self): - return 1 - - @myparam.constraints # optional - def myparam(self): - return 1, 0, 0 - """ - - def __init__(self, fget=None, fset=None, fdel=None, doc=""): - if fget is not None: - fget = self._param_getter(fget) - if fset is not None: - fset = self._param_setter(fset) - super().__init__(fget=fget, fset=fset, fdel=fdel, doc=doc) - self.fcount = self._fcount_default() - self.fconstraints = self._fconstraints_default() - - def getter(self, fget): - if fget is not None: - fget = self._param_getter(fget) - return super().getter(fget) - - def setter(self, fset): - if fset is not None: - fset = self._param_setter(fset) - return super().setter(fset) - - def counter(self, fcount): - self.fcount = fcount - return self - - def constraints(self, fconstraints): - self.fconstraints = fconstraints - return self - - @classmethod - def _param_getter(cls, fget): - @functools.wraps(fget) - def wrapper(self): - return self._get_parameter(fget) - - return wrapper - - @classmethod - def _param_setter(cls, fset): - @functools.wraps(fset) - def wrapper(self, value): - return self._set_parameter(fset, value) - - return wrapper - - def _fcount_default(self): - def fcount(wself): - try: - return len(self.fget(wself)) - except TypeError: - return 1 - - return fcount - - def _fconstraints_default(self): - def fconstraints(wself): - return numpy.zeros((self.fcount(wself), 3)) - - return fconstraints - - -class linear_parameter(parameter): - pass - - -class ModelInterface(Cached, ModelUserInterface): - """Interface for all fit models (Model and ConcatModel derived classes).""" - - @property - def parameter_group_names(self): - raise AttributeError from NotImplementedError - - @property - def linear_parameter_group_names(self): - raise AttributeError from NotImplementedError - - def _parameter_groups(self, linear_only=False): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - raise NotImplementedError - - def _get_parameters(self, linear_only=False, fitting=True): - """ - :param bool linear_only: - :param bool fitting: - :returns array: - """ - raise NotImplementedError - - def _get_constraints(self, linear_only=False): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - raise NotImplementedError - - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - raise NotImplementedError - - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - raise NotImplementedError - - @property - def ndata(self): - raise AttributeError from NotImplementedError - - @property - def yfitdata(self): - raise AttributeError from NotImplementedError - - @property - def yfitstd(self): - raise AttributeError from NotImplementedError - - @property - def yfullmodel(self): - """Model of ydata""" - return self.evaluate_fullmodel() - - @property - def yfitmodel(self): - """Model of yfitdata""" - return self.evaluate_fitmodel() - - @property - def parameters(self): - return self._get_parameters(fitting=False) - - @property - def linear_parameters(self): - return self._get_parameters(linear_only=True, fitting=False) - - @property - def fit_parameters(self): - return self._get_parameters(fitting=True) - - @property - def linear_fit_parameters(self): - return self._get_parameters(linear_only=True, fitting=True) - - @property - def constraints(self): - return self._get_constraints() - - @property - def linear_constraints(self): - return self._get_constraints(linear_only=True) - - @parameters.setter - def parameters(self, values): - return self._set_parameters(values, fitting=False) - - @fit_parameters.setter - def fit_parameters(self, values): - return self._set_parameters(values, fitting=True) - - @linear_parameters.setter - def linear_parameters(self, values): - return self._set_parameters(values, linear_only=True, fitting=False) - - @linear_fit_parameters.setter - def linear_fit_parameters(self, values): - return self._set_parameters(values, linear_only=True, fitting=True) - - @property - def nparameters(self): - return sum(n for _, n in self._parameter_groups()) - - @property - def nlinear_parameters(self): - return sum(n for _, n in self._parameter_groups(linear_only=True)) - - @property - def parameter_names(self): - return list(self._iter_parameter_names()) - - @property - def linear_parameter_names(self): - return list(self._iter_parameter_names(linear_only=True)) - - def _iter_parameter_names(self, linear_only=False): - for name, n in self._parameter_groups(linear_only=linear_only): - if n > 1: - for i in range(n): - yield name + str(i) - else: - yield name - - def fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - if self.linear: - return self.linear_fit(full_output=full_output) - else: - return self.nonlinear_fit(full_output=full_output) - - def linear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self._linear_fit_context(): - b = self.yfitdata # ndata - for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives_fitmodel().T # ndata, nparams - result = lstsq( - A, - b.copy(), - uncertainties=True, - covariances=False, - digested_output=True, - ) - if self.niter_non_leastsquares: - self.linear_fit_parameters = result["parameters"] - self.non_leastsquares_increment() - result["linear"] = True - result["parameters"] = self._fit_to_linear_parameters(result["parameters"]) - result["uncertainties"] = self._fit_to_linear_uncertainties( - result["uncertainties"] - ) - result.pop("svd") - return result - - def nonlinear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self._nonlinear_fit_context(): - constraints = self.constraints.T - xdata = self.xdata - ydata = self.yfitdata - ystd = self.yfitstd - for i in range(max(self.niter_non_leastsquares, 1)): - result = Gefit.LeastSquaresFit( - self._gefit_evaluate_fitmodel, - self.fit_parameters, - model_deriv=self._gefit_derivative_fitmodel, - xdata=xdata, - ydata=ydata, - sigmadata=ystd, - constrains=constraints, - maxiter=self.maxiter, - weightflag=self.weightflag, - deltachi=self.deltachi, - fulloutput=full_output, - ) - if self.niter_non_leastsquares: - self.fit_parameters = result[0] - self.non_leastsquares_increment() - ret = { - "linear": False, - "parameters": self._fit_to_parameters(result[0]), - "uncertainties": self._fit_to_uncertainties(result[2]), - "chi2_red": result[1], - } - if full_output: - ret["niter"] = result[3] - ret["lastdeltachi"] = result[4] - return ret - - @property - def maxiter(self): - return 100 - - @property - def deltachi(self): - return None - - @property - def weightflag(self): - return 0 - - @property - def niter_non_leastsquares(self): - return 0 - - @contextmanager - def _linear_fit_context(self): - with self.cachingContext("fit"): - with self._linear_context(True): - with self._cache_parameters_context(): - yield - - @contextmanager - def _nonlinear_fit_context(self): - with self.cachingContext("fit"): - with self._linear_context(False): - with self._cache_parameters_context(): - yield - - @contextmanager - def _cache_parameters_context(self): - with self.cachingContext("parameters"): - yield - - @contextmanager - def _linear_context(self, linear): - keep = self.linear - self.linear = linear - try: - yield - finally: - self.linear = keep - - def _gefit_evaluate_fitmodel(self, parameters, xdata): - """Update parameters and evaluate model - - :param array parameters: length nparams - :param array xdata: length nxdata - :returns array: nxdata - """ - self.fit_parameters = parameters - return self.evaluate_fitmodel(xdata=xdata) - - def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): - """Update parameters and return derivate to a specific parameter - - :param array parameters: length nparams - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - self.fit_parameters = parameters - return self.derivative_fitmodel(param_idx, xdata=xdata) - - def linear_decomposition_fitmodel(self, xdata=None): - """Linear decomposition of the fit model. - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - return self.linear_parameters[:, numpy.newaxis] * derivatives - - def derivatives_fitmodel(self, xdata=None): - """Derivates to all parameters of the fit model. - - :param array xdata: length nxdata - :returns list(array): nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - return [ - self.derivative_fitmodel(i, xdata=xdata) for i in range(self.nparameters) - ] - - def use_fit_result(self, result): - """ - :param dict result: - """ - if result["linear"]: - self.linear_parameters = result["parameters"] - else: - self.parameters = result["parameters"] - - @contextmanager - def use_fit_result_context(self, result): - with self._linear_context(result["linear"]): - with self._cache_parameters_context(): - self.use_fit_result(result) - yield - - def _parameters_to_fit(self, params): - return params - - def _linear_parameters_to_fit(self, params): - return params - - def _fit_to_parameters(self, params): - return params - - def _fit_to_linear_parameters(self, params): - return params - - def _fit_to_linear_uncertainties(self, uncertainties): - return uncertainties - - def _fit_to_uncertainties(self, uncertainties): - return uncertainties - - -class Model(ModelInterface): +class Model(ModelInterface, ModelParameterInterface): """Evaluation and derivatives of a model to be used in least-squares fitting. Derived classes: @@ -620,13 +42,13 @@ class Model(ModelInterface): def __init__(self): self._included_parameters = None # for ConcatModel self._excluded_parameters = None # for ConcatModel - super(Model, self).__init__() + super().__init__() def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) allp = cls._PARAMETER_GROUP_NAMES = list() linp = cls._LINEAR_PARAMETER_GROUP_NAMES = list() - for name in sorted(dir(cls)): + for name in sorted(dir(cls)): # TODO: keep order if declaration? attr = getattr(cls, name) if isinstance(attr, parameter): allp.append(name) @@ -651,44 +73,71 @@ def _filter_parameter_names(self, names): continue yield name - def _get_parameters(self, linear_only=False, fitting=True): + def _get_constraints(self, linear_only=None): + """ + :param bool linear_only: + :returns array: nparams x 3 + """ + if linear_only is None: + linear_only = self.linear + if linear_only: + nparams = self.nlinear_parameters + else: + nparams = self.nparameters + codes = numpy.zeros((nparams, 3)) + for group_name, idx in self._parameter_group_indices(linear_only=linear_only): + paramprop = getattr(self.__class__, group_name) + codes[idx] = paramprop.fconstraints(self) + return codes + + def _get_parameters(self, linear_only=None): """ :param bool linear_only: - :param bool fitting: :returns array: """ cache = self.getCache("parameters") if cache is None: - return self._get_parameters_notcached( - linear_only=linear_only, fitting=fitting - ) + return self._get_parameters_notcached(linear_only=linear_only) key = self._parameters_cache_key() parameters = cache.get(key, None) if parameters is None: parameters = cache[key] = self._get_parameters_notcached( - linear_only=linear_only, fitting=fitting + linear_only=linear_only ) return parameters - def _get_parameters_notcached(self, linear_only=False, fitting=True): + def _set_parameters(self, params, linear_only=None): + """ + :param bool linear_only: + """ + cache = self.getCache("parameters") + if cache is None: + self._set_parameters_notcached(params, linear_only=linear_only) + else: + key = self._parameters_cache_key() + cache[key] = params + + def _get_parameters_notcached(self, linear_only=None): """Helper for `_get_parameters`""" + if linear_only is None: + linear_only = self.linear if linear_only: nparams = self.nlinear_parameters else: nparams = self.nparameters params = numpy.zeros(nparams) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - params[i : i + n] = getattr(self, name) - i += n - if fitting: - if linear_only: - return self._linear_parameters_to_fit(params) - else: - return self._parameters_to_fit(params) - else: - return params + for group_name, idx in self._parameter_group_indices(linear_only=linear_only): + params[idx] = getattr(self, group_name) + return params + + def _set_parameters_notcached(self, params, linear_only=None): + """Helper of `_set_parameters` + + :param bool linear_only: + """ + for group_name, idx in self._parameter_group_indices(linear_only=linear_only): + setattr(self, group_name, params[idx]) def _get_parameter(self, fget): """Helper for parameter getters.""" @@ -701,58 +150,11 @@ def _get_parameter(self, fget): if parameters is None: return fget(self) - idx = self._parameter_slice_from_groupname(fget.__name__) + idx = self._parameter_group_index(fget.__name__) + if idx is None: + return fget(self) return parameters[idx] - def _get_constraints(self, linear_only=False): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - if linear_only: - nparams = self.nlinear_parameters - else: - nparams = self.nparameters - codes = numpy.zeros((nparams, 3), numpy.float64) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - codes[i : i + n] = getattr(self.__class__, name).fconstraints(self) - i += n - return codes - - def _set_parameters(self, params, linear_only=False, fitting=False): - """ - :param bool linear_only: - :param bool fitting: - """ - cache = self.getCache("parameters") - if cache is None: - self._set_parameters_notcached( - params, linear_only=linear_only, fitting=fitting - ) - else: - key = self._parameters_cache_key() - cache[key] = params - - def _set_parameters_notcached(self, params, linear_only=False, fitting=False): - """Helper of `_set_parameters` - - :param bool linear_only: - :param bool fitting: - """ - if fitting: - if linear_only: - params = self._fit_to_linear_parameters(params) - else: - params = self._fit_to_parameters(params) - i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): - if n > 1: - getattr(self, name)[:] = params[i : i + n] - elif n == 1: - setattr(self, name, params[i]) - i += n - def _set_parameter(self, fset, value): """Helper for parameter setters""" parameters = self.getCache("parameters") @@ -764,10 +166,12 @@ def _set_parameter(self, fset, value): if parameters is None: return fset(self, value) - idx = self._parameter_slice_from_groupname(fset.__name__) + idx = self._parameter_group_index(fset.__name__) + if idx is None: + return fset(self, value) parameters[idx] = value - def _parameter_groups(self, linear_only=False): + def _parameter_groups(self, linear_only=None): """Yield name and count of enabled parameter groups :param bool linear_only: @@ -795,45 +199,56 @@ def _parameters_cache_key(self): b = tuple(sorted(b)) return a, b - def _parameter_groups_notcached(self, linear_only=False): + def _parameter_groups_notcached(self, linear_only=None): """Helper for `_parameter_groups`. :param bool linear_only: :yields str, int: group name, nb. parameters in the group """ + if linear_only is None: + linear_only = self.linear if linear_only: names = self.linear_parameter_group_names else: names = self.parameter_group_names for name in names: - param = getattr(self.__class__, name) - n = param.fcount(self) + paramprop = getattr(self.__class__, name) + n = paramprop.fcount(self) if n: yield name, n - def _parameter_name_from_index(self, idx, linear_only=False): + def _parameter_name_from_index(self, idx, linear_only=None): """Parameter index to group name and group index :returns str, int: group name, index in parameter group """ i = 0 - for name, n in self._parameter_groups(linear_only=linear_only): + for group_name, n in self._parameter_groups(linear_only=linear_only): if idx >= i and idx < (i + n): - return name, idx - i + return group_name, idx - i i += n - def _parameter_slice_from_groupname(self, name, linear_only=False): + def _parameter_group_index(self, name, linear_only=None): """Parameter group name to index range - :returns int or slice: slice of parameter group in all parameters + :returns int or slice or None: index of parameter group in all parameters + """ + for group_name, idx in self._parameter_group_indices(linear_only=linear_only): + if name == group_name: + return idx + return None + + def _parameter_group_indices(self, linear_only=None): + """Parameter indices for each group + + :yields int or slice: index of parameter group in all parameters """ i = 0 - for _name, n in self._parameter_groups(linear_only=linear_only): - if name == _name: - if n == 1: - return i - else: - return slice(i, i + n) + for group_name, n in self._parameter_groups(linear_only=linear_only): + if n == 1: + yield group_name, i + else: + yield group_name, slice(i, i + n) i += n @property @@ -842,20 +257,20 @@ def ndata(self): @property def yfitdata(self): - return self._ydata_to_fit(self.ydata) + return self._y_full_to_fit(self.ydata) @property def yfitstd(self): - return self._ystd_to_fit(self.ystd) + return self._ystd_full_to_fit(self.ystd) - def _ydata_to_fit(self, ydata, xdata=None): - return ydata + def _y_full_to_fit(self, y, xdata=None): + return y - def _ystd_to_fit(self, ystd, xdata=None): + def _ystd_full_to_fit(self, ystd, xdata=None): return ystd - def _fit_to_ydata(self, yfit, xdata=None): - return yfit + def _y_fit_to_full(self, y, xdata=None): + return y def evaluate_fullmodel(self, xdata=None): """Evaluate the full model. @@ -864,7 +279,7 @@ def evaluate_fullmodel(self, xdata=None): :returns array: nxdata """ y = self.evaluate_fitmodel(xdata=xdata) - return self._fit_to_ydata(y, xdata=xdata) + return self._y_fit_to_full(y, xdata=xdata) def evaluate_linear_fullmodel(self, xdata=None): """Evaluate the full model. @@ -873,7 +288,7 @@ def evaluate_linear_fullmodel(self, xdata=None): :returns array: n x nxdata """ y = self.evaluate_linear_fitmodel(xdata=xdata) - return self._fit_to_ydata(y, xdata=xdata) + return self._y_fit_to_full(y, xdata=xdata) def evaluate_linear_fitmodel(self, xdata=None): """Evaluate the fit model. @@ -884,499 +299,99 @@ def evaluate_linear_fitmodel(self, xdata=None): derivatives = self.linear_derivatives_fitmodel(xdata=xdata) return self.linear_parameters.dot(derivatives) - -class ConcatModel(ModelInterface): - """Concatenated model with shared parameters""" - - def __init__(self, models, shared_attributes=None): - if not isinstance(models, Sequence): - models = [models] - for model in models: - if not isinstance(model, Model): - raise ValueError("'models' must be a list of type 'Model'") - if len(set(type(m) for m in models)) > 1: - raise ValueError("Multiple model types are currently not supported") - self._models = models - self.__fixed_shared_attributes = { - "linear", - "niter_non_leastsquares", - "_included_parameters", - "_excluded_parameters", - } - self.shared_attributes = shared_attributes - super().__init__() - - @contextmanager - def cachingContext(self, cachename): - """Enter the same caching context for all models""" - with ExitStack() as stack: - ctx = super().cachingContext(cachename) - stack.enter_context(ctx) - for m in self._models: - stack.enter_context(m.cachingContext(cachename)) - yield - - @property - def nmodels(self): - return len(self._models) - - @property - def shared_model(self): - """Model used to get/set shared attributes""" - return self._models[0] - - @property - def _all_other_models(self): - """All models except for `shared_model`""" - return self._models[1:] - - def __getattr__(self, name): - """Get shared attribute""" - if self.nmodels and name in self.shared_attributes: - return getattr(self.shared_model, name) - raise AttributeError(name) - - def __setattr__(self, name, value): - """Set the attributes of all models when shared""" - if ( - name != "_models" - and self.nmodels - and hasattr(self.shared_model, name) - and name in self.shared_attributes - ): - for m in self._models: - setattr(m, name, value) - else: - super().__setattr__(name, value) - - @property - def shared_attributes(self): - """Attributes shared between the fit models (they should have the same value)""" - return self._shared_attributes - - @shared_attributes.setter - def shared_attributes(self, shared_attributes): - """ - :param Sequence(str) shared_attributes: - """ - if shared_attributes is None: - shared_attributes = set() - else: - shared_attributes = set(shared_attributes) - shared_attributes |= self.__fixed_shared_attributes - if self.nmodels <= 1: - self._shared_attributes = shared_attributes - return - self.share_attributes(shared_attributes) - self.validate_shared_attributes(shared_attributes) - self._shared_attributes = shared_attributes - - def validate_shared_attributes(self, shared_attributes=None): - """Check whether attributes are shared - - :param Sequence(str) shared_attributes: - :raises AssertionError: - """ - if self.nmodels <= 1: - return - if shared_attributes is None: - shared_attributes = self._shared_attributes - for name in shared_attributes: - value = getattr(self.shared_model, name) - if isinstance(value, (Sequence, MutableMapping, numpy.ndarray)): - for m in self._all_other_models: - assert id(value) == id(getattr(m, name)), name - else: - for m in self._all_other_models: - assert value == getattr(m, name), name - - def share_attributes(self, shared_attributes=None): - """Ensure attributes are shared - - :param Sequence(str) shared_attributes: - """ - if self.nmodels <= 1: - return - if shared_attributes is None: - shared_attributes = self._shared_attributes - model = self.shared_model - adict = {name: getattr(model, name) for name in shared_attributes} - for model in self._all_other_models: - for name, value in adict.items(): - try: - setattr(model, name, value) - except AttributeError: - pass # no setter - - @property - def ndata(self): - nmodels = self.nmodels - if nmodels == 0: - return 0 - else: - return sum(m.ndata for m in self._models) - - @property - def xdata(self): - return self._get_data("xdata") - - @xdata.setter - def xdata(self, values): - self._set_data("xdata", values) - - @property - def ydata(self): - return self._get_data("ydata") - - @ydata.setter - def ydata(self, values): - self._set_data("ydata", values) - - @property - def ystd(self): - return self._get_data("ystd") - - @ystd.setter - def ystd(self, values): - self._set_data("ystd", values) - - @property - def yfitdata(self): - return self._get_data("yfitdata") - - @property - def yfitstd(self): - return self._get_data("yfitstd") - - def _get_data(self, attr): - """ - :param str attr: - :returns array: - """ - nmodels = self.nmodels - if nmodels == 0: - return None - elif nmodels == 1: - return getattr(self.shared_model, attr) - elif getattr(self.shared_model, attr) is None: - return None - else: - return numpy.concatenate([getattr(m, attr) for m in self._models]) - - def _set_data(self, attr, values): - """ - :param str attr: - :param array values: - """ - if len(values) != self.ndata: - raise ValueError("Not the expected number of channels") - for idx, model in self._iter_model_data_slices(values): - setattr(model, attr, values[idx]) - - @contextmanager - def _filter_parameter_context(self, shared=True): - keepex = self._excluded_parameters # shared between all models - keepin = self._included_parameters # shared between all models - try: - if shared: - if keepin: - self._included_parameters = list( - set(keepin) - set(self.shared_attributes) - ) - else: - self._included_parameters = self.shared_attributes - else: - if keepex: - self._excluded_parameters.extend(self.shared_attributes) - else: - self._excluded_parameters = self.shared_attributes - yield - finally: - self._excluded_parameters = keepex - self._included_parameters = keepin - - def _iter_parameter_models(self): - """Yields models which are in such a state that they have - either shared or non-shared parameters enabled. - """ - with self._filter_parameter_context(shared=True): - yield self.shared_model - with self._filter_parameter_context(shared=False): - for m in self._models: - yield m - - def _iter_model_data_slices_types(self): - modeltypes = set() - for model in self._models: - modeltype = type(model) - if modeltype not in modeltypes: - modeltypes.add(modeltype) - yield model - - @property - def nshared_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.nparameters - - @property - def nshared_linear_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.nlinear_parameters - - def _get_parameters(self, linear_only=False, fitting=True): - """ - :param bool linear_only: - :returns array: - """ - return numpy.concatenate( - [ - m._get_parameters(linear_only=linear_only, fitting=fitting) - for m in self._iter_parameter_models() - ] - ) - - def _set_parameters(self, values, linear_only=False, fitting=False): - """ - :paramm array values: - :param bool linear_only: - """ - i = 0 - for m in self._iter_parameter_models(): - if linear_only: - n = m.nlinear_parameters - else: - n = m.nparameters - if n: - m._set_parameters( - values[i : i + n], linear_only=linear_only, fitting=fitting - ) - i += n - - def _get_constraints(self, linear_only=False): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - return numpy.concatenate( - [ - m._get_constraints(linear_only=linear_only) - for m in self._iter_parameter_models() - ] - ) - - def _parameter_groups(self, linear_only=False): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - with self._filter_parameter_context(shared=True): - for item in self.shared_model._parameter_groups(linear_only=linear_only): - yield item - with self._filter_parameter_context(shared=False): - for i, m in enumerate(self._models): - for name, n in self.shared_model._parameter_groups( - linear_only=linear_only - ): - yield name + str(i), n - - def _parameter_model_index(self, idx, linear_only=False): - """Convert parameter index of ConcatModel to a parameter indices - of the underlying models (only one when parameter is not shared). - - :param bool linear_only: - :param int idx: - :yields (int, int): model index, parameter index in this model - """ - cache = self.getCache("fit", "parameter_model_index") - if cache is None: - yield from self._iter_parameter_index(idx, linear_only=linear_only) - return - - it = cache.get(idx) - if it is None: - it = cache[idx] = list( - self._iter_parameter_index(idx, linear_only=linear_only) - ) - yield from it - - def _iter_parameter_index(self, idx, linear_only=False): - """Convert parameter index of ConcatModel to a parameter indices - of the underlying models (only one when parameter is not shared). - - :param bool linear_only: - :param int idx: - :yields (int, int): model index, parameter index in this model - """ - if linear_only: - nshared = self.nshared_linear_parameters - else: - nshared = self.nshared_parameters - shared_attributes = self.shared_attributes - if idx < nshared: - for i, m in enumerate(self._models): - iglobal = 0 - imodel = 0 - for name, n in m._parameter_groups(linear_only=linear_only): - if name in shared_attributes: - if idx >= iglobal and idx < (iglobal + n): - yield i, imodel + idx - iglobal - iglobal += n - imodel += n - else: - iglobal = nshared - for i, m in enumerate(self._models): - imodel = 0 - for name, n in m._parameter_groups(linear_only=linear_only): - if name not in shared_attributes: - if idx >= iglobal and idx < (iglobal + n): - yield i, imodel + idx - iglobal - return - iglobal += n - imodel += n - - @property - def shared_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.parameters - - @shared_parameters.setter - def shared_parameters(self, values): - with self._filter_parameter_context(shared=True): - self.shared_model.parameters = values - - @property - def shared_linear_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.linear_parameters - - @shared_linear_parameters.setter - def shared_linear_parameters(self, values): - with self._filter_parameter_context(shared=True): - self.shared_model.linear_parameters = values - - def _concatenate_evaluation(self, funcname, xdata=None): - """Evaluate model + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters :param array xdata: length nxdata - :returns array: nxdata + :returns array: nparams x nxdata """ - if xdata is None: - xdata = self.xdata - ret = xdata * 0.0 - for idx, model in self._iter_model_data_slices(xdata): - func = getattr(model, funcname) - ret[idx] = func(xdata=xdata[idx]) - return ret + with self._linear_context(True): + return numpy.array( + [ + self.derivative_fitmodel(i, xdata=xdata) + for i in range(self.nlinear_parameters) + ] + ) - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. + :param int param_idx: :param array xdata: length nxdata :returns array: nxdata """ - return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) - - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) + return self.numerical_derivative_fitmodel(param_idx, xdata=xdata) - def evaluate_fitmodel(self, xdata=None): - """Evaluate the fit model. + def numerical_derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. + :param int param_idx: :param array xdata: length nxdata :returns array: nxdata """ - return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) + linear = self.linear + if not linear: + name, _ = self._parameter_name_from_index(param_idx) + linear = name in self._LINEAR_PARAMETER_GROUP_NAMES - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. + keep = parameters = self.fit_parameters + parameters = parameters.copy() + try: + if linear: + return self._numerical_derivative_linear_param(parameters, param_idx, xdata=xdata) + else: + return self._numerical_derivative_nonlinear_param(parameters, param_idx, xdata=xdata) + finally: + self.fit_parameters = keep - :param array xdata: length nxdata - :returns array: n x nxdata + def _numerical_derivative_linear_param(self, parameters, param_idx, xdata=None): + """The numerical derivative to a linear parameter is exact so + far as the calculation of the fit model itself is exact. """ - return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. + # y(x) = p0*f0(x) + ... + pi*fi(x) + ... + # dy/dpi(x) = fi(x) + if self.linear: + # All of them are linear parameters + parameters = numpy.zeros_like(parameters) + else: + # Only some of them are linear parameters + for name, idx in self._parameter_group_indices(): + if name in self._LINEAR_PARAMETER_GROUP_NAMES: + parameters[idx] = 0 + parameters[param_idx] = 1 + self.fit_parameters = parameters + return self.evaluate_fitmodel(xdata=xdata) - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata + def _numerical_derivative_nonlinear_param(self, parameters, param_idx, xdata=None): + """The numerical derivative to a non-linear parameter is an approximation """ - if xdata is None: - xdata = self.xdata - ret = xdata * 0.0 - model_data_slices = self._model_data_slices(len(xdata)) - for model_idx, param_idx in self._parameter_model_index(param_idx): - idx = model_data_slices[model_idx] - model = self._models[model_idx] - ret[idx] = model.derivative_fitmodel(param_idx, xdata=xdata[idx]) - return ret + # Choose delta to be a small fraction of the + # parameter value but not too small, otherwise + # the derivative is zero. + p0 = parameters[param_idx] + delta = p0 * 1e-5 + if delta < 0: + delta = min(delta, -1e-12) + else: + delta = max(delta, 1e-12) - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters + parameters[param_idx] = p0 + delta + self.fit_parameters = parameters + f1 = self.evaluate_fitmodel(xdata=xdata) - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - ret = numpy.empty((self.nlinear_parameters, xdata.size)) - for idx, model in self._iter_model_data_slices(xdata): - ret[:, idx] = model.linear_derivatives_fitmodel(xdata=xdata[idx]) - return ret - - def _iter_model_data_slices(self, xdata): - """ - :param array xdata: - :yields (slice, Model): - """ - for item in zip(self._model_data_slices(len(xdata)), self._models): - yield item + parameters[param_idx] = p0 - delta + self.fit_parameters = parameters + f2 = self.evaluate_fitmodel(xdata=xdata) + + return (f1 - f2) / (2.0 * delta) - def _model_data_slices(self, nconcat): - """Slice of each model in the concatenated data + def compare_derivatives(self, xdata=None): + """Compare analytical and numerical derivatives. Useful to + validate the user defined `derivative_fitmodel`. - :param int nconcat: - :returns list(slice): - """ - cache = self.getCache("fit", "model_data_slices") - if cache is None: - return list(self._generate_model_data_slices(nconcat)) - else: - if nconcat != cache.get("nconcat"): - cache["idx"] = list(self._generate_model_data_slices(nconcat)) - cache["nconcat"] = nconcat - return cache["idx"] - - def _generate_model_data_slices(self, nconcat, stride=None): - """Yield slice of the concatenated data for each model. - The concatenated data could be sliced as `xdata[::stride]`. + :yields str, array, array: parameter name, analytical, numerical """ - ndata = [m.ndata for m in self._models] - if not stride: - stride, remain = divmod(sum(ndata), nconcat) - stride += remain > 0 - start = 0 - offset = 0 - i = 0 - for n in ndata: - # Index of model in concatenated xdata due to slicing - stop = start + n - lst = list(range(start + offset, stop, stride)) - nlst = len(lst) - # Index of model in concatenated xdata after slicing - idx = slice(i, i + nlst) - i += nlst - # Prepare for next model - if lst: - offset = lst[-1] + stride - stop - else: - offset -= n - start = stop - yield idx + for param_idx, name in enumerate(self.fit_parameter_names): + ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata) + ynumderiv = self.numerical_derivative_fitmodel(param_idx, xdata=xdata) + yield name, ycalderiv, ynumderiv diff --git a/PyMca5/PyMcaMath/fitting/ModelInterface.py b/PyMca5/PyMcaMath/fitting/ModelInterface.py new file mode 100644 index 000000000..f7b80652f --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/ModelInterface.py @@ -0,0 +1,426 @@ +import numpy +from PyMca5.PyMcaMath.linalg import lstsq +from PyMca5.PyMcaMath.fitting import Gefit + +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface + + +class ModelUserInterface: + """The part of the interface for all fit models that needs + to be implemented by all classes derived from `Model`. + """ + + @property + def xdata(self): + raise AttributeError from NotImplementedError + + @xdata.setter + def xdata(self, value): + raise AttributeError from NotImplementedError + + @property + def ydata(self): + raise AttributeError from NotImplementedError + + @ydata.setter + def ydata(self, value): + raise AttributeError from NotImplementedError + + @property + def ystd(self): + raise AttributeError from NotImplementedError + + @ystd.setter + def ystd(self, value): + raise AttributeError from NotImplementedError + + @property + def linear(self): + raise AttributeError from NotImplementedError + + @linear.setter + def linear(self, value): + raise AttributeError from NotImplementedError + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def derivative_fitmodel(self, param_idx, xdata=None): + """Derivate to a specific parameter of the fit model. + + The call is forwarded to `numerical_derivative_fitmodel` + in `Model` so it is not strictly necessary to implement + this method. Note that the numerical derivative to a + non-linear parameter is an approximation. + + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def non_leastsquares_increment(self): + raise NotImplementedError + + + + + +class ModelInterface(ModelParameterInterface, ModelUserInterface): + """Interface for all fit models (Model and ConcatModel derived classes).""" + + @property + def parameter_group_names(self): + raise AttributeError from NotImplementedError + + @property + def linear_parameter_group_names(self): + raise AttributeError from NotImplementedError + + def _parameter_groups(self, linear_only=None): + """Yield name and count of enabled parameter groups + + :param bool linear_only: + :yields str, int: group name, nb. parameters in the group + """ + raise NotImplementedError + + def _get_parameters(self, linear_only=None): + """ + :param bool linear_only: + :returns array: + """ + raise NotImplementedError + + def _get_constraints(self, linear_only=None): + """ + :param bool linear_only: + :returns array: nparams x 3 + """ + raise NotImplementedError + + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: nxdata + """ + raise NotImplementedError + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + raise NotImplementedError + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: length nxdata + :returns array: n x nxdata + """ + raise NotImplementedError + + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + raise NotImplementedError + + @property + def ndata(self): + raise AttributeError from NotImplementedError + + @property + def yfitdata(self): + raise AttributeError from NotImplementedError + + @property + def yfitstd(self): + raise AttributeError from NotImplementedError + + @property + def yfullmodel(self): + """Model of ydata""" + return self.evaluate_fullmodel() + + @property + def yfitmodel(self): + """Model of yfitdata""" + return self.evaluate_fitmodel() + + @property + def parameters(self): + return self._get_parameters(linear_only=False) + + @property + def linear_parameters(self): + return self._get_parameters(linear_only=True) + + @property + def fit_parameters(self): + """`parameters` when `linear=False` or `linear_parameters` when `linear=True`""" + return self._get_parameters() + + @property + def constraints(self): + return self._get_constraints(linear_only=False) + + @property + def linear_constraints(self): + return self._get_constraints(linear_only=True) + + @property + def fit_constraints(self): + return self._get_constraints() + + @parameters.setter + def parameters(self, values): + return self._set_parameters(values, linear_only=False) + + @linear_parameters.setter + def linear_parameters(self, values): + return self._set_parameters(values, linear_only=True) + + @fit_parameters.setter + def fit_parameters(self, values): + return self._set_parameters(values) + + @property + def nparameters(self): + return sum(n for _, n in self._parameter_groups(linear_only=False)) + + @property + def nlinear_parameters(self): + return sum(n for _, n in self._parameter_groups(linear_only=True)) + + @property + def nfit_parameters(self): + return sum(n for _, n in self._parameter_groups()) + + @property + def parameter_names(self): + return list(self._iter_parameter_names(linear_only=False)) + + @property + def linear_parameter_names(self): + return list(self._iter_parameter_names(linear_only=True)) + + @property + def fit_parameter_names(self): + return list(self._iter_parameter_names()) + + def _iter_parameter_names(self, linear_only=None): + for group_name, n in self._parameter_groups(linear_only=linear_only): + if n > 1: + for i in range(n): + yield group_name + str(i) + else: + yield group_name + + def fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + if self.linear: + return self.linear_fit(full_output=full_output) + else: + return self.nonlinear_fit(full_output=full_output) + + def linear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + with self.__linear_fit_context(): + b = self.yfitdata # ndata + for i in range(max(self.niter_non_leastsquares, 1)): + A = self.linear_derivatives_fitmodel().T # ndata, nparams + result = lstsq( + A, + b.copy(), + uncertainties=True, + covariances=False, + digested_output=True, + ) + if self.niter_non_leastsquares: + self.linear_parameters = result["parameters"] + self.non_leastsquares_increment() + result["linear"] = True + result["parameters"] = self._fit_to_linear_parameters(result["parameters"]) + result["uncertainties"] = self._fit_to_linear_uncertainties( + result["uncertainties"] + ) + result.pop("svd") + return result + + def nonlinear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + with self._nonlinear_fit_context(): + constraints = self.constraints.T + xdata = self.xdata + ydata = self.yfitdata + ystd = self.yfitstd + for i in range(max(self.niter_non_leastsquares, 1)): + result = Gefit.LeastSquaresFit( + self._gefit_evaluate_fitmodel, + self.parameters, + model_deriv=self._gefit_derivative_fitmodel, + xdata=xdata, + ydata=ydata, + sigmadata=ystd, + constrains=constraints, + maxiter=self.maxiter, + weightflag=self.weightflag, + deltachi=self.deltachi, + fulloutput=full_output, + ) + if self.niter_non_leastsquares: + self.parameters = result[0] + self.non_leastsquares_increment() + ret = { + "linear": False, + "parameters": self._fit_to_parameters(result[0]), + "uncertainties": self._fit_to_uncertainties(result[2]), + "chi2_red": result[1], + } + if full_output: + ret["niter"] = result[3] + ret["lastdeltachi"] = result[4] + return ret + + @property + def maxiter(self): + return 100 + + @property + def deltachi(self): + return None + + @property + def weightflag(self): + return 0 + + @property + def niter_non_leastsquares(self): + return 0 + + @contextmanager + def __linear_fit_context(self): + with ExitStack() as stack: + ctx = self.cachingContext("fit") + stack.enter_context(ctx) + ctx = self._linear_context(True) + stack.enter_context(ctx) + ctx = self.cachingContext("parameters") + stack.enter_context(ctx) + ctx = self._linear_fit_context() + yield + + @contextmanager + def __nonlinear_fit_context(self): + with ExitStack() as stack: + ctx = self.cachingContext("fit") + stack.enter_context(ctx) + ctx = self._linear_context(False) + stack.enter_context(ctx) + ctx = self.cachingContext("parameters") + stack.enter_context(ctx) + ctx = self._nonlinear_fit_context() + yield + + @contextmanager + def _linear_fit_context(self): + """To allow derived classes to add context""" + yield + + @contextmanager + def _nonlinear_fit_context(self): + """To allow derived classes to add context""" + yield + + @contextmanager + def _linear_context(self, linear): + keep = self.linear + self.linear = linear + try: + yield + finally: + self.linear = keep + + def _gefit_evaluate_fitmodel(self, parameters, xdata): + """Update parameters and evaluate model + + :param array parameters: length nparams + :param array xdata: length nxdata + :returns array: nxdata + """ + self.parameters = parameters + return self.evaluate_fitmodel(xdata=xdata) + + def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): + """Update parameters and return derivate to a specific parameter + + :param array parameters: length nparams + :param int param_idx: + :param array xdata: length nxdata + :returns array: nxdata + """ + self.parameters = parameters + return self.derivative_fitmodel(param_idx, xdata=xdata) + + def use_fit_result(self, result): + """ + :param dict result: + """ + if result["linear"]: + self.linear_parameters = result["parameters"] + else: + self.parameters = result["parameters"] + + @contextmanager + def use_fit_result_context(self, result): + with self._linear_context(result["linear"]): + with self.cachingContext("parameters"): + self.use_fit_result(result) + yield + + def _parameters_to_fit(self, params): + return params + + def _linear_parameters_to_fit(self, params): + return params + + def _fit_to_parameters(self, params): + return params + + def _fit_to_linear_parameters(self, params): + return params + + def _fit_to_linear_uncertainties(self, uncertainties): + return uncertainties + + def _fit_to_uncertainties(self, uncertainties): + return uncertainties + + def linear_decomposition_fitmodel(self, xdata=None): + """Linear decomposition of the fit model. + + :param array xdata: length nxdata + :returns array: nparams x nxdata + """ + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + return self.linear_parameters[:, numpy.newaxis] * derivatives diff --git a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py new file mode 100644 index 000000000..4e87016ff --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py @@ -0,0 +1,353 @@ +from contextlib import contextmanager +import numpy +from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedInterface +from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface +from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_property +from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_contextmanager + + +class parameter_group(linked_property): + """Usage: + + .. highlight:: python + .. code-block:: python + + class MyClass(Model): + + def __init__(self): + self._myparam = 0. + + @parameter_group + def myparam(self): + return self._myparam + + @myparam.setter # optional + def myparam(self, value): + self._myparam = value + + @myparam.counter # optional + def myparam(self): + return 1 + + @myparam.constraints # optional + def myparam(self): + return 1, 0, 0 + """ + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.fcount = self._fcount_default() + self.fconstraints = self._fconstraints_default() + + def counter(self, fcount): + self.fcount = fcount + return self + + def constraints(self, fconstraints): + self.fconstraints = fconstraints + return self + + def _fcount_default(self): + def fcount(oself): + try: + return len(self.fget(oself)) + except TypeError: + return 1 + + return fcount + + def _fconstraints_default(self): + def fconstraints(oself): + return numpy.zeros((self.fcount(oself), 3)) + + return fconstraints + + +class linear_parameter_group(parameter_group): + pass + + +class ModelParameterInterfaceBase: + def __init__(self): + self.__cache = dict() + self.linear = False + super().__init__() + + @linked_contextmanager + def cachingContext(self, cachename): + reset = not self.cachingEnabled(cachename) + if reset: + self.__cache[cachename] = dict() + try: + yield + finally: + if reset: + del self.__cache[cachename] + + def cachingEnabled(self, cachename): + return cachename in self.__cache + + def getCache(self, cachename, *subnames): + if cachename in self.__cache: + ret = self.__cache[cachename] + for cachename in subnames: + try: + ret = ret[cachename] + except KeyError: + ret[cachename] = dict() + ret = ret[cachename] + return ret + else: + return None + + @contextmanager + def linear_context(self, linear): + keep = self.linear + self.linear = linear + try: + yield + finally: + self.linear = keep + + @classmethod + def parameter_group_is_linear(cls, name): + return isinstance(getattr(cls, name, None), linear_parameter_group) + + def get_parameter_group_names(self, **paramtype): + return tuple(self.iter_parameter_group_names(**paramtype)) + + def get_parameter_names(self, **paramtype): + return tuple(self.iter_parameter_names(**paramtype)) + + def get_n_parameters(self, **paramtype): + return sum(n for _, n in self.iter_parameter_groups(**paramtype)) + + def iter_parameter_names(self, **paramtype): + for group_name, n in self.iter_parameter_groups(**paramtype): + if n > 1: + for i in range(n): + yield group_name + str(i) + else: + yield group_name + + def iter_parameter_groups(self, **paramtype): + """Yield name and count of enabled parameter groups + + :param bool linear_only: + :yields str, int: group name, nb. parameters in the group + """ + cache = self.getCache("iter_parameter_groups") + if cache is None: + yield from self._parameter_groups_notcached(**paramtype) + return + + key = self._parameters_cache_key(**paramtype) + it = cache.get(key) + if it is None: + it = cache[key] = list( + self._parameter_groups_notcached(**paramtype) + ) + yield from it + + def _parameter_groups_notcached(self, **paramtype): + """Helper for `iter_parameter_groups`. + + :yields str, int: group name, nb. parameters in the group + """ + names = self.iter_parameter_group_names(**paramtype) + for name in names: + paramprop = getattr(self.__class__, name) + n = paramprop.fcount(self) + if n: + yield name, n + + def _parameters_cache_key(self, linear=None, linked=None): + if linear is None: + linear = self.linear + return linear, linked + + def get_parameter_values(self, **paramtype): + """All parameters values in one numpy array + + :returns array: + """ + raise NotImplementedError + + def set_parameter_values(self, values, **paramtype): + """ + :returns array: + """ + raise NotImplementedError + + def iter_parameter_group_names(self, **paramtype): + """ + :yield str: + """ + raise NotImplementedError + + +class ModelParameterInterface(LinkedInterface, ModelParameterInterfaceBase): + _PARAMETERS = tuple() + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + allp = list() + for name in sorted(dir(cls)): # TODO: keep order if declaration? + attr = getattr(cls, name) + if isinstance(attr, parameter_group): + allp.append(name) + cls._PARAMETERS = tuple(allp) + + @linked_property + def linear(self): + return self.__linear + + @linear.setter + def linear(self, value): + self.__linear = value + + def get_parameter_values(self, **paramtype): + """All parameters values in one numpy array + + :returns array: + """ + cache = self.getCache("_parameters") + if cache is None: + return self._get_parameter_values_notcached(**paramtype) + + key = self._parameters_cache_key() + parameters = cache.get(key, None) + if parameters is None: + parameters = cache[key] = self._get_parameter_values_notcached(**paramtype) + return parameters + + def set_parameter_values(self, values, **paramtype): + """ + :returns array: + """ + cache = self.getCache("_parameters") + if cache is None: + self._set_parameter_values_notcached(values, **paramtype) + else: + key = self._parameters_cache_key(**paramtype) + cache[key] = values + + def _get_parameter_values_notcached(self, **paramtype): + """Merge all parameters values in one numpy array + + :returns array: + """ + nvalues = self._n_parameters(**paramtype) + values = numpy.zeros(nvalues) + for group_name, idx in self._parameter_group_indices(**paramtype): + values[idx] = getattr(self, group_name) + return values + + def _set_parameter_values_notcached(self, values, **paramtype): + for group_name, idx in self._parameter_group_indices(**paramtype): + setattr(self, group_name, values[idx]) + + def _get_parameter(self, fget): + parameters = self.getCache("_parameters") + if parameters is None: + return fget(self) + + key = self._parameters_cache_key() + parameters = parameters.get(key, None) + if parameters is None: + return fget(self) + + idx = self._parameter_group_index(fget.__name__) + if idx is None: + return fget(self) + return parameters[idx] + + def _set_parameter(self, fset, value): + parameters = self.getCache("_parameters") + if parameters is None: + return fset(self, value) + + key = self._parameters_cache_key() + parameters = parameters.get(key, None) + if parameters is None: + return fset(self, value) + + idx = self._parameter_group_index(fset.__name__) + if idx is None: + return fset(self, value) + parameters[idx] = value + + def _parameter_group_index(self, name, **paramtype): + """Parameter group index in the parameter sequence + + :returns int or slice or None: + """ + for group_name, idx in self._parameter_group_indices(**paramtype): + if name == group_name: + return idx + return None + + def _parameter_group_indices(self, **paramtype): + """Index of each parameter group in the parameter sequence + + :yields int or slice: index of parameter group in all parameters + """ + i = 0 + for group_name, n in self.iter_parameter_groups(**paramtype): + if n == 1: + yield group_name, i + else: + yield group_name, slice(i, i + n) + i += n + + def iter_parameter_group_names(self, linear=None): + for name in self._PARAMETERS: + if linear is not None: + is_linear = self.parameter_group_is_linear(name) + if linear == is_linear: + yield name + + +class ConcatModelParameterInterface(LinkedContainerInterface, ModelParameterInterfaceBase): + @property + def models(self): + return self.linked_instances + + @property + def nmodels(self): + return len(self.linked_instances) + + def iter_parameter_group_names(self, **paramtype): + """ + :yield str: + """ + encountered = set() + for i, instance in enumerate(self.linked_instances): + for name in instance.iter_parameter_group_names(**paramtype): + if instance.property_is_linked(name): + if name not in encountered: + encountered.add(name) + yield name + else: + yield f"model{i}_{name}" + + def get_parameter_values(self, **paramtype): + """All parameters values in one numpy array + + :returns array: + """ + values = list() + for instance in self.linked_instances: + ivalues = instance.get_parameter_values(**paramtype) + values.append(ivalues) + return numpy.concatenate(values) + + def set_parameter_values(self, values, **paramtype): + """ + :returns array: + """ + i = 0 + for instance in self.linked_instances: + n = instance.get_n_parameters(**paramtype) + instance.set_parameter_values(values[i: i+n], **paramtype) + i += n diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/PolynomialModels.py index eacba31eb..7038c518f 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/PolynomialModels.py @@ -33,6 +33,7 @@ import numpy from PyMca5.PyMcaMath.fitting.Model import Model +from PyMca5.PyMcaMath.fitting.Model import parameter from PyMca5.PyMcaMath.fitting.Model import linear_parameter @@ -44,7 +45,7 @@ def __init__(self, degree=0, maxiter=100): self._linear = True self.degree = degree self.maxiter = maxiter - super(PolynomialModel, self).__init__() + super().__init__() @property def degree(self): @@ -56,7 +57,7 @@ def degree(self, n): raise ValueError("degree must be a positive integer") self._coefficients = numpy.zeros(n + 1) - @linear_parameter + @property def coefficients(self): return self._coefficients @@ -104,6 +105,14 @@ def maxiter(self, value): class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" + @linear_parameter + def fitmodel_coefficients(self): + return self.coefficients + + @fitmodel_coefficients.setter + def fitmodel_coefficients(self, values): + self.coefficients = values + def evaluate_fitmodel(self, xdata=None): """Evaluate the fit model, not the full model. @@ -112,24 +121,12 @@ def evaluate_fitmodel(self, xdata=None): """ if xdata is None: xdata = self.xdata - coeff = self.fit_parameters + coeff = numpy.atleast_1d(self.fitmodel_coefficients) y = coeff[0] * numpy.ones_like(xdata) for i in range(1, len(coeff)): y += coeff[i] * (xdata ** i) return y - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - return numpy.array( - [self.derivative_fitmodel(i, xdata=xdata) for i in range(self.degree + 1)] - ) - def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter @@ -150,24 +147,20 @@ class ExponentialPolynomialModel(LinearPolynomialModel): yfit = log(y) = log(c1) + c1*x + c2*x^2 + ... """ - def _ydata_to_fit(self, ydata, xdata=None): - return numpy.log(ydata) - - def _fit_to_ydata(self, yfit, xdata=None): - return numpy.exp(yfit) - - def _parameters_to_fit(self, parameters): - parameters = parameters.copy() - parameters[0] = numpy.log(parameters[0]) - return parameters - - def _fit_to_parameters(self, parameters): - parameters = parameters.copy() - parameters[0] = numpy.exp(parameters[0]) - return parameters - - def _linear_parameters_to_fit(self, parameters): - return self._parameters_to_fit(parameters) - - def _fit_to_linear_parameters(self, parameters): - return self._fit_to_parameters(parameters) + @linear_parameter + def fitmodel_coefficients(self): + coefficients = self.coefficients.copy() + coefficients[0] = numpy.log(coefficients[0]) + return coefficients + + @fitmodel_coefficients.setter + def fitmodel_coefficients(self, values): + values = numpy.atleast_1d(values).copy() + values[0] = numpy.exp(values[0]) + self.coefficients = values + + def _y_full_to_fit(self, y, xdata=None): + return numpy.log(y) + + def _y_fit_to_full(self, y, xdata=None): + return numpy.exp(y) diff --git a/PyMca5/PyMcaMath/fitting/PropertyUtils.py b/PyMca5/PyMcaMath/fitting/PropertyUtils.py new file mode 100644 index 000000000..fa81c337f --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/PropertyUtils.py @@ -0,0 +1,57 @@ +class wrapped_property(property): + """Property that prepares fget, fset and fdel wrappers + for derived property classes. + """ + + def __init__( + self, + fget=None, + fset=None, + fdel=None, + doc=None, + ) -> None: + if fget is not None: + fget = self._wrap_getter(fget) + if fset is not None: + fset = self._wrap_setter(fset) + if fdel is not None: + fget = self._wrap_deleter(fdel) + super().__init__( + fget=fget, + fset=fset, + fdel=fdel, + doc=doc, + ) + + def getter(self, fget): + """Decorator to change fget after property instantiation + """ + if fget is not None: + fget = self._wrap_getter(fget) + return super().getter(fget) + + def setter(self, fset): + """Decorator to change fset after property instantiation + """ + if fset is not None: + fset = self._wrap_setter(fset) + return super().setter(fset) + + def deleter(self, fdel): + """Decorator to change fdel after property instantiation + """ + if fdel is not None: + fget = self._wrap_deleter(fdel) + return super().deleter(fdel) + + def _wrap_getter(self, fget): + """Intended for derived property classes""" + return fget + + def _wrap_setter(self, fset): + """Intended for derived property classes""" + return fset + + def _wrap_deleter(self, fdel): + """Intended for derived property classes""" + return fdel diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 3ae3389af..21e3d34ee 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -1974,25 +1974,25 @@ def ypileup(self, ymodel, xdata=None): ymodel, min(xdata), self.zero, self.gain, 0 ) - def _ydata_to_fit(self, ydata, xdata=None): + def _y_full_to_fit(self, y, xdata=None): """The fitting is done after subtracting the numerical background""" if self.hasNumBkg: - ydata = ydata - self.ynumbkg(xdata=xdata) + y = y - self.ynumbkg(xdata=xdata) if self.linear and self.hasPileUp: ymodel = self.mcatheory(xdata=xdata, summing=False) - ydata = ydata - self.ypileup(ymodel, xdata=xdata) - return ydata + y = y - self.ypileup(ymodel, xdata=xdata) + return y @property def hasPileUp(self): return bool(self.sum) - def _fit_to_ydata(self, yfit, xdata=None): + def _y_fit_to_full(self, y, xdata=None): """The numerical background is not included in the fit model""" if self.hasNumBkg: - return yfit + self.ynumbkg(xdata=xdata) + return y + self.ynumbkg(xdata=xdata) else: - return yfit + return y def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachedInterfaceTest.py new file mode 100644 index 000000000..ea377c8af --- /dev/null +++ b/PyMca5/tests/CachedInterfaceTest.py @@ -0,0 +1,127 @@ +import unittest +import numpy +from collections import Counter +from PyMca5.PyMcaMath.fitting.CachedInterface import CachedInterface +from PyMca5.PyMcaMath.fitting.CachedInterface import cached_property + + +class Cached(CachedInterface): + def __init__(self): + super().__init__() + self._cfg = {"var1": 1, "var2": 2} + self.reset_counters() + + def reset_counters(self): + self.get_counter = Counter() + self.set_counter = Counter() + + @cached_property + def var1(self): + self.get_counter["var1"] += 1 + return self._cfg.get("var1") + + @var1.setter + def var1(self, value): + self.set_counter["var1"] += 1 + self._cfg["var1"] = value + + @cached_property + def var2(self): + self.get_counter["var2"] += 1 + return self._cfg.get("var2") + + @var2.setter + def var2(self, value): + self.set_counter["var2"] += 1 + self._cfg["var2"] = value + + def _property_cache_index(self, name): + if name == "var1": + return 0 + else: + return 1 + + def _property_cache_key(self, **_): + return None + + def _create_empty_cache(self, key, **_): + return numpy.zeros(2, dtype=float) + + +class testCachedInterface(unittest.TestCase): + def setUp(self): + self.cached_object = Cached() + + def _assertGetSetCount(self, name, getcount, setcount): + self.assertEqual(self.cached_object.get_counter[name], getcount) + self.assertEqual(self.cached_object.set_counter[name], setcount) + + def _assertVarValues(self, v1, v2): + self.assertEqual(self.cached_object.var1, v1) + self.assertEqual(self.cached_object.var2, v2) + + def test_get_without_caching(self): + for i in range(5): + self.assertEqual(self.cached_object.var1, 1) + self.assertEqual(self.cached_object.var2, 2) + self._assertGetSetCount("var1", i + 1, 0) + self._assertGetSetCount("var2", i + 1, 0) + + def test_set_without_caching(self): + for i in range(5): + self.cached_object.var1 = 100 + self.cached_object.var2 = 200 + self._assertGetSetCount("var1", 0, i + 1) + self._assertGetSetCount("var2", 0, i + 1) + self._assertVarValues(100, 200) + + def test_get_with_caching(self): + with self.cached_object.propertyCachingContext() as cache: + for i in range(5): + self.assertEqual(self.cached_object.var1, 1) + self.assertEqual(self.cached_object.var2, 2) + self.assertEqual(cache.tolist(), [1, 2]) + self._assertGetSetCount("var1", 1, 0) + self._assertGetSetCount("var2", 1, 0) + + def test_set_with_caching(self): + with self.cached_object.propertyCachingContext() as cache: + for i in range(5): + self.cached_object.var1 = 100 + self.cached_object.var2 = 200 + self.assertEqual(self.cached_object.var1, 100) + self.assertEqual(self.cached_object.var2, 200) + self.assertEqual(cache.tolist(), [100, 200]) + self._assertGetSetCount("var1", 1, 0) + self._assertGetSetCount("var2", 1, 0) + self._assertVarValues(1, 2) + + def test_set_with_persistent_caching(self): + with self.cached_object.propertyCachingContext(persist=True) as cache: + for i in range(5): + self.cached_object.var1 = 100 + self.cached_object.var2 = 200 + self.assertEqual(self.cached_object.var1, 100) + self.assertEqual(self.cached_object.var2, 200) + self.assertEqual(cache.tolist(), [100, 200]) + self._assertGetSetCount("var1", 1, 1) + self._assertGetSetCount("var2", 1, 1) + self._assertVarValues(100, 200) + + def test_start_cache(self): + with self.cached_object.propertyCachingContext(start_cache=[100, 200]) as cache: + self.assertEqual(cache, [100, 200]) + self.assertEqual(self.cached_object.var1, 100) + self.assertEqual(self.cached_object.var2, 200) + self._assertGetSetCount("var1", 0, 0) + self._assertGetSetCount("var2", 0, 0) + self._assertVarValues(1, 2) + + def test_persistent_start_cache(self): + with self.cached_object.propertyCachingContext( + start_cache=[100, 200], persist=True + ) as cache: + self.assertEqual(cache, [100, 200]) + self._assertGetSetCount("var1", 0, 1) + self._assertGetSetCount("var2", 0, 1) + self._assertVarValues(100, 200) diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitModelTest.py index f29aab279..5d2b3a0f1 100644 --- a/PyMca5/tests/FitModelTest.py +++ b/PyMca5/tests/FitModelTest.py @@ -66,7 +66,7 @@ def create_model(self, nmodels): self.fitmodel.ydata = ydata numpy.testing.assert_array_equal(self.fitmodel.ydata, ydata) numpy.testing.assert_array_equal(self.fitmodel.yfullmodel, ydata) - numpy.testing.assert_allclose(self.fitmodel.yfitmodel, ydata - 10) + numpy.testing.assert_allclose(self.fitmodel.yfitmodel, ydata - 10, atol=1e-12) self.validate_model() def init_random(self, **kw): @@ -102,7 +102,6 @@ def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): def modify_random(self, only_linear=False): self._modify_random(only_linear=only_linear) self.validate_model() - # self.plot() def _modify_random(self, only_linear=False): porg = self.fitmodel.parameters.copy() @@ -134,10 +133,15 @@ def _modify_random(self, only_linear=False): def validate_model(self): self._validate_model(self.fitmodel, self.is_concat) if self.is_concat: - for model in self.fitmodel._models: + for i, model in enumerate(self.fitmodel._models): self._validate_model(model, False) + self.fitmodel.share_attributes() # TODO: find a better way of sharing + self._validate_model(self.fitmodel, self.is_concat) def _validate_model(self, model, is_concat): + keep_parameters = model.parameters.copy() + keep_linear_parameters = model.linear_parameters.copy() + if not is_concat: # Alphabetic order expected = ["concentrations", "gain", "wgain", "wzero", "zero"] @@ -160,6 +164,7 @@ def _validate_model(self, model, is_concat): arr2 = model.evaluate_linear_fitmodel() arr3 = model.yfitmodel arr4 = sum(model.linear_decomposition_fitmodel()) + numpy.testing.assert_allclose(arr1, arr2) numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) @@ -187,22 +192,19 @@ def _validate_model(self, model, is_concat): self.assertEqual(model.parameter_names, names) self.assertEqual(model.linear_parameter_names, lin_names) - def plot(self): - import matplotlib.pyplot as plt - - m = self.fitmodel - derivatives = m.derivatives() - names = m.parameter_names - plt.figure() - plt.plot(m.ydata, label="data") - plt.plot(m.yfitmodel, label="model") - plt.legend() - plt.figure() - for y, name in zip(derivatives, names): - plt.plot(y, label=name) - plt.title("Derivatives") - plt.legend() - plt.show() + if not is_concat: + for linear in [not model.linear, model.linear]: + model.linear = linear + for param_name, calc, numerical in model.compare_derivatives(): + err_msg = "[linear={}] Analytical and numerical derivative of {} are not equal".format( + linear, repr(param_name) + ) + numpy.testing.assert_allclose( + calc, numerical, err_msg=err_msg, rtol=1e-3, atol=1e-6 + ) + + numpy.testing.assert_array_equal(keep_parameters, model.parameters) + numpy.testing.assert_array_equal(keep_linear_parameters, model.linear_parameters) @with_model(1) def testLinearFit(self): @@ -218,7 +220,11 @@ def _testLinearFit(self): self.modify_random(only_linear=True) result = self.fitmodel.fit() - self.assert_result(result, expected) + try: + self.assert_result(result, expected) + except Exception: + breakpoint() + raise self.assertTrue( not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) ) @@ -255,11 +261,28 @@ def _testNonLinearFit(self): self.assertTrue(not numpy.allclose(self.fitmodel.parameters, expected1)) self.assertTrue(not numpy.allclose(self.fitmodel.linear_parameters, expected2)) + if False: + import matplotlib.pyplot as plt + + plt.plot(self.fitmodel.ydata) + plt.plot(self.fitmodel.yfullmodel) + plt.show() + self.fitmodel.use_fit_result(result) - # self.plot() - self.assert_ymodel() + if self.is_concat: + self.fitmodel.share_attributes() + # TODO: non-linear parameters not precise # numpy.testing.assert_allclose(self.fitmodel.parameters, expected1) + if True: + import matplotlib.pyplot as plt + + plt.plot(self.fitmodel.ydata) + plt.plot(self.fitmodel.yfullmodel) + plt.show() + numpy.testing.assert_allclose( + self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 + ) numpy.testing.assert_allclose( self.fitmodel.linear_parameters, expected2, rtol=1e-3 ) @@ -271,13 +294,6 @@ def assert_result(self, result, expected): ul = p + 3 * pstd self.assertTrue(all((expected >= ll) & (expected <= ul))) - def assert_ymodel(self): - a = self.fitmodel.ydata - b = self.fitmodel.yfullmodel - mask = (a > 1) & (b > 1) - self.assertTrue(mask.any()) - numpy.testing.assert_allclose(a[mask], b[mask], rtol=1e-3) - @with_model(8) def testParameterIndex(self): # Test parameter index conversion from concatenated model to single model @@ -292,9 +308,7 @@ def testParameterIndex(self): imodels = [] iparams = [] for param_idx, param_name in enumerate(self.fitmodel.parameter_names): - lst = list( - self.fitmodel._parameter_model_index(param_idx, linear_only=linear) - ) + lst = list(self.fitmodel._parameter_model_index(param_idx)) # imodel: model indices # iparam: index of the parameter in the corresponing models if lst: diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 62232890f..ef9dfe194 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -52,8 +52,10 @@ def testLinearPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected - self.assertEqual(model.parameter_group_names, ["coefficients"]) - self.assertEqual(model.linear_parameter_group_names, ["coefficients"]) + self.assertEqual(model.parameter_group_names, ["fitmodel_coefficients"]) + self.assertEqual( + model.linear_parameter_group_names, ["fitmodel_coefficients"] + ) numpy.testing.assert_array_equal(model.parameters, expected) fitmodel.ydata = model.yfullmodel @@ -81,6 +83,7 @@ def testExpPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected + expected[0] = numpy.log(expected[0]) numpy.testing.assert_array_equal(model.parameters, expected) fitmodel.ydata = model.yfullmodel diff --git a/PyMca5/tests/LinkedInterfaceTest.py b/PyMca5/tests/LinkedInterfaceTest.py new file mode 100644 index 000000000..37186ca82 --- /dev/null +++ b/PyMca5/tests/LinkedInterfaceTest.py @@ -0,0 +1,211 @@ +import unittest +from collections import Counter +from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedInterface +from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface +from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_contextmanager +from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_property + + +class ModelBase(LinkedInterface): + def __init__(self): + super().__init__() + self.context_counter = 0 + + @linked_contextmanager + def context(self): + self.context_counter += 1 + yield + + def fit(self): + with self.context(): + pass + + +class Model1(ModelBase): + def __init__(self, cfg): + super().__init__() + self._cfg = cfg + self.reset_counters() + + def reset_counters(self): + self.get_counter = Counter() + self.set_counter = Counter() + + @linked_property + def var1(self): + self.get_counter["var1"] += 1 + return self._cfg.get("var1") + + @var1.setter + def var1(self, value): + self.set_counter["var1"] += 1 + self._cfg["var1"] = value + + +class Model2(Model1): + @linked_property + def var2(self): + self.get_counter["var2"] += 1 + return self._cfg.get("var2") + + @var2.setter + def var2(self, value): + self.set_counter["var2"] += 1 + self._var2 = value + self._cfg["var2"] = value + + +class ConcatModel(LinkedContainerInterface, ModelBase): + def __init__(self): + cfg1a = {"var1": 1} + cfg1b = {"var1": 2} + cfg2a = {"var1": 3, "var2": 4} + cfg2b = {"var1": 5, "var2": 6} + super().__init__([Model1(cfg1a), Model1(cfg1b), + Model2(cfg2a), Model2(cfg2b)]) + + def reset_counters(self): + for m in self.linked_instances: + m.reset_counters() + + def link(self): + self.enable_property_link("var1", "var2") + self.reset_counters() + + def unlink(self): + self.disable_property_link("var1", "var2") + self.reset_counters() + + +class testLinkedInterface(unittest.TestCase): + def setUp(self): + self.concat_model = ConcatModel() + + def test_instances(self): + """establish links + """ + nlinked = len(self.concat_model.linked_instances) - 1 + for m in self.concat_model.linked_instances: + self.assertEqual(len(m.linked_instances), nlinked) + + def test_init_properties(self): + """initial property values + """ + self.assert_property_values("var1", 1, [1, 2, 3, 5]) + self.assert_property_values("var2", 4, [4, 6]) + + def test_enable_property_link_syncing(self): + """maximal 1 get/set of a linked property when enabling linking + """ + self.concat_model.enable_property_link("var1", "var2") + for name in ("var1", "var2"): + for m in self.concat_model.linked_instances: + self.assertTrue(m.get_counter[name] <= 1) + self.assertTrue(m.set_counter[name] <= 1) + self.assert_synced_values() + + def test_contexts_concat(self): + """entering a linked context manager of a container + """ + self.concat_model.fit() + self.assertEqual(self.concat_model.context_counter, 1) + for m in self.concat_model.linked_instances: + m.context_counter, 1 + + def test_contexts_single(self): + """entering a linked context manager of a single instance + """ + model = self.concat_model.linked_instances[2] + model.fit() + self.assertEqual(model.context_counter, 1) + for m in self.concat_model.linked_instances: + m.context_counter, 1 + + def test_get_var1_concat(self): + """getting a linked property (present in all models) from a container + """ + self.concat_model.link() + self.assertEqual(self.concat_model.get_linked_property("var1"), 1) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var1"], 0, msg=f"model{i}") + + def test_get_var1_single(self): + """getting a linked property (present in all models) from a single instance + """ + self.concat_model.link() + model = self.concat_model.linked_instances[2] + self.assertEqual(model.var1, 1) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var1"], 0, msg=f"model{i}") + + def test_get_var2_concat(self): + """getting a linked property (present in some models) from a container + """ + self.concat_model.link() + self.assertEqual(self.concat_model.get_linked_property("var2"), 4) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var2"], 0, msg=f"model{i}") + + def test_get_var2_single(self): + """getting a linked property (present in some models) from a single instance + """ + self.concat_model.link() + model = self.concat_model.linked_instances[2] + self.assertEqual(model.var2, 4) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var2"], 0, msg=f"model{i}") + + def test_set_var1_concat(self): + """setting a linked property (present in all models) from a container + """ + self.concat_model.link() + self.concat_model.set_linked_property("var1", 100) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var1"], 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var1"], 1, msg=f"model{i}") + self.assert_synced_values(var1=100) + + def test_set_var1_single(self): + """setting a linked property (present in all models) from a single instance + """ + self.concat_model.link() + model = self.concat_model.linked_instances[2] + model.var1 = 1 + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var1"], 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var1"], 1, msg=f"model{i}") + + def test_set_var2_concat(self): + """setting a linked property (present in some models) from a container + """ + self.concat_model.link() + self.concat_model.set_linked_property("var2", 100) + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var2"], 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") + self.assert_synced_values(var2=100) + + def test_set_var2_single(self): + """setting a linked property (present in some models) from a single instance + """ + self.concat_model.link() + model = self.concat_model.linked_instances[2] + model.var2 = 100 + for i, m in enumerate(self.concat_model.linked_instances): + self.assertEqual(m.get_counter["var2"], 0, msg=f"model{i}") + self.assertEqual(m.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") + + def assert_synced_values(self, var1=1, var2=4): + self.assert_property_values("var1", var1) + self.assert_property_values("var2", var2) + + def assert_property_values(self, name, value, values=None): + self.assertEqual(self.concat_model.get_linked_property(name), value, msg=name) + if values is None: + values = [value] * len(self.concat_model.linked_instances) + for m, v in zip(self.concat_model.instances_with_linked_property(name), values): + self.assertEqual(getattr(m, name), v, msg=name) diff --git a/PyMca5/tests/ModelParameterInterfaceTest.py b/PyMca5/tests/ModelParameterInterfaceTest.py new file mode 100644 index 000000000..d18f86b7d --- /dev/null +++ b/PyMca5/tests/ModelParameterInterfaceTest.py @@ -0,0 +1,92 @@ +import unittest +from collections import Counter +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ( + ConcatModelParameterInterface, +) +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import parameter_group +from PyMca5.PyMcaMath.fitting.ModelParameterInterface import linear_parameter_group + + +class Model(ModelParameterInterface): + def __init__(self, cfg): + super().__init__() + self._cfg = cfg + self._shared_param = 2 + self._shared_linear_param = 3 + self._param = 4 + self._linear_param = 5 + self.reset_counters() + + def reset_counters(self): + self.get_counter = Counter() + self.set_counter = Counter() + + @parameter_group + def shared_param(self): + self.get_counter["shared_param"] += 1 + return self._cfg.get("shared_param", None) + + @shared_param.setter + def shared_param(self, value): + self.set_counter["shared_param"] += 1 + self._cfg["shared_param"] = value + + @linear_parameter_group + def shared_linear_param(self): + self.get_counter["shared_linear_param"] += 1 + return self._cfg.get("shared_linear_param", None) + + @shared_linear_param.setter + def shared_linear_param(self, value): + self.set_counter["shared_linear_param"] += 1 + self._cfg["shared_linear_param"] = value + + @parameter_group + def param(self): + self.get_counter["param"] += 1 + return self._cfg.get("param", None) + + @param.setter + def param(self, value): + self.set_counter["param"] += 1 + self._cfg["param"] = value + + @linear_parameter_group + def linear_param(self): + self.get_counter["linear_param"] += 1 + return self._cfg.get("linear_param", None) + + @linear_param.setter + def linear_param(self, value): + self.set_counter["linear_param"] += 1 + self._cfg["linear_param"] = value + + +class ConcatModel(ConcatModelParameterInterface): + def __init__(self): + cfgs = list() + for i in range(2): + off = i * 4 + cfg = { + "shared_param": off + 1, + "shared_linear_param": off + 2, + "param": off + 3, + "linear_param": off + 4, + } + cfgs.append(cfg) + super().__init__([Model(cfg) for cfg in cfgs]) + self.enable_property_link("shared_param", "shared_linear_param") + self.reset_counters() + + def reset_counters(self): + for m in self.linked_instances: + m.reset_counters() + + +class testModelParameterInterface(unittest.TestCase): + def setUp(self): + self.concat_model = ConcatModel() + + def test_parameter_names(self): + self.assertEqual(self.concat_model.get_parameter_names(), ("shared_param",)) diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index f48adcafa..0b5232aad 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -43,6 +43,8 @@ class SimpleModel(Model): """Model MCA data using a fixed list of peak positions and efficiencies""" + SIGMA_TO_FWHM = 2 * numpy.sqrt(2 * numpy.log(2)) + def __init__(self): self.config = { "detector": {"zero": 0.0, "gain": 1.0, "wzero": 0.0, "wgain": 1.0}, @@ -55,8 +57,7 @@ def __init__(self): self.ydata_raw = None self.ystd_raw = None self.ybkg = 0 - self.sigma_to_fwhm = 2 * numpy.sqrt(2 * numpy.log(2)) - super(SimpleModel, self).__init__() + super().__init__() def __str__(self): return "{}(npeaks={}, zero={}, gain={}, wzero={}, wgain={})".format( @@ -114,7 +115,7 @@ def positions(self, value): @property def fwhms(self): - return self.zero + self.gain * self.positions + return self.wzero + self.wgain * self.positions @property def areas(self): @@ -155,11 +156,11 @@ def xdata(self, values): def xenergy(self): return self.zero + self.gain * self.xdata - def _ydata_to_fit(self, ydata, xdata=None): - return ydata - self.ybkg + def _y_full_to_fit(self, y, xdata=None): + return y - self.ybkg - def _fit_to_ydata(self, yfit, xdata=None): - return yfit + self.ybkg + def _y_fit_to_full(self, y, xdata=None): + return y + self.ybkg @property def ydata(self): @@ -201,20 +202,7 @@ def evaluate_fitmodel(self, xdata=None): xdata = self.xdata x = self.zero + self.gain * xdata p = list(zip(self.areas, self.positions, self.fwhms)) - y = SpecfitFuns.agauss(p, x) - return y - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - x = self.zero + self.gain * xdata - it = zip(self.efficiency, self.positions, self.fwhms) - return numpy.array([SpecfitFuns.agauss([a, p, w], x) for a, p, w in it]) + return SpecfitFuns.agauss(p, x) def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter @@ -226,30 +214,33 @@ def derivative_fitmodel(self, param_idx, xdata=None): if xdata is None: xdata = self.xdata x = self.zero + self.gain * xdata + name, i = self._parameter_name_from_index(param_idx) if name == "concentrations": p = self.positions[i] a = self.efficiency[i] w = self.wzero + self.wgain * p - y = SpecfitFuns.agauss([a, p, w], x) - else: - fwhms = self.fwhms - sigmas = fwhms / self.sigma_to_fwhm - y = x * 0.0 - for p, a, w, s in zip(self.positions, self.areas, fwhms, sigmas): - if name in ("zero", "gain"): - # Derivative to position - m = -(x - p) / s ** 2 - # Derivative to position param - if name == "gain": - m *= xdata - else: - # Derivative to FWHM - m = ((x - p) ** 2 / s ** 2 - 1) / (self.sigma_to_fwhm * s) - # Derivative to FWHM param - if name == "wgain": - m *= p - y += m * SpecfitFuns.agauss([a, p, w], x) + return SpecfitFuns.agauss([a, p, w], x) + + fwhms = self.fwhms + sigmas = fwhms / self.SIGMA_TO_FWHM + y = x * 0.0 + for p, a, w, s in zip(self.positions, self.areas, fwhms, sigmas): + if name in ("zero", "gain"): + # Derivative to position + m = -(x - p) / s ** 2 + # Derivative to position param + if name == "gain": + m *= xdata + elif name in ("wzero", "wgain"): + # Derivative to FWHM + m = ((x - p) ** 2 / s ** 2 - 1) / (self.SIGMA_TO_FWHM * s) + # Derivative to FWHM param + if name == "wgain": + m *= p + else: + raise ValueError(name) + y += m * SpecfitFuns.agauss([a, p, w], x) return y @@ -257,6 +248,4 @@ class SimpleConcatModel(ConcatModel): def __init__(self, ndetectors=1): models = [SimpleModel() for i in range(ndetectors)] shared_attributes = ["concentrations", "positions"] - super(SimpleConcatModel, self).__init__( - models, shared_attributes=shared_attributes - ) + super().__init__(models, shared_attributes=shared_attributes) diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index e0190d7ee..d8aab532f 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -547,6 +547,7 @@ def testStainlessSteelDataFit(self): def testCompareLegacyMcaTheory(self): x, y, configuration = self._readTrainingData() self._testCompareLegacyMcaTheory(x, y, configuration) + return x, y, configuration = self._readStainlessSteelData() @@ -615,8 +616,10 @@ def _testCompareLegacyMcaTheory(self, x, y, configuration): plt.legend() plt.show() - if False: + if True: for i, name in enumerate(mcaFit.parameter_names): + if "linegroup" in name: + continue yd = mcaFit.derivative_fitmodel(i) yd_num = mcaFit._numerical_derivative(i) plt.plot(yd, label="yd") From 435104503d82b3e566b01fda9e33665b23819d5b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 22 Jun 2021 19:44:52 +0200 Subject: [PATCH 32/74] fixup --- PyMca5/PyMcaMath/fitting/CachedInterface.py | 125 +++++++++++--------- PyMca5/tests/CachedInterfaceTest.py | 41 +++++-- 2 files changed, 103 insertions(+), 63 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachedInterface.py b/PyMca5/PyMcaMath/fitting/CachedInterface.py index d7ff7cb28..eb4084572 100644 --- a/PyMca5/PyMcaMath/fitting/CachedInterface.py +++ b/PyMca5/PyMcaMath/fitting/CachedInterface.py @@ -3,62 +3,51 @@ from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property -class cached_property(wrapped_property): - def _wrap_getter(self, fget): - fget = super()._wrap_getter(fget) - - @functools.wraps(fget) - def wrapper(oself): - return oself._cached_property_fget(fget) - - return wrapper - - def _wrap_setter(self, fset): - fset = super()._wrap_setter(fset) +class CacheManagerInterface: + """Object that manages a cache""" - @functools.wraps(fset) - def wrapper(oself, value): - return oself._cached_property_fset(fset, value) + def __init__(self): + self._cache_root = dict() + super().__init__() - return wrapper + def _create_empty_cache(self, key, **cacheoptions): + # By default the property cache is a dictionary + return dict() + def _property_cache_index(self, name): + # By default the property cache index is its name + return name -class CachedInterface: - _CACHED_PROPERTIES = tuple() + def _property_cache_key(self, **cacheoptions): + # By default we only manage 1 cache (None) + return None - def __init_subclass__(subcls, **kwargs): - super().__init_subclass__(**kwargs) - allp = list() - for name, attr in vars(subcls).items(): - if isinstance(attr, cached_property): - allp.append(name) - subcls._CACHED_PROPERTIES = subcls._CACHED_PROPERTIES + tuple(allp) - @classmethod - def _cached_properties(self): - return self._CACHED_PROPERTIES +class CacheInterface(CacheManagerInterface): + """Object that manages and uses an internal cache (default) or + uses an external cache. + """ def __init__(self): - self._cache_object = None - self._cache_object._cache_root = dict() + self.cache_object = None super().__init__() @property - def _cache_object(self): + def cache_object(self): if self.__external_cache_object is None: return self else: return self.__external_cache_object - @_cache_object.setter - def _cache_object(self, obj): - if obj is not None and not isinstance(obj, CachedInterface): + @cache_object.setter + def cache_object(self, obj): + if obj is not None and not isinstance(obj, CacheManagerInterface): raise TypeError(obj, type(obj)) self.__external_cache_object = obj @contextmanager def cachingContext(self, cachename): - cache_root = self._cache_object._cache_root + cache_root = self.cache_object._cache_root new_context_entry = cachename not in cache_root if new_context_entry: cache_root[cachename] = dict() @@ -69,10 +58,10 @@ def cachingContext(self, cachename): del cache_root[cachename] def cachingEnabled(self, cachename): - return cachename in self._cache_object._cache_root + return cachename in self.cache_object._cache_root def getCache(self, cachename, *subnames): - cache_root = self._cache_object._cache_root + cache_root = self.cache_object._cache_root if cachename not in cache_root: return None ret = cache_root[cachename] @@ -84,6 +73,44 @@ def getCache(self, cachename, *subnames): ret = ret[cachename] return ret + +class cached_property(wrapped_property): + def _wrap_getter(self, fget): + fget = super()._wrap_getter(fget) + + @functools.wraps(fget) + def wrapper(oself): + return oself._cached_property_fget(fget) + + return wrapper + + def _wrap_setter(self, fset): + fset = super()._wrap_setter(fset) + + @functools.wraps(fset) + def wrapper(oself, value): + return oself._cached_property_fset(fset, value) + + return wrapper + + +class CachedPropertiesInterface(CacheInterface): + """Object with cached properties when enabled.""" + + _CACHED_PROPERTIES = tuple() + + def __init_subclass__(subcls, **kwargs): + super().__init_subclass__(**kwargs) + allp = list() + for name, attr in vars(subcls).items(): + if isinstance(attr, cached_property): + allp.append(name) + subcls._CACHED_PROPERTIES = subcls._CACHED_PROPERTIES + tuple(allp) + + @classmethod + def _cached_properties(self): + return self._CACHED_PROPERTIES + @contextmanager def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): values_cache = self._get_property_values_cache(**cacheoptions) @@ -94,7 +121,7 @@ def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions if start_cache is None: # Fill and empty cache with property values - cache_object = self._cache_object + cache_object = self.cache_object key = cache_object._property_cache_key(**cacheoptions) values_cache = cache_object._create_empty_cache(key, **cacheoptions) nameindexmap = dict() @@ -114,7 +141,7 @@ def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions if persist: # Set property values to the cached values if nameindexmap is None: - cache_object = self._cache_object + cache_object = self.cache_object for name in self._cached_properties(): index = cache_object._property_cache_index(name) setattr(self, name, values_cache[index]) @@ -126,14 +153,14 @@ def _get_property_values_cache(self, **cacheoptions): caches = self.getCache("_cached_properties") if caches is None: return None - key = self._cache_object._property_cache_key(**cacheoptions) + key = self.cache_object._property_cache_key(**cacheoptions) return caches.get(key, None) def _set_property_values_cache(self, values_cache, **cacheoptions): caches = self.getCache("_cached_properties") if caches is None: return - cache_object = self._cache_object + cache_object = self.cache_object key = cache_object._property_cache_key(**cacheoptions) caches[key] = values_cache @@ -141,24 +168,12 @@ def _cached_property_fget(self, fget): values_cache = self._get_property_values_cache() if values_cache is None: return fget(self) - index = self._cache_object._property_cache_index(fget.__name__) + index = self.cache_object._property_cache_index(fget.__name__) return values_cache[index] def _cached_property_fset(self, fset, value): values_cache = self._get_property_values_cache() if values_cache is None: return fset(self, value) - index = self._cache_object._property_cache_index(fset.__name__) + index = self.cache_object._property_cache_index(fset.__name__) values_cache[index] = value - - def _create_empty_cache(self, key, **cacheoptions): - # By default the property cache is a dictionary - return dict() - - def _property_cache_index(self, name): - # By default the property cache index is its name - return name - - def _property_cache_key(self, **cacheoptions): - # By default we only manage 1 cache (None) - return None diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachedInterfaceTest.py index ea377c8af..94e93ace3 100644 --- a/PyMca5/tests/CachedInterfaceTest.py +++ b/PyMca5/tests/CachedInterfaceTest.py @@ -1,11 +1,12 @@ import unittest import numpy from collections import Counter -from PyMca5.PyMcaMath.fitting.CachedInterface import CachedInterface +from PyMca5.PyMcaMath.fitting.CachedInterface import CachedPropertiesInterface +from PyMca5.PyMcaMath.fitting.CachedInterface import CacheInterface from PyMca5.PyMcaMath.fitting.CachedInterface import cached_property -class Cached(CachedInterface): +class Cached(CachedPropertiesInterface): def __init__(self): super().__init__() self._cfg = {"var1": 1, "var2": 2} @@ -41,13 +42,23 @@ def _property_cache_index(self, name): else: return 1 - def _property_cache_key(self, **_): - return None - def _create_empty_cache(self, key, **_): return numpy.zeros(2, dtype=float) +class ExternalCached(CacheInterface): + def _property_cache_index(self, name): + if name == "var1": + return 0 + elif name == "var2": + return 1 + else: + return 2 + + def _create_empty_cache(self, key, **_): + return numpy.zeros(3, dtype=float) + + class testCachedInterface(unittest.TestCase): def setUp(self): self.cached_object = Cached() @@ -60,6 +71,12 @@ def _assertVarValues(self, v1, v2): self.assertEqual(self.cached_object.var1, v1) self.assertEqual(self.cached_object.var2, v2) + def _assertCache(self, cache, values): + if self.cached_object.cache_object is self.cached_object: + self.assertEqual(cache.tolist(), values) + else: + self.assertEqual(cache.tolist(), values + [0]) + def test_get_without_caching(self): for i in range(5): self.assertEqual(self.cached_object.var1, 1) @@ -76,11 +93,19 @@ def test_set_without_caching(self): self._assertVarValues(100, 200) def test_get_with_caching(self): + with self.subTest("no external cache"): + self._test_get_with_caching() + with self.subTest("with external cache"): + self.cached_object = Cached() + self.cached_object.cache_object = ExternalCached() + self._test_get_with_caching() + + def _test_get_with_caching(self): with self.cached_object.propertyCachingContext() as cache: + self._assertCache(cache, [1, 2]) for i in range(5): self.assertEqual(self.cached_object.var1, 1) self.assertEqual(self.cached_object.var2, 2) - self.assertEqual(cache.tolist(), [1, 2]) self._assertGetSetCount("var1", 1, 0) self._assertGetSetCount("var2", 1, 0) @@ -91,7 +116,7 @@ def test_set_with_caching(self): self.cached_object.var2 = 200 self.assertEqual(self.cached_object.var1, 100) self.assertEqual(self.cached_object.var2, 200) - self.assertEqual(cache.tolist(), [100, 200]) + self._assertCache(cache, [100, 200]) self._assertGetSetCount("var1", 1, 0) self._assertGetSetCount("var2", 1, 0) self._assertVarValues(1, 2) @@ -103,7 +128,7 @@ def test_set_with_persistent_caching(self): self.cached_object.var2 = 200 self.assertEqual(self.cached_object.var1, 100) self.assertEqual(self.cached_object.var2, 200) - self.assertEqual(cache.tolist(), [100, 200]) + self._assertCache(cache, [100, 200]) self._assertGetSetCount("var1", 1, 1) self._assertGetSetCount("var2", 1, 1) self._assertVarValues(100, 200) From 4d9fb80d3e908bab26488885548f88e40dd9e64a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 22 Jun 2021 19:52:11 +0200 Subject: [PATCH 33/74] fixup --- PyMca5/tests/CachedInterfaceTest.py | 57 ++++++++++++++++++----------- PyMca5/tests/LinkedInterfaceTest.py | 2 +- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachedInterfaceTest.py index 94e93ace3..21dfae5d7 100644 --- a/PyMca5/tests/CachedInterfaceTest.py +++ b/PyMca5/tests/CachedInterfaceTest.py @@ -1,5 +1,6 @@ import unittest import numpy +import functools from collections import Counter from PyMca5.PyMcaMath.fitting.CachedInterface import CachedPropertiesInterface from PyMca5.PyMcaMath.fitting.CachedInterface import CacheInterface @@ -59,24 +60,24 @@ def _create_empty_cache(self, key, **_): return numpy.zeros(3, dtype=float) -class testCachedInterface(unittest.TestCase): - def setUp(self): - self.cached_object = Cached() +def external_subtests(method): + @functools.wraps(method) + def wrapper(self): + with self.subTest("no external cache"): + method(self) + with self.subTest("with external cache"): + self.cached_object = Cached() + self.cached_object.cache_object = ExternalCached() + method(self) - def _assertGetSetCount(self, name, getcount, setcount): - self.assertEqual(self.cached_object.get_counter[name], getcount) - self.assertEqual(self.cached_object.set_counter[name], setcount) + return wrapper - def _assertVarValues(self, v1, v2): - self.assertEqual(self.cached_object.var1, v1) - self.assertEqual(self.cached_object.var2, v2) - def _assertCache(self, cache, values): - if self.cached_object.cache_object is self.cached_object: - self.assertEqual(cache.tolist(), values) - else: - self.assertEqual(cache.tolist(), values + [0]) +class testCachedInterface(unittest.TestCase): + def setUp(self): + self.cached_object = Cached() + @external_subtests def test_get_without_caching(self): for i in range(5): self.assertEqual(self.cached_object.var1, 1) @@ -84,6 +85,7 @@ def test_get_without_caching(self): self._assertGetSetCount("var1", i + 1, 0) self._assertGetSetCount("var2", i + 1, 0) + @external_subtests def test_set_without_caching(self): for i in range(5): self.cached_object.var1 = 100 @@ -92,15 +94,8 @@ def test_set_without_caching(self): self._assertGetSetCount("var2", 0, i + 1) self._assertVarValues(100, 200) + @external_subtests def test_get_with_caching(self): - with self.subTest("no external cache"): - self._test_get_with_caching() - with self.subTest("with external cache"): - self.cached_object = Cached() - self.cached_object.cache_object = ExternalCached() - self._test_get_with_caching() - - def _test_get_with_caching(self): with self.cached_object.propertyCachingContext() as cache: self._assertCache(cache, [1, 2]) for i in range(5): @@ -109,6 +104,7 @@ def _test_get_with_caching(self): self._assertGetSetCount("var1", 1, 0) self._assertGetSetCount("var2", 1, 0) + @external_subtests def test_set_with_caching(self): with self.cached_object.propertyCachingContext() as cache: for i in range(5): @@ -121,6 +117,7 @@ def test_set_with_caching(self): self._assertGetSetCount("var2", 1, 0) self._assertVarValues(1, 2) + @external_subtests def test_set_with_persistent_caching(self): with self.cached_object.propertyCachingContext(persist=True) as cache: for i in range(5): @@ -133,6 +130,7 @@ def test_set_with_persistent_caching(self): self._assertGetSetCount("var2", 1, 1) self._assertVarValues(100, 200) + @external_subtests def test_start_cache(self): with self.cached_object.propertyCachingContext(start_cache=[100, 200]) as cache: self.assertEqual(cache, [100, 200]) @@ -142,6 +140,7 @@ def test_start_cache(self): self._assertGetSetCount("var2", 0, 0) self._assertVarValues(1, 2) + @external_subtests def test_persistent_start_cache(self): with self.cached_object.propertyCachingContext( start_cache=[100, 200], persist=True @@ -150,3 +149,17 @@ def test_persistent_start_cache(self): self._assertGetSetCount("var1", 0, 1) self._assertGetSetCount("var2", 0, 1) self._assertVarValues(100, 200) + + def _assertGetSetCount(self, name, getcount, setcount): + self.assertEqual(self.cached_object.get_counter[name], getcount) + self.assertEqual(self.cached_object.set_counter[name], setcount) + + def _assertVarValues(self, v1, v2): + self.assertEqual(self.cached_object.var1, v1) + self.assertEqual(self.cached_object.var2, v2) + + def _assertCache(self, cache, values): + if self.cached_object.cache_object is self.cached_object: + self.assertEqual(cache.tolist(), values) + else: + self.assertEqual(cache.tolist(), values + [0]) diff --git a/PyMca5/tests/LinkedInterfaceTest.py b/PyMca5/tests/LinkedInterfaceTest.py index 37186ca82..fd17c9698 100644 --- a/PyMca5/tests/LinkedInterfaceTest.py +++ b/PyMca5/tests/LinkedInterfaceTest.py @@ -81,7 +81,7 @@ class testLinkedInterface(unittest.TestCase): def setUp(self): self.concat_model = ConcatModel() - def test_instances(self): + def test_links(self): """establish links """ nlinked = len(self.concat_model.linked_instances) - 1 From 08dc5f626bd157bc3f7d98b4838de627e525b37b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 23 Jun 2021 10:16:37 +0200 Subject: [PATCH 34/74] fixup --- PyMca5/tests/CachedInterfaceTest.py | 49 +++++++++++++++-------------- appveyor.yml => appveyor.yml.bak | 0 2 files changed, 26 insertions(+), 23 deletions(-) rename appveyor.yml => appveyor.yml.bak (100%) diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachedInterfaceTest.py index 21dfae5d7..a2f91a278 100644 --- a/PyMca5/tests/CachedInterfaceTest.py +++ b/PyMca5/tests/CachedInterfaceTest.py @@ -50,14 +50,14 @@ def _create_empty_cache(self, key, **_): class ExternalCached(CacheInterface): def _property_cache_index(self, name): if name == "var1": - return 0 + return 2 elif name == "var2": - return 1 + return 3 else: - return 2 + raise ValueError(name) def _create_empty_cache(self, key, **_): - return numpy.zeros(3, dtype=float) + return numpy.zeros(4, dtype=float) def external_subtests(method): @@ -81,18 +81,16 @@ def setUp(self): def test_get_without_caching(self): for i in range(5): self.assertEqual(self.cached_object.var1, 1) - self.assertEqual(self.cached_object.var2, 2) self._assertGetSetCount("var1", i + 1, 0) - self._assertGetSetCount("var2", i + 1, 0) + self._assertGetSetCount("var2", 0, 0) @external_subtests def test_set_without_caching(self): for i in range(5): self.cached_object.var1 = 100 - self.cached_object.var2 = 200 self._assertGetSetCount("var1", 0, i + 1) - self._assertGetSetCount("var2", 0, i + 1) - self._assertVarValues(100, 200) + self._assertGetSetCount("var2", 0, 0) + self._assertVarValues(100, 2) @external_subtests def test_get_with_caching(self): @@ -100,7 +98,6 @@ def test_get_with_caching(self): self._assertCache(cache, [1, 2]) for i in range(5): self.assertEqual(self.cached_object.var1, 1) - self.assertEqual(self.cached_object.var2, 2) self._assertGetSetCount("var1", 1, 0) self._assertGetSetCount("var2", 1, 0) @@ -109,10 +106,8 @@ def test_set_with_caching(self): with self.cached_object.propertyCachingContext() as cache: for i in range(5): self.cached_object.var1 = 100 - self.cached_object.var2 = 200 self.assertEqual(self.cached_object.var1, 100) - self.assertEqual(self.cached_object.var2, 200) - self._assertCache(cache, [100, 200]) + self._assertCache(cache, [100, 2]) self._assertGetSetCount("var1", 1, 0) self._assertGetSetCount("var2", 1, 0) self._assertVarValues(1, 2) @@ -122,18 +117,18 @@ def test_set_with_persistent_caching(self): with self.cached_object.propertyCachingContext(persist=True) as cache: for i in range(5): self.cached_object.var1 = 100 - self.cached_object.var2 = 200 self.assertEqual(self.cached_object.var1, 100) - self.assertEqual(self.cached_object.var2, 200) - self._assertCache(cache, [100, 200]) + self._assertCache(cache, [100, 2]) self._assertGetSetCount("var1", 1, 1) self._assertGetSetCount("var2", 1, 1) - self._assertVarValues(100, 200) + self._assertVarValues(100, 2) @external_subtests def test_start_cache(self): - with self.cached_object.propertyCachingContext(start_cache=[100, 200]) as cache: - self.assertEqual(cache, [100, 200]) + with self.cached_object.propertyCachingContext( + start_cache=self._start_cache([100, 200]), + ) as cache: + self._assertCache(cache, [100, 200]) self.assertEqual(self.cached_object.var1, 100) self.assertEqual(self.cached_object.var2, 200) self._assertGetSetCount("var1", 0, 0) @@ -143,9 +138,9 @@ def test_start_cache(self): @external_subtests def test_persistent_start_cache(self): with self.cached_object.propertyCachingContext( - start_cache=[100, 200], persist=True + start_cache=self._start_cache([100, 200]), persist=True ) as cache: - self.assertEqual(cache, [100, 200]) + self._assertCache(cache, [100, 200]) self._assertGetSetCount("var1", 0, 1) self._assertGetSetCount("var2", 0, 1) self._assertVarValues(100, 200) @@ -159,7 +154,15 @@ def _assertVarValues(self, v1, v2): self.assertEqual(self.cached_object.var2, v2) def _assertCache(self, cache, values): + if not isinstance(cache, list): + cache = cache.tolist() + if self.cached_object.cache_object is self.cached_object: + self.assertEqual(cache, values) + else: + self.assertEqual(cache, [0, 0] + values) + + def _start_cache(self, start_cache): if self.cached_object.cache_object is self.cached_object: - self.assertEqual(cache.tolist(), values) + return start_cache else: - self.assertEqual(cache.tolist(), values + [0]) + return [0, 0] + start_cache diff --git a/appveyor.yml b/appveyor.yml.bak similarity index 100% rename from appveyor.yml rename to appveyor.yml.bak From 82f27b4f832809bef1cd0ea299fafe6625e2b640 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 14:56:02 +0200 Subject: [PATCH 35/74] fixup --- PyMca5/PyMcaMath/fitting/LinkedInterface.py | 76 +++++++------ PyMca5/tests/CachedInterfaceTest.py | 14 +++ PyMca5/tests/LinkedInterfaceTest.py | 115 +++++++++++--------- 3 files changed, 120 insertions(+), 85 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/LinkedInterface.py b/PyMca5/PyMcaMath/fitting/LinkedInterface.py index 5db33444b..082a40281 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedInterface.py +++ b/PyMca5/PyMcaMath/fitting/LinkedInterface.py @@ -7,19 +7,24 @@ class linked_property(wrapped_property): """Setting a linked property of one object will set that property for all linked objects """ + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.propagate = False + def _wrap_setter(self, fset): propname = fset.__name__ fset = super()._wrap_setter(fset) @functools.wraps(fset) def wrapper(oself, value): - ret = fset(oself, value) - if oself.propagation_is_enabled(propname): - for instance in oself._filter_class_has_linked_property( + fset(oself, value) + if not self.propagate: + return + for instance in oself._filter_class_has_linked_property( oself._non_propagating_instances, propname ): setattr(instance, propname, value) - return ret return wrapper @@ -51,28 +56,41 @@ class LinkedInterface: to derived from this class. """ def __init__(self): - self.__enabled_linked_properties = dict() self.__linked_instances = list() self.__propagate = True super().__init__() - def propagation_is_enabled(self, name): - if not self.__linked_instances: - return False - return self.property_is_linked(name) - - def property_is_linked(self, name): - return self.__enabled_linked_properties.get(name, False) + @classmethod + def get_linked_property(cls, prop_name): + prop = getattr(cls, prop_name, None) + if isinstance(prop, linked_property): + return prop + return None - def disable_property_link(self, *names): - for name in names: - if self.has_linked_property(name): - self.__enabled_linked_properties[name] = False + @classmethod + def has_linked_property(cls, prop_name): + return cls.get_linked_property(prop_name) is not None - def enable_property_link(self, *names): - for name in names: - if self.has_linked_property(name): - self.__enabled_linked_properties[name] = True + @classmethod + def property_is_linked(cls, prop_name): + prop = cls.get_linked_property(prop_name) + if prop is None: + return None + return prop.propagate + + @classmethod + def disable_property_link(cls, *prop_names): + for prop_name in prop_names: + prop = cls.get_linked_property(prop_name) + if prop is not None: + prop.propagate = False + + @classmethod + def enable_property_link(cls, *prop_names): + for prop_name in prop_names: + prop = cls.get_linked_property(prop_name) + if prop is not None: + prop.propagate = True @property def linked_instances(self): @@ -80,9 +98,6 @@ def linked_instances(self): @linked_instances.setter def linked_instances(self, instances): - self._propagated_linked_instances_setter(instances) - - def _propagated_linked_instances_setter(self, instances): others = list() for instance in instances: if instance is self: @@ -114,11 +129,6 @@ def _disable_propagation(self): finally: self.__propagate = keep - @classmethod - def has_linked_property(cls, prop_name): - prop = getattr(cls, prop_name, None) - return isinstance(prop, linked_property) - @staticmethod def _filter_class_has_linked_property(instances, prop_name): for instance in instances: @@ -151,13 +161,13 @@ def instance_with_linked_property(self, prop_name): return instance return None - def get_linked_property(self, prop_name): + def get_linked_property_value(self, prop_name): instance = self.instance_with_linked_property(prop_name) if instance is None: raise ValueError(f"No instance has linked property {repr(prop_name)}") return getattr(instance, prop_name) - def set_linked_property(self, prop_name, value): + def set_linked_property_value(self, prop_name, value): instance = self.instance_with_linked_property(prop_name) if instance is None: raise ValueError(f"No instance has linked property {repr(prop_name)}") @@ -170,7 +180,7 @@ def disable_property_link(self, *names): def enable_property_link(self, *names): for name in names: - value = self.get_linked_property(name) - for i, instance in enumerate(self.instances_with_linked_property(name)): + value = self.get_linked_property_value(name) + for instance in self.instances_with_linked_property(name): instance.enable_property_link(name) - self.set_linked_property(name, value) + self.set_linked_property_value(name, value) diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachedInterfaceTest.py index a2f91a278..f4c0ed585 100644 --- a/PyMca5/tests/CachedInterfaceTest.py +++ b/PyMca5/tests/CachedInterfaceTest.py @@ -79,6 +79,8 @@ def setUp(self): @external_subtests def test_get_without_caching(self): + """verify property get/set count when getting a cached property value + """ for i in range(5): self.assertEqual(self.cached_object.var1, 1) self._assertGetSetCount("var1", i + 1, 0) @@ -86,6 +88,8 @@ def test_get_without_caching(self): @external_subtests def test_set_without_caching(self): + """verify property get/set count when setting a cached property value + """ for i in range(5): self.cached_object.var1 = 100 self._assertGetSetCount("var1", 0, i + 1) @@ -94,6 +98,8 @@ def test_set_without_caching(self): @external_subtests def test_get_with_caching(self): + """verify property get/set count when getting a cached property value + """ with self.cached_object.propertyCachingContext() as cache: self._assertCache(cache, [1, 2]) for i in range(5): @@ -103,6 +109,8 @@ def test_get_with_caching(self): @external_subtests def test_set_with_caching(self): + """verify property get/set count when setting a cached property value + """ with self.cached_object.propertyCachingContext() as cache: for i in range(5): self.cached_object.var1 = 100 @@ -114,6 +122,8 @@ def test_set_with_caching(self): @external_subtests def test_set_with_persistent_caching(self): + """verify cache persistency + """ with self.cached_object.propertyCachingContext(persist=True) as cache: for i in range(5): self.cached_object.var1 = 100 @@ -125,6 +135,8 @@ def test_set_with_persistent_caching(self): @external_subtests def test_start_cache(self): + """verify cache initialization + """ with self.cached_object.propertyCachingContext( start_cache=self._start_cache([100, 200]), ) as cache: @@ -137,6 +149,8 @@ def test_start_cache(self): @external_subtests def test_persistent_start_cache(self): + """verify cache persistency + """ with self.cached_object.propertyCachingContext( start_cache=self._start_cache([100, 200]), persist=True ) as cache: diff --git a/PyMca5/tests/LinkedInterfaceTest.py b/PyMca5/tests/LinkedInterfaceTest.py index fd17c9698..e6ccf9aae 100644 --- a/PyMca5/tests/LinkedInterfaceTest.py +++ b/PyMca5/tests/LinkedInterfaceTest.py @@ -65,8 +65,8 @@ def __init__(self): Model2(cfg2a), Model2(cfg2b)]) def reset_counters(self): - for m in self.linked_instances: - m.reset_counters() + for model in self.linked_instances: + model.reset_counters() def link(self): self.enable_property_link("var1", "var2") @@ -85,8 +85,8 @@ def test_links(self): """establish links """ nlinked = len(self.concat_model.linked_instances) - 1 - for m in self.concat_model.linked_instances: - self.assertEqual(len(m.linked_instances), nlinked) + for model in self.concat_model.linked_instances: + self.assertEqual(len(model.linked_instances), nlinked) def test_init_properties(self): """initial property values @@ -97,20 +97,35 @@ def test_init_properties(self): def test_enable_property_link_syncing(self): """maximal 1 get/set of a linked property when enabling linking """ - self.concat_model.enable_property_link("var1", "var2") - for name in ("var1", "var2"): - for m in self.concat_model.linked_instances: - self.assertTrue(m.get_counter[name] <= 1) - self.assertTrue(m.set_counter[name] <= 1) - self.assert_synced_values() + for i, model in enumerate(self.concat_model.linked_instances): + model.var1 = 100 + i + if model.has_linked_property("var2"): + model.var2 = 200 + i + self.assert_property_values("var1", 100, [100, 101, 102, 103]) + self.assert_property_values("var2", 202, [202, 203]) + self.concat_model.reset_counters() + + self.concat_model.enable_property_link("var1") + getmodel = self.concat_model.instance_with_linked_property("var1") + for model in self.concat_model.linked_instances: + if model is getmodel: + self.assertEqual(model.get_counter["var1"], 1) + self.assertTrue(model.set_counter["var1"] <= 1) + else: + self.assertEqual(model.get_counter["var1"], 0) + self.assertEqual(model.set_counter["var1"], 1) + self.assertEqual(model.get_counter["var2"], 0) + self.assertEqual(model.set_counter["var2"], 0) + self.assert_property_values("var1", 100) + self.assert_property_values("var2", 202, [202, 203]) def test_contexts_concat(self): """entering a linked context manager of a container """ self.concat_model.fit() self.assertEqual(self.concat_model.context_counter, 1) - for m in self.concat_model.linked_instances: - m.context_counter, 1 + for model in self.concat_model.linked_instances: + model.context_counter, 1 def test_contexts_single(self): """entering a linked context manager of a single instance @@ -118,17 +133,17 @@ def test_contexts_single(self): model = self.concat_model.linked_instances[2] model.fit() self.assertEqual(model.context_counter, 1) - for m in self.concat_model.linked_instances: - m.context_counter, 1 + for model in self.concat_model.linked_instances: + model.context_counter, 1 def test_get_var1_concat(self): """getting a linked property (present in all models) from a container """ self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property("var1"), 1) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var1"], 0, msg=f"model{i}") + self.assertEqual(self.concat_model.get_linked_property_value("var1"), 1) + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var1"], 0, msg=f"model{i}") def test_get_var1_single(self): """getting a linked property (present in all models) from a single instance @@ -136,18 +151,18 @@ def test_get_var1_single(self): self.concat_model.link() model = self.concat_model.linked_instances[2] self.assertEqual(model.var1, 1) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var1"], 0, msg=f"model{i}") + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var1"], 0, msg=f"model{i}") def test_get_var2_concat(self): """getting a linked property (present in some models) from a container """ self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property("var2"), 4) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var2"], 0, msg=f"model{i}") + self.assertEqual(self.concat_model.get_linked_property_value("var2"), 4) + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var2"], 0, msg=f"model{i}") def test_get_var2_single(self): """getting a linked property (present in some models) from a single instance @@ -155,19 +170,19 @@ def test_get_var2_single(self): self.concat_model.link() model = self.concat_model.linked_instances[2] self.assertEqual(model.var2, 4) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var2"], 0, msg=f"model{i}") + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var2"], 0, msg=f"model{i}") def test_set_var1_concat(self): """setting a linked property (present in all models) from a container """ self.concat_model.link() - self.concat_model.set_linked_property("var1", 100) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var1"], 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var1"], 1, msg=f"model{i}") - self.assert_synced_values(var1=100) + self.concat_model.set_linked_property_value("var1", 100) + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var1"], 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var1"], 1, msg=f"model{i}") + self.assert_property_values("var1", 100) def test_set_var1_single(self): """setting a linked property (present in all models) from a single instance @@ -175,19 +190,19 @@ def test_set_var1_single(self): self.concat_model.link() model = self.concat_model.linked_instances[2] model.var1 = 1 - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var1"], 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var1"], 1, msg=f"model{i}") + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var1"], 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var1"], 1, msg=f"model{i}") def test_set_var2_concat(self): """setting a linked property (present in some models) from a container """ self.concat_model.link() - self.concat_model.set_linked_property("var2", 100) - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var2"], 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") - self.assert_synced_values(var2=100) + self.concat_model.set_linked_property_value("var2", 100) + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") + self.assert_property_values("var2", 100) def test_set_var2_single(self): """setting a linked property (present in some models) from a single instance @@ -195,17 +210,13 @@ def test_set_var2_single(self): self.concat_model.link() model = self.concat_model.linked_instances[2] model.var2 = 100 - for i, m in enumerate(self.concat_model.linked_instances): - self.assertEqual(m.get_counter["var2"], 0, msg=f"model{i}") - self.assertEqual(m.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") - - def assert_synced_values(self, var1=1, var2=4): - self.assert_property_values("var1", var1) - self.assert_property_values("var2", var2) + for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") + self.assertEqual(model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") def assert_property_values(self, name, value, values=None): - self.assertEqual(self.concat_model.get_linked_property(name), value, msg=name) - if values is None: + self.assertEqual(self.concat_model.get_linked_property_value(name), value, msg=name) + if not isinstance(values, list): values = [value] * len(self.concat_model.linked_instances) - for m, v in zip(self.concat_model.instances_with_linked_property(name), values): - self.assertEqual(getattr(m, name), v, msg=name) + for model, v in zip(self.concat_model.instances_with_linked_property(name), values): + self.assertEqual(getattr(model, name), v, msg=name) From b328f6971995ead49e1a5b46655fe45e7af16f2e Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 15:11:33 +0200 Subject: [PATCH 36/74] fixup --- PyMca5/PyMcaMath/fitting/ConcatModel.py | 4 +- .../{LinkedInterface.py => LinkedModel.py} | 24 ++-- .../fitting/ModelParameterInterface.py | 18 ++- ...kedInterfaceTest.py => LinkedModelTest.py} | 106 +++++++++--------- 4 files changed, 79 insertions(+), 73 deletions(-) rename PyMca5/PyMcaMath/fitting/{LinkedInterface.py => LinkedModel.py} (91%) rename PyMca5/tests/{LinkedInterfaceTest.py => LinkedModelTest.py} (75%) diff --git a/PyMca5/PyMcaMath/fitting/ConcatModel.py b/PyMca5/PyMcaMath/fitting/ConcatModel.py index 4b1dd49c6..d0132f921 100644 --- a/PyMca5/PyMcaMath/fitting/ConcatModel.py +++ b/PyMca5/PyMcaMath/fitting/ConcatModel.py @@ -1,11 +1,11 @@ import numpy from collections.abc import Sequence, MutableMapping -from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface from PyMca5.PyMcaMath.fitting.Model import Model -class ConcatModel(LinkedContainerInterface, ModelInterface): +class ConcatModel(LinkedModelContainer, ModelInterface): """Concatenated model with shared parameters""" def __init__(self, models, shared_attributes=None): diff --git a/PyMca5/PyMcaMath/fitting/LinkedInterface.py b/PyMca5/PyMcaMath/fitting/LinkedModel.py similarity index 91% rename from PyMca5/PyMcaMath/fitting/LinkedInterface.py rename to PyMca5/PyMcaMath/fitting/LinkedModel.py index 082a40281..142ceab9d 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedInterface.py +++ b/PyMca5/PyMcaMath/fitting/LinkedModel.py @@ -22,9 +22,9 @@ def wrapper(oself, value): if not self.propagate: return for instance in oself._filter_class_has_linked_property( - oself._non_propagating_instances, propname - ): - setattr(instance, propname, value) + oself._non_propagating_instances, propname + ): + setattr(instance, propname, value) return wrapper @@ -51,10 +51,11 @@ def wrapper(self, *args, **kw): return contextmanager(wrapper) -class LinkedInterface: +class LinkedModel: """Every class that uses the link decorators needs to derived from this class. """ + def __init__(self): self.__linked_instances = list() self.__propagate = True @@ -102,8 +103,10 @@ def linked_instances(self, instances): for instance in instances: if instance is self: continue - if not isinstance(instance, LinkedInterface): - raise TypeError(type(instance), "can only link objects of the 'LinkedInterface' type") + if not isinstance(instance, LinkedModel): + raise TypeError( + type(instance), "can only link objects of the 'LinkedModel' type" + ) others.append(instance) self.__linked_instances = others for instance in others: @@ -136,10 +139,11 @@ def _filter_class_has_linked_property(instances, prop_name): yield instance -class LinkedContainerInterface: - """Classes that manage LinkedInterface objects should +class LinkedModelContainer: + """Classes that manage LinkedModel objects should derive from this class. """ + def __init__(self, linked_instances): self.linked_instances = linked_instances super().__init__() @@ -154,7 +158,9 @@ def linked_instances(self, linked_instances): self.__linked_instances = linked_instances def instances_with_linked_property(self, prop_name): - yield from LinkedInterface._filter_class_has_linked_property(self.linked_instances, prop_name) + yield from LinkedModel._filter_class_has_linked_property( + self.linked_instances, prop_name + ) def instance_with_linked_property(self, prop_name): for instance in self.instances_with_linked_property(prop_name): diff --git a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py index 4e87016ff..2b0f1c5b4 100644 --- a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py +++ b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py @@ -1,9 +1,9 @@ from contextlib import contextmanager import numpy -from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedInterface -from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface -from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_property -from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_contextmanager +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.LinkedModel import linked_contextmanager class parameter_group(linked_property): @@ -144,9 +144,7 @@ def iter_parameter_groups(self, **paramtype): key = self._parameters_cache_key(**paramtype) it = cache.get(key) if it is None: - it = cache[key] = list( - self._parameter_groups_notcached(**paramtype) - ) + it = cache[key] = list(self._parameter_groups_notcached(**paramtype)) yield from it def _parameter_groups_notcached(self, **paramtype): @@ -186,7 +184,7 @@ def iter_parameter_group_names(self, **paramtype): raise NotImplementedError -class ModelParameterInterface(LinkedInterface, ModelParameterInterfaceBase): +class ModelParameterInterface(LinkedModel, ModelParameterInterfaceBase): _PARAMETERS = tuple() def __init_subclass__(cls, **kwargs): @@ -308,7 +306,7 @@ def iter_parameter_group_names(self, linear=None): yield name -class ConcatModelParameterInterface(LinkedContainerInterface, ModelParameterInterfaceBase): +class ConcatModelParameterInterface(LinkedModelContainer, ModelParameterInterfaceBase): @property def models(self): return self.linked_instances @@ -349,5 +347,5 @@ def set_parameter_values(self, values, **paramtype): i = 0 for instance in self.linked_instances: n = instance.get_n_parameters(**paramtype) - instance.set_parameter_values(values[i: i+n], **paramtype) + instance.set_parameter_values(values[i : i + n], **paramtype) i += n diff --git a/PyMca5/tests/LinkedInterfaceTest.py b/PyMca5/tests/LinkedModelTest.py similarity index 75% rename from PyMca5/tests/LinkedInterfaceTest.py rename to PyMca5/tests/LinkedModelTest.py index e6ccf9aae..699a326f8 100644 --- a/PyMca5/tests/LinkedInterfaceTest.py +++ b/PyMca5/tests/LinkedModelTest.py @@ -1,12 +1,12 @@ import unittest from collections import Counter -from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedInterface -from PyMca5.PyMcaMath.fitting.LinkedInterface import LinkedContainerInterface -from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_contextmanager -from PyMca5.PyMcaMath.fitting.LinkedInterface import linked_property +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import linked_contextmanager +from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property -class ModelBase(LinkedInterface): +class ModelBase(LinkedModel): def __init__(self): super().__init__() self.context_counter = 0 @@ -55,14 +55,13 @@ def var2(self, value): self._cfg["var2"] = value -class ConcatModel(LinkedContainerInterface, ModelBase): +class ConcatModel(LinkedModelContainer, ModelBase): def __init__(self): cfg1a = {"var1": 1} cfg1b = {"var1": 2} cfg2a = {"var1": 3, "var2": 4} cfg2b = {"var1": 5, "var2": 6} - super().__init__([Model1(cfg1a), Model1(cfg1b), - Model2(cfg2a), Model2(cfg2b)]) + super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) def reset_counters(self): for model in self.linked_instances: @@ -77,26 +76,23 @@ def unlink(self): self.reset_counters() -class testLinkedInterface(unittest.TestCase): +class testLinkedModel(unittest.TestCase): def setUp(self): self.concat_model = ConcatModel() def test_links(self): - """establish links - """ + """establish links""" nlinked = len(self.concat_model.linked_instances) - 1 for model in self.concat_model.linked_instances: - self.assertEqual(len(model.linked_instances), nlinked) + self.assertEqual(len(model.linked_instances), nlinked) def test_init_properties(self): - """initial property values - """ + """initial property values""" self.assert_property_values("var1", 1, [1, 2, 3, 5]) self.assert_property_values("var2", 4, [4, 6]) def test_enable_property_link_syncing(self): - """maximal 1 get/set of a linked property when enabling linking - """ + """maximal 1 get/set of a linked property when enabling linking""" for i, model in enumerate(self.concat_model.linked_instances): model.var1 = 100 + i if model.has_linked_property("var2"): @@ -120,63 +116,64 @@ def test_enable_property_link_syncing(self): self.assert_property_values("var2", 202, [202, 203]) def test_contexts_concat(self): - """entering a linked context manager of a container - """ + """entering a linked context manager of a container""" self.concat_model.fit() - self.assertEqual(self.concat_model.context_counter, 1) + self.assertEqual(self.concat_model.context_counter, 1) for model in self.concat_model.linked_instances: - model.context_counter, 1 + model.context_counter, 1 def test_contexts_single(self): - """entering a linked context manager of a single instance - """ + """entering a linked context manager of a single instance""" model = self.concat_model.linked_instances[2] model.fit() - self.assertEqual(model.context_counter, 1) + self.assertEqual(model.context_counter, 1) for model in self.concat_model.linked_instances: - model.context_counter, 1 + model.context_counter, 1 def test_get_var1_concat(self): - """getting a linked property (present in all models) from a container - """ + """getting a linked property (present in all models) from a container""" self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property_value("var1"), 1) + self.assertEqual(self.concat_model.get_linked_property_value("var1"), 1) for i, model in enumerate(self.concat_model.linked_instances): - self.assertEqual(model.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}") + self.assertEqual( + model.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}" + ) self.assertEqual(model.set_counter["var1"], 0, msg=f"model{i}") def test_get_var1_single(self): - """getting a linked property (present in all models) from a single instance - """ + """getting a linked property (present in all models) from a single instance""" self.concat_model.link() model = self.concat_model.linked_instances[2] self.assertEqual(model.var1, 1) for i, model in enumerate(self.concat_model.linked_instances): - self.assertEqual(model.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual( + model.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}" + ) self.assertEqual(model.set_counter["var1"], 0, msg=f"model{i}") def test_get_var2_concat(self): - """getting a linked property (present in some models) from a container - """ + """getting a linked property (present in some models) from a container""" self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property_value("var2"), 4) + self.assertEqual(self.concat_model.get_linked_property_value("var2"), 4) for i, model in enumerate(self.concat_model.linked_instances): - self.assertEqual(model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual( + model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}" + ) self.assertEqual(model.set_counter["var2"], 0, msg=f"model{i}") def test_get_var2_single(self): - """getting a linked property (present in some models) from a single instance - """ + """getting a linked property (present in some models) from a single instance""" self.concat_model.link() model = self.concat_model.linked_instances[2] self.assertEqual(model.var2, 4) for i, model in enumerate(self.concat_model.linked_instances): - self.assertEqual(model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}") + self.assertEqual( + model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}" + ) self.assertEqual(model.set_counter["var2"], 0, msg=f"model{i}") def test_set_var1_concat(self): - """setting a linked property (present in all models) from a container - """ + """setting a linked property (present in all models) from a container""" self.concat_model.link() self.concat_model.set_linked_property_value("var1", 100) for i, model in enumerate(self.concat_model.linked_instances): @@ -185,8 +182,7 @@ def test_set_var1_concat(self): self.assert_property_values("var1", 100) def test_set_var1_single(self): - """setting a linked property (present in all models) from a single instance - """ + """setting a linked property (present in all models) from a single instance""" self.concat_model.link() model = self.concat_model.linked_instances[2] model.var1 = 1 @@ -195,28 +191,34 @@ def test_set_var1_single(self): self.assertEqual(model.set_counter["var1"], 1, msg=f"model{i}") def test_set_var2_concat(self): - """setting a linked property (present in some models) from a container - """ + """setting a linked property (present in some models) from a container""" self.concat_model.link() - self.concat_model.set_linked_property_value("var2", 100) + self.concat_model.set_linked_property_value("var2", 100) for i, model in enumerate(self.concat_model.linked_instances): self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") - self.assertEqual(model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") + self.assertEqual( + model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}" + ) self.assert_property_values("var2", 100) def test_set_var2_single(self): - """setting a linked property (present in some models) from a single instance - """ + """setting a linked property (present in some models) from a single instance""" self.concat_model.link() model = self.concat_model.linked_instances[2] - model.var2 = 100 + model.var2 = 100 for i, model in enumerate(self.concat_model.linked_instances): self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") - self.assertEqual(model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}") + self.assertEqual( + model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}" + ) def assert_property_values(self, name, value, values=None): - self.assertEqual(self.concat_model.get_linked_property_value(name), value, msg=name) + self.assertEqual( + self.concat_model.get_linked_property_value(name), value, msg=name + ) if not isinstance(values, list): values = [value] * len(self.concat_model.linked_instances) - for model, v in zip(self.concat_model.instances_with_linked_property(name), values): + for model, v in zip( + self.concat_model.instances_with_linked_property(name), values + ): self.assertEqual(getattr(model, name), v, msg=name) From 86f8c3453a4fcfac24f6d7bf7774e592d15059b9 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 15:16:00 +0200 Subject: [PATCH 37/74] fixup --- .../{CachedInterface.py => CachingModel.py} | 8 ++--- ...edInterfaceTest.py => CachingModelTest.py} | 31 +++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) rename PyMca5/PyMcaMath/fitting/{CachedInterface.py => CachingModel.py} (96%) rename PyMca5/tests/{CachedInterfaceTest.py => CachingModelTest.py} (90%) diff --git a/PyMca5/PyMcaMath/fitting/CachedInterface.py b/PyMca5/PyMcaMath/fitting/CachingModel.py similarity index 96% rename from PyMca5/PyMcaMath/fitting/CachedInterface.py rename to PyMca5/PyMcaMath/fitting/CachingModel.py index eb4084572..43d11b0ac 100644 --- a/PyMca5/PyMcaMath/fitting/CachedInterface.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -3,7 +3,7 @@ from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property -class CacheManagerInterface: +class CacheManager: """Object that manages a cache""" def __init__(self): @@ -23,7 +23,7 @@ def _property_cache_key(self, **cacheoptions): return None -class CacheInterface(CacheManagerInterface): +class CachingModel(CacheManager): """Object that manages and uses an internal cache (default) or uses an external cache. """ @@ -41,7 +41,7 @@ def cache_object(self): @cache_object.setter def cache_object(self, obj): - if obj is not None and not isinstance(obj, CacheManagerInterface): + if obj is not None and not isinstance(obj, CacheManager): raise TypeError(obj, type(obj)) self.__external_cache_object = obj @@ -94,7 +94,7 @@ def wrapper(oself, value): return wrapper -class CachedPropertiesInterface(CacheInterface): +class CachedPropertiesModel(CachingModel): """Object with cached properties when enabled.""" _CACHED_PROPERTIES = tuple() diff --git a/PyMca5/tests/CachedInterfaceTest.py b/PyMca5/tests/CachingModelTest.py similarity index 90% rename from PyMca5/tests/CachedInterfaceTest.py rename to PyMca5/tests/CachingModelTest.py index f4c0ed585..6bf20a0cb 100644 --- a/PyMca5/tests/CachedInterfaceTest.py +++ b/PyMca5/tests/CachingModelTest.py @@ -2,12 +2,12 @@ import numpy import functools from collections import Counter -from PyMca5.PyMcaMath.fitting.CachedInterface import CachedPropertiesInterface -from PyMca5.PyMcaMath.fitting.CachedInterface import CacheInterface -from PyMca5.PyMcaMath.fitting.CachedInterface import cached_property +from PyMca5.PyMcaMath.fitting.CachingModel import CachedPropertiesModel +from PyMca5.PyMcaMath.fitting.CachingModel import CachingModel +from PyMca5.PyMcaMath.fitting.CachingModel import cached_property -class Cached(CachedPropertiesInterface): +class Cached(CachedPropertiesModel): def __init__(self): super().__init__() self._cfg = {"var1": 1, "var2": 2} @@ -47,7 +47,7 @@ def _create_empty_cache(self, key, **_): return numpy.zeros(2, dtype=float) -class ExternalCached(CacheInterface): +class ExternalCached(CachingModel): def _property_cache_index(self, name): if name == "var1": return 2 @@ -79,8 +79,7 @@ def setUp(self): @external_subtests def test_get_without_caching(self): - """verify property get/set count when getting a cached property value - """ + """verify property get/set count when getting a cached property value""" for i in range(5): self.assertEqual(self.cached_object.var1, 1) self._assertGetSetCount("var1", i + 1, 0) @@ -88,8 +87,7 @@ def test_get_without_caching(self): @external_subtests def test_set_without_caching(self): - """verify property get/set count when setting a cached property value - """ + """verify property get/set count when setting a cached property value""" for i in range(5): self.cached_object.var1 = 100 self._assertGetSetCount("var1", 0, i + 1) @@ -98,8 +96,7 @@ def test_set_without_caching(self): @external_subtests def test_get_with_caching(self): - """verify property get/set count when getting a cached property value - """ + """verify property get/set count when getting a cached property value""" with self.cached_object.propertyCachingContext() as cache: self._assertCache(cache, [1, 2]) for i in range(5): @@ -109,8 +106,7 @@ def test_get_with_caching(self): @external_subtests def test_set_with_caching(self): - """verify property get/set count when setting a cached property value - """ + """verify property get/set count when setting a cached property value""" with self.cached_object.propertyCachingContext() as cache: for i in range(5): self.cached_object.var1 = 100 @@ -122,8 +118,7 @@ def test_set_with_caching(self): @external_subtests def test_set_with_persistent_caching(self): - """verify cache persistency - """ + """verify cache persistency""" with self.cached_object.propertyCachingContext(persist=True) as cache: for i in range(5): self.cached_object.var1 = 100 @@ -135,8 +130,7 @@ def test_set_with_persistent_caching(self): @external_subtests def test_start_cache(self): - """verify cache initialization - """ + """verify cache initialization""" with self.cached_object.propertyCachingContext( start_cache=self._start_cache([100, 200]), ) as cache: @@ -149,8 +143,7 @@ def test_start_cache(self): @external_subtests def test_persistent_start_cache(self): - """verify cache persistency - """ + """verify cache persistency""" with self.cached_object.propertyCachingContext( start_cache=self._start_cache([100, 200]), persist=True ) as cache: From f582ff17109f1913b705ca4677ee83d232a8e482 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 15:29:32 +0200 Subject: [PATCH 38/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 56 +++++++++---------- PyMca5/PyMcaMath/fitting/ConcatModel.py | 4 +- PyMca5/PyMcaMath/fitting/Model.py | 10 ++-- PyMca5/PyMcaMath/fitting/ModelInterface.py | 10 ++-- .../fitting/ModelParameterInterface.py | 18 +++--- PyMca5/tests/CachingModelTest.py | 16 +++--- 6 files changed, 57 insertions(+), 57 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index 43d11b0ac..cdf37e472 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -29,25 +29,25 @@ class CachingModel(CacheManager): """ def __init__(self): - self.cache_object = None + self._cache_manager = None super().__init__() @property - def cache_object(self): - if self.__external_cache_object is None: + def _cache_manager(self): + if self.__external_cache_manager is None: return self else: - return self.__external_cache_object + return self.__external_cache_manager - @cache_object.setter - def cache_object(self, obj): + @_cache_manager.setter + def _cache_manager(self, obj): if obj is not None and not isinstance(obj, CacheManager): raise TypeError(obj, type(obj)) - self.__external_cache_object = obj + self.__external_cache_manager = obj @contextmanager - def cachingContext(self, cachename): - cache_root = self.cache_object._cache_root + def _cachingContext(self, cachename): + cache_root = self._cache_manager._cache_root new_context_entry = cachename not in cache_root if new_context_entry: cache_root[cachename] = dict() @@ -57,11 +57,11 @@ def cachingContext(self, cachename): if new_context_entry: del cache_root[cachename] - def cachingEnabled(self, cachename): - return cachename in self.cache_object._cache_root + def _cachingEnabled(self, cachename): + return cachename in self._cache_manager._cache_root - def getCache(self, cachename, *subnames): - cache_root = self.cache_object._cache_root + def _getCache(self, cachename, *subnames): + cache_root = self._cache_manager._cache_root if cachename not in cache_root: return None ret = cache_root[cachename] @@ -112,7 +112,7 @@ def _cached_properties(self): return self._CACHED_PROPERTIES @contextmanager - def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): + def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): values_cache = self._get_property_values_cache(**cacheoptions) if values_cache is not None: # Re-entering this context should not affect anything @@ -121,19 +121,19 @@ def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions if start_cache is None: # Fill and empty cache with property values - cache_object = self.cache_object - key = cache_object._property_cache_key(**cacheoptions) - values_cache = cache_object._create_empty_cache(key, **cacheoptions) + _cache_manager = self._cache_manager + key = _cache_manager._property_cache_key(**cacheoptions) + values_cache = _cache_manager._create_empty_cache(key, **cacheoptions) nameindexmap = dict() for name in self._cached_properties(): - index = cache_object._property_cache_index(name) + index = _cache_manager._property_cache_index(name) nameindexmap[name] = index values_cache[index] = getattr(self, name) else: values_cache = start_cache nameindexmap = None - with self.cachingContext("_cached_properties"): + with self._cachingContext("_cached_properties"): # Initialize the property values cache self._set_property_values_cache(values_cache, **cacheoptions) yield values_cache @@ -141,39 +141,39 @@ def propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions if persist: # Set property values to the cached values if nameindexmap is None: - cache_object = self.cache_object + _cache_manager = self._cache_manager for name in self._cached_properties(): - index = cache_object._property_cache_index(name) + index = _cache_manager._property_cache_index(name) setattr(self, name, values_cache[index]) else: for name, index in nameindexmap.items(): setattr(self, name, values_cache[index]) def _get_property_values_cache(self, **cacheoptions): - caches = self.getCache("_cached_properties") + caches = self._getCache("_cached_properties") if caches is None: return None - key = self.cache_object._property_cache_key(**cacheoptions) + key = self._cache_manager._property_cache_key(**cacheoptions) return caches.get(key, None) def _set_property_values_cache(self, values_cache, **cacheoptions): - caches = self.getCache("_cached_properties") + caches = self._getCache("_cached_properties") if caches is None: return - cache_object = self.cache_object - key = cache_object._property_cache_key(**cacheoptions) + _cache_manager = self._cache_manager + key = _cache_manager._property_cache_key(**cacheoptions) caches[key] = values_cache def _cached_property_fget(self, fget): values_cache = self._get_property_values_cache() if values_cache is None: return fget(self) - index = self.cache_object._property_cache_index(fget.__name__) + index = self._cache_manager._property_cache_index(fget.__name__) return values_cache[index] def _cached_property_fset(self, fset, value): values_cache = self._get_property_values_cache() if values_cache is None: return fset(self, value) - index = self.cache_object._property_cache_index(fset.__name__) + index = self._cache_manager._property_cache_index(fset.__name__) values_cache[index] = value diff --git a/PyMca5/PyMcaMath/fitting/ConcatModel.py b/PyMca5/PyMcaMath/fitting/ConcatModel.py index d0132f921..5fcbaacf4 100644 --- a/PyMca5/PyMcaMath/fitting/ConcatModel.py +++ b/PyMca5/PyMcaMath/fitting/ConcatModel.py @@ -305,7 +305,7 @@ def _parameter_model_index(self, idx, linear_only=None): :param int idx: :yields (int, int): model index, parameter index in this model """ - cache = self.getCache("fit", "parameter_model_index") + cache = self._getCache("fit", "parameter_model_index") if cache is None: yield from self._iter_parameter_index(idx, linear_only=linear_only) return @@ -464,7 +464,7 @@ def _model_data_slices(self, nconcat): :param int nconcat: :returns list(slice): """ - cache = self.getCache("fit", "model_data_slices") + cache = self._getCache("fit", "model_data_slices") if cache is None: return list(self._generate_model_data_slices(nconcat)) else: diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py index 2e4264f82..0bcc2d6e9 100644 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ b/PyMca5/PyMcaMath/fitting/Model.py @@ -95,7 +95,7 @@ def _get_parameters(self, linear_only=None): :param bool linear_only: :returns array: """ - cache = self.getCache("parameters") + cache = self._getCache("parameters") if cache is None: return self._get_parameters_notcached(linear_only=linear_only) @@ -111,7 +111,7 @@ def _set_parameters(self, params, linear_only=None): """ :param bool linear_only: """ - cache = self.getCache("parameters") + cache = self._getCache("parameters") if cache is None: self._set_parameters_notcached(params, linear_only=linear_only) else: @@ -141,7 +141,7 @@ def _set_parameters_notcached(self, params, linear_only=None): def _get_parameter(self, fget): """Helper for parameter getters.""" - parameters = self.getCache("parameters") + parameters = self._getCache("parameters") if parameters is None: return fget(self) @@ -157,7 +157,7 @@ def _get_parameter(self, fget): def _set_parameter(self, fset, value): """Helper for parameter setters""" - parameters = self.getCache("parameters") + parameters = self._getCache("parameters") if parameters is None: return fset(self, value) @@ -177,7 +177,7 @@ def _parameter_groups(self, linear_only=None): :param bool linear_only: :yields str, int: group name, nb. parameters in the group """ - cache = self.getCache("fit", "parameter_groups") + cache = self._getCache("fit", "parameter_groups") if cache is None: yield from self._parameter_groups_notcached(linear_only=linear_only) return diff --git a/PyMca5/PyMcaMath/fitting/ModelInterface.py b/PyMca5/PyMcaMath/fitting/ModelInterface.py index f7b80652f..e23f83872 100644 --- a/PyMca5/PyMcaMath/fitting/ModelInterface.py +++ b/PyMca5/PyMcaMath/fitting/ModelInterface.py @@ -321,11 +321,11 @@ def niter_non_leastsquares(self): @contextmanager def __linear_fit_context(self): with ExitStack() as stack: - ctx = self.cachingContext("fit") + ctx = self._cachingContext("fit") stack.enter_context(ctx) ctx = self._linear_context(True) stack.enter_context(ctx) - ctx = self.cachingContext("parameters") + ctx = self._cachingContext("parameters") stack.enter_context(ctx) ctx = self._linear_fit_context() yield @@ -333,11 +333,11 @@ def __linear_fit_context(self): @contextmanager def __nonlinear_fit_context(self): with ExitStack() as stack: - ctx = self.cachingContext("fit") + ctx = self._cachingContext("fit") stack.enter_context(ctx) ctx = self._linear_context(False) stack.enter_context(ctx) - ctx = self.cachingContext("parameters") + ctx = self._cachingContext("parameters") stack.enter_context(ctx) ctx = self._nonlinear_fit_context() yield @@ -394,7 +394,7 @@ def use_fit_result(self, result): @contextmanager def use_fit_result_context(self, result): with self._linear_context(result["linear"]): - with self.cachingContext("parameters"): + with self._cachingContext("parameters"): self.use_fit_result(result) yield diff --git a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py index 2b0f1c5b4..d1f060c64 100644 --- a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py +++ b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py @@ -74,8 +74,8 @@ def __init__(self): super().__init__() @linked_contextmanager - def cachingContext(self, cachename): - reset = not self.cachingEnabled(cachename) + def _cachingContext(self, cachename): + reset = not self._cachingEnabled(cachename) if reset: self.__cache[cachename] = dict() try: @@ -84,10 +84,10 @@ def cachingContext(self, cachename): if reset: del self.__cache[cachename] - def cachingEnabled(self, cachename): + def _cachingEnabled(self, cachename): return cachename in self.__cache - def getCache(self, cachename, *subnames): + def _getCache(self, cachename, *subnames): if cachename in self.__cache: ret = self.__cache[cachename] for cachename in subnames: @@ -136,7 +136,7 @@ def iter_parameter_groups(self, **paramtype): :param bool linear_only: :yields str, int: group name, nb. parameters in the group """ - cache = self.getCache("iter_parameter_groups") + cache = self._getCache("iter_parameter_groups") if cache is None: yield from self._parameter_groups_notcached(**paramtype) return @@ -209,7 +209,7 @@ def get_parameter_values(self, **paramtype): :returns array: """ - cache = self.getCache("_parameters") + cache = self._getCache("_parameters") if cache is None: return self._get_parameter_values_notcached(**paramtype) @@ -223,7 +223,7 @@ def set_parameter_values(self, values, **paramtype): """ :returns array: """ - cache = self.getCache("_parameters") + cache = self._getCache("_parameters") if cache is None: self._set_parameter_values_notcached(values, **paramtype) else: @@ -246,7 +246,7 @@ def _set_parameter_values_notcached(self, values, **paramtype): setattr(self, group_name, values[idx]) def _get_parameter(self, fget): - parameters = self.getCache("_parameters") + parameters = self._getCache("_parameters") if parameters is None: return fget(self) @@ -261,7 +261,7 @@ def _get_parameter(self, fget): return parameters[idx] def _set_parameter(self, fset, value): - parameters = self.getCache("_parameters") + parameters = self._getCache("_parameters") if parameters is None: return fset(self, value) diff --git a/PyMca5/tests/CachingModelTest.py b/PyMca5/tests/CachingModelTest.py index 6bf20a0cb..9e270d9f0 100644 --- a/PyMca5/tests/CachingModelTest.py +++ b/PyMca5/tests/CachingModelTest.py @@ -67,7 +67,7 @@ def wrapper(self): method(self) with self.subTest("with external cache"): self.cached_object = Cached() - self.cached_object.cache_object = ExternalCached() + self.cached_object._cache_manager = ExternalCached() method(self) return wrapper @@ -97,7 +97,7 @@ def test_set_without_caching(self): @external_subtests def test_get_with_caching(self): """verify property get/set count when getting a cached property value""" - with self.cached_object.propertyCachingContext() as cache: + with self.cached_object._propertyCachingContext() as cache: self._assertCache(cache, [1, 2]) for i in range(5): self.assertEqual(self.cached_object.var1, 1) @@ -107,7 +107,7 @@ def test_get_with_caching(self): @external_subtests def test_set_with_caching(self): """verify property get/set count when setting a cached property value""" - with self.cached_object.propertyCachingContext() as cache: + with self.cached_object._propertyCachingContext() as cache: for i in range(5): self.cached_object.var1 = 100 self.assertEqual(self.cached_object.var1, 100) @@ -119,7 +119,7 @@ def test_set_with_caching(self): @external_subtests def test_set_with_persistent_caching(self): """verify cache persistency""" - with self.cached_object.propertyCachingContext(persist=True) as cache: + with self.cached_object._propertyCachingContext(persist=True) as cache: for i in range(5): self.cached_object.var1 = 100 self.assertEqual(self.cached_object.var1, 100) @@ -131,7 +131,7 @@ def test_set_with_persistent_caching(self): @external_subtests def test_start_cache(self): """verify cache initialization""" - with self.cached_object.propertyCachingContext( + with self.cached_object._propertyCachingContext( start_cache=self._start_cache([100, 200]), ) as cache: self._assertCache(cache, [100, 200]) @@ -144,7 +144,7 @@ def test_start_cache(self): @external_subtests def test_persistent_start_cache(self): """verify cache persistency""" - with self.cached_object.propertyCachingContext( + with self.cached_object._propertyCachingContext( start_cache=self._start_cache([100, 200]), persist=True ) as cache: self._assertCache(cache, [100, 200]) @@ -163,13 +163,13 @@ def _assertVarValues(self, v1, v2): def _assertCache(self, cache, values): if not isinstance(cache, list): cache = cache.tolist() - if self.cached_object.cache_object is self.cached_object: + if self.cached_object._cache_manager is self.cached_object: self.assertEqual(cache, values) else: self.assertEqual(cache, [0, 0] + values) def _start_cache(self, start_cache): - if self.cached_object.cache_object is self.cached_object: + if self.cached_object._cache_manager is self.cached_object: return start_cache else: return [0, 0] + start_cache From cb070c49b40eda30c9f3e7c5501ab27adaa2ccff Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 15:35:13 +0200 Subject: [PATCH 39/74] fixup --- PyMca5/PyMcaMath/fitting/LinkedModel.py | 72 +++++++++---------- .../fitting/ModelParameterInterface.py | 12 ++-- PyMca5/tests/LinkedModelTest.py | 66 ++++++++--------- PyMca5/tests/ModelParameterInterfaceTest.py | 4 +- 4 files changed, 77 insertions(+), 77 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/LinkedModel.py b/PyMca5/PyMcaMath/fitting/LinkedModel.py index 142ceab9d..107697b19 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/LinkedModel.py @@ -62,43 +62,43 @@ def __init__(self): super().__init__() @classmethod - def get_linked_property(cls, prop_name): + def _get_linked_property(cls, prop_name): prop = getattr(cls, prop_name, None) if isinstance(prop, linked_property): return prop return None @classmethod - def has_linked_property(cls, prop_name): - return cls.get_linked_property(prop_name) is not None + def _has_linked_property(cls, prop_name): + return cls._get_linked_property(prop_name) is not None @classmethod - def property_is_linked(cls, prop_name): - prop = cls.get_linked_property(prop_name) + def _property_is_linked(cls, prop_name): + prop = cls._get_linked_property(prop_name) if prop is None: return None return prop.propagate @classmethod - def disable_property_link(cls, *prop_names): + def _disable_property_link(cls, *prop_names): for prop_name in prop_names: - prop = cls.get_linked_property(prop_name) + prop = cls._get_linked_property(prop_name) if prop is not None: prop.propagate = False @classmethod - def enable_property_link(cls, *prop_names): + def _enable_property_link(cls, *prop_names): for prop_name in prop_names: - prop = cls.get_linked_property(prop_name) + prop = cls._get_linked_property(prop_name) if prop is not None: prop.propagate = True @property - def linked_instances(self): + def _linked_instances(self): return self.__linked_instances - @linked_instances.setter - def linked_instances(self, instances): + @_linked_instances.setter + def _linked_instances(self, instances): others = list() for instance in instances: if instance is self: @@ -119,7 +119,7 @@ def _unpropagated_linked_instances_setter(self, instances): def _non_propagating_instances(self): if not self.__propagate: return - for instance in self.linked_instances: + for instance in self._linked_instances: with instance._disable_propagation(): yield instance @@ -135,7 +135,7 @@ def _disable_propagation(self): @staticmethod def _filter_class_has_linked_property(instances, prop_name): for instance in instances: - if instance.has_linked_property(prop_name): + if instance._has_linked_property(prop_name): yield instance @@ -145,48 +145,48 @@ class LinkedModelContainer: """ def __init__(self, linked_instances): - self.linked_instances = linked_instances + self._linked_instances = linked_instances super().__init__() @property - def linked_instances(self): + def _linked_instances(self): return self.__linked_instances - @linked_instances.setter - def linked_instances(self, linked_instances): - linked_instances[0].linked_instances = linked_instances - self.__linked_instances = linked_instances + @_linked_instances.setter + def _linked_instances(self, _linked_instances): + _linked_instances[0]._linked_instances = _linked_instances + self.__linked_instances = _linked_instances - def instances_with_linked_property(self, prop_name): + def _instances_with_linked_property(self, prop_name): yield from LinkedModel._filter_class_has_linked_property( - self.linked_instances, prop_name + self._linked_instances, prop_name ) - def instance_with_linked_property(self, prop_name): - for instance in self.instances_with_linked_property(prop_name): + def _instance_with_linked_property(self, prop_name): + for instance in self._instances_with_linked_property(prop_name): return instance return None - def get_linked_property_value(self, prop_name): - instance = self.instance_with_linked_property(prop_name) + def _get_linked_property_value(self, prop_name): + instance = self._instance_with_linked_property(prop_name) if instance is None: raise ValueError(f"No instance has linked property {repr(prop_name)}") return getattr(instance, prop_name) - def set_linked_property_value(self, prop_name, value): - instance = self.instance_with_linked_property(prop_name) + def _set_linked_property_value(self, prop_name, value): + instance = self._instance_with_linked_property(prop_name) if instance is None: raise ValueError(f"No instance has linked property {repr(prop_name)}") setattr(instance, prop_name, value) - def disable_property_link(self, *names): + def _disable_property_link(self, *names): for name in names: - for instance in self.instances_with_linked_property(name): - instance.disable_property_link(name) + for instance in self._instances_with_linked_property(name): + instance._disable_property_link(name) - def enable_property_link(self, *names): + def _enable_property_link(self, *names): for name in names: - value = self.get_linked_property_value(name) - for instance in self.instances_with_linked_property(name): - instance.enable_property_link(name) - self.set_linked_property_value(name, value) + value = self._get_linked_property_value(name) + for instance in self._instances_with_linked_property(name): + instance._enable_property_link(name) + self._set_linked_property_value(name, value) diff --git a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py index d1f060c64..7864c7349 100644 --- a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py +++ b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py @@ -309,20 +309,20 @@ def iter_parameter_group_names(self, linear=None): class ConcatModelParameterInterface(LinkedModelContainer, ModelParameterInterfaceBase): @property def models(self): - return self.linked_instances + return self._linked_instances @property def nmodels(self): - return len(self.linked_instances) + return len(self._linked_instances) def iter_parameter_group_names(self, **paramtype): """ :yield str: """ encountered = set() - for i, instance in enumerate(self.linked_instances): + for i, instance in enumerate(self._linked_instances): for name in instance.iter_parameter_group_names(**paramtype): - if instance.property_is_linked(name): + if instance._property_is_linked(name): if name not in encountered: encountered.add(name) yield name @@ -335,7 +335,7 @@ def get_parameter_values(self, **paramtype): :returns array: """ values = list() - for instance in self.linked_instances: + for instance in self._linked_instances: ivalues = instance.get_parameter_values(**paramtype) values.append(ivalues) return numpy.concatenate(values) @@ -345,7 +345,7 @@ def set_parameter_values(self, values, **paramtype): :returns array: """ i = 0 - for instance in self.linked_instances: + for instance in self._linked_instances: n = instance.get_n_parameters(**paramtype) instance.set_parameter_values(values[i : i + n], **paramtype) i += n diff --git a/PyMca5/tests/LinkedModelTest.py b/PyMca5/tests/LinkedModelTest.py index 699a326f8..9fec509c2 100644 --- a/PyMca5/tests/LinkedModelTest.py +++ b/PyMca5/tests/LinkedModelTest.py @@ -64,15 +64,15 @@ def __init__(self): super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) def reset_counters(self): - for model in self.linked_instances: + for model in self._linked_instances: model.reset_counters() def link(self): - self.enable_property_link("var1", "var2") + self._enable_property_link("var1", "var2") self.reset_counters() def unlink(self): - self.disable_property_link("var1", "var2") + self._disable_property_link("var1", "var2") self.reset_counters() @@ -82,9 +82,9 @@ def setUp(self): def test_links(self): """establish links""" - nlinked = len(self.concat_model.linked_instances) - 1 - for model in self.concat_model.linked_instances: - self.assertEqual(len(model.linked_instances), nlinked) + nlinked = len(self.concat_model._linked_instances) - 1 + for model in self.concat_model._linked_instances: + self.assertEqual(len(model._linked_instances), nlinked) def test_init_properties(self): """initial property values""" @@ -93,17 +93,17 @@ def test_init_properties(self): def test_enable_property_link_syncing(self): """maximal 1 get/set of a linked property when enabling linking""" - for i, model in enumerate(self.concat_model.linked_instances): + for i, model in enumerate(self.concat_model._linked_instances): model.var1 = 100 + i - if model.has_linked_property("var2"): + if model._has_linked_property("var2"): model.var2 = 200 + i self.assert_property_values("var1", 100, [100, 101, 102, 103]) self.assert_property_values("var2", 202, [202, 203]) self.concat_model.reset_counters() - self.concat_model.enable_property_link("var1") - getmodel = self.concat_model.instance_with_linked_property("var1") - for model in self.concat_model.linked_instances: + self.concat_model._enable_property_link("var1") + getmodel = self.concat_model._instance_with_linked_property("var1") + for model in self.concat_model._linked_instances: if model is getmodel: self.assertEqual(model.get_counter["var1"], 1) self.assertTrue(model.set_counter["var1"] <= 1) @@ -119,22 +119,22 @@ def test_contexts_concat(self): """entering a linked context manager of a container""" self.concat_model.fit() self.assertEqual(self.concat_model.context_counter, 1) - for model in self.concat_model.linked_instances: + for model in self.concat_model._linked_instances: model.context_counter, 1 def test_contexts_single(self): """entering a linked context manager of a single instance""" - model = self.concat_model.linked_instances[2] + model = self.concat_model._linked_instances[2] model.fit() self.assertEqual(model.context_counter, 1) - for model in self.concat_model.linked_instances: + for model in self.concat_model._linked_instances: model.context_counter, 1 def test_get_var1_concat(self): """getting a linked property (present in all models) from a container""" self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property_value("var1"), 1) - for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(self.concat_model._get_linked_property_value("var1"), 1) + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( model.get_counter["var1"], 1 if i == 0 else 0, msg=f"model{i}" ) @@ -143,9 +143,9 @@ def test_get_var1_concat(self): def test_get_var1_single(self): """getting a linked property (present in all models) from a single instance""" self.concat_model.link() - model = self.concat_model.linked_instances[2] + model = self.concat_model._linked_instances[2] self.assertEqual(model.var1, 1) - for i, model in enumerate(self.concat_model.linked_instances): + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( model.get_counter["var1"], 1 if i == 2 else 0, msg=f"model{i}" ) @@ -154,8 +154,8 @@ def test_get_var1_single(self): def test_get_var2_concat(self): """getting a linked property (present in some models) from a container""" self.concat_model.link() - self.assertEqual(self.concat_model.get_linked_property_value("var2"), 4) - for i, model in enumerate(self.concat_model.linked_instances): + self.assertEqual(self.concat_model._get_linked_property_value("var2"), 4) + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}" ) @@ -164,9 +164,9 @@ def test_get_var2_concat(self): def test_get_var2_single(self): """getting a linked property (present in some models) from a single instance""" self.concat_model.link() - model = self.concat_model.linked_instances[2] + model = self.concat_model._linked_instances[2] self.assertEqual(model.var2, 4) - for i, model in enumerate(self.concat_model.linked_instances): + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( model.get_counter["var2"], 1 if i == 2 else 0, msg=f"model{i}" ) @@ -175,8 +175,8 @@ def test_get_var2_single(self): def test_set_var1_concat(self): """setting a linked property (present in all models) from a container""" self.concat_model.link() - self.concat_model.set_linked_property_value("var1", 100) - for i, model in enumerate(self.concat_model.linked_instances): + self.concat_model._set_linked_property_value("var1", 100) + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var1"], 0, msg=f"model{i}") self.assertEqual(model.set_counter["var1"], 1, msg=f"model{i}") self.assert_property_values("var1", 100) @@ -184,17 +184,17 @@ def test_set_var1_concat(self): def test_set_var1_single(self): """setting a linked property (present in all models) from a single instance""" self.concat_model.link() - model = self.concat_model.linked_instances[2] + model = self.concat_model._linked_instances[2] model.var1 = 1 - for i, model in enumerate(self.concat_model.linked_instances): + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var1"], 0, msg=f"model{i}") self.assertEqual(model.set_counter["var1"], 1, msg=f"model{i}") def test_set_var2_concat(self): """setting a linked property (present in some models) from a container""" self.concat_model.link() - self.concat_model.set_linked_property_value("var2", 100) - for i, model in enumerate(self.concat_model.linked_instances): + self.concat_model._set_linked_property_value("var2", 100) + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") self.assertEqual( model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}" @@ -204,9 +204,9 @@ def test_set_var2_concat(self): def test_set_var2_single(self): """setting a linked property (present in some models) from a single instance""" self.concat_model.link() - model = self.concat_model.linked_instances[2] + model = self.concat_model._linked_instances[2] model.var2 = 100 - for i, model in enumerate(self.concat_model.linked_instances): + for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") self.assertEqual( model.set_counter["var2"], 1 if i > 1 else 0, msg=f"model{i}" @@ -214,11 +214,11 @@ def test_set_var2_single(self): def assert_property_values(self, name, value, values=None): self.assertEqual( - self.concat_model.get_linked_property_value(name), value, msg=name + self.concat_model._get_linked_property_value(name), value, msg=name ) if not isinstance(values, list): - values = [value] * len(self.concat_model.linked_instances) + values = [value] * len(self.concat_model._linked_instances) for model, v in zip( - self.concat_model.instances_with_linked_property(name), values + self.concat_model._instances_with_linked_property(name), values ): self.assertEqual(getattr(model, name), v, msg=name) diff --git a/PyMca5/tests/ModelParameterInterfaceTest.py b/PyMca5/tests/ModelParameterInterfaceTest.py index d18f86b7d..dd32cda3e 100644 --- a/PyMca5/tests/ModelParameterInterfaceTest.py +++ b/PyMca5/tests/ModelParameterInterfaceTest.py @@ -76,11 +76,11 @@ def __init__(self): } cfgs.append(cfg) super().__init__([Model(cfg) for cfg in cfgs]) - self.enable_property_link("shared_param", "shared_linear_param") + self._enable_property_link("shared_param", "shared_linear_param") self.reset_counters() def reset_counters(self): - for m in self.linked_instances: + for m in self._linked_instances: m.reset_counters() From 905fa0eeb26bb386c95400d71dc1e944d4365ddc Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 18:17:14 +0200 Subject: [PATCH 40/74] fixup --- PyMca5/PyMcaCore/DataObject.py | 63 ++-- PyMca5/PyMcaMath/fitting/CachingModel.py | 71 ++-- PyMca5/PyMcaMath/fitting/LinkedModel.py | 14 +- .../fitting/ModelParameterInterface.py | 351 ------------------ PyMca5/PyMcaMath/fitting/ParameterModel.py | 267 +++++++++++++ PyMca5/tests/ParameterModelTest.py | 99 +++++ 6 files changed, 449 insertions(+), 416 deletions(-) delete mode 100644 PyMca5/PyMcaMath/fitting/ModelParameterInterface.py create mode 100644 PyMca5/PyMcaMath/fitting/ParameterModel.py create mode 100644 PyMca5/tests/ParameterModelTest.py diff --git a/PyMca5/PyMcaCore/DataObject.py b/PyMca5/PyMcaCore/DataObject.py index 20494681b..ceb7655b7 100644 --- a/PyMca5/PyMcaCore/DataObject.py +++ b/PyMca5/PyMcaCore/DataObject.py @@ -1,4 +1,4 @@ -#/*########################################################################## +# /*########################################################################## # # The PyMca X-Ray Fluorescence Toolkit # @@ -32,8 +32,9 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import numpy + class DataObject(object): - ''' + """ Simple container of an array and associated information. Basically it has the members: info: A dictionary @@ -47,15 +48,16 @@ class DataObject(object): x: A list containing arrays to be considered axes y: A list of data to be considered as signals m: A list containing the monitor data - ''' + """ + GETINFO_DEPRECATION_WARNING = True GETDATA_DEPRECATION_WARNING = True SELECT_DEPRECATION_WARNING = True def __init__(self): - ''' + """ Default Constructor - ''' + """ self.info = {} self.data = numpy.array([]) @@ -68,7 +70,7 @@ def getInfo(self): """ if DataObject.GETINFO_DEPRECATION_WARNING: print("DEPRECATION WARNING: DataObject.getInfo()") - DataObject.GETINFO_DEPRECATION_WARNING = False + DataObject.GETINFO_DEPRECATION_WARNING = False return self.info def getData(self): @@ -77,7 +79,7 @@ def getData(self): """ if DataObject.GETDATA_DEPRECATION_WARNING: print("DEPRECATION WARNING: DataObject.getData()") - DataObject.GETDATA_DEPRECATION_WARNING = False + DataObject.GETDATA_DEPRECATION_WARNING = False return self.data def select(self, selection=None): @@ -89,78 +91,79 @@ def select(self, selection=None): DataObject.SELECT_DEPRECATION_WARNING = False dataObject = DataObject() dataObject.info = self.info - dataObject.info['selection'] = selection + dataObject.info["selection"] = selection if selection is None: dataObject.data = self.data return dataObject if type(selection) == dict: - #dataObject.data = self.data #should I set it to none??? + # dataObject.data = self.data #should I set it to none??? dataObject.data = None - if 'rows' in selection: + if "rows" in selection: dataObject.x = None dataObject.y = None dataObject.m = None - if 'x' in selection['rows']: - for rownumber in selection['rows']['x']: + if "x" in selection["rows"]: + for rownumber in selection["rows"]["x"]: if rownumber is None: continue if dataObject.x is None: dataObject.x = [] dataObject.x.append(self.data[rownumber, :]) - if 'y' in selection['rows']: - for rownumber in selection['rows']['y']: + if "y" in selection["rows"]: + for rownumber in selection["rows"]["y"]: if rownumber is None: continue if dataObject.y is None: dataObject.y = [] dataObject.y.append(self.data[rownumber, :]) - if 'm' in selection['rows']: - for rownumber in selection['rows']['m']: + if "m" in selection["rows"]: + for rownumber in selection["rows"]["m"]: if rownumber is None: continue if dataObject.m is None: dataObject.m = [] dataObject.m.append(self.data[rownumber, :]) - elif ('cols' in selection) or ('columns' in selection): - if 'cols' in selection: - key = 'cols' + elif ("cols" in selection) or ("columns" in selection): + if "cols" in selection: + key = "cols" else: - key = 'columns' + key = "columns" dataObject.x = None dataObject.y = None dataObject.m = None - if 'x' in selection[key]: - for rownumber in selection[key]['x']: + if "x" in selection[key]: + for rownumber in selection[key]["x"]: if rownumber is None: continue if dataObject.x is None: dataObject.x = [] dataObject.x.append(self.data[:, rownumber]) - if 'y' in selection[key]: - for rownumber in selection[key]['y']: + if "y" in selection[key]: + for rownumber in selection[key]["y"]: if rownumber is None: continue if dataObject.y is None: dataObject.y = [] dataObject.y.append(self.data[:, rownumber]) - if 'm' in selection[key]: - for rownumber in selection[key]['m']: + if "m" in selection[key]: + for rownumber in selection[key]["m"]: if rownumber is None: continue if dataObject.m is None: dataObject.m = [] dataObject.m.append(self.data[:, rownumber]) if dataObject.x is None: - if 'Channel0' in dataObject.info: - ch0 = int(dataObject.info['Channel0']) + if "Channel0" in dataObject.info: + ch0 = int(dataObject.info["Channel0"]) else: ch0 = 0 - dataObject.x = [numpy.arange(ch0, - ch0 + len(dataObject.y[0])).astype(numpy.float64)] + dataObject.x = [ + numpy.arange(ch0, ch0 + len(dataObject.y[0])).astype(numpy.float64) + ] if not ("selectiontype" in dataObject.info): dataObject.info["selectiontype"] = "%dD" % len(dataObject.y) return dataObject diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index cdf37e472..56ac3f582 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -6,15 +6,15 @@ class CacheManager: """Object that manages a cache""" - def __init__(self): + def __init__(self, *args, **kw): self._cache_root = dict() - super().__init__() + super().__init__(*args, **kw) def _create_empty_cache(self, key, **cacheoptions): # By default the property cache is a dictionary return dict() - def _property_cache_index(self, name): + def _property_cache_index(self, name, **cacheoptions): # By default the property cache index is its name return name @@ -28,9 +28,9 @@ class CachingModel(CacheManager): uses an external cache. """ - def __init__(self): + def __init__(self, *args, **kw): + super().__init__(*args, **kw) self._cache_manager = None - super().__init__() @property def _cache_manager(self): @@ -119,34 +119,26 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption yield values_cache return - if start_cache is None: - # Fill and empty cache with property values + with self._cachingContext("_cached_property_indices") as nameindexmap: _cache_manager = self._cache_manager - key = _cache_manager._property_cache_key(**cacheoptions) - values_cache = _cache_manager._create_empty_cache(key, **cacheoptions) - nameindexmap = dict() - for name in self._cached_properties(): - index = _cache_manager._property_cache_index(name) - nameindexmap[name] = index - values_cache[index] = getattr(self, name) - else: - values_cache = start_cache - nameindexmap = None + if start_cache is None: + # Fill and empty cache with property values + key = _cache_manager._property_cache_key(**cacheoptions) + values_cache = _cache_manager._create_empty_cache(key, **cacheoptions) + for name in self._cached_properties(): + index = self._get_property_cache_index(name, **cacheoptions) + values_cache[index] = getattr(self, name) + else: + values_cache = start_cache - with self._cachingContext("_cached_properties"): - # Initialize the property values cache - self._set_property_values_cache(values_cache, **cacheoptions) - yield values_cache + with self._cachingContext("_cached_properties"): + # Initialize the property values cache + self._set_property_values_cache(values_cache, **cacheoptions) + yield values_cache - if persist: - # Set property values to the cached values - if nameindexmap is None: - _cache_manager = self._cache_manager + if persist: for name in self._cached_properties(): - index = _cache_manager._property_cache_index(name) - setattr(self, name, values_cache[index]) - else: - for name, index in nameindexmap.items(): + index = self._get_property_cache_index(name, **cacheoptions) setattr(self, name, values_cache[index]) def _get_property_values_cache(self, **cacheoptions): @@ -164,16 +156,33 @@ def _set_property_values_cache(self, values_cache, **cacheoptions): key = _cache_manager._property_cache_key(**cacheoptions) caches[key] = values_cache + def _get_property_indices_cache(self, **cacheoptions): + caches = self._getCache("_cached_property_indices") + if caches is None: + return None + key = self._cache_manager._property_cache_key(**cacheoptions) + return caches.get(key, None) + def _cached_property_fget(self, fget): values_cache = self._get_property_values_cache() if values_cache is None: return fget(self) - index = self._cache_manager._property_cache_index(fget.__name__) + index = self._get_property_cache_index(fget.__name__) return values_cache[index] def _cached_property_fset(self, fset, value): values_cache = self._get_property_values_cache() if values_cache is None: return fset(self, value) - index = self._cache_manager._property_cache_index(fset.__name__) + index = self._get_property_cache_index(fset.__name__) values_cache[index] = value + + def _get_property_cache_index(self, name, **cacheoptions): + name_to_index = self._get_property_indices_cache(**cacheoptions) + if name_to_index is None: + return self._cache_manager._property_cache_index(name, **cacheoptions) + if name in name_to_index: + return self.name_to_index[name] + index = self._cache_manager._property_cache_index(name, **cacheoptions) + self.name_to_index[name] = index + return index diff --git a/PyMca5/PyMcaMath/fitting/LinkedModel.py b/PyMca5/PyMcaMath/fitting/LinkedModel.py index 107697b19..d16bf238f 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/LinkedModel.py @@ -56,10 +56,10 @@ class LinkedModel: to derived from this class. """ - def __init__(self): + def __init__(self, *args, **kw): self.__linked_instances = list() self.__propagate = True - super().__init__() + super().__init__(*args, **kw) @classmethod def _get_linked_property(cls, prop_name): @@ -144,9 +144,9 @@ class LinkedModelContainer: derive from this class. """ - def __init__(self, linked_instances): + def __init__(self, linked_instances=tuple(), *args, **kw): self._linked_instances = linked_instances - super().__init__() + super().__init__(*args, **kw) @property def _linked_instances(self): @@ -167,6 +167,12 @@ def _instance_with_linked_property(self, prop_name): return instance return None + def _get_linked_property(self, prop_name): + instance = self._instance_with_linked_property(prop_name) + if instance is None: + raise ValueError(f"No instance has linked property {repr(prop_name)}") + return getattr(type(instance), prop_name) + def _get_linked_property_value(self, prop_name): instance = self._instance_with_linked_property(prop_name) if instance is None: diff --git a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py b/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py deleted file mode 100644 index 7864c7349..000000000 --- a/PyMca5/PyMcaMath/fitting/ModelParameterInterface.py +++ /dev/null @@ -1,351 +0,0 @@ -from contextlib import contextmanager -import numpy -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer -from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property -from PyMca5.PyMcaMath.fitting.LinkedModel import linked_contextmanager - - -class parameter_group(linked_property): - """Usage: - - .. highlight:: python - .. code-block:: python - - class MyClass(Model): - - def __init__(self): - self._myparam = 0. - - @parameter_group - def myparam(self): - return self._myparam - - @myparam.setter # optional - def myparam(self, value): - self._myparam = value - - @myparam.counter # optional - def myparam(self): - return 1 - - @myparam.constraints # optional - def myparam(self): - return 1, 0, 0 - """ - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self.fcount = self._fcount_default() - self.fconstraints = self._fconstraints_default() - - def counter(self, fcount): - self.fcount = fcount - return self - - def constraints(self, fconstraints): - self.fconstraints = fconstraints - return self - - def _fcount_default(self): - def fcount(oself): - try: - return len(self.fget(oself)) - except TypeError: - return 1 - - return fcount - - def _fconstraints_default(self): - def fconstraints(oself): - return numpy.zeros((self.fcount(oself), 3)) - - return fconstraints - - -class linear_parameter_group(parameter_group): - pass - - -class ModelParameterInterfaceBase: - def __init__(self): - self.__cache = dict() - self.linear = False - super().__init__() - - @linked_contextmanager - def _cachingContext(self, cachename): - reset = not self._cachingEnabled(cachename) - if reset: - self.__cache[cachename] = dict() - try: - yield - finally: - if reset: - del self.__cache[cachename] - - def _cachingEnabled(self, cachename): - return cachename in self.__cache - - def _getCache(self, cachename, *subnames): - if cachename in self.__cache: - ret = self.__cache[cachename] - for cachename in subnames: - try: - ret = ret[cachename] - except KeyError: - ret[cachename] = dict() - ret = ret[cachename] - return ret - else: - return None - - @contextmanager - def linear_context(self, linear): - keep = self.linear - self.linear = linear - try: - yield - finally: - self.linear = keep - - @classmethod - def parameter_group_is_linear(cls, name): - return isinstance(getattr(cls, name, None), linear_parameter_group) - - def get_parameter_group_names(self, **paramtype): - return tuple(self.iter_parameter_group_names(**paramtype)) - - def get_parameter_names(self, **paramtype): - return tuple(self.iter_parameter_names(**paramtype)) - - def get_n_parameters(self, **paramtype): - return sum(n for _, n in self.iter_parameter_groups(**paramtype)) - - def iter_parameter_names(self, **paramtype): - for group_name, n in self.iter_parameter_groups(**paramtype): - if n > 1: - for i in range(n): - yield group_name + str(i) - else: - yield group_name - - def iter_parameter_groups(self, **paramtype): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - cache = self._getCache("iter_parameter_groups") - if cache is None: - yield from self._parameter_groups_notcached(**paramtype) - return - - key = self._parameters_cache_key(**paramtype) - it = cache.get(key) - if it is None: - it = cache[key] = list(self._parameter_groups_notcached(**paramtype)) - yield from it - - def _parameter_groups_notcached(self, **paramtype): - """Helper for `iter_parameter_groups`. - - :yields str, int: group name, nb. parameters in the group - """ - names = self.iter_parameter_group_names(**paramtype) - for name in names: - paramprop = getattr(self.__class__, name) - n = paramprop.fcount(self) - if n: - yield name, n - - def _parameters_cache_key(self, linear=None, linked=None): - if linear is None: - linear = self.linear - return linear, linked - - def get_parameter_values(self, **paramtype): - """All parameters values in one numpy array - - :returns array: - """ - raise NotImplementedError - - def set_parameter_values(self, values, **paramtype): - """ - :returns array: - """ - raise NotImplementedError - - def iter_parameter_group_names(self, **paramtype): - """ - :yield str: - """ - raise NotImplementedError - - -class ModelParameterInterface(LinkedModel, ModelParameterInterfaceBase): - _PARAMETERS = tuple() - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - allp = list() - for name in sorted(dir(cls)): # TODO: keep order if declaration? - attr = getattr(cls, name) - if isinstance(attr, parameter_group): - allp.append(name) - cls._PARAMETERS = tuple(allp) - - @linked_property - def linear(self): - return self.__linear - - @linear.setter - def linear(self, value): - self.__linear = value - - def get_parameter_values(self, **paramtype): - """All parameters values in one numpy array - - :returns array: - """ - cache = self._getCache("_parameters") - if cache is None: - return self._get_parameter_values_notcached(**paramtype) - - key = self._parameters_cache_key() - parameters = cache.get(key, None) - if parameters is None: - parameters = cache[key] = self._get_parameter_values_notcached(**paramtype) - return parameters - - def set_parameter_values(self, values, **paramtype): - """ - :returns array: - """ - cache = self._getCache("_parameters") - if cache is None: - self._set_parameter_values_notcached(values, **paramtype) - else: - key = self._parameters_cache_key(**paramtype) - cache[key] = values - - def _get_parameter_values_notcached(self, **paramtype): - """Merge all parameters values in one numpy array - - :returns array: - """ - nvalues = self._n_parameters(**paramtype) - values = numpy.zeros(nvalues) - for group_name, idx in self._parameter_group_indices(**paramtype): - values[idx] = getattr(self, group_name) - return values - - def _set_parameter_values_notcached(self, values, **paramtype): - for group_name, idx in self._parameter_group_indices(**paramtype): - setattr(self, group_name, values[idx]) - - def _get_parameter(self, fget): - parameters = self._getCache("_parameters") - if parameters is None: - return fget(self) - - key = self._parameters_cache_key() - parameters = parameters.get(key, None) - if parameters is None: - return fget(self) - - idx = self._parameter_group_index(fget.__name__) - if idx is None: - return fget(self) - return parameters[idx] - - def _set_parameter(self, fset, value): - parameters = self._getCache("_parameters") - if parameters is None: - return fset(self, value) - - key = self._parameters_cache_key() - parameters = parameters.get(key, None) - if parameters is None: - return fset(self, value) - - idx = self._parameter_group_index(fset.__name__) - if idx is None: - return fset(self, value) - parameters[idx] = value - - def _parameter_group_index(self, name, **paramtype): - """Parameter group index in the parameter sequence - - :returns int or slice or None: - """ - for group_name, idx in self._parameter_group_indices(**paramtype): - if name == group_name: - return idx - return None - - def _parameter_group_indices(self, **paramtype): - """Index of each parameter group in the parameter sequence - - :yields int or slice: index of parameter group in all parameters - """ - i = 0 - for group_name, n in self.iter_parameter_groups(**paramtype): - if n == 1: - yield group_name, i - else: - yield group_name, slice(i, i + n) - i += n - - def iter_parameter_group_names(self, linear=None): - for name in self._PARAMETERS: - if linear is not None: - is_linear = self.parameter_group_is_linear(name) - if linear == is_linear: - yield name - - -class ConcatModelParameterInterface(LinkedModelContainer, ModelParameterInterfaceBase): - @property - def models(self): - return self._linked_instances - - @property - def nmodels(self): - return len(self._linked_instances) - - def iter_parameter_group_names(self, **paramtype): - """ - :yield str: - """ - encountered = set() - for i, instance in enumerate(self._linked_instances): - for name in instance.iter_parameter_group_names(**paramtype): - if instance._property_is_linked(name): - if name not in encountered: - encountered.add(name) - yield name - else: - yield f"model{i}_{name}" - - def get_parameter_values(self, **paramtype): - """All parameters values in one numpy array - - :returns array: - """ - values = list() - for instance in self._linked_instances: - ivalues = instance.get_parameter_values(**paramtype) - values.append(ivalues) - return numpy.concatenate(values) - - def set_parameter_values(self, values, **paramtype): - """ - :returns array: - """ - i = 0 - for instance in self._linked_instances: - n = instance.get_n_parameters(**paramtype) - instance.set_parameter_values(values[i : i + n], **paramtype) - i += n diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/ParameterModel.py new file mode 100644 index 000000000..a8105e00f --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/ParameterModel.py @@ -0,0 +1,267 @@ +from contextlib import contextmanager +import numpy +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.CachingModel import CachedPropertiesModel +from PyMca5.PyMcaMath.fitting.CachingModel import cached_property + + +class parameter_group(cached_property, linked_property): + """Usage: + + .. highlight:: python + .. code-block:: python + + class MyClass(Model): + + def __init__(self): + self._myparam = 0. + + @parameter_group + def myparam(self): + return self._myparam + + @myparam.setter # optional + def myparam(self, value): + self._myparam = value + + @myparam.counter # optional + def myparam(self): + return 1 + + @myparam.constraints # optional + def myparam(self): + return 1, 0, 0 + """ + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.fcount = self._fcount_default() + self.fconstraints = self._fconstraints_default() + + def counter(self, fcount): + self.fcount = fcount + return self + + def constraints(self, fconstraints): + self.fconstraints = fconstraints + return self + + def _fcount_default(self): + def fcount(oself): + try: + return len(self.fget(oself)) + except TypeError: + return 1 + + return fcount + + def _fconstraints_default(self): + def fconstraints(oself): + return numpy.zeros((self.fcount(oself), 3)) + + return fconstraints + + +class linear_parameter_group(parameter_group): + pass + + +class ParameterModelBase(CachedPropertiesModel): + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self._linear = False + + @property + def linear(self): + return self._linear + + @linear.setter + def linear(self, value): + self._linear = value + + @contextmanager + def linear_context(self, linear): + keep = self.linear + self.linear = linear + try: + yield + finally: + self.linear = keep + + def _property_cache_key(self, linear=None): + if linear is None: + linear = self.linear + return linear + + def _create_empty_cache(self, key, **paramtype): + return numpy.zeros(self.get_n_parameters(**paramtype)) + + def _property_cache_index(self, group_name, **paramtype): + return self._parameter_group_index(group_name, **paramtype) + + def _parameter_group_index(self, group_name, **paramtype): + """Parameter group index in the parameter sequence + + :returns int or slice or None: + """ + for name, idx in self._parameter_group_indices(**paramtype): + if group_name == name: + return idx + return None + + def _parameter_group_indices(self, **paramtype): + """Index of each parameter group in the parameter sequence + + :yields int or slice: index of parameter group in all parameters + """ + i = 0 + for group_name, n in self._iter_parameter_group_count(**paramtype): + if n == 1: + yield group_name, i + else: + yield group_name, slice(i, i + n) + i += n + + def get_parameter_group_names(self, **paramtype): + return tuple(self._iter_parameter_group_names(**paramtype)) + + def get_parameter_names(self, **paramtype): + return tuple(self._iter_parameter_names(**paramtype)) + + def get_n_parameters(self, **paramtype): + return sum(n for _, n in self._iter_parameter_group_count(**paramtype)) + + def _iter_parameter_names(self, **paramtype): + for group_name, n in self._iter_parameter_group_count(**paramtype): + if n > 1: + for i in range(n): + yield group_name + str(i) + else: + yield group_name + + def _iter_parameter_group_count(self, **paramtype): + """Yield name and count of enabled parameter groups + + :param bool linear_only: + :yields str, int: group name, nb. parameters in the group + """ + group_names = self._iter_parameter_group_names(**paramtype) + for group_name in group_names: + paramprop = self._get_parameter_group_property(group_name) + n = paramprop.fcount(self) + if n: + yield group_name, n + + def _get_parameter_group_property(self, group_name): + """ + :yield parameter_group: + """ + raise NotImplementedError + + def _iter_parameter_group_names(self, **paramtype): + """ + :yield str: + """ + raise NotImplementedError + + +class ParameterModel(ParameterModelBase, LinkedModel): + _PARAMETER_GROUP_NAMES = tuple() + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + allp = list() + for group_name in sorted(dir(cls)): # TODO: how to keep order of declaration? + attr = getattr(cls, group_name) + if isinstance(attr, parameter_group): + allp.append(group_name) + cls._PARAMETER_GROUP_NAMES = tuple(allp) + + def _iter_parameter_group_names(self, linear=None, linked=None): + for group_name in self._PARAMETER_GROUP_NAMES: + if linked is not None: + if self._property_is_linked(group_name) is not linked: + continue + if linear is None: + linear = self.linear + if linear: + if self._parameter_group_is_linear(group_name): + yield group_name + else: + yield group_name + + def _get_parameter_group_property(self, group_name): + """ + :yield parameter_group: + """ + return getattr(type(self), group_name) + + @classmethod + def _parameter_group_is_linear(cls, group_name): + return isinstance(getattr(cls, group_name, None), linear_parameter_group) + + @linked_property + def linear(self): + return self._linear + + @linear.setter + def linear(self, value): + self._linear = value + + +class ParameterModelContainer(ParameterModelBase, LinkedModelContainer): + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self._enable_property_link("linear") + for model in self.models: + model._cache_manager = self + + @property + def models(self): + return self._linked_instances + + @property + def nmodels(self): + return len(self._linked_instances) + + @property + def linear(self): + return self._get_linked_property_value("linear") + + @linear.setter + def linear(self, value): + self._set_linked_property_value("linear", value) + + def _iter_parameter_group_names(self, **paramtype): + """ + :yield str: + """ + # Shared parameters + encountered = set() + for model in self.models: + for group_name in model._iter_parameter_group_names( + linked=True, **paramtype + ): + if group_name in encountered: + continue + encountered.add(group_name) + yield group_name + # Non-shared parameters + for i, model in enumerate(self.models): + for group_name in model._iter_parameter_group_names( + linked=False, **paramtype + ): + yield f"model{i}:{group_name}" + + def _get_parameter_group_property(self, group_name): + """ + :yield parameter_group: + """ + if ":" in group_name: + model_name, group_name = group_name.split(":") + i = int(model_name.replace("model", "")) + return self.models[i]._get_parameter_group_property(group_name) + else: + return self._get_linked_property(group_name) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py new file mode 100644 index 000000000..8d4ec19e6 --- /dev/null +++ b/PyMca5/tests/ParameterModelTest.py @@ -0,0 +1,99 @@ +import unittest +from collections import Counter +from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModel +from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModelContainer +from PyMca5.PyMcaMath.fitting.ParameterModel import parameter_group +from PyMca5.PyMcaMath.fitting.ParameterModel import linear_parameter_group + + +class Model1(ParameterModel): + def __init__(self, cfg): + super().__init__() + self._cfg = cfg + + def reset_counters(self): + self.get_counter = Counter() + self.set_counter = Counter() + + @parameter_group + def var1_nonlin(self): + return self._cfg.get("var1_nonlin") + + @var1_nonlin.setter + def var1_nonlin(self, value): + self._cfg["var1_nonlin"] = value + + @linear_parameter_group + def var1_lin(self): + return self._cfg.get("var1_lin") + + @var1_lin.setter + def var1_lin(self, value): + self._cfg["var1_lin"] = value + + +class Model2(Model1): + @parameter_group + def var2_nonlin(self): + return self._cfg.get("var2_nonlin") + + @var2_nonlin.setter + def var2_nonlin(self, value): + self._cfg["var2_nonlin"] = value + + @linear_parameter_group + def var2_lin(self): + return self._cfg.get("var2_lin") + + @var2_lin.setter + def var2_lin(self, value): + self._cfg["var2_lin"] = value + + +class ConcatModel(ParameterModelContainer): + def __init__(self): + cfg1a = {"var1_lin": 11, "var1_nonlin": 12} + cfg1b = {"var1_lin": 21, "var1_nonlin": 22} + cfg2a = {"var1_lin": 31, "var1_nonlin": 32, "var2_lin": 41, "var2_nonlin": 42} + cfg2b = {"var1_lin": 51, "var1_nonlin": 12, "var2_lin": 61, "var2_nonlin": 62} + super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) + self._enable_property_link("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") + + +class testParameterModel(unittest.TestCase): + def setUp(self): + self.concat_model = ConcatModel() + + def test_instantiation(self): + self.assertFalse(self.concat_model.linear) + self.assertEqual(self.concat_model.nmodels, 4) + + def test_linear_context(self): + with self.concat_model.linear_context(True): + self.assertTrue(self.concat_model.linear) + for model in self.concat_model.models: + self.assertTrue(model.linear) + + self.assertFalse(self.concat_model.linear) + for model in self.concat_model.models: + self.assertFalse(model.linear) + + def test_parameter_group_names(self): + names = self.concat_model.models[0].get_parameter_group_names() + self.assertEqual(names, ("var1_lin", "var1_nonlin")) + names = self.concat_model.models[-1].get_parameter_group_names() + self.assertEqual(names, ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin")) + names = self.concat_model.get_parameter_group_names() + self.assertEqual(names, ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin")) + + def test_linear_parameter_group_names(self): + self.concat_model.linear = True + names = self.concat_model.models[0].get_parameter_group_names() + self.assertEqual(names, ("var1_lin",)) + names = self.concat_model.models[-1].get_parameter_group_names() + self.assertEqual(names, ("var1_lin", "var2_lin")) + names = self.concat_model.get_parameter_group_names() + self.assertEqual(names, ("var1_lin", "var2_lin")) + + def test_parameter_names(self): + pass # print(self.concat_model.get_parameter_names()) From 61403b4b4967490b7db696924c0c6dde79210694 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 18:58:57 +0200 Subject: [PATCH 41/74] fixup --- PyMca5/PyMcaMath/fitting/LinkedModel.py | 9 +- PyMca5/PyMcaMath/fitting/ParameterModel.py | 29 ++-- PyMca5/tests/ParameterModelTest.py | 184 ++++++++++++++++++--- 3 files changed, 178 insertions(+), 44 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/LinkedModel.py b/PyMca5/PyMcaMath/fitting/LinkedModel.py index d16bf238f..7bfed68e6 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/LinkedModel.py @@ -108,7 +108,7 @@ def _linked_instances(self, instances): type(instance), "can only link objects of the 'LinkedModel' type" ) others.append(instance) - self.__linked_instances = others + self.__linked_instances = tuple(others) for instance in others: instance._unpropagated_linked_instances_setter(instances) @@ -153,9 +153,10 @@ def _linked_instances(self): return self.__linked_instances @_linked_instances.setter - def _linked_instances(self, _linked_instances): - _linked_instances[0]._linked_instances = _linked_instances - self.__linked_instances = _linked_instances + def _linked_instances(self, linked_instances): + linked_instances = tuple(linked_instances) + linked_instances[0]._linked_instances = linked_instances + self.__linked_instances = linked_instances def _instances_with_linked_property(self, prop_name): yield from LinkedModel._filter_class_has_linked_property( diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/ParameterModel.py index a8105e00f..fa5751827 100644 --- a/PyMca5/PyMcaMath/fitting/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/ParameterModel.py @@ -149,12 +149,11 @@ def _iter_parameter_group_count(self, **paramtype): """ group_names = self._iter_parameter_group_names(**paramtype) for group_name in group_names: - paramprop = self._get_parameter_group_property(group_name) - n = paramprop.fcount(self) + n = self._get_parameter_group_count(group_name) if n: yield group_name, n - def _get_parameter_group_property(self, group_name): + def _get_parameter_group_count(self, group_name): """ :yield parameter_group: """ @@ -192,11 +191,11 @@ def _iter_parameter_group_names(self, linear=None, linked=None): else: yield group_name - def _get_parameter_group_property(self, group_name): + def _get_parameter_group_count(self, group_name): """ - :yield parameter_group: + :returns int: """ - return getattr(type(self), group_name) + return getattr(type(self), group_name).fcount(self) @classmethod def _parameter_group_is_linear(cls, group_name): @@ -240,14 +239,14 @@ def _iter_parameter_group_names(self, **paramtype): """ # Shared parameters encountered = set() - for model in self.models: + for i, model in enumerate(self.models): for group_name in model._iter_parameter_group_names( linked=True, **paramtype ): if group_name in encountered: continue encountered.add(group_name) - yield group_name + yield f"model{i}:{group_name}" # Non-shared parameters for i, model in enumerate(self.models): for group_name in model._iter_parameter_group_names( @@ -255,13 +254,11 @@ def _iter_parameter_group_names(self, **paramtype): ): yield f"model{i}:{group_name}" - def _get_parameter_group_property(self, group_name): + def _get_parameter_group_count(self, group_name): """ - :yield parameter_group: + :returns int: """ - if ":" in group_name: - model_name, group_name = group_name.split(":") - i = int(model_name.replace("model", "")) - return self.models[i]._get_parameter_group_property(group_name) - else: - return self._get_linked_property(group_name) + model_name, group_name = group_name.split(":") + i = int(model_name.replace("model", "")) + model = self.models[i] + return model._get_parameter_group_count(group_name) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 8d4ec19e6..e5e0ff62c 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -1,5 +1,4 @@ import unittest -from collections import Counter from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModel from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModelContainer from PyMca5.PyMcaMath.fitting.ParameterModel import parameter_group @@ -11,13 +10,9 @@ def __init__(self, cfg): super().__init__() self._cfg = cfg - def reset_counters(self): - self.get_counter = Counter() - self.set_counter = Counter() - @parameter_group def var1_nonlin(self): - return self._cfg.get("var1_nonlin") + return self._cfg["var1_nonlin"] @var1_nonlin.setter def var1_nonlin(self, value): @@ -25,17 +20,21 @@ def var1_nonlin(self, value): @linear_parameter_group def var1_lin(self): - return self._cfg.get("var1_lin") + return self._cfg["var1_lin"] @var1_lin.setter def var1_lin(self, value): self._cfg["var1_lin"] = value + @var1_lin.counter + def var1_lin(self): + return 2 + class Model2(Model1): @parameter_group def var2_nonlin(self): - return self._cfg.get("var2_nonlin") + return self._cfg["var2_nonlin"] @var2_nonlin.setter def var2_nonlin(self, value): @@ -43,7 +42,7 @@ def var2_nonlin(self, value): @linear_parameter_group def var2_lin(self): - return self._cfg.get("var2_lin") + return self._cfg["var2_lin"] @var2_lin.setter def var2_lin(self, value): @@ -52,10 +51,20 @@ def var2_lin(self, value): class ConcatModel(ParameterModelContainer): def __init__(self): - cfg1a = {"var1_lin": 11, "var1_nonlin": 12} - cfg1b = {"var1_lin": 21, "var1_nonlin": 22} - cfg2a = {"var1_lin": 31, "var1_nonlin": 32, "var2_lin": 41, "var2_nonlin": 42} - cfg2b = {"var1_lin": 51, "var1_nonlin": 12, "var2_lin": 61, "var2_nonlin": 62} + cfg1a = {"var1_lin": [11, 11], "var1_nonlin": 12} + cfg1b = {"var1_lin": [21, 21], "var1_nonlin": 22} + cfg2a = { + "var1_lin": [31, 31], + "var1_nonlin": 32, + "var2_lin": 41, + "var2_nonlin": 42, + } + cfg2b = { + "var1_lin": [51, 51], + "var1_nonlin": 12, + "var2_lin": 61, + "var2_nonlin": 62, + } super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) self._enable_property_link("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") @@ -80,20 +89,147 @@ def test_linear_context(self): def test_parameter_group_names(self): names = self.concat_model.models[0].get_parameter_group_names() - self.assertEqual(names, ("var1_lin", "var1_nonlin")) + expected = ("var1_lin", "var1_nonlin") + self.assertEqual(set(names), set(expected)) + names = self.concat_model.models[-1].get_parameter_group_names() - self.assertEqual(names, ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin")) + expected = ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") + self.assertEqual(set(names), set(expected)) + names = self.concat_model.get_parameter_group_names() - self.assertEqual(names, ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin")) + expected = ( + "model0:var1_lin", + "model0:var1_nonlin", + "model2:var2_lin", + "model2:var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) + + self.concat_model._disable_property_link("var1_lin") + names = self.concat_model.get_parameter_group_names() + expected = ( + "model0:var1_lin", + "model0:var1_nonlin", + "model1:var1_lin", + "model2:var1_lin", + "model2:var2_lin", + "model2:var2_nonlin", + "model3:var1_lin", + ) + self.assertEqual(set(names), set(expected)) def test_linear_parameter_group_names(self): - self.concat_model.linear = True - names = self.concat_model.models[0].get_parameter_group_names() - self.assertEqual(names, ("var1_lin",)) - names = self.concat_model.models[-1].get_parameter_group_names() - self.assertEqual(names, ("var1_lin", "var2_lin")) - names = self.concat_model.get_parameter_group_names() - self.assertEqual(names, ("var1_lin", "var2_lin")) + names = self.concat_model.models[0].get_parameter_group_names(linear=True) + expected = ("var1_lin",) + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.models[-1].get_parameter_group_names(linear=True) + expected = ("var1_lin", "var2_lin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_group_names(linear=True) + expected = ("model0:var1_lin", "model2:var2_lin") + self.assertEqual(set(names), set(expected)) + + self.concat_model._disable_property_link("var1_lin") + names = self.concat_model.get_parameter_group_names(linear=True) + expected = ( + "model0:var1_lin", + "model1:var1_lin", + "model2:var1_lin", + "model2:var2_lin", + "model3:var1_lin", + ) + self.assertEqual(set(names), set(expected)) def test_parameter_names(self): - pass # print(self.concat_model.get_parameter_names()) + names = self.concat_model.models[0].get_parameter_names() + expected = ("var1_lin0", "var1_lin1", "var1_nonlin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.models[-1].get_parameter_names() + expected = ("var1_lin0", "var1_lin1", "var1_nonlin", "var2_lin", "var2_nonlin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_names() + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model0:var1_nonlin", + "model2:var2_lin", + "model2:var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) + + self.concat_model._disable_property_link("var1_lin") + names = self.concat_model.get_parameter_names() + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model0:var1_nonlin", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model2:var2_lin", + "model2:var2_nonlin", + "model3:var1_lin0", + "model3:var1_lin1", + ) + self.assertEqual(set(names), set(expected)) + + def test_linear_parameter_names(self): + names = self.concat_model.models[0].get_parameter_names(linear=True) + expected = ("var1_lin0", "var1_lin1") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.models[-1].get_parameter_names(linear=True) + expected = ("var1_lin0", "var1_lin1", "var2_lin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_names(linear=True) + expected = ("model0:var1_lin0", "model0:var1_lin1", "model2:var2_lin") + self.assertEqual(set(names), set(expected)) + + self.concat_model._disable_property_link("var1_lin") + names = self.concat_model.get_parameter_names(linear=True) + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model2:var2_lin", + "model3:var1_lin0", + "model3:var1_lin1", + ) + self.assertEqual(set(names), set(expected)) + + def test_n_parameter(self): + n = self.concat_model.models[0].get_n_parameters() + self.assertEqual(n, 3) + + n = self.concat_model.models[-1].get_n_parameters() + self.assertEqual(n, 5) + + n = self.concat_model.get_n_parameters() + self.assertEqual(n, 5) + + self.concat_model._disable_property_link("var1_lin") + n = self.concat_model.get_n_parameters() + self.assertEqual(n, 11) + + def test_n_linear_parameter(self): + n = self.concat_model.models[0].get_n_parameters(linear=True) + self.assertEqual(n, 2) + + n = self.concat_model.models[-1].get_n_parameters(linear=True) + self.assertEqual(n, 3) + + n = self.concat_model.get_n_parameters(linear=True) + self.assertEqual(n, 3) + + self.concat_model._disable_property_link("var1_lin") + n = self.concat_model.get_n_parameters(linear=True) + self.assertEqual(n, 9) From 4603c3ca2d981b15780eb3532cfe29e7794e865a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 19:05:52 +0200 Subject: [PATCH 42/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 25 +++++++++++++--------- PyMca5/PyMcaMath/fitting/ParameterModel.py | 2 +- PyMca5/tests/CachingModelTest.py | 4 ++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index 56ac3f582..fbb5729d8 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -10,7 +10,7 @@ def __init__(self, *args, **kw): self._cache_root = dict() super().__init__(*args, **kw) - def _create_empty_cache(self, key, **cacheoptions): + def _create_empty_property_values_cache(self, key, **cacheoptions): # By default the property cache is a dictionary return dict() @@ -119,20 +119,13 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption yield values_cache return - with self._cachingContext("_cached_property_indices") as nameindexmap: - _cache_manager = self._cache_manager + with self._cachingContext("_cached_property_indices"): if start_cache is None: - # Fill and empty cache with property values - key = _cache_manager._property_cache_key(**cacheoptions) - values_cache = _cache_manager._create_empty_cache(key, **cacheoptions) - for name in self._cached_properties(): - index = self._get_property_cache_index(name, **cacheoptions) - values_cache[index] = getattr(self, name) + values_cache = self._create_start_property_values_cache(**cacheoptions) else: values_cache = start_cache with self._cachingContext("_cached_properties"): - # Initialize the property values cache self._set_property_values_cache(values_cache, **cacheoptions) yield values_cache @@ -156,6 +149,18 @@ def _set_property_values_cache(self, values_cache, **cacheoptions): key = _cache_manager._property_cache_key(**cacheoptions) caches[key] = values_cache + def _create_start_property_values_cache(self, **cacheoptions): + # Fill and empty cache with property values + _cache_manager = self._cache_manager + key = _cache_manager._property_cache_key(**cacheoptions) + values_cache = _cache_manager._create_empty_property_values_cache( + key, **cacheoptions + ) + for name in self._cached_properties(): + index = self._get_property_cache_index(name, **cacheoptions) + values_cache[index] = getattr(self, name) + return values_cache + def _get_property_indices_cache(self, **cacheoptions): caches = self._getCache("_cached_property_indices") if caches is None: diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/ParameterModel.py index fa5751827..04ac8330e 100644 --- a/PyMca5/PyMcaMath/fitting/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/ParameterModel.py @@ -95,7 +95,7 @@ def _property_cache_key(self, linear=None): linear = self.linear return linear - def _create_empty_cache(self, key, **paramtype): + def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) def _property_cache_index(self, group_name, **paramtype): diff --git a/PyMca5/tests/CachingModelTest.py b/PyMca5/tests/CachingModelTest.py index 9e270d9f0..5fa6078f2 100644 --- a/PyMca5/tests/CachingModelTest.py +++ b/PyMca5/tests/CachingModelTest.py @@ -43,7 +43,7 @@ def _property_cache_index(self, name): else: return 1 - def _create_empty_cache(self, key, **_): + def _create_empty_property_values_cache(self, key, **_): return numpy.zeros(2, dtype=float) @@ -56,7 +56,7 @@ def _property_cache_index(self, name): else: raise ValueError(name) - def _create_empty_cache(self, key, **_): + def _create_empty_property_values_cache(self, key, **_): return numpy.zeros(4, dtype=float) From 91062c150b760ff4a89d735e9224eda6c5f877e7 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 19:13:58 +0200 Subject: [PATCH 43/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index fbb5729d8..0f93619b4 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -130,9 +130,7 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption yield values_cache if persist: - for name in self._cached_properties(): - index = self._get_property_cache_index(name, **cacheoptions) - setattr(self, name, values_cache[index]) + self._persist_property_values(values_cache, **cacheoptions) def _get_property_values_cache(self, **cacheoptions): caches = self._getCache("_cached_properties") @@ -144,10 +142,11 @@ def _get_property_values_cache(self, **cacheoptions): def _set_property_values_cache(self, values_cache, **cacheoptions): caches = self._getCache("_cached_properties") if caches is None: - return + return False _cache_manager = self._cache_manager key = _cache_manager._property_cache_key(**cacheoptions) caches[key] = values_cache + return True def _create_start_property_values_cache(self, **cacheoptions): # Fill and empty cache with property values @@ -161,6 +160,22 @@ def _create_start_property_values_cache(self, **cacheoptions): values_cache[index] = getattr(self, name) return values_cache + def _get_property_values(self, **cacheoptions): + values = self._get_property_values_cache(**cacheoptions) + if values is None: + return self._create_start_property_values_cache(**cacheoptions) + return values + + def _set_property_values(self, values, **cacheoptions): + success = self._set_property_values_cache(values, **cacheoptions) + if not success: + self._persist_property_values(values) + + def _persist_property_values(self, values, **cacheoptions): + for name in self._cached_properties(): + index = self._get_property_cache_index(name, **cacheoptions) + setattr(self, name, values[index]) + def _get_property_indices_cache(self, **cacheoptions): caches = self._getCache("_cached_property_indices") if caches is None: From 4ada2f70652ff2217aeed6ff5d5c4af5c4f22f84 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 28 Jun 2021 19:27:14 +0200 Subject: [PATCH 44/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index 0f93619b4..19a97087d 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -143,8 +143,7 @@ def _set_property_values_cache(self, values_cache, **cacheoptions): caches = self._getCache("_cached_properties") if caches is None: return False - _cache_manager = self._cache_manager - key = _cache_manager._property_cache_key(**cacheoptions) + key = self._cache_manager._property_cache_key(**cacheoptions) caches[key] = values_cache return True From f059f06901592d1bcd432bd11f3a4e84f77e5234 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 29 Jun 2021 16:40:20 +0200 Subject: [PATCH 45/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 72 ++++-- PyMca5/PyMcaMath/fitting/ConcatModel.py | 4 +- PyMca5/PyMcaMath/fitting/LinkedModel.py | 82 ++++--- PyMca5/PyMcaMath/fitting/ParameterModel.py | 262 ++++++++++++--------- PyMca5/tests/LinkedModelTest.py | 27 ++- PyMca5/tests/ParameterModelTest.py | 106 ++++++--- 6 files changed, 359 insertions(+), 194 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index 19a97087d..72f86970c 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -101,15 +101,25 @@ class CachedPropertiesModel(CachingModel): def __init_subclass__(subcls, **kwargs): super().__init_subclass__(**kwargs) - allp = list() + allp = list(subcls._CACHED_PROPERTIES) for name, attr in vars(subcls).items(): - if isinstance(attr, cached_property): + if isinstance(attr, cached_property) and name not in allp: allp.append(name) - subcls._CACHED_PROPERTIES = subcls._CACHED_PROPERTIES + tuple(allp) + subcls._CACHED_PROPERTIES = tuple(sorted(allp)) @classmethod - def _cached_properties(self): - return self._CACHED_PROPERTIES + def _cached_property_names(cls): + return cls._CACHED_PROPERTIES + + def _cached_properties(self, **cacheoptions): + return self._cached_property_names() + + def __iter_cached_property_names(self, **cacheoptions): + name_to_index = self._get_property_indices_cache(**cacheoptions) + if name_to_index is None: + yield from self._cached_properties(**cacheoptions) + else: + yield from name_to_index.keys() @contextmanager def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): @@ -148,48 +158,65 @@ def _set_property_values_cache(self, values_cache, **cacheoptions): return True def _create_start_property_values_cache(self, **cacheoptions): - # Fill and empty cache with property values + """Fill an empty cache with property values""" _cache_manager = self._cache_manager key = _cache_manager._property_cache_key(**cacheoptions) values_cache = _cache_manager._create_empty_property_values_cache( key, **cacheoptions ) - for name in self._cached_properties(): + for name in self.__iter_cached_property_names(): index = self._get_property_cache_index(name, **cacheoptions) - values_cache[index] = getattr(self, name) + values_cache[index] = self._get_noncached_property_value(name) return values_cache def _get_property_values(self, **cacheoptions): + """Get the cache when enabled, get instance property values when disabled""" values = self._get_property_values_cache(**cacheoptions) if values is None: return self._create_start_property_values_cache(**cacheoptions) return values def _set_property_values(self, values, **cacheoptions): + """Set the cache when enabled, set instance property values when disabled""" success = self._set_property_values_cache(values, **cacheoptions) if not success: self._persist_property_values(values) def _persist_property_values(self, values, **cacheoptions): - for name in self._cached_properties(): + for name in self.__iter_cached_property_names(): index = self._get_property_cache_index(name, **cacheoptions) - setattr(self, name, values[index]) + self._set_noncached_property_value(name, values[index]) - def _get_property_indices_cache(self, **cacheoptions): - caches = self._getCache("_cached_property_indices") - if caches is None: - return None - key = self._cache_manager._property_cache_key(**cacheoptions) - return caches.get(key, None) + def _get_property_value(self, name, **cacheoptions): + """Get the value from the cache or from the property""" + values_cache = self._get_property_values_cache(**cacheoptions) + if values_cache is None: + return self._get_noncached_property_value(name) + index = self._get_property_cache_index(name, **cacheoptions) + return values_cache[index] def _cached_property_fget(self, fget): + """Same as _get_property_value but well have the property object + instead of the name + """ values_cache = self._get_property_values_cache() if values_cache is None: return fget(self) index = self._get_property_cache_index(fget.__name__) return values_cache[index] + def _set_property_value(self, name, value, **cacheoptions): + """Set the value in the cache or the property""" + values_cache = self._get_property_values_cache(**cacheoptions) + if values_cache is None: + return self._set_noncached_property_value(name, value) + index = self._get_property_cache_index(name, **cacheoptions) + values_cache[index] = value + def _cached_property_fset(self, fset, value): + """Same as _set_property_value but well have the property object + instead of the name + """ values_cache = self._get_property_values_cache() if values_cache is None: return fset(self, value) @@ -205,3 +232,16 @@ def _get_property_cache_index(self, name, **cacheoptions): index = self._cache_manager._property_cache_index(name, **cacheoptions) self.name_to_index[name] = index return index + + def _get_noncached_property_value(self, name): + return getattr(self, name) + + def _set_noncached_property_value(self, name, value): + setattr(self, name, value) + + def _get_property_indices_cache(self, **cacheoptions): + caches = self._getCache("_cached_property_indices") + if caches is None: + return None + key = self._cache_manager._property_cache_key(**cacheoptions) + return caches.get(key, None) diff --git a/PyMca5/PyMcaMath/fitting/ConcatModel.py b/PyMca5/PyMcaMath/fitting/ConcatModel.py index 5fcbaacf4..f968e665d 100644 --- a/PyMca5/PyMcaMath/fitting/ConcatModel.py +++ b/PyMca5/PyMcaMath/fitting/ConcatModel.py @@ -1,11 +1,11 @@ import numpy from collections.abc import Sequence, MutableMapping -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface from PyMca5.PyMcaMath.fitting.Model import Model -class ConcatModel(LinkedModelContainer, ModelInterface): +class ConcatModel(LinkedModelManager, ModelInterface): """Concatenated model with shared parameters""" def __init__(self, models, shared_attributes=None): diff --git a/PyMca5/PyMcaMath/fitting/LinkedModel.py b/PyMca5/PyMcaMath/fitting/LinkedModel.py index 7bfed68e6..11beacf0a 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/LinkedModel.py @@ -1,5 +1,7 @@ import functools from contextlib import ExitStack, contextmanager +from collections.abc import Mapping +from typing import Type from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property @@ -57,7 +59,7 @@ class LinkedModel: """ def __init__(self, *args, **kw): - self.__linked_instances = list() + self._link_manager = None self.__propagate = True super().__init__(*args, **kw) @@ -94,26 +96,28 @@ def _enable_property_link(cls, *prop_names): prop.propagate = True @property - def _linked_instances(self): - return self.__linked_instances + def _link_manager(self): + return self.__link_manager - @_linked_instances.setter - def _linked_instances(self, instances): - others = list() - for instance in instances: - if instance is self: - continue - if not isinstance(instance, LinkedModel): - raise TypeError( - type(instance), "can only link objects of the 'LinkedModel' type" - ) - others.append(instance) - self.__linked_instances = tuple(others) - for instance in others: - instance._unpropagated_linked_instances_setter(instances) + @_link_manager.setter + def _link_manager(self, obj): + if obj is not None and not isinstance(obj, LinkedModelManager): + raise TypeError(obj, type(obj)) + self.__link_manager = obj + + @property + def _linked_instances(self): + if self._link_manager is None: + return + for instance in self._link_manager._linked_instances: + if instance is not self: + yield instance - def _unpropagated_linked_instances_setter(self, instances): - self.__linked_instances = [i for i in instances if i is not self] + @property + def _linked_instance_to_key(self): + if self._link_manager is None: + return None + return self._link_manager._linked_instance_to_key(self) @property def _non_propagating_instances(self): @@ -139,24 +143,46 @@ def _filter_class_has_linked_property(instances, prop_name): yield instance -class LinkedModelContainer: +class LinkedModelManager: """Classes that manage LinkedModel objects should derive from this class. """ - def __init__(self, linked_instances=tuple(), *args, **kw): - self._linked_instances = linked_instances + def __init__(self, linked_instances=None, *args, **kw): super().__init__(*args, **kw) + self._linked_instance_mapping = linked_instances @property def _linked_instances(self): - return self.__linked_instances + return self.__linked_instance_mapping.values() + + @property + def _linked_instance_mapping(self): + return self.__linked_instance_mapping + + @_linked_instance_mapping.setter + def _linked_instance_mapping(self, instances): + if instances is None: + instances = dict() + elif not isinstance(instances, Mapping): + raise TypeError(instances, "Linked instance must be a 'Mapping'") + for instance in instances.values(): + if not isinstance(instance, LinkedModel): + raise TypeError( + instance, "Linked instance must be of type 'LinkedModel'" + ) + self.__linked_instance_mapping = instances + for instance in instances.values(): + instance._link_manager = self + + def _linked_instance_to_key(self, instance): + for name, _instance in self._linked_instance_mapping.items(): + if _instance is instance: + return name + return None - @_linked_instances.setter - def _linked_instances(self, linked_instances): - linked_instances = tuple(linked_instances) - linked_instances[0]._linked_instances = linked_instances - self.__linked_instances = linked_instances + def _linked_key_to_instance(self, name): + return self._linked_instance_mapping[name] def _instances_with_linked_property(self, prop_name): yield from LinkedModel._filter_class_has_linked_property( diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/ParameterModel.py index 04ac8330e..8ceabf175 100644 --- a/PyMca5/PyMcaMath/fitting/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/ParameterModel.py @@ -1,7 +1,9 @@ +import typing +from dataclasses import dataclass, field from contextlib import contextmanager import numpy from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property from PyMca5.PyMcaMath.fitting.CachingModel import CachedPropertiesModel from PyMca5.PyMcaMath.fitting.CachingModel import cached_property @@ -68,6 +70,25 @@ class linear_parameter_group(parameter_group): pass +@dataclass(frozen=True, eq=True) +class ParameterGroupId: + name: str + linear: bool = field(compare=False, hash=False) + linked: bool = field(compare=False, hash=False) + count: int = field(compare=False, hash=False) + start_index: int = field(compare=False, hash=False) + index: "typing.Any" = field(compare=False, hash=False) + property_name: str = field(compare=False, hash=False) + instance_key: "typing.Any" = field(compare=False, hash=False) + + def _iter_parameter_names(self): + if self.count > 1: + for i in range(self.count): + yield self.name + str(i) + elif self.count == 1: + yield self.name + + class ParameterModelBase(CachedPropertiesModel): def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -98,108 +119,117 @@ def _property_cache_key(self, linear=None): def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) - def _property_cache_index(self, group_name, **paramtype): - return self._parameter_group_index(group_name, **paramtype) - - def _parameter_group_index(self, group_name, **paramtype): - """Parameter group index in the parameter sequence + def _property_cache_index(self, group, **paramtype): + return group.index - :returns int or slice or None: - """ - for name, idx in self._parameter_group_indices(**paramtype): - if group_name == name: - return idx - return None + def get_parameter_names(self, **paramtype): + return tuple(self._iter_parameter_names(**paramtype)) - def _parameter_group_indices(self, **paramtype): - """Index of each parameter group in the parameter sequence + def _iter_parameter_names(self, **paramtype): + for group in self._iter_parameter_group_ids(**paramtype): + yield from group._iter_parameter_names() - :yields int or slice: index of parameter group in all parameters - """ - i = 0 - for group_name, n in self._iter_parameter_group_count(**paramtype): - if n == 1: - yield group_name, i - else: - yield group_name, slice(i, i + n) - i += n + def get_n_parameters(self, **paramtype): + return sum(group.count for group in self._iter_parameter_group_ids(**paramtype)) - def get_parameter_group_names(self, **paramtype): - return tuple(self._iter_parameter_group_names(**paramtype)) + def get_parameter_values(self, **paramtype): + return self._get_property_values(**paramtype) - def get_parameter_names(self, **paramtype): - return tuple(self._iter_parameter_names(**paramtype)) + def set_parameter_values(self, values, **paramtype): + self._set_property_values(values, **paramtype) - def get_n_parameters(self, **paramtype): - return sum(n for _, n in self._iter_parameter_group_count(**paramtype)) + def get_parameter_group_value(self, group, **paramtype): + return self._get_property_value(group) - def _iter_parameter_names(self, **paramtype): - for group_name, n in self._iter_parameter_group_count(**paramtype): - if n > 1: - for i in range(n): - yield group_name + str(i) - else: - yield group_name + def set_parameter_group_value(self, group, value, **paramtype): + self._set_property_value(group, value) - def _iter_parameter_group_count(self, **paramtype): - """Yield name and count of enabled parameter groups + def get_parameter_groups(self, **paramtype): + return tuple(self._iter_parameter_group_ids(**paramtype)) - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - group_names = self._iter_parameter_group_names(**paramtype) - for group_name in group_names: - n = self._get_parameter_group_count(group_name) - if n: - yield group_name, n + def get_parameter_group_names(self, **paramtype): + return tuple( + group.name for group in self._iter_parameter_group_ids(**paramtype) + ) - def _get_parameter_group_count(self, group_name): + def _iter_parameter_group_ids(self, **paramtype): """ - :yield parameter_group: + :yield ParameterGroupId: """ raise NotImplementedError - def _iter_parameter_group_names(self, **paramtype): - """ - :yield str: - """ - raise NotImplementedError + def _cached_properties(self): + yield from self._iter_parameter_group_ids() class ParameterModel(ParameterModelBase, LinkedModel): - _PARAMETER_GROUP_NAMES = tuple() - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - allp = list() - for group_name in sorted(dir(cls)): # TODO: how to keep order of declaration? - attr = getattr(cls, group_name) - if isinstance(attr, parameter_group): - allp.append(group_name) - cls._PARAMETER_GROUP_NAMES = tuple(allp) - - def _iter_parameter_group_names(self, linear=None, linked=None): - for group_name in self._PARAMETER_GROUP_NAMES: + @classmethod + def _get_parameter_property(cls, property_name): + prop = getattr(cls, property_name, None) + if isinstance(prop, parameter_group): + return prop + return None + + def _iter_parameter_group_ids(self, linear=None, linked=None, tracker=None): + """ + :yields ParameterGroupId: + """ + count = None + index = None + if tracker is None: + start_index = 0 + else: + start_index = tracker.start_index + for property_name in self._cached_property_names(): + prop = self._get_parameter_property(property_name) + if prop is None: + continue + + group_is_linked = prop.propagate if linked is not None: - if self._property_is_linked(group_name) is not linked: + if group_is_linked is not linked: continue + + group_is_linear = isinstance(prop, linear_parameter_group) if linear is None: linear = self.linear if linear: - if self._parameter_group_is_linear(group_name): - yield group_name - else: - yield group_name + if not group_is_linear: + continue - def _get_parameter_group_count(self, group_name): - """ - :returns int: - """ - return getattr(type(self), group_name).fcount(self) + count = prop.fcount(self) + if not count: + continue - @classmethod - def _parameter_group_is_linear(cls, group_name): - return isinstance(getattr(cls, group_name, None), linear_parameter_group) + if count > 1: + index = slice(start_index, start_index + count) + elif count == 1: + index = start_index + else: + index = None + + instance_key = self._linked_instance_to_key + if group_is_linked: + name = property_name + else: + name = f"{instance_key}:{property_name}" + + group = ParameterGroupId( + name=name, + linear=group_is_linear, + linked=group_is_linked, + property_name=property_name, + instance_key=instance_key, + count=count, + start_index=start_index, + index=index, + ) + if tracker is None: + yield group + start_index += count + elif tracker.is_new_group(group): + yield group + start_index = tracker.start_index @linked_property def linear(self): @@ -209,8 +239,31 @@ def linear(self): def linear(self, value): self._linear = value + def _get_noncached_property_value(self, group): + return getattr(self, group.property_name) + + def _set_noncached_property_value(self, group, value): + setattr(self, group.property_name, value) + -class ParameterModelContainer(ParameterModelBase, LinkedModelContainer): +class Tracker: + def __init__(self): + self._start_index = 0 + self._encountered = set() + + @property + def start_index(self): + return self._start_index + + def is_new_group(self, group): + if group in self._encountered: + return False + self._encountered.add(group) + self._start_index += group.count + return True + + +class ParameterModelContainer(ParameterModelBase, LinkedModelManager): def __init__(self, *args, **kw): super().__init__(*args, **kw) self._enable_property_link("linear") @@ -221,9 +274,13 @@ def __init__(self, *args, **kw): def models(self): return self._linked_instances + @property + def model_mapping(self): + return self._linked_instance_mapping + @property def nmodels(self): - return len(self._linked_instances) + return len(self.model_mapping) @property def linear(self): @@ -233,32 +290,27 @@ def linear(self): def linear(self, value): self._set_linked_property_value("linear", value) - def _iter_parameter_group_names(self, **paramtype): + def _iter_parameter_group_ids(self, **paramtype): """ - :yield str: + :yields ParameterGroupId: """ # Shared parameters - encountered = set() - for i, model in enumerate(self.models): - for group_name in model._iter_parameter_group_names( - linked=True, **paramtype - ): - if group_name in encountered: - continue - encountered.add(group_name) - yield f"model{i}:{group_name}" + tracker = Tracker() + start_index = 0 + for model in self.models: + yield from model._iter_parameter_group_ids( + linked=True, tracker=tracker, **paramtype + ) # Non-shared parameters - for i, model in enumerate(self.models): - for group_name in model._iter_parameter_group_names( - linked=False, **paramtype - ): - yield f"model{i}:{group_name}" + for model in self.models: + yield from model._iter_parameter_group_ids( + linked=False, tracker=tracker, **paramtype + ) - def _get_parameter_group_count(self, group_name): - """ - :returns int: - """ - model_name, group_name = group_name.split(":") - i = int(model_name.replace("model", "")) - model = self.models[i] - return model._get_parameter_group_count(group_name) + def _get_noncached_property_value(self, group): + instance = self._linked_key_to_instance(group.instance_key) + return getattr(instance, group.property_name) + + def _set_noncached_property_value(self, group, value): + instance = self._linked_key_to_instance(group.instance_key) + setattr(instance, group.property_name, value) diff --git a/PyMca5/tests/LinkedModelTest.py b/PyMca5/tests/LinkedModelTest.py index 9fec509c2..9d638cf8f 100644 --- a/PyMca5/tests/LinkedModelTest.py +++ b/PyMca5/tests/LinkedModelTest.py @@ -1,7 +1,7 @@ import unittest from collections import Counter from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelContainer +from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager from PyMca5.PyMcaMath.fitting.LinkedModel import linked_contextmanager from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property @@ -55,13 +55,22 @@ def var2(self, value): self._cfg["var2"] = value -class ConcatModel(LinkedModelContainer, ModelBase): +class ConcatModel(LinkedModelManager, ModelBase): def __init__(self): cfg1a = {"var1": 1} cfg1b = {"var1": 2} cfg2a = {"var1": 3, "var2": 4} cfg2b = {"var1": 5, "var2": 6} - super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) + instances = { + 0: Model1(cfg1a), + 1: Model1(cfg1b), + 2: Model2(cfg2a), + 3: Model2(cfg2b), + } + super().__init__(instances) + + def __getitem__(self, index): + return self._linked_instance_mapping[index] def reset_counters(self): for model in self._linked_instances: @@ -84,7 +93,7 @@ def test_links(self): """establish links""" nlinked = len(self.concat_model._linked_instances) - 1 for model in self.concat_model._linked_instances: - self.assertEqual(len(model._linked_instances), nlinked) + self.assertEqual(len(list(model._linked_instances)), nlinked) def test_init_properties(self): """initial property values""" @@ -124,7 +133,7 @@ def test_contexts_concat(self): def test_contexts_single(self): """entering a linked context manager of a single instance""" - model = self.concat_model._linked_instances[2] + model = self.concat_model[2] model.fit() self.assertEqual(model.context_counter, 1) for model in self.concat_model._linked_instances: @@ -143,7 +152,7 @@ def test_get_var1_concat(self): def test_get_var1_single(self): """getting a linked property (present in all models) from a single instance""" self.concat_model.link() - model = self.concat_model._linked_instances[2] + model = self.concat_model[2] self.assertEqual(model.var1, 1) for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( @@ -164,7 +173,7 @@ def test_get_var2_concat(self): def test_get_var2_single(self): """getting a linked property (present in some models) from a single instance""" self.concat_model.link() - model = self.concat_model._linked_instances[2] + model = self.concat_model[2] self.assertEqual(model.var2, 4) for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual( @@ -184,7 +193,7 @@ def test_set_var1_concat(self): def test_set_var1_single(self): """setting a linked property (present in all models) from a single instance""" self.concat_model.link() - model = self.concat_model._linked_instances[2] + model = self.concat_model[2] model.var1 = 1 for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var1"], 0, msg=f"model{i}") @@ -204,7 +213,7 @@ def test_set_var2_concat(self): def test_set_var2_single(self): """setting a linked property (present in some models) from a single instance""" self.concat_model.link() - model = self.concat_model._linked_instances[2] + model = self.concat_model[2] model.var2 = 100 for i, model in enumerate(self.concat_model._linked_instances): self.assertEqual(model.get_counter["var2"], 0, msg=f"model{i}") diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index e5e0ff62c..9b818b49a 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -30,6 +30,9 @@ def var1_lin(self, value): def var1_lin(self): return 2 + def __getitem__(self, index): + return self._linked_instance_mapping[index] + class Model2(Model1): @parameter_group @@ -65,9 +68,20 @@ def __init__(self): "var2_lin": 61, "var2_nonlin": 62, } - super().__init__([Model1(cfg1a), Model1(cfg1b), Model2(cfg2a), Model2(cfg2b)]) + models = { + "model0": Model1(cfg1a), + "model1": Model1(cfg1b), + "model2": Model2(cfg2a), + "model3": Model2(cfg2b), + } + super().__init__(models) self._enable_property_link("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") + def __getitem__(self, index): + while index < 0: + index += len(self._linked_instance_mapping) + return self._linked_instance_mapping[f"model{index}"] + class testParameterModel(unittest.TestCase): def setUp(self): @@ -88,20 +102,20 @@ def test_linear_context(self): self.assertFalse(model.linear) def test_parameter_group_names(self): - names = self.concat_model.models[0].get_parameter_group_names() + names = self.concat_model[0].get_parameter_group_names() expected = ("var1_lin", "var1_nonlin") self.assertEqual(set(names), set(expected)) - names = self.concat_model.models[-1].get_parameter_group_names() + names = self.concat_model[-1].get_parameter_group_names() expected = ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") self.assertEqual(set(names), set(expected)) names = self.concat_model.get_parameter_group_names() expected = ( - "model0:var1_lin", - "model0:var1_nonlin", - "model2:var2_lin", - "model2:var2_nonlin", + "var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", ) self.assertEqual(set(names), set(expected)) @@ -109,26 +123,26 @@ def test_parameter_group_names(self): names = self.concat_model.get_parameter_group_names() expected = ( "model0:var1_lin", - "model0:var1_nonlin", "model1:var1_lin", "model2:var1_lin", - "model2:var2_lin", - "model2:var2_nonlin", "model3:var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", ) self.assertEqual(set(names), set(expected)) def test_linear_parameter_group_names(self): - names = self.concat_model.models[0].get_parameter_group_names(linear=True) + names = self.concat_model[0].get_parameter_group_names(linear=True) expected = ("var1_lin",) self.assertEqual(set(names), set(expected)) - names = self.concat_model.models[-1].get_parameter_group_names(linear=True) + names = self.concat_model[-1].get_parameter_group_names(linear=True) expected = ("var1_lin", "var2_lin") self.assertEqual(set(names), set(expected)) names = self.concat_model.get_parameter_group_names(linear=True) - expected = ("model0:var1_lin", "model2:var2_lin") + expected = ("var1_lin", "var2_lin") self.assertEqual(set(names), set(expected)) self.concat_model._disable_property_link("var1_lin") @@ -137,28 +151,22 @@ def test_linear_parameter_group_names(self): "model0:var1_lin", "model1:var1_lin", "model2:var1_lin", - "model2:var2_lin", "model3:var1_lin", + "var2_lin", ) self.assertEqual(set(names), set(expected)) def test_parameter_names(self): - names = self.concat_model.models[0].get_parameter_names() + names = self.concat_model[0].get_parameter_names() expected = ("var1_lin0", "var1_lin1", "var1_nonlin") self.assertEqual(set(names), set(expected)) - names = self.concat_model.models[-1].get_parameter_names() + names = self.concat_model[-1].get_parameter_names() expected = ("var1_lin0", "var1_lin1", "var1_nonlin", "var2_lin", "var2_nonlin") self.assertEqual(set(names), set(expected)) names = self.concat_model.get_parameter_names() - expected = ( - "model0:var1_lin0", - "model0:var1_lin1", - "model0:var1_nonlin", - "model2:var2_lin", - "model2:var2_nonlin", - ) + expected = ("var1_lin0", "var1_lin1", "var1_nonlin", "var2_lin", "var2_nonlin") self.assertEqual(set(names), set(expected)) self.concat_model._disable_property_link("var1_lin") @@ -166,29 +174,29 @@ def test_parameter_names(self): expected = ( "model0:var1_lin0", "model0:var1_lin1", - "model0:var1_nonlin", "model1:var1_lin0", "model1:var1_lin1", "model2:var1_lin0", "model2:var1_lin1", - "model2:var2_lin", - "model2:var2_nonlin", "model3:var1_lin0", "model3:var1_lin1", + "var1_nonlin", + "var2_lin", + "var2_nonlin", ) self.assertEqual(set(names), set(expected)) def test_linear_parameter_names(self): - names = self.concat_model.models[0].get_parameter_names(linear=True) + names = self.concat_model[0].get_parameter_names(linear=True) expected = ("var1_lin0", "var1_lin1") self.assertEqual(set(names), set(expected)) - names = self.concat_model.models[-1].get_parameter_names(linear=True) + names = self.concat_model[-1].get_parameter_names(linear=True) expected = ("var1_lin0", "var1_lin1", "var2_lin") self.assertEqual(set(names), set(expected)) names = self.concat_model.get_parameter_names(linear=True) - expected = ("model0:var1_lin0", "model0:var1_lin1", "model2:var2_lin") + expected = ("var1_lin0", "var1_lin1", "var2_lin") self.assertEqual(set(names), set(expected)) self.concat_model._disable_property_link("var1_lin") @@ -200,17 +208,17 @@ def test_linear_parameter_names(self): "model1:var1_lin1", "model2:var1_lin0", "model2:var1_lin1", - "model2:var2_lin", "model3:var1_lin0", "model3:var1_lin1", + "var2_lin", ) self.assertEqual(set(names), set(expected)) def test_n_parameter(self): - n = self.concat_model.models[0].get_n_parameters() + n = self.concat_model[0].get_n_parameters() self.assertEqual(n, 3) - n = self.concat_model.models[-1].get_n_parameters() + n = self.concat_model[-1].get_n_parameters() self.assertEqual(n, 5) n = self.concat_model.get_n_parameters() @@ -221,10 +229,10 @@ def test_n_parameter(self): self.assertEqual(n, 11) def test_n_linear_parameter(self): - n = self.concat_model.models[0].get_n_parameters(linear=True) + n = self.concat_model[0].get_n_parameters(linear=True) self.assertEqual(n, 2) - n = self.concat_model.models[-1].get_n_parameters(linear=True) + n = self.concat_model[-1].get_n_parameters(linear=True) self.assertEqual(n, 3) n = self.concat_model.get_n_parameters(linear=True) @@ -233,3 +241,33 @@ def test_n_linear_parameter(self): self.concat_model._disable_property_link("var1_lin") n = self.concat_model.get_n_parameters(linear=True) self.assertEqual(n, 9) + + def test_get_parameter_values(self): + values = self.concat_model[0].get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 0, 0]) + + values = self.concat_model[-1].get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + values = self.concat_model.get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + self.concat_model._disable_property_link("var1_lin") + values = self.concat_model.get_parameter_values() + self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) + + def test_get_parameter_values_in_caching_context(self): + with self.concat_model._propertyCachingContext(): + values = self.concat_model[0].get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + values = self.concat_model[-1].get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + values = self.concat_model.get_parameter_values() + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + self.concat_model._disable_property_link("var1_lin") + with self.concat_model._propertyCachingContext(): + values = self.concat_model.get_parameter_values() + self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) From 0d8104ac3e9e892ef1f9d15783de00f8b14029ad Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 09:30:20 +0200 Subject: [PATCH 46/74] fixup --- PyMca5/PyMcaMath/fitting/CachingModel.py | 21 +- PyMca5/PyMcaMath/fitting/ParameterModel.py | 49 ++- PyMca5/tests/ParameterModelTest.py | 363 +++++++++++++-------- 3 files changed, 252 insertions(+), 181 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/CachingModel.py index 72f86970c..fa2894b5f 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/CachingModel.py @@ -109,15 +109,20 @@ def __init_subclass__(subcls, **kwargs): @classmethod def _cached_property_names(cls): + """All property names for this class""" return cls._CACHED_PROPERTIES - def _cached_properties(self, **cacheoptions): + def _instance_cached_property_names(self, **cacheoptions): + """All property names for this instance and the provided options""" return self._cached_property_names() - def __iter_cached_property_names(self, **cacheoptions): + def _iter_cached_property_names(self, **cacheoptions): + """To be used when iterating over all property names + of this instance. + """ name_to_index = self._get_property_indices_cache(**cacheoptions) if name_to_index is None: - yield from self._cached_properties(**cacheoptions) + yield from self._instance_cached_property_names(**cacheoptions) else: yield from name_to_index.keys() @@ -135,7 +140,7 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption else: values_cache = start_cache - with self._cachingContext("_cached_properties"): + with self._cachingContext("_instance_cached_property_names"): self._set_property_values_cache(values_cache, **cacheoptions) yield values_cache @@ -143,14 +148,14 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption self._persist_property_values(values_cache, **cacheoptions) def _get_property_values_cache(self, **cacheoptions): - caches = self._getCache("_cached_properties") + caches = self._getCache("_instance_cached_property_names") if caches is None: return None key = self._cache_manager._property_cache_key(**cacheoptions) return caches.get(key, None) def _set_property_values_cache(self, values_cache, **cacheoptions): - caches = self._getCache("_cached_properties") + caches = self._getCache("_instance_cached_property_names") if caches is None: return False key = self._cache_manager._property_cache_key(**cacheoptions) @@ -164,7 +169,7 @@ def _create_start_property_values_cache(self, **cacheoptions): values_cache = _cache_manager._create_empty_property_values_cache( key, **cacheoptions ) - for name in self.__iter_cached_property_names(): + for name in self._iter_cached_property_names(**cacheoptions): index = self._get_property_cache_index(name, **cacheoptions) values_cache[index] = self._get_noncached_property_value(name) return values_cache @@ -183,7 +188,7 @@ def _set_property_values(self, values, **cacheoptions): self._persist_property_values(values) def _persist_property_values(self, values, **cacheoptions): - for name in self.__iter_cached_property_names(): + for name in self._iter_cached_property_names(**cacheoptions): index = self._get_property_cache_index(name, **cacheoptions) self._set_noncached_property_value(name, values[index]) diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/ParameterModel.py index 8ceabf175..2456e69d2 100644 --- a/PyMca5/PyMcaMath/fitting/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/ParameterModel.py @@ -111,7 +111,7 @@ def linear_context(self, linear): finally: self.linear = keep - def _property_cache_key(self, linear=None): + def _property_cache_key(self, linear=None, **paramtype): if linear is None: linear = self.linear return linear @@ -126,11 +126,11 @@ def get_parameter_names(self, **paramtype): return tuple(self._iter_parameter_names(**paramtype)) def _iter_parameter_names(self, **paramtype): - for group in self._iter_parameter_group_ids(**paramtype): + for group in self._iter_parameter_groups(**paramtype): yield from group._iter_parameter_names() def get_n_parameters(self, **paramtype): - return sum(group.count for group in self._iter_parameter_group_ids(**paramtype)) + return sum(group.count for group in self._iter_parameter_groups(**paramtype)) def get_parameter_values(self, **paramtype): return self._get_property_values(**paramtype) @@ -139,38 +139,26 @@ def set_parameter_values(self, values, **paramtype): self._set_property_values(values, **paramtype) def get_parameter_group_value(self, group, **paramtype): - return self._get_property_value(group) + return self._get_property_value(group, **paramtype) def set_parameter_group_value(self, group, value, **paramtype): - self._set_property_value(group, value) + self._set_property_value(group, value, **paramtype) def get_parameter_groups(self, **paramtype): - return tuple(self._iter_parameter_group_ids(**paramtype)) + return tuple(self._iter_parameter_groups(**paramtype)) def get_parameter_group_names(self, **paramtype): - return tuple( - group.name for group in self._iter_parameter_group_ids(**paramtype) - ) + return tuple(group.name for group in self._iter_parameter_groups(**paramtype)) - def _iter_parameter_group_ids(self, **paramtype): + def _iter_parameter_groups(self, **paramtype): """ - :yield ParameterGroupId: + :yields ParameterGroupId: """ - raise NotImplementedError - - def _cached_properties(self): - yield from self._iter_parameter_group_ids() + yield from self._iter_cached_property_names(**paramtype) class ParameterModel(ParameterModelBase, LinkedModel): - @classmethod - def _get_parameter_property(cls, property_name): - prop = getattr(cls, property_name, None) - if isinstance(prop, parameter_group): - return prop - return None - - def _iter_parameter_group_ids(self, linear=None, linked=None, tracker=None): + def _instance_cached_property_names(self, linear=None, linked=None, tracker=None): """ :yields ParameterGroupId: """ @@ -180,10 +168,13 @@ def _iter_parameter_group_ids(self, linear=None, linked=None, tracker=None): start_index = 0 else: start_index = tracker.start_index + cls = type(self) for property_name in self._cached_property_names(): - prop = self._get_parameter_property(property_name) - if prop is None: - continue + prop = getattr(cls, property_name) + if not isinstance(prop, parameter_group): + raise TypeError( + "Currently only 'parameter_group' properties support caching" + ) group_is_linked = prop.propagate if linked is not None: @@ -290,7 +281,7 @@ def linear(self): def linear(self, value): self._set_linked_property_value("linear", value) - def _iter_parameter_group_ids(self, **paramtype): + def _instance_cached_property_names(self, **paramtype): """ :yields ParameterGroupId: """ @@ -298,12 +289,12 @@ def _iter_parameter_group_ids(self, **paramtype): tracker = Tracker() start_index = 0 for model in self.models: - yield from model._iter_parameter_group_ids( + yield from model._iter_parameter_groups( linked=True, tracker=tracker, **paramtype ) # Non-shared parameters for model in self.models: - yield from model._iter_parameter_group_ids( + yield from model._iter_parameter_groups( linked=False, tracker=tracker, **paramtype ) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 9b818b49a..1af77042b 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -1,4 +1,5 @@ import unittest +from contextlib import contextmanager from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModel from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModelContainer from PyMca5.PyMcaMath.fitting.ParameterModel import parameter_group @@ -102,172 +103,246 @@ def test_linear_context(self): self.assertFalse(model.linear) def test_parameter_group_names(self): - names = self.concat_model[0].get_parameter_group_names() - expected = ("var1_lin", "var1_nonlin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model[-1].get_parameter_group_names() - expected = ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model.get_parameter_group_names() - expected = ( - "var1_lin", - "var1_nonlin", - "var2_lin", - "var2_nonlin", - ) - self.assertEqual(set(names), set(expected)) - - self.concat_model._disable_property_link("var1_lin") - names = self.concat_model.get_parameter_group_names() - expected = ( - "model0:var1_lin", - "model1:var1_lin", - "model2:var1_lin", - "model3:var1_lin", - "var1_nonlin", - "var2_lin", - "var2_nonlin", - ) - self.assertEqual(set(names), set(expected)) + for cacheoptions in self._parameterize_nonlinear_test(): + names = self.concat_model[0].get_parameter_group_names(**cacheoptions) + expected = ("var1_lin", "var1_nonlin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model[-1].get_parameter_group_names(**cacheoptions) + expected = ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ( + "var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) + + with self._unlink_var1_lin(): + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ( + "model0:var1_lin", + "model1:var1_lin", + "model2:var1_lin", + "model3:var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) def test_linear_parameter_group_names(self): - names = self.concat_model[0].get_parameter_group_names(linear=True) - expected = ("var1_lin",) - self.assertEqual(set(names), set(expected)) - - names = self.concat_model[-1].get_parameter_group_names(linear=True) - expected = ("var1_lin", "var2_lin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model.get_parameter_group_names(linear=True) - expected = ("var1_lin", "var2_lin") - self.assertEqual(set(names), set(expected)) - - self.concat_model._disable_property_link("var1_lin") - names = self.concat_model.get_parameter_group_names(linear=True) - expected = ( - "model0:var1_lin", - "model1:var1_lin", - "model2:var1_lin", - "model3:var1_lin", - "var2_lin", - ) - self.assertEqual(set(names), set(expected)) + for cacheoptions in self._parameterize_linear_test(): + names = self.concat_model[0].get_parameter_group_names(**cacheoptions) + expected = ("var1_lin",) + self.assertEqual(set(names), set(expected)) + + names = self.concat_model[-1].get_parameter_group_names(**cacheoptions) + expected = ("var1_lin", "var2_lin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ("var1_lin", "var2_lin") + self.assertEqual(set(names), set(expected)) + + with self._unlink_var1_lin(): + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ( + "model0:var1_lin", + "model1:var1_lin", + "model2:var1_lin", + "model3:var1_lin", + "var2_lin", + ) + self.assertEqual(set(names), set(expected)) def test_parameter_names(self): - names = self.concat_model[0].get_parameter_names() - expected = ("var1_lin0", "var1_lin1", "var1_nonlin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model[-1].get_parameter_names() - expected = ("var1_lin0", "var1_lin1", "var1_nonlin", "var2_lin", "var2_nonlin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model.get_parameter_names() - expected = ("var1_lin0", "var1_lin1", "var1_nonlin", "var2_lin", "var2_nonlin") - self.assertEqual(set(names), set(expected)) - - self.concat_model._disable_property_link("var1_lin") - names = self.concat_model.get_parameter_names() - expected = ( - "model0:var1_lin0", - "model0:var1_lin1", - "model1:var1_lin0", - "model1:var1_lin1", - "model2:var1_lin0", - "model2:var1_lin1", - "model3:var1_lin0", - "model3:var1_lin1", - "var1_nonlin", - "var2_lin", - "var2_nonlin", - ) - self.assertEqual(set(names), set(expected)) + for cacheoptions in self._parameterize_nonlinear_test(): + names = self.concat_model[0].get_parameter_names(**cacheoptions) + expected = ("var1_lin0", "var1_lin1", "var1_nonlin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model[-1].get_parameter_names(**cacheoptions) + expected = ( + "var1_lin0", + "var1_lin1", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ( + "var1_lin0", + "var1_lin1", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) + + with self._unlink_var1_lin(): + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model3:var1_lin0", + "model3:var1_lin1", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) def test_linear_parameter_names(self): - names = self.concat_model[0].get_parameter_names(linear=True) - expected = ("var1_lin0", "var1_lin1") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model[-1].get_parameter_names(linear=True) - expected = ("var1_lin0", "var1_lin1", "var2_lin") - self.assertEqual(set(names), set(expected)) - - names = self.concat_model.get_parameter_names(linear=True) - expected = ("var1_lin0", "var1_lin1", "var2_lin") - self.assertEqual(set(names), set(expected)) - - self.concat_model._disable_property_link("var1_lin") - names = self.concat_model.get_parameter_names(linear=True) - expected = ( - "model0:var1_lin0", - "model0:var1_lin1", - "model1:var1_lin0", - "model1:var1_lin1", - "model2:var1_lin0", - "model2:var1_lin1", - "model3:var1_lin0", - "model3:var1_lin1", - "var2_lin", - ) - self.assertEqual(set(names), set(expected)) + for cacheoptions in self._parameterize_linear_test(): + names = self.concat_model[0].get_parameter_names(**cacheoptions) + expected = ("var1_lin0", "var1_lin1") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model[-1].get_parameter_names(**cacheoptions) + expected = ("var1_lin0", "var1_lin1", "var2_lin") + self.assertEqual(set(names), set(expected)) + + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ("var1_lin0", "var1_lin1", "var2_lin") + self.assertEqual(set(names), set(expected)) + + with self._unlink_var1_lin(): + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model3:var1_lin0", + "model3:var1_lin1", + "var2_lin", + ) + self.assertEqual(set(names), set(expected)) def test_n_parameter(self): - n = self.concat_model[0].get_n_parameters() - self.assertEqual(n, 3) + for cacheoptions in self._parameterize_nonlinear_test(): + n = self.concat_model[0].get_n_parameters(**cacheoptions) + self.assertEqual(n, 3) - n = self.concat_model[-1].get_n_parameters() - self.assertEqual(n, 5) + n = self.concat_model[-1].get_n_parameters(**cacheoptions) + self.assertEqual(n, 5) - n = self.concat_model.get_n_parameters() - self.assertEqual(n, 5) + n = self.concat_model.get_n_parameters(**cacheoptions) + self.assertEqual(n, 5) - self.concat_model._disable_property_link("var1_lin") - n = self.concat_model.get_n_parameters() - self.assertEqual(n, 11) + with self._unlink_var1_lin(): + n = self.concat_model.get_n_parameters(**cacheoptions) + self.assertEqual(n, 11) def test_n_linear_parameter(self): - n = self.concat_model[0].get_n_parameters(linear=True) - self.assertEqual(n, 2) + for cacheoptions in self._parameterize_linear_test(): + n = self.concat_model[0].get_n_parameters(**cacheoptions) + self.assertEqual(n, 2) - n = self.concat_model[-1].get_n_parameters(linear=True) - self.assertEqual(n, 3) + n = self.concat_model[-1].get_n_parameters(**cacheoptions) + self.assertEqual(n, 3) - n = self.concat_model.get_n_parameters(linear=True) - self.assertEqual(n, 3) + n = self.concat_model.get_n_parameters(**cacheoptions) + self.assertEqual(n, 3) - self.concat_model._disable_property_link("var1_lin") - n = self.concat_model.get_n_parameters(linear=True) - self.assertEqual(n, 9) + with self._unlink_var1_lin(): + n = self.concat_model.get_n_parameters(**cacheoptions) + self.concat_model._enable_property_link("var1_lin") + self.assertEqual(n, 9) def test_get_parameter_values(self): - values = self.concat_model[0].get_parameter_values() - self.assertEqual(values.tolist(), [11, 11, 12, 0, 0]) - - values = self.concat_model[-1].get_parameter_values() - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - - values = self.concat_model.get_parameter_values() - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + for cacheoptions in self._parameterize_nonlinear_test(): + values = self.concat_model[0].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 12, 0, 0]) - self.concat_model._disable_property_link("var1_lin") - values = self.concat_model.get_parameter_values() - self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) - - def test_get_parameter_values_in_caching_context(self): - with self.concat_model._propertyCachingContext(): - values = self.concat_model[0].get_parameter_values() + values = self.concat_model[-1].get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - values = self.concat_model[-1].get_parameter_values() + values = self.concat_model.get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - values = self.concat_model.get_parameter_values() - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + with self._unlink_var1_lin(): + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) + def test_get_parameter_values_in_caching_context(self): + for cacheoptions in self._parameterize_nonlinear_test(): + with self.concat_model._propertyCachingContext(**cacheoptions): + values = self.concat_model[0].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + values = self.concat_model[-1].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + + with self._unlink_var1_lin(): + with self.concat_model._propertyCachingContext(**cacheoptions): + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) + + def test_get_linear_parameter_values(self): + for cacheoptions in self._parameterize_linear_test(): + values = self.concat_model[0].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 0]) + + values = self.concat_model[-1].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 41]) + + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 41]) + + with self._unlink_var1_lin(): + values = self.concat_model.get_parameter_values(**cacheoptions) + self.concat_model._enable_property_link("var1_lin") + self.assertEqual(values.tolist(), [41] + [11] * 8) + + def test_get_linear_parameter_values_in_caching_context(self): + for cacheoptions in self._parameterize_linear_test(): + with self.concat_model._propertyCachingContext(**cacheoptions): + values = self.concat_model[0].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 41]) + + values = self.concat_model[-1].get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 41]) + + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [11, 11, 41]) + + with self._unlink_var1_lin(): + with self.concat_model._propertyCachingContext(**cacheoptions): + values = self.concat_model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), [41] + [11] * 8) + + def _parameterize_linear_test(self): + for local_linear, global_linear in [[True, False], [None, True]]: + with self.subTest(local_linear=local_linear, global_linear=global_linear): + self.concat_model.linear = global_linear + yield {"linear": local_linear} + + def _parameterize_nonlinear_test(self): + for local_linear, global_linear in [[False, True], [None, False]]: + with self.subTest(local_linear=local_linear, global_linear=global_linear): + self.concat_model.linear = global_linear + yield {"linear": local_linear} + + @contextmanager + def _unlink_var1_lin(self): self.concat_model._disable_property_link("var1_lin") - with self.concat_model._propertyCachingContext(): - values = self.concat_model.get_parameter_values() - self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) + try: + yield + finally: + self.concat_model._enable_property_link("var1_lin") From dc2dc75b79d98809d1bc47abc0ba61a861910903 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 12:53:33 +0200 Subject: [PATCH 47/74] fixup --- PyMca5/PyMcaMath/fitting/ConcatModel.py | 501 -------------- PyMca5/PyMcaMath/fitting/Model.py | 397 ------------ PyMca5/PyMcaMath/fitting/ModelInterface.py | 426 ------------ .../fitting/{ => model}/CachingModel.py | 12 +- .../fitting/model/LeastSquaresFitModel.py | 612 ++++++++++++++++++ .../fitting/{ => model}/LinkedModel.py | 11 +- .../fitting/{ => model}/ParameterModel.py | 50 +- .../fitting/{ => model}/PropertyUtils.py | 9 +- PyMca5/PyMcaMath/fitting/model/__init__.py | 9 + PyMca5/tests/CachingModelTest.py | 6 +- PyMca5/tests/LinkedModelTest.py | 8 +- PyMca5/tests/ModelParameterInterfaceTest.py | 92 --- PyMca5/tests/ParameterModelTest.py | 41 +- PyMca5/tests/SimpleModel.py | 30 +- setup.py | 1 + 15 files changed, 733 insertions(+), 1472 deletions(-) delete mode 100644 PyMca5/PyMcaMath/fitting/ConcatModel.py delete mode 100644 PyMca5/PyMcaMath/fitting/Model.py delete mode 100644 PyMca5/PyMcaMath/fitting/ModelInterface.py rename PyMca5/PyMcaMath/fitting/{ => model}/CachingModel.py (96%) create mode 100644 PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py rename PyMca5/PyMcaMath/fitting/{ => model}/LinkedModel.py (96%) rename PyMca5/PyMcaMath/fitting/{ => model}/ParameterModel.py (84%) rename PyMca5/PyMcaMath/fitting/{ => model}/PropertyUtils.py (97%) create mode 100644 PyMca5/PyMcaMath/fitting/model/__init__.py delete mode 100644 PyMca5/tests/ModelParameterInterfaceTest.py diff --git a/PyMca5/PyMcaMath/fitting/ConcatModel.py b/PyMca5/PyMcaMath/fitting/ConcatModel.py deleted file mode 100644 index f968e665d..000000000 --- a/PyMca5/PyMcaMath/fitting/ConcatModel.py +++ /dev/null @@ -1,501 +0,0 @@ -import numpy -from collections.abc import Sequence, MutableMapping -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager -from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface -from PyMca5.PyMcaMath.fitting.Model import Model - - -class ConcatModel(LinkedModelManager, ModelInterface): - """Concatenated model with shared parameters""" - - def __init__(self, models, shared_attributes=None): - if not isinstance(models, Sequence): - models = [models] - for model in models: - if not isinstance(model, Model): - raise ValueError("'models' must be a list of type 'Model'") - - super().__init__(models) - - self.__fixed_shared_attributes = { - "linear", - "niter_non_leastsquares", - "_included_parameters", - "_excluded_parameters", - } - self.shared_attributes = shared_attributes - - def _iter_context_managers(self, context_name): - ctxmgr = getattr(super(), context_name, None) - if ctxmgr is not None: - yield ctxmgr - for model in self._models: - yield getattr(model, context_name) - - @property - def nmodels(self): - return len(self._models) - - @property - def shared_model(self): - """Model used to get/set shared attributes""" - return self._models[0] - - @property - def _all_other_models(self): - """All models except for `shared_model`""" - return self._models[1:] - - def __getattr__(self, name): - """Get shared attribute""" - if self.nmodels and name in self.shared_attributes: - return getattr(self.shared_model, name) - raise AttributeError(name) - - def __setattr__(self, name, value): - """Set the attributes of all models when shared""" - if ( - name != "_models" - and self.nmodels - and hasattr(self.shared_model, name) - and name in self.shared_attributes - ): - for model in self._models: - setattr(model, name, value) - else: - super().__setattr__(name, value) - - @property - def shared_attributes(self): - """Attributes shared between the fit models (they should have the same value)""" - return self._shared_attributes - - @shared_attributes.setter - def shared_attributes(self, shared_attributes): - """ - :param Sequence(str) shared_attributes: - """ - if shared_attributes is None: - shared_attributes = set() - else: - shared_attributes = set(shared_attributes) - shared_attributes |= self.__fixed_shared_attributes - if self.nmodels <= 1: - self._shared_attributes = shared_attributes - return - self.share_attributes(shared_attributes) - self.validate_shared_attributes(shared_attributes) - self._shared_attributes = shared_attributes - - def validate_shared_attributes(self, shared_attributes=None): - """Check whether attributes are shared - - :param Sequence(str) shared_attributes: - :raises AssertionError: - """ - if self.nmodels <= 1: - return - if shared_attributes is None: - shared_attributes = self._shared_attributes - for name in shared_attributes: - value = getattr(self.shared_model, name) - if isinstance(value, (Sequence, MutableMapping, numpy.ndarray)): - for model in self._all_other_models: - assert id(value) == id(getattr(model, name)), name - else: - for model in self._all_other_models: - assert value == getattr(model, name), name - - def share_attributes(self, shared_attributes=None): - """Ensure attributes are shared - - :param Sequence(str) shared_attributes: - """ - if self.nmodels <= 1: - return - if shared_attributes is None: - shared_attributes = self._shared_attributes - model = self.shared_model - adict = {name: getattr(model, name) for name in shared_attributes} - for model in self._all_other_models: - for name, value in adict.items(): - try: - setattr(model, name, value) - except AttributeError: - pass # no setter - - @property - def ndata(self): - nmodels = self.nmodels - if nmodels == 0: - return 0 - else: - return sum(model.ndata for model in self._models) - - @property - def xdata(self): - return self._get_data("xdata") - - @xdata.setter - def xdata(self, values): - self._set_data("xdata", values) - - @property - def ydata(self): - return self._get_data("ydata") - - @ydata.setter - def ydata(self, values): - self._set_data("ydata", values) - - @property - def ystd(self): - return self._get_data("ystd") - - @ystd.setter - def ystd(self, values): - self._set_data("ystd", values) - - @property - def yfitdata(self): - return self._get_data("yfitdata") - - @property - def yfitstd(self): - return self._get_data("yfitstd") - - def _get_data(self, attr): - """ - :param str attr: - :returns array: - """ - nmodels = self.nmodels - if nmodels == 0: - return None - elif nmodels == 1: - return getattr(self.shared_model, attr) - elif getattr(self.shared_model, attr) is None: - return None - else: - return numpy.concatenate([getattr(model, attr) for model in self._models]) - - def _set_data(self, attr, values): - """ - :param str attr: - :param array values: - """ - if len(values) != self.ndata: - raise ValueError("Not the expected number of channels") - for idx, model in self._iter_model_data_slices(values): - setattr(model, attr, values[idx]) - - @contextmanager - def _filter_parameter_context(self, shared=True): - keepex = self._excluded_parameters # shared between all models - keepin = self._included_parameters # shared between all models - try: - if shared: - if keepin: - self._included_parameters = list( - set(keepin) - set(self.shared_attributes) - ) - else: - self._included_parameters = self.shared_attributes - else: - if keepex: - self._excluded_parameters.extend(self.shared_attributes) - else: - self._excluded_parameters = self.shared_attributes - yield - finally: - self._excluded_parameters = keepex - self._included_parameters = keepin - - def _iter_parameter_models(self): - """Yields models which are in such a state that they have - either shared or non-shared parameters enabled. - """ - with self._filter_parameter_context(shared=True): - yield self.shared_model - with self._filter_parameter_context(shared=False): - for model in self._models: - yield model - - def _iter_model_data_slices_types(self): - modeltypes = set() - for model in self._models: - modeltype = type(model) - if modeltype not in modeltypes: - modeltypes.add(modeltype) - yield model - - @property - def nshared_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.nparameters - - @property - def nshared_linear_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.nlinear_parameters - - def _get_parameters(self, linear_only=None): - """ - :param bool linear_only: - :returns array: - """ - return numpy.concatenate( - [ - model._get_parameters(linear_only=linear_only) - for model in self._iter_parameter_models() - ] - ) - - def _set_parameters(self, values, linear_only=None): - """ - :paramm array values: - :param bool linear_only: - """ - if linear_only is None: - linear_only = self.linear - i = 0 - for model in self._iter_parameter_models(): - if linear_only: - n = model.nlinear_parameters - else: - n = model.nparameters - if n: - model._set_parameters(values[i : i + n], linear_only=linear_only) - i += n - self.share_attributes() # TODO: find a better way to share parameters - - def _get_constraints(self, linear_only=None): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - return numpy.concatenate( - [ - model._get_constraints(linear_only=linear_only) - for model in self._iter_parameter_models() - ] - ) - - def _parameter_groups(self, linear_only=None): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - with self._filter_parameter_context(shared=True): - for item in self.shared_model._parameter_groups(linear_only=linear_only): - yield item - with self._filter_parameter_context(shared=False): - for i, model in enumerate(self._models): - for name, n in self.shared_model._parameter_groups( - linear_only=linear_only - ): - yield name + str(i), n - - def _parameter_model_index(self, idx, linear_only=None): - """Convert parameter index of ConcatModel to a parameter indices - of the underlying models (only one when parameter is not shared). - - :param bool linear_only: - :param int idx: - :yields (int, int): model index, parameter index in this model - """ - cache = self._getCache("fit", "parameter_model_index") - if cache is None: - yield from self._iter_parameter_index(idx, linear_only=linear_only) - return - - it = cache.get(idx) - if it is None: - it = cache[idx] = list( - self._iter_parameter_index(idx, linear_only=linear_only) - ) - yield from it - - def _iter_parameter_index(self, idx, linear_only=None): - """Convert parameter index of ConcatModel to a parameter indices - of the underlying models (only one when parameter is not shared). - - :param bool linear_only: - :param int idx: - :yields (int, int): model index, parameter index in this model - """ - if linear_only is None: - linear_only = self.linear - if linear_only: - nshared = self.nshared_linear_parameters - else: - nshared = self.nshared_parameters - shared_attributes = self.shared_attributes - if idx < nshared: - for i, model in enumerate(self._models): - iglobal = 0 - imodel = 0 - for name, n in model._parameter_groups(linear_only=linear_only): - if name in shared_attributes: - if idx >= iglobal and idx < (iglobal + n): - yield i, imodel + idx - iglobal - iglobal += n - imodel += n - else: - iglobal = nshared - for i, model in enumerate(self._models): - imodel = 0 - for name, n in model._parameter_groups(linear_only=linear_only): - if name not in shared_attributes: - if idx >= iglobal and idx < (iglobal + n): - yield i, imodel + idx - iglobal - return - iglobal += n - imodel += n - - @property - def shared_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.parameters - - @shared_parameters.setter - def shared_parameters(self, values): - with self._filter_parameter_context(shared=True): - self.shared_model.parameters = values - - @property - def shared_linear_parameters(self): - with self._filter_parameter_context(shared=True): - return self.shared_model.linear_parameters - - @shared_linear_parameters.setter - def shared_linear_parameters(self, values): - with self._filter_parameter_context(shared=True): - self.shared_model.linear_parameters = values - - def _concatenate_evaluation(self, funcname, xdata=None): - """Evaluate model - - :param array xdata: length nxdata - :returns array: nxdata - """ - if xdata is None: - xdata = self.xdata - ret = numpy.empty(len(xdata)) - for idx, model in self._iter_model_data_slices(xdata): - func = getattr(model, funcname) - ret[idx] = func(xdata=xdata[idx]) - return ret - - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) - - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) - - def evaluate_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) - - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - if xdata is None: - xdata = self.xdata - ret = numpy.empty(len(xdata)) - model_data_slices = self._model_data_slices(len(xdata)) - for model_idx, param_idx in self._parameter_model_index(param_idx): - idx = model_data_slices[model_idx] - model = self._models[model_idx] - ret[idx] = model.derivative_fitmodel(param_idx, xdata=xdata[idx]) - return ret - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - if xdata is None: - xdata = self.xdata - ret = numpy.empty((self.nlinear_parameters, len(xdata))) - for idx, model in self._iter_model_data_slices(xdata): - ret[:, idx] = model.linear_derivatives_fitmodel(xdata=xdata[idx]) - return ret - - def _iter_model_data_slices(self, xdata): - """ - :param array xdata: - :yields (slice, Model): - """ - for item in zip(self._model_data_slices(len(xdata)), self._models): - yield item - - def _model_data_slices(self, nconcat): - """Slice of each model in the concatenated data - - :param int nconcat: - :returns list(slice): - """ - cache = self._getCache("fit", "model_data_slices") - if cache is None: - return list(self._generate_model_data_slices(nconcat)) - else: - if nconcat != cache.get("nconcat"): - cache["idx"] = list(self._generate_model_data_slices(nconcat)) - cache["nconcat"] = nconcat - return cache["idx"] - - def _generate_model_data_slices(self, nconcat, stride=None): - """Yield slice of the concatenated data for each model. - The concatenated data could be sliced as `xdata[::stride]`. - """ - ndata = [model.ndata for model in self._models] - if not stride: - stride, remain = divmod(sum(ndata), nconcat) - stride += remain > 0 - start = 0 - offset = 0 - i = 0 - for n in ndata: - # Index of model in concatenated xdata due to slicing - stop = start + n - lst = list(range(start + offset, stop, stride)) - nlst = len(lst) - # Index of model in concatenated xdata after slicing - idx = slice(i, i + nlst) - i += nlst - # Prepare for next model - if lst: - offset = lst[-1] + stride - stop - else: - offset -= n - start = stop - yield idx diff --git a/PyMca5/PyMcaMath/fitting/Model.py b/PyMca5/PyMcaMath/fitting/Model.py deleted file mode 100644 index 0bcc2d6e9..000000000 --- a/PyMca5/PyMcaMath/fitting/Model.py +++ /dev/null @@ -1,397 +0,0 @@ -import numpy -from PyMca5.PyMcaMath.fitting.ModelInterface import ModelInterface -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface - - -class Model(ModelInterface, ModelParameterInterface): - """Evaluation and derivatives of a model to be used in least-squares fitting. - - Derived classes: - - * implement the ModelUserInterface. - * add parameter like a python property by using the `parameter` or - `linear_parameter` decorators instead of the `property` decortor. - - There is a "fit model" and a "full model". The full model describes the data, - the fit model describes the pre-processed data (for example smoothed, - numerical back subtracted, ...). By default the full model and the fit model - are identical. - - Fitting is done with linear least-squares optimization (not iterative) - or non-linear least-squares optimization (iterative). An outer loop of - non-least-squares optimization can be enabled (iterative). - - Example: - - .. code-block:: python - - model = MyModel() - model.xdata = xdata - model.ydata = ydata - model.ystd = ydata**0.5 - - plt.plot(model.xdata, model.ydata, label="data") - plt.plot(model.xdata, model.ymodel, label="initial") - - result = model.fit() - model.use_fit_result(result) - - plt.plot(model.xdata, model.ymodel, label="fit") - """ - - def __init__(self): - self._included_parameters = None # for ConcatModel - self._excluded_parameters = None # for ConcatModel - super().__init__() - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - allp = cls._PARAMETER_GROUP_NAMES = list() - linp = cls._LINEAR_PARAMETER_GROUP_NAMES = list() - for name in sorted(dir(cls)): # TODO: keep order if declaration? - attr = getattr(cls, name) - if isinstance(attr, parameter): - allp.append(name) - if isinstance(attr, linear_parameter): - linp.append(name) - - @property - def parameter_group_names(self): - return list(self._filter_parameter_names(self._PARAMETER_GROUP_NAMES)) - - @property - def linear_parameter_group_names(self): - return list(self._filter_parameter_names(self._LINEAR_PARAMETER_GROUP_NAMES)) - - def _filter_parameter_names(self, names): - included = self._included_parameters - excluded = self._excluded_parameters - for name in names: - if included is not None and name not in included: - continue - if excluded is not None and name in excluded: - continue - yield name - - def _get_constraints(self, linear_only=None): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - if linear_only is None: - linear_only = self.linear - if linear_only: - nparams = self.nlinear_parameters - else: - nparams = self.nparameters - codes = numpy.zeros((nparams, 3)) - for group_name, idx in self._parameter_group_indices(linear_only=linear_only): - paramprop = getattr(self.__class__, group_name) - codes[idx] = paramprop.fconstraints(self) - return codes - - def _get_parameters(self, linear_only=None): - """ - :param bool linear_only: - :returns array: - """ - cache = self._getCache("parameters") - if cache is None: - return self._get_parameters_notcached(linear_only=linear_only) - - key = self._parameters_cache_key() - parameters = cache.get(key, None) - if parameters is None: - parameters = cache[key] = self._get_parameters_notcached( - linear_only=linear_only - ) - return parameters - - def _set_parameters(self, params, linear_only=None): - """ - :param bool linear_only: - """ - cache = self._getCache("parameters") - if cache is None: - self._set_parameters_notcached(params, linear_only=linear_only) - else: - key = self._parameters_cache_key() - cache[key] = params - - def _get_parameters_notcached(self, linear_only=None): - """Helper for `_get_parameters`""" - if linear_only is None: - linear_only = self.linear - if linear_only: - nparams = self.nlinear_parameters - else: - nparams = self.nparameters - params = numpy.zeros(nparams) - for group_name, idx in self._parameter_group_indices(linear_only=linear_only): - params[idx] = getattr(self, group_name) - return params - - def _set_parameters_notcached(self, params, linear_only=None): - """Helper of `_set_parameters` - - :param bool linear_only: - """ - for group_name, idx in self._parameter_group_indices(linear_only=linear_only): - setattr(self, group_name, params[idx]) - - def _get_parameter(self, fget): - """Helper for parameter getters.""" - parameters = self._getCache("parameters") - if parameters is None: - return fget(self) - - key = self._parameters_cache_key() - parameters = parameters.get(key, None) - if parameters is None: - return fget(self) - - idx = self._parameter_group_index(fget.__name__) - if idx is None: - return fget(self) - return parameters[idx] - - def _set_parameter(self, fset, value): - """Helper for parameter setters""" - parameters = self._getCache("parameters") - if parameters is None: - return fset(self, value) - - key = self._parameters_cache_key() - parameters = parameters.get(key, None) - if parameters is None: - return fset(self, value) - - idx = self._parameter_group_index(fset.__name__) - if idx is None: - return fset(self, value) - parameters[idx] = value - - def _parameter_groups(self, linear_only=None): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - cache = self._getCache("fit", "parameter_groups") - if cache is None: - yield from self._parameter_groups_notcached(linear_only=linear_only) - return - - key = self._parameters_cache_key() - it = cache.get(key) - if it is None: - it = cache[key] = list( - self._parameter_groups_notcached(linear_only=linear_only) - ) - yield from it - - def _parameters_cache_key(self): - a = self._included_parameters - b = self._excluded_parameters - if a is not None: - a = tuple(sorted(a)) - if b is not None: - b = tuple(sorted(b)) - return a, b - - def _parameter_groups_notcached(self, linear_only=None): - """Helper for `_parameter_groups`. - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - if linear_only is None: - linear_only = self.linear - if linear_only: - names = self.linear_parameter_group_names - else: - names = self.parameter_group_names - for name in names: - paramprop = getattr(self.__class__, name) - n = paramprop.fcount(self) - if n: - yield name, n - - def _parameter_name_from_index(self, idx, linear_only=None): - """Parameter index to group name and group index - - :returns str, int: group name, index in parameter group - """ - i = 0 - for group_name, n in self._parameter_groups(linear_only=linear_only): - if idx >= i and idx < (i + n): - return group_name, idx - i - i += n - - def _parameter_group_index(self, name, linear_only=None): - """Parameter group name to index range - - :returns int or slice or None: index of parameter group in all parameters - """ - for group_name, idx in self._parameter_group_indices(linear_only=linear_only): - if name == group_name: - return idx - return None - - def _parameter_group_indices(self, linear_only=None): - """Parameter indices for each group - - :yields int or slice: index of parameter group in all parameters - """ - i = 0 - for group_name, n in self._parameter_groups(linear_only=linear_only): - if n == 1: - yield group_name, i - else: - yield group_name, slice(i, i + n) - i += n - - @property - def ndata(self): - return len(self.xdata) - - @property - def yfitdata(self): - return self._y_full_to_fit(self.ydata) - - @property - def yfitstd(self): - return self._ystd_full_to_fit(self.ystd) - - def _y_full_to_fit(self, y, xdata=None): - return y - - def _ystd_full_to_fit(self, ystd, xdata=None): - return ystd - - def _y_fit_to_full(self, y, xdata=None): - return y - - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - y = self.evaluate_fitmodel(xdata=xdata) - return self._y_fit_to_full(y, xdata=xdata) - - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - y = self.evaluate_linear_fitmodel(xdata=xdata) - return self._y_fit_to_full(y, xdata=xdata) - - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - return self.linear_parameters.dot(derivatives) - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - with self._linear_context(True): - return numpy.array( - [ - self.derivative_fitmodel(i, xdata=xdata) - for i in range(self.nlinear_parameters) - ] - ) - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - return self.numerical_derivative_fitmodel(param_idx, xdata=xdata) - - def numerical_derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - linear = self.linear - if not linear: - name, _ = self._parameter_name_from_index(param_idx) - linear = name in self._LINEAR_PARAMETER_GROUP_NAMES - - keep = parameters = self.fit_parameters - parameters = parameters.copy() - try: - if linear: - return self._numerical_derivative_linear_param(parameters, param_idx, xdata=xdata) - else: - return self._numerical_derivative_nonlinear_param(parameters, param_idx, xdata=xdata) - finally: - self.fit_parameters = keep - - def _numerical_derivative_linear_param(self, parameters, param_idx, xdata=None): - """The numerical derivative to a linear parameter is exact so - far as the calculation of the fit model itself is exact. - """ - # y(x) = p0*f0(x) + ... + pi*fi(x) + ... - # dy/dpi(x) = fi(x) - if self.linear: - # All of them are linear parameters - parameters = numpy.zeros_like(parameters) - else: - # Only some of them are linear parameters - for name, idx in self._parameter_group_indices(): - if name in self._LINEAR_PARAMETER_GROUP_NAMES: - parameters[idx] = 0 - parameters[param_idx] = 1 - self.fit_parameters = parameters - return self.evaluate_fitmodel(xdata=xdata) - - def _numerical_derivative_nonlinear_param(self, parameters, param_idx, xdata=None): - """The numerical derivative to a non-linear parameter is an approximation - """ - # Choose delta to be a small fraction of the - # parameter value but not too small, otherwise - # the derivative is zero. - p0 = parameters[param_idx] - delta = p0 * 1e-5 - if delta < 0: - delta = min(delta, -1e-12) - else: - delta = max(delta, 1e-12) - - parameters[param_idx] = p0 + delta - self.fit_parameters = parameters - f1 = self.evaluate_fitmodel(xdata=xdata) - - parameters[param_idx] = p0 - delta - self.fit_parameters = parameters - f2 = self.evaluate_fitmodel(xdata=xdata) - - return (f1 - f2) / (2.0 * delta) - - def compare_derivatives(self, xdata=None): - """Compare analytical and numerical derivatives. Useful to - validate the user defined `derivative_fitmodel`. - - :yields str, array, array: parameter name, analytical, numerical - """ - for param_idx, name in enumerate(self.fit_parameter_names): - ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata) - ynumderiv = self.numerical_derivative_fitmodel(param_idx, xdata=xdata) - yield name, ycalderiv, ynumderiv diff --git a/PyMca5/PyMcaMath/fitting/ModelInterface.py b/PyMca5/PyMcaMath/fitting/ModelInterface.py deleted file mode 100644 index e23f83872..000000000 --- a/PyMca5/PyMcaMath/fitting/ModelInterface.py +++ /dev/null @@ -1,426 +0,0 @@ -import numpy -from PyMca5.PyMcaMath.linalg import lstsq -from PyMca5.PyMcaMath.fitting import Gefit - -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface - - -class ModelUserInterface: - """The part of the interface for all fit models that needs - to be implemented by all classes derived from `Model`. - """ - - @property - def xdata(self): - raise AttributeError from NotImplementedError - - @xdata.setter - def xdata(self, value): - raise AttributeError from NotImplementedError - - @property - def ydata(self): - raise AttributeError from NotImplementedError - - @ydata.setter - def ydata(self, value): - raise AttributeError from NotImplementedError - - @property - def ystd(self): - raise AttributeError from NotImplementedError - - @ystd.setter - def ystd(self, value): - raise AttributeError from NotImplementedError - - @property - def linear(self): - raise AttributeError from NotImplementedError - - @linear.setter - def linear(self, value): - raise AttributeError from NotImplementedError - - def evaluate_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter of the fit model. - - The call is forwarded to `numerical_derivative_fitmodel` - in `Model` so it is not strictly necessary to implement - this method. Note that the numerical derivative to a - non-linear parameter is an approximation. - - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def non_leastsquares_increment(self): - raise NotImplementedError - - - - - -class ModelInterface(ModelParameterInterface, ModelUserInterface): - """Interface for all fit models (Model and ConcatModel derived classes).""" - - @property - def parameter_group_names(self): - raise AttributeError from NotImplementedError - - @property - def linear_parameter_group_names(self): - raise AttributeError from NotImplementedError - - def _parameter_groups(self, linear_only=None): - """Yield name and count of enabled parameter groups - - :param bool linear_only: - :yields str, int: group name, nb. parameters in the group - """ - raise NotImplementedError - - def _get_parameters(self, linear_only=None): - """ - :param bool linear_only: - :returns array: - """ - raise NotImplementedError - - def _get_constraints(self, linear_only=None): - """ - :param bool linear_only: - :returns array: nparams x 3 - """ - raise NotImplementedError - - def evaluate_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: nxdata - """ - raise NotImplementedError - - def evaluate_linear_fullmodel(self, xdata=None): - """Evaluate the full model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - raise NotImplementedError - - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: length nxdata - :returns array: n x nxdata - """ - raise NotImplementedError - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - raise NotImplementedError - - @property - def ndata(self): - raise AttributeError from NotImplementedError - - @property - def yfitdata(self): - raise AttributeError from NotImplementedError - - @property - def yfitstd(self): - raise AttributeError from NotImplementedError - - @property - def yfullmodel(self): - """Model of ydata""" - return self.evaluate_fullmodel() - - @property - def yfitmodel(self): - """Model of yfitdata""" - return self.evaluate_fitmodel() - - @property - def parameters(self): - return self._get_parameters(linear_only=False) - - @property - def linear_parameters(self): - return self._get_parameters(linear_only=True) - - @property - def fit_parameters(self): - """`parameters` when `linear=False` or `linear_parameters` when `linear=True`""" - return self._get_parameters() - - @property - def constraints(self): - return self._get_constraints(linear_only=False) - - @property - def linear_constraints(self): - return self._get_constraints(linear_only=True) - - @property - def fit_constraints(self): - return self._get_constraints() - - @parameters.setter - def parameters(self, values): - return self._set_parameters(values, linear_only=False) - - @linear_parameters.setter - def linear_parameters(self, values): - return self._set_parameters(values, linear_only=True) - - @fit_parameters.setter - def fit_parameters(self, values): - return self._set_parameters(values) - - @property - def nparameters(self): - return sum(n for _, n in self._parameter_groups(linear_only=False)) - - @property - def nlinear_parameters(self): - return sum(n for _, n in self._parameter_groups(linear_only=True)) - - @property - def nfit_parameters(self): - return sum(n for _, n in self._parameter_groups()) - - @property - def parameter_names(self): - return list(self._iter_parameter_names(linear_only=False)) - - @property - def linear_parameter_names(self): - return list(self._iter_parameter_names(linear_only=True)) - - @property - def fit_parameter_names(self): - return list(self._iter_parameter_names()) - - def _iter_parameter_names(self, linear_only=None): - for group_name, n in self._parameter_groups(linear_only=linear_only): - if n > 1: - for i in range(n): - yield group_name + str(i) - else: - yield group_name - - def fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - if self.linear: - return self.linear_fit(full_output=full_output) - else: - return self.nonlinear_fit(full_output=full_output) - - def linear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self.__linear_fit_context(): - b = self.yfitdata # ndata - for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives_fitmodel().T # ndata, nparams - result = lstsq( - A, - b.copy(), - uncertainties=True, - covariances=False, - digested_output=True, - ) - if self.niter_non_leastsquares: - self.linear_parameters = result["parameters"] - self.non_leastsquares_increment() - result["linear"] = True - result["parameters"] = self._fit_to_linear_parameters(result["parameters"]) - result["uncertainties"] = self._fit_to_linear_uncertainties( - result["uncertainties"] - ) - result.pop("svd") - return result - - def nonlinear_fit(self, full_output=False): - """ - :param bool full_output: add statistics to fitted parameters - :returns dict: - """ - with self._nonlinear_fit_context(): - constraints = self.constraints.T - xdata = self.xdata - ydata = self.yfitdata - ystd = self.yfitstd - for i in range(max(self.niter_non_leastsquares, 1)): - result = Gefit.LeastSquaresFit( - self._gefit_evaluate_fitmodel, - self.parameters, - model_deriv=self._gefit_derivative_fitmodel, - xdata=xdata, - ydata=ydata, - sigmadata=ystd, - constrains=constraints, - maxiter=self.maxiter, - weightflag=self.weightflag, - deltachi=self.deltachi, - fulloutput=full_output, - ) - if self.niter_non_leastsquares: - self.parameters = result[0] - self.non_leastsquares_increment() - ret = { - "linear": False, - "parameters": self._fit_to_parameters(result[0]), - "uncertainties": self._fit_to_uncertainties(result[2]), - "chi2_red": result[1], - } - if full_output: - ret["niter"] = result[3] - ret["lastdeltachi"] = result[4] - return ret - - @property - def maxiter(self): - return 100 - - @property - def deltachi(self): - return None - - @property - def weightflag(self): - return 0 - - @property - def niter_non_leastsquares(self): - return 0 - - @contextmanager - def __linear_fit_context(self): - with ExitStack() as stack: - ctx = self._cachingContext("fit") - stack.enter_context(ctx) - ctx = self._linear_context(True) - stack.enter_context(ctx) - ctx = self._cachingContext("parameters") - stack.enter_context(ctx) - ctx = self._linear_fit_context() - yield - - @contextmanager - def __nonlinear_fit_context(self): - with ExitStack() as stack: - ctx = self._cachingContext("fit") - stack.enter_context(ctx) - ctx = self._linear_context(False) - stack.enter_context(ctx) - ctx = self._cachingContext("parameters") - stack.enter_context(ctx) - ctx = self._nonlinear_fit_context() - yield - - @contextmanager - def _linear_fit_context(self): - """To allow derived classes to add context""" - yield - - @contextmanager - def _nonlinear_fit_context(self): - """To allow derived classes to add context""" - yield - - @contextmanager - def _linear_context(self, linear): - keep = self.linear - self.linear = linear - try: - yield - finally: - self.linear = keep - - def _gefit_evaluate_fitmodel(self, parameters, xdata): - """Update parameters and evaluate model - - :param array parameters: length nparams - :param array xdata: length nxdata - :returns array: nxdata - """ - self.parameters = parameters - return self.evaluate_fitmodel(xdata=xdata) - - def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): - """Update parameters and return derivate to a specific parameter - - :param array parameters: length nparams - :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata - """ - self.parameters = parameters - return self.derivative_fitmodel(param_idx, xdata=xdata) - - def use_fit_result(self, result): - """ - :param dict result: - """ - if result["linear"]: - self.linear_parameters = result["parameters"] - else: - self.parameters = result["parameters"] - - @contextmanager - def use_fit_result_context(self, result): - with self._linear_context(result["linear"]): - with self._cachingContext("parameters"): - self.use_fit_result(result) - yield - - def _parameters_to_fit(self, params): - return params - - def _linear_parameters_to_fit(self, params): - return params - - def _fit_to_parameters(self, params): - return params - - def _fit_to_linear_parameters(self, params): - return params - - def _fit_to_linear_uncertainties(self, uncertainties): - return uncertainties - - def _fit_to_uncertainties(self, uncertainties): - return uncertainties - - def linear_decomposition_fitmodel(self, xdata=None): - """Linear decomposition of the fit model. - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - return self.linear_parameters[:, numpy.newaxis] * derivatives diff --git a/PyMca5/PyMcaMath/fitting/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py similarity index 96% rename from PyMca5/PyMcaMath/fitting/CachingModel.py rename to PyMca5/PyMcaMath/fitting/model/CachingModel.py index fa2894b5f..34f1cab15 100644 --- a/PyMca5/PyMcaMath/fitting/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -1,6 +1,6 @@ import functools from contextlib import contextmanager -from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property +from PyMca5.PyMcaMath.fitting.model.PropertyUtils import wrapped_property class CacheManager: @@ -24,8 +24,8 @@ def _property_cache_key(self, **cacheoptions): class CachingModel(CacheManager): - """Object that manages and uses an internal cache (default) or - uses an external cache. + """Model that manages and uses an internal cache (default) + or uses an external cache. """ def __init__(self, *args, **kw): @@ -75,6 +75,10 @@ def _getCache(self, cachename, *subnames): class cached_property(wrapped_property): + """Property getter/setter may get/set from + a cache when enabled. + """ + def _wrap_getter(self, fget): fget = super()._wrap_getter(fget) @@ -95,7 +99,7 @@ def wrapper(oself, value): class CachedPropertiesModel(CachingModel): - """Object with cached properties when enabled.""" + """Model that implements cached properties""" _CACHED_PROPERTIES = tuple() diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py new file mode 100644 index 000000000..038e8c732 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -0,0 +1,612 @@ +from contextlib import contextmanager, ExitStack +import numpy +from PyMca5.PyMcaMath.linalg import lstsq +from PyMca5.PyMcaMath.fitting import Gefit + +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelBase +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModel +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelManager + + +class LeastSquaresFitModelInterface: + """All classes derived from `LeastSquaresFitModel` must implement + this interface. + """ + + @property + def xdata(self): + raise NotImplementedError + + @xdata.setter + def xdata(self, value): + raise NotImplementedError + + @property + def ydata(self): + raise NotImplementedError + + @ydata.setter + def ydata(self, value): + raise NotImplementedError + + @property + def ystd(self): + raise NotImplementedError + + @ystd.setter + def ystd(self, value): + raise NotImplementedError + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + raise NotImplementedError + + def derivative_fitmodel(self, param_idx, xdata=None, linear=None): + """Derivate to a specific parameter of the fit model. + + Only required when you want to implement analytical derivatives + of the fit model. Numerical derivatives are used by default. + + Note that the numerical derivatives for non-linear fitting + are approximations. They are exact for linear fitting. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + raise NotImplementedError + + def non_leastsquares_increment(self): + """Only required when niter_non_leastsquares > 0""" + raise NotImplementedError + + +class LeastSquaresFitModelBase(LeastSquaresFitModelInterface, ParameterModelBase): + """A parameter model with least-squares optimization""" + + @property + def ndata(self): + raise NotImplementedError + + @property + def yfitdata(self): + raise NotImplementedError + + @property + def yfitstd(self): + raise NotImplementedError + + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + raise NotImplementedError + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) + :returns array: n x ndata + """ + raise NotImplementedError + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: shape (ndata,) + :returns array: n x ndata + """ + raise NotImplementedError + + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: shape (ndata,) + :returns array: nparams x ndata + """ + raise NotImplementedError + + def linear_decomposition_fitmodel(self, xdata=None): + """Linear decomposition of the fit model. + + :param array xdata: shape (ndata,) + :returns array: nparams x ndata + """ + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + parameters = self.get_parameter_values(linear=True) + return parameters[:, numpy.newaxis] * derivatives + + @property + def yfullmodel(self): + """Model of ydata""" + return self.evaluate_fullmodel() + + @property + def yfitmodel(self): + """Model of yfitdata""" + return self.evaluate_fitmodel() + + def fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + if self.linear: + return self.linear_fit(full_output=full_output) + else: + return self.nonlinear_fit(full_output=full_output) + + def linear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + with self.__linear_fit_context(): + b = self.yfitdata + for i in range(max(self.niter_non_leastsquares, 1)): + A = self.linear_derivatives_fitmodel() + result = lstsq( + A.T, # ndata, nparams + b.copy(), # ndata + uncertainties=True, + covariances=False, + digested_output=True, + ) + if self.niter_non_leastsquares: + self.set_parameter_values(result["parameters"]) + self.non_leastsquares_increment() + result["linear"] = True + result["parameters"] = result["parameters"] + result["uncertainties"] = result["uncertainties"] + result.pop("svd") + return result + + def nonlinear_fit(self, full_output=False): + """ + :param bool full_output: add statistics to fitted parameters + :returns dict: + """ + with self.__nonlinear_fit_context(): + constraints = self.get_parameter_constraints().T + xdata = self.xdata + ydata = self.yfitdata + ystd = self.yfitstd + for i in range(max(self.niter_non_leastsquares, 1)): + parameters = self.get_parameter_values() + result = Gefit.LeastSquaresFit( + self._gefit_evaluate_fitmodel, + parameters, + model_deriv=self._gefit_derivative_fitmodel, + xdata=xdata, + ydata=ydata, + sigmadata=ystd, + constrains=constraints, + maxiter=self.maxiter, + weightflag=self.weightflag, + deltachi=self.deltachi, + fulloutput=full_output, + ) + if self.niter_non_leastsquares: + self.set_parameter_values(result[0]) + self.non_leastsquares_increment() + ret = { + "linear": False, + "parameters": result[0], + "uncertainties": result[2], + "chi2_red": result[1], + } + if full_output: + ret["niter"] = result[3] + ret["lastdeltachi"] = result[4] + return ret + + @property + def maxiter(self): + return 100 + + @property + def deltachi(self): + return None + + @property + def weightflag(self): + return 0 + + @property + def niter_non_leastsquares(self): + return 0 + + @contextmanager + def __linear_fit_context(self): + with ExitStack() as stack: + ctx = self._linear_context(True) + stack.enter_context(ctx) + ctx = self._propertyCachingContext() + stack.enter_context(ctx) + ctx = self._linear_fit_context() + stack.enter_context(ctx) + yield + + @contextmanager + def __nonlinear_fit_context(self): + with ExitStack() as stack: + ctx = self._linear_context(False) + stack.enter_context(ctx) + ctx = self._propertyCachingContext() + stack.enter_context(ctx) + ctx = self._nonlinear_fit_context() + stack.enter_context(ctx) + yield + + @contextmanager + def _linear_fit_context(self): + """To allow derived classes to add context""" + yield + + @contextmanager + def _nonlinear_fit_context(self): + """To allow derived classes to add context""" + yield + + def _gefit_evaluate_fitmodel(self, parameters, xdata): + """Update parameters and evaluate model + + :param array parameters: shape (nparams,) + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + self.set_parameter_values(parameters) + return self.evaluate_fitmodel(xdata=xdata) + + def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): + """Update parameters and return derivate to a specific parameter + + :param array parameters: shape (nparams,) + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + self.set_parameter_values(parameters) + return self.derivative_fitmodel(param_idx, xdata=xdata, linear=False) + + def use_fit_result(self, result): + """ + :param dict result: + """ + self.set_parameter_values(result["parameters"], linear=result["linear"]) + + @contextmanager + def use_fit_result_context(self, result): + """Changes the parameters only for the duration of this context + + :param dict result: + """ + with self._linear_context(result["linear"]): + with self._propertyCachingContext(): + self.use_fit_result(result) + yield + + +class LeastSquaresFitModel(LeastSquaresFitModelBase, ParameterModel): + """A least-squares parameter model which implement the fit model + + Derived classes: + + * implement the LeastSquaresFitModelInterface. + * add parameter like a python property by using the `parameter_group` or + `linear_parameter_group` decorators instead of the `property` decorator. + + There is a "fit model" and a "full model". The full model describes the data, + the fit model describes the pre-processed data (for example smoothed, + numerical back subtracted, ...). By default the full model and the fit model + are identical. + + Fitting is done with linear least-squares optimization (not iterative) + or non-linear least-squares optimization (iterative). An outer loop of + non-least-squares optimization can be enabled (iterative). + + Example: + + .. code-block:: python + + model = MyModel() + model.xdata = xdata + model.ydata = ydata + model.ystd = ydata**0.5 + + plt.plot(model.xdata, model.ydata, label="data") + plt.plot(model.xdata, model.ymodel, label="initial") + + result = model.fit() + model.use_fit_result(result) + + plt.plot(model.xdata, model.ymodel, label="fit") + """ + + @property + def ndata(self): + return len(self.xdata) + + @property + def yfitdata(self): + return self._y_full_to_fit(self.ydata) + + @property + def yfitstd(self): + return self._ystd_full_to_fit(self.ystd) + + def _y_full_to_fit(self, y, xdata=None): + return y + + def _ystd_full_to_fit(self, ystd, xdata=None): + return ystd + + def _y_fit_to_full(self, y, xdata=None): + return y + + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + y = self.evaluate_fitmodel(xdata=xdata) + return self._y_fit_to_full(y, xdata=xdata) + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + y = self.evaluate_linear_fitmodel(xdata=xdata) + return self._y_fit_to_full(y, xdata=xdata) + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + parameters = self.get_parameter_values(linear=True) + return parameters.dot(derivatives) + + def linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all linear parameters + + :param array xdata: shape (ndata,) + :returns array: shape (nparams, ndata) + """ + nparams = self.get_n_parameters(linear=True) + return numpy.array( + [ + self.derivative_fitmodel(i, xdata=xdata, linear=True) + for i in range(nparams) + ] + ) + + def derivative_fitmodel(self, param_idx, xdata=None, linear=None): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + return self.numerical_derivative_fitmodel(param_idx, xdata=xdata, linear=linear) + + def numerical_derivative_fitmodel(self, param_idx, xdata=None, linear=None): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + if linear is None: + linear = self.linear + parameters = self.get_parameter_values(linear=linear) + try: + if linear: + return self._numerical_derivative_linear_param( + parameters, param_idx, xdata=xdata + ) + else: + return self._numerical_derivative_nonlinear_param( + parameters, param_idx, xdata=xdata + ) + finally: + self.set_parameter_values(parameters, linear=linear) + + def _numerical_derivative_linear_param(self, parameters, param_idx, xdata=None): + """The numerical derivative to a linear parameter is exact so + far as the calculation of the fit model itself is exact. + """ + # y(x) = p0*f0(x) + ... + pi*fi(x) + ... + # dy/dpi(x) = fi(x) + parameters = numpy.zeros_like(parameters) + parameters[param_idx] = 1 + self.set_parameter_values(parameters) + return self.evaluate_fitmodel(xdata=xdata) + + def _numerical_derivative_nonlinear_param(self, parameters, param_idx, xdata=None): + """The numerical derivative to a non-linear parameter is an approximation""" + # Choose delta to be a small fraction of the parameter value but not too small, + # otherwise the derivative is zero. + p0 = parameters[param_idx] + delta = p0 * 1e-5 + if delta < 0: + delta = min(delta, -1e-12) + else: + delta = max(delta, 1e-12) + + parameters = parameters.copy() + parameters[param_idx] = p0 + delta + self.set_parameter_values(parameters) + f1 = self.evaluate_fitmodel(xdata=xdata) + + parameters[param_idx] = p0 - delta + self.set_parameter_values(parameters) + f2 = self.evaluate_fitmodel(xdata=xdata) + + return (f1 - f2) / (2.0 * delta) + + def compare_derivatives(self, xdata=None, linear=None): + """Compare analytical and numerical derivatives. Useful to + validate the user defined `derivative_fitmodel`. + + :yields str, array, array: parameter name, analytical, numerical + """ + for param_idx, name in enumerate(self.fit_parameter_names): + ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata, linear=linear) + ynumderiv = self.numerical_derivative_fitmodel( + param_idx, xdata=xdata, linear=linear + ) + yield name, ycalderiv, ynumderiv + + +class LeastSquaresCombinedFitModel(LeastSquaresFitModelBase, ParameterModelManager): + """A least-squares parameter model which manages models that implement the fit model""" + + @property + def ndata(self): + return sum(model.ndata for model in self.models) + + @property + def xdata(self): + return self._get_concatenated_data("xdata") + + @xdata.setter + def xdata(self, values): + self._set_concatenated_data("xdata", values) + + @property + def ydata(self): + return self._get_concatenated_data("ydata") + + @ydata.setter + def ydata(self, values): + self._set_concatenated_data("ydata", values) + + @property + def ystd(self): + return self._get_concatenated_data("ystd") + + @ystd.setter + def ystd(self, values): + self._set_concatenated_data("ystd", values) + + @property + def yfitdata(self): + return self._get_concatenated_data("yfitdata") + + @property + def yfitstd(self): + return self._get_concatenated_data("yfitstd") + + def evaluate_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) or (nmodels, ndatai) + :returns array: shape (ndata,) or (sum(ndatai),) + """ + return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) + + def evaluate_linear_fullmodel(self, xdata=None): + """Evaluate the full model. + + :param array xdata: shape (ndata,) or (nmodels, ndatai) + :returns array: shape (ndata,) or (sum(ndatai),) + """ + return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) + + def evaluate_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: shape (ndata,) or (nmodels, ndatai) + :returns array: shape (ndata,) or (sum(ndatai),) + """ + return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) + + def evaluate_linear_fitmodel(self, xdata=None): + """Evaluate the fit model. + + :param array xdata: shape (ndata,) or (nmodels, ndatai) + :returns array: shape (ndata,) or (sum(ndatai),) + """ + return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) + + def _get_concatenated_data(self, attr): + """ + :param str attr: + :returns array: + """ + if self.nmodels == 0: + return None + return numpy.concatenate([getattr(model, attr) for model in self.models]) + + def _set_concatenated_data(self, attr, values): + """ + :param str attr: + :param array values: + """ + if len(values) != self.ndata: + raise ValueError("Not the expected number of channels") + for model, values, _ in self._iter_model_data_slices(values): + setattr(model, attr, values) + + def _concatenate_evaluation(self, funcname, xdata=None): + """Evaluate model + + :param array xdata: shape (ndata,) or (nmodels, ndatai) + :returns array: shape (ndata,) or (sum(ndatai),) + """ + ret = numpy.empty(self._get_ndata(xdata)) + for model, xdata, idx in self._iter_model_data_slices(xdata): + func = getattr(model, funcname) + ret[idx] = func(xdata=xdata) + return ret + + def _get_ndata(self, data): + """ + :param array data: shape (ndata,) or (nmodels, ndatai) + """ + ndata = self.ndata + if data is None: + return ndata + elif len(data) == ndata: + return ndata + else: + if len(data) != self.nmodels: + raise ValueError(f"Expected {self.nmodels} data arrays") + return sum(len(x) for x in data) + + def _iter_model_data_slices(self, data): + """ + :param array data: shape (ndata,) or (nmodels, ndatai) + :yields tuple: model, datai, slice + """ + i = 0 + if data is None: + for model in self.models: + n = model.ndata + yield model, model.xdata, slice(i, i + n) + i += n + elif len(data) == self.ndata: + for model in self.models: + n = model.ndata + idx = slice(i, i + n) + yield model, data[idx], idx + i += n + else: + if len(data) != self.nmodels: + raise ValueError(f"Expected {self.nmodels} data arrays") + for model, xdata_model in zip(self.models, data): + n = len(xdata_model) + yield model, xdata_model, slice(i, i + n) + i += n diff --git a/PyMca5/PyMcaMath/fitting/LinkedModel.py b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py similarity index 96% rename from PyMca5/PyMcaMath/fitting/LinkedModel.py rename to PyMca5/PyMcaMath/fitting/model/LinkedModel.py index 11beacf0a..be947149d 100644 --- a/PyMca5/PyMcaMath/fitting/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py @@ -1,8 +1,7 @@ import functools from contextlib import ExitStack, contextmanager from collections.abc import Mapping -from typing import Type -from PyMca5.PyMcaMath.fitting.PropertyUtils import wrapped_property +from PyMca5.PyMcaMath.fitting.model.PropertyUtils import wrapped_property class linked_property(wrapped_property): @@ -54,8 +53,8 @@ def wrapper(self, *args, **kw): class LinkedModel: - """Every class that uses the link decorators needs - to derived from this class. + """Model with properties and context's that are linked to other + LinkedModel instances. """ def __init__(self, *args, **kw): @@ -144,9 +143,7 @@ def _filter_class_has_linked_property(instances, prop_name): class LinkedModelManager: - """Classes that manage LinkedModel objects should - derive from this class. - """ + """Model that manages linked LinkedModel objects""" def __init__(self, linked_instances=None, *args, **kw): super().__init__(*args, **kw) diff --git a/PyMca5/PyMcaMath/fitting/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py similarity index 84% rename from PyMca5/PyMcaMath/fitting/ParameterModel.py rename to PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 2456e69d2..39f919a23 100644 --- a/PyMca5/PyMcaMath/fitting/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -1,12 +1,12 @@ -import typing +from typing import Any from dataclasses import dataclass, field from contextlib import contextmanager import numpy -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager -from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property -from PyMca5.PyMcaMath.fitting.CachingModel import CachedPropertiesModel -from PyMca5.PyMcaMath.fitting.CachingModel import cached_property +from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModelManager +from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel +from PyMca5.PyMcaMath.fitting.model.CachingModel import cached_property class parameter_group(cached_property, linked_property): @@ -76,10 +76,12 @@ class ParameterGroupId: linear: bool = field(compare=False, hash=False) linked: bool = field(compare=False, hash=False) count: int = field(compare=False, hash=False) + constraints: Any = field(compare=False, hash=False) start_index: int = field(compare=False, hash=False) - index: "typing.Any" = field(compare=False, hash=False) + stop_index: int = field(compare=False, hash=False) + index: Any = field(compare=False, hash=False) property_name: str = field(compare=False, hash=False) - instance_key: "typing.Any" = field(compare=False, hash=False) + instance_key: Any = field(compare=False, hash=False) def _iter_parameter_names(self): if self.count > 1: @@ -90,6 +92,8 @@ def _iter_parameter_names(self): class ParameterModelBase(CachedPropertiesModel): + """Interface for all models that manage fit parameters""" + def __init__(self, *args, **kw): super().__init__(*args, **kw) self._linear = False @@ -138,6 +142,14 @@ def get_parameter_values(self, **paramtype): def set_parameter_values(self, values, **paramtype): self._set_property_values(values, **paramtype) + def get_parameter_constraints(self, **paramtype): + """ + :returns array: nparams x 3 + """ + return numpy.vstack( + group.constraints for group in self._iter_parameter_groups(**paramtype) + ) + def get_parameter_group_value(self, group, **paramtype): return self._get_property_value(group, **paramtype) @@ -156,8 +168,15 @@ def _iter_parameter_groups(self, **paramtype): """ yield from self._iter_cached_property_names(**paramtype) + def _group_from_parameter_index(self, param_idx, **paramtype): + for group in self._iter_parameter_groups(**paramtype): + if group.start_index <= param_idx < group.stop_index: + return group + class ParameterModel(ParameterModelBase, LinkedModel): + """Model that implements fit parameters""" + def _instance_cached_property_names(self, linear=None, linked=None, tracker=None): """ :yields ParameterGroupId: @@ -192,13 +211,16 @@ def _instance_cached_property_names(self, linear=None, linked=None, tracker=None if not count: continue + stop_index = start_index + count if count > 1: - index = slice(start_index, start_index + count) + index = slice(start_index, stop_index) elif count == 1: index = start_index else: index = None + constraints = prop.fconstraints(self) + instance_key = self._linked_instance_to_key if group_is_linked: name = property_name @@ -213,7 +235,9 @@ def _instance_cached_property_names(self, linear=None, linked=None, tracker=None instance_key=instance_key, count=count, start_index=start_index, + stop_index=stop_index, index=index, + constraints=constraints, ) if tracker is None: yield group @@ -237,7 +261,7 @@ def _set_noncached_property_value(self, group, value): setattr(self, group.property_name, value) -class Tracker: +class _IterGroupTracker: def __init__(self): self._start_index = 0 self._encountered = set() @@ -254,7 +278,9 @@ def is_new_group(self, group): return True -class ParameterModelContainer(ParameterModelBase, LinkedModelManager): +class ParameterModelManager(ParameterModelBase, LinkedModelManager): + """Model that manages linked models that implement fit parameters.""" + def __init__(self, *args, **kw): super().__init__(*args, **kw) self._enable_property_link("linear") @@ -286,7 +312,7 @@ def _instance_cached_property_names(self, **paramtype): :yields ParameterGroupId: """ # Shared parameters - tracker = Tracker() + tracker = _IterGroupTracker() start_index = 0 for model in self.models: yield from model._iter_parameter_groups( diff --git a/PyMca5/PyMcaMath/fitting/PropertyUtils.py b/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py similarity index 97% rename from PyMca5/PyMcaMath/fitting/PropertyUtils.py rename to PyMca5/PyMcaMath/fitting/model/PropertyUtils.py index fa81c337f..240fa60c4 100644 --- a/PyMca5/PyMcaMath/fitting/PropertyUtils.py +++ b/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py @@ -24,22 +24,19 @@ def __init__( ) def getter(self, fget): - """Decorator to change fget after property instantiation - """ + """Decorator to change fget after property instantiation""" if fget is not None: fget = self._wrap_getter(fget) return super().getter(fget) def setter(self, fset): - """Decorator to change fset after property instantiation - """ + """Decorator to change fset after property instantiation""" if fset is not None: fset = self._wrap_setter(fset) return super().setter(fset) def deleter(self, fdel): - """Decorator to change fdel after property instantiation - """ + """Decorator to change fdel after property instantiation""" if fdel is not None: fget = self._wrap_deleter(fdel) return super().deleter(fdel) diff --git a/PyMca5/PyMcaMath/fitting/model/__init__.py b/PyMca5/PyMcaMath/fitting/model/__init__.py new file mode 100644 index 000000000..02bab89ac --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/model/__init__.py @@ -0,0 +1,9 @@ +"""Base classes for fit models and combined fit models +""" + +from PyMca5.PyMcaMath.fitting.model.ParameterModel import parameter_group +from PyMca5.PyMcaMath.fitting.model.ParameterModel import linear_parameter_group +from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import LeastSquaresFitModel +from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import ( + LeastSquaresCombinedFitModel, +) diff --git a/PyMca5/tests/CachingModelTest.py b/PyMca5/tests/CachingModelTest.py index 5fa6078f2..5a5bfefd0 100644 --- a/PyMca5/tests/CachingModelTest.py +++ b/PyMca5/tests/CachingModelTest.py @@ -2,9 +2,9 @@ import numpy import functools from collections import Counter -from PyMca5.PyMcaMath.fitting.CachingModel import CachedPropertiesModel -from PyMca5.PyMcaMath.fitting.CachingModel import CachingModel -from PyMca5.PyMcaMath.fitting.CachingModel import cached_property +from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel +from PyMca5.PyMcaMath.fitting.model.CachingModel import CachingModel +from PyMca5.PyMcaMath.fitting.model.CachingModel import cached_property class Cached(CachedPropertiesModel): diff --git a/PyMca5/tests/LinkedModelTest.py b/PyMca5/tests/LinkedModelTest.py index 9d638cf8f..40db322ef 100644 --- a/PyMca5/tests/LinkedModelTest.py +++ b/PyMca5/tests/LinkedModelTest.py @@ -1,9 +1,9 @@ import unittest from collections import Counter -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.LinkedModel import LinkedModelManager -from PyMca5.PyMcaMath.fitting.LinkedModel import linked_contextmanager -from PyMca5.PyMcaMath.fitting.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModelManager +from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_contextmanager +from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property class ModelBase(LinkedModel): diff --git a/PyMca5/tests/ModelParameterInterfaceTest.py b/PyMca5/tests/ModelParameterInterfaceTest.py deleted file mode 100644 index dd32cda3e..000000000 --- a/PyMca5/tests/ModelParameterInterfaceTest.py +++ /dev/null @@ -1,92 +0,0 @@ -import unittest -from collections import Counter -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ModelParameterInterface -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import ( - ConcatModelParameterInterface, -) -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import parameter_group -from PyMca5.PyMcaMath.fitting.ModelParameterInterface import linear_parameter_group - - -class Model(ModelParameterInterface): - def __init__(self, cfg): - super().__init__() - self._cfg = cfg - self._shared_param = 2 - self._shared_linear_param = 3 - self._param = 4 - self._linear_param = 5 - self.reset_counters() - - def reset_counters(self): - self.get_counter = Counter() - self.set_counter = Counter() - - @parameter_group - def shared_param(self): - self.get_counter["shared_param"] += 1 - return self._cfg.get("shared_param", None) - - @shared_param.setter - def shared_param(self, value): - self.set_counter["shared_param"] += 1 - self._cfg["shared_param"] = value - - @linear_parameter_group - def shared_linear_param(self): - self.get_counter["shared_linear_param"] += 1 - return self._cfg.get("shared_linear_param", None) - - @shared_linear_param.setter - def shared_linear_param(self, value): - self.set_counter["shared_linear_param"] += 1 - self._cfg["shared_linear_param"] = value - - @parameter_group - def param(self): - self.get_counter["param"] += 1 - return self._cfg.get("param", None) - - @param.setter - def param(self, value): - self.set_counter["param"] += 1 - self._cfg["param"] = value - - @linear_parameter_group - def linear_param(self): - self.get_counter["linear_param"] += 1 - return self._cfg.get("linear_param", None) - - @linear_param.setter - def linear_param(self, value): - self.set_counter["linear_param"] += 1 - self._cfg["linear_param"] = value - - -class ConcatModel(ConcatModelParameterInterface): - def __init__(self): - cfgs = list() - for i in range(2): - off = i * 4 - cfg = { - "shared_param": off + 1, - "shared_linear_param": off + 2, - "param": off + 3, - "linear_param": off + 4, - } - cfgs.append(cfg) - super().__init__([Model(cfg) for cfg in cfgs]) - self._enable_property_link("shared_param", "shared_linear_param") - self.reset_counters() - - def reset_counters(self): - for m in self._linked_instances: - m.reset_counters() - - -class testModelParameterInterface(unittest.TestCase): - def setUp(self): - self.concat_model = ConcatModel() - - def test_parameter_names(self): - self.assertEqual(self.concat_model.get_parameter_names(), ("shared_param",)) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 1af77042b..5e8b44210 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -1,9 +1,9 @@ import unittest from contextlib import contextmanager -from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModel -from PyMca5.PyMcaMath.fitting.ParameterModel import ParameterModelContainer -from PyMca5.PyMcaMath.fitting.ParameterModel import parameter_group -from PyMca5.PyMcaMath.fitting.ParameterModel import linear_parameter_group +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModel +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelManager +from PyMca5.PyMcaMath.fitting.model.ParameterModel import parameter_group +from PyMca5.PyMcaMath.fitting.model.ParameterModel import linear_parameter_group class Model1(ParameterModel): @@ -53,7 +53,7 @@ def var2_lin(self, value): self._cfg["var2_lin"] = value -class ConcatModel(ParameterModelContainer): +class ConcatModel(ParameterModelManager): def __init__(self): cfg1a = {"var1_lin": [11, 11], "var1_nonlin": 12} cfg1b = {"var1_lin": [21, 21], "var1_nonlin": 22} @@ -262,6 +262,37 @@ def test_n_linear_parameter(self): self.concat_model._enable_property_link("var1_lin") self.assertEqual(n, 9) + def test_parameter_constraints(self): + for cacheoptions in self._parameterize_nonlinear_test(): + arr = self.concat_model[0].get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (3, 3)) + + arr = self.concat_model[-1].get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (5, 3)) + + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (5, 3)) + + with self._unlink_var1_lin(): + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (11, 3)) + + def test_linear_parameter_contraints(self): + for cacheoptions in self._parameterize_linear_test(): + arr = self.concat_model[0].get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (2, 3)) + + arr = self.concat_model[-1].get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (3, 3)) + + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (3, 3)) + + with self._unlink_var1_lin(): + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.concat_model._enable_property_link("var1_lin") + self.assertEqual(arr.shape, (9, 3)) + def test_get_parameter_values(self): for cacheoptions in self._parameterize_nonlinear_test(): values = self.concat_model[0].get_parameter_values(**cacheoptions) diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 0b5232aad..af7772b53 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -34,13 +34,13 @@ import numpy from PyMca5.PyMcaMath.fitting import SpecfitFuns -from PyMca5.PyMcaMath.fitting.Model import Model -from PyMca5.PyMcaMath.fitting.Model import ConcatModel -from PyMca5.PyMcaMath.fitting.Model import parameter -from PyMca5.PyMcaMath.fitting.Model import linear_parameter +from PyMca5.PyMcaMath.fitting.model import parameter_group +from PyMca5.PyMcaMath.fitting.model import linear_parameter_group +from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel +from PyMca5.PyMcaMath.fitting.model import LeastSquaresCombinedFitModel -class SimpleModel(Model): +class SimpleModel(LeastSquaresFitModel): """Model MCA data using a fixed list of peak positions and efficiencies""" SIGMA_TO_FWHM = 2 * numpy.sqrt(2 * numpy.log(2)) @@ -64,7 +64,7 @@ def __str__(self): self.__class__, self.npeaks, self.zero, self.gain, self.wzero, self.wgain ) - @parameter + @parameter_group def zero(self): return self.config["detector"]["zero"] @@ -72,7 +72,7 @@ def zero(self): def zero(self, value): self.config["detector"]["zero"] = value - @parameter + @parameter_group def gain(self): return self.config["detector"]["gain"] @@ -80,7 +80,7 @@ def gain(self): def gain(self, value): self.config["detector"]["gain"] = value - @parameter + @parameter_group def wzero(self): return self.config["detector"]["wzero"] @@ -88,7 +88,7 @@ def wzero(self): def wzero(self, value): self.config["detector"]["wzero"] = value - @parameter + @parameter_group def wgain(self): return self.config["detector"]["wgain"] @@ -121,7 +121,7 @@ def fwhms(self): def areas(self): return self.efficiency * self.concentrations - @linear_parameter + @linear_parameter_group def concentrations(self): return self.config["matrix"]["concentrations"] @@ -205,7 +205,7 @@ def evaluate_fitmodel(self, xdata=None): return SpecfitFuns.agauss(p, x) def derivative_fitmodel(self, param_idx, xdata=None): - """Derivate to a specific parameter + """Derivate to a specific parameter_group :param int param_idx: :param array xdata: length nxdata @@ -244,8 +244,8 @@ def derivative_fitmodel(self, param_idx, xdata=None): return y -class SimpleConcatModel(ConcatModel): +class SimpleConcatModel(LeastSquaresCombinedFitModel): def __init__(self, ndetectors=1): - models = [SimpleModel() for i in range(ndetectors)] - shared_attributes = ["concentrations", "positions"] - super().__init__(models, shared_attributes=shared_attributes) + models = {f"detector{i}":SimpleModel() for i in range(ndetectors)} + super().__init__(models) + self._enable_property_link("concentrations", "positions") diff --git a/setup.py b/setup.py index 1085aeb8c..bd236c827 100644 --- a/setup.py +++ b/setup.py @@ -176,6 +176,7 @@ def use_gui(): 'PyMca5.PyMcaMisc', 'PyMca5.PyMcaMath', 'PyMca5.PyMcaMath.fitting', + 'PyMca5.PyMcaMath.fitting.model', 'PyMca5.PyMcaMath.mva', 'PyMca5.PyMcaMath.mva.py_nnma', 'PyMca5.PyMcaGraph', 'PyMca5.PyMcaGraph.backends', From d8dbe4652e2627aae83e36e43dffbb91bab6cd10 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 13:08:01 +0200 Subject: [PATCH 48/74] fixup --- .../fitting/{ => model}/PolynomialModels.py | 20 +++++++++---------- PyMca5/tests/FitPolModelTest.py | 2 +- ...{FitModelTest.py => FitSimpleModelTest.py} | 0 3 files changed, 11 insertions(+), 11 deletions(-) rename PyMca5/PyMcaMath/fitting/{ => model}/PolynomialModels.py (90%) rename PyMca5/tests/{FitModelTest.py => FitSimpleModelTest.py} (100%) diff --git a/PyMca5/PyMcaMath/fitting/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py similarity index 90% rename from PyMca5/PyMcaMath/fitting/PolynomialModels.py rename to PyMca5/PyMcaMath/fitting/model/PolynomialModels.py index 7038c518f..a847381da 100644 --- a/PyMca5/PyMcaMath/fitting/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py @@ -32,12 +32,12 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import numpy -from PyMca5.PyMcaMath.fitting.Model import Model -from PyMca5.PyMcaMath.fitting.Model import parameter -from PyMca5.PyMcaMath.fitting.Model import linear_parameter +from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel +from PyMca5.PyMcaMath.fitting.model import parameter_group +from PyMca5.PyMcaMath.fitting.model import linear_parameter_group -class PolynomialModel(Model): +class PolynomialModel(LeastSquaresFitModel): def __init__(self, degree=0, maxiter=100): self._xdata = None self._ydata = None @@ -105,7 +105,7 @@ def maxiter(self, value): class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" - @linear_parameter + @linear_parameter_group def fitmodel_coefficients(self): return self.coefficients @@ -116,8 +116,8 @@ def fitmodel_coefficients(self, values): def evaluate_fitmodel(self, xdata=None): """Evaluate the fit model, not the full model. - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ if xdata is None: xdata = self.xdata @@ -131,8 +131,8 @@ def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ if xdata is None: xdata = self.xdata @@ -147,7 +147,7 @@ class ExponentialPolynomialModel(LinearPolynomialModel): yfit = log(y) = log(c1) + c1*x + c2*x^2 + ... """ - @linear_parameter + @linear_parameter_group def fitmodel_coefficients(self): coefficients = self.coefficients.copy() coefficients[0] = numpy.log(coefficients[0]) diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index ef9dfe194..7f7f228c4 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -33,7 +33,7 @@ import unittest import numpy -from PyMca5.PyMcaMath.fitting import PolynomialModels +from PyMca5.PyMcaMath.fitting.model import PolynomialModels class testFitPolModel(unittest.TestCase): diff --git a/PyMca5/tests/FitModelTest.py b/PyMca5/tests/FitSimpleModelTest.py similarity index 100% rename from PyMca5/tests/FitModelTest.py rename to PyMca5/tests/FitSimpleModelTest.py From adb95912e9a5220c497ed90eb729dbaf0a79e3cf Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 13:09:35 +0200 Subject: [PATCH 49/74] fixup --- PyMca5/PyMcaMath/fitting/model/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/__init__.py b/PyMca5/PyMcaMath/fitting/model/__init__.py index 02bab89ac..5e54e6598 100644 --- a/PyMca5/PyMcaMath/fitting/model/__init__.py +++ b/PyMca5/PyMcaMath/fitting/model/__init__.py @@ -1,9 +1,11 @@ """Base classes for fit models and combined fit models """ -from PyMca5.PyMcaMath.fitting.model.ParameterModel import parameter_group -from PyMca5.PyMcaMath.fitting.model.ParameterModel import linear_parameter_group -from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import LeastSquaresFitModel +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ( + parameter_group, + linear_parameter_group, +) from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import ( + LeastSquaresFitModel, LeastSquaresCombinedFitModel, ) From dbd9dbf99ca97c6791fc3e263dd9743cc2a540ce Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 13:44:42 +0200 Subject: [PATCH 50/74] fixup --- PyMca5/tests/FitSimpleModelTest.py | 342 +++++++++++------------------ PyMca5/tests/ParameterModelTest.py | 4 +- PyMca5/tests/SimpleModel.py | 2 +- 3 files changed, 135 insertions(+), 213 deletions(-) diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 5d2b3a0f1..04aecae20 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -33,7 +33,8 @@ import unittest import numpy -from PyMca5.tests import SimpleModel +from PyMca5.tests.SimpleModel import SimpleModel +from PyMca5.tests.SimpleModel import SimpleCombinedModel def with_model(nmodels): @@ -53,13 +54,95 @@ class testFitModel(unittest.TestCase): def setUp(self): self.random_state = numpy.random.RandomState(seed=100) + def testLinearFit(self): + with self._fit_model_subtests(): + self.fitmodel.linear = True + expected = self.fitmodel.get_parameter_values().copy() + self.modify_random(only_linear=True) + + result = self.fitmodel.fit() + self._assert_fit_result(result, expected) + self.assertTrue( + not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + ) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + self.assertTrue(not numpy.allclose(parameters, expected)) + + self.fitmodel.use_fit_result(result) + numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + numpy.testing.assert_allclose(parameters, expected) + + def testNonLinearFit(self): + with self._fit_model_subtests(): + self.fitmodel.linear = False + expected_nonlin = self.fitmodel.get_parameter_values(only_linear=False).copy() + expected_lin = self.fitmodel.get_parameter_values(only_linear=True).copy() + self.modify_random(only_linear=False) + + # from PyMca5.PyMcaMisc.ProfilingUtils import profile + # filename = "testNonLinearFit{}.pyprof".format(self.nmodels) + # with profile(memory=False, filename=filename): + result = self.fitmodel.fit(full_output=True) + + # TODO: non-linear parameters not precise + # self._assert_fit_result(result, expected_nonlin) + self.assertTrue( + not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + ) + parameters = self.fitmodel.get_parameter_values(only_linear=False) + self.assertTrue(not numpy.allclose(parameters, expected_nonlin)) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + self.assertTrue(not numpy.allclose(parameters, expected_lin)) + + if False: + import matplotlib.pyplot as plt + + plt.plot(self.fitmodel.ydata) + plt.plot(self.fitmodel.yfullmodel) + plt.show() + + self.fitmodel.use_fit_result(result) + if self.is_combined_model: + self.fitmodel.share_attributes() + + # TODO: non-linear parameters not precise + # numpy.testing.assert_allclose(self.fitmodel.parameters, expected_nonlin) + if False: + import matplotlib.pyplot as plt + + plt.plot(self.fitmodel.ydata) + plt.plot(self.fitmodel.yfullmodel) + plt.show() + numpy.testing.assert_allclose( + self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 + ) + numpy.testing.assert_allclose( + self.fitmodel.linear_parameters, expected_lin, rtol=1e-3 + ) + + def _assert_fit_result(self, result, expected): + p = numpy.asarray(result["parameters"]) + pstd = numpy.asarray(result["uncertainties"]) + ll = p - 3 * pstd + ul = p + 3 * pstd + self.assertTrue(all((expected >= ll) & (expected <= ul))) + + def _fit_model_subtests(self): + for nmodels in (1, 8): + with self.subTest(nmodels=nmodels): + self.create_model(nmodels=nmodels) + self.validate_model() + yield + self.validate_model() + def create_model(self, nmodels): self.nmodels = nmodels - self.is_concat = nmodels != 1 + self.is_combined_model = nmodels != 1 if nmodels == 1: - self.fitmodel = SimpleModel.SimpleModel() + self.fitmodel = SimpleModel() else: - self.fitmodel = SimpleModel.SimpleConcatModel(ndetectors=nmodels) + self.fitmodel = SimpleCombinedModel(ndetectors=nmodels) self.assertTrue(not self.fitmodel.linear) self.init_random() ydata = self.fitmodel.yfullmodel.copy() @@ -67,20 +150,20 @@ def create_model(self, nmodels): numpy.testing.assert_array_equal(self.fitmodel.ydata, ydata) numpy.testing.assert_array_equal(self.fitmodel.yfullmodel, ydata) numpy.testing.assert_allclose(self.fitmodel.yfitmodel, ydata - 10, atol=1e-12) - self.validate_model() def init_random(self, **kw): - if self.is_concat: - for model in self.fitmodel._models: - self._init_random(model, **kw) - self.fitmodel.shared_attributes = self.fitmodel.shared_attributes + self.npeaks = 10 # concentrations + self.nshapeparams = 4 # zero, gain, wzero, wgain + self.nchannels = 2048 + self.border = 0.1 # peak positions not within this border fraction + if self.is_combined_model: + for model in self.fitmodel.models: + self._init_random(model) else: - self._init_random(self.fitmodel, **kw) + self._init_random(self.fitmodel) - def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): - """Peaks close to the border will cause the nlls to fail""" - self.npeaks = npeaks # concentrations - self.nshapeparams = 4 # zero, gain, wzero, wgain + def _init_random(self, model): + nchannels = self.nchannels model.xdata_raw = numpy.arange(nchannels) model.ydata_raw = numpy.full(nchannels, numpy.nan) model.ybkg = 10 @@ -88,12 +171,16 @@ def _init_random(self, model, npeaks=10, nchannels=2048, border=0.1): model.xmax = self.random_state.randint(low=nchannels - 10, high=nchannels) model.zero = self.random_state.uniform(low=1, high=1.5) model.gain = self.random_state.uniform(low=10e-3, high=11e-3) + + # Peaks too close to the border will cause numerical checking to fail a = model.zero b = model.zero + model.gain * nchannels - border = border * (b - a) + border = self.border * (b - a) a += border b -= border + npeaks = self.npeaks model.positions = numpy.linspace(a, b, npeaks) + model.wzero = self.random_state.uniform(low=0.0, high=0.01) model.wgain = self.random_state.uniform(low=0.05, high=0.1) model.concentrations = self.random_state.uniform(low=0.5, high=1, size=npeaks) @@ -104,45 +191,46 @@ def modify_random(self, only_linear=False): self.validate_model() def _modify_random(self, only_linear=False): - porg = self.fitmodel.parameters.copy() - plinorg = self.fitmodel.linear_parameters.copy() + porg = self.fitmodel.get_parameter_values(only_linear=False).copy() + plinorg = self.fitmodel.get_parameter_values(only_linear=True).copy() if only_linear: - plin = self.fitmodel.linear_parameters + plin = self.fitmodel.get_parameter_values(only_linear=True) plin *= self.random_state.uniform(0.5, 0.8, len(plin)) - self.fitmodel.linear_parameters = plin - numpy.testing.assert_array_equal(self.fitmodel.linear_parameters, plin) - p = self.fitmodel.parameters + self.fitmodel.set_parameter_values(plin, only_linear=True) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + numpy.testing.assert_array_equal(parameters, plin) + p = self.fitmodel.get_parameter_values(only_linear=False) else: - p = self.fitmodel.parameters + p = self.fitmodel.get_parameter_values(only_linear=False) p *= self.random_state.uniform(0.95, 1, len(p)) - self.fitmodel.parameters = p - numpy.testing.assert_array_equal(self.fitmodel.parameters, p) - plin = self.fitmodel.linear_parameters - - for param_idx, param_name in enumerate(self.fitmodel.parameter_names): - if "concentration" in param_name or not only_linear: - self.assertNotEqual(p[param_idx], porg[param_idx], msg=param_name) + self.fitmodel.set_parameter_values(p, only_linear=False) + parameters = self.fitmodel.get_parameter_values(only_linear=False) + numpy.testing.assert_array_equal(parameters, p) + plin = self.fitmodel.get_parameter_values(only_linear=True) + + for group in self.fitmodel.get_parameter_groups(only_linear=False): + if only_linear and not group.linear: + self.assertEqual(p[group.index], porg[group.index], msg=group.name) else: - self.assertEqual(p[param_idx], porg[param_idx], msg=param_name) + self.assertNotEqual(p[group.index], porg[group.index], msg=group.name) - for param_idx, param_name in enumerate(self.fitmodel.linear_parameter_names): - self.assertNotEqual(plin[param_idx], plinorg[param_idx], msg=param_name) + for group in self.fitmodel.get_parameter_groups(only_linear=True): + self.assertNotEqual(plin[group.index], plinorg[group.index], msg=group.name) return p def validate_model(self): - self._validate_model(self.fitmodel, self.is_concat) - if self.is_concat: - for i, model in enumerate(self.fitmodel._models): + self._validate_model(self.fitmodel, self.is_combined_model) + if self.is_combined_model: + for model in self.fitmodel.models: self._validate_model(model, False) - self.fitmodel.share_attributes() # TODO: find a better way of sharing - self._validate_model(self.fitmodel, self.is_concat) + self._validate_model(self.fitmodel, self.is_combined_model) - def _validate_model(self, model, is_concat): - keep_parameters = model.parameters.copy() - keep_linear_parameters = model.linear_parameters.copy() + def _validate_model(self, model, is_combined_model): + keep_parameters = model.get_parameter_values(only_linear=False).copy() + keep_linear_parameters = model.get_parameter_values(only_linear=True).copy() - if not is_concat: + if not is_combined_model: # Alphabetic order expected = ["concentrations", "gain", "wgain", "wzero", "zero"] self.assertEqual(model.parameter_group_names, expected) @@ -173,7 +261,7 @@ def _validate_model(self, model, is_concat): nonlin_names = ["gain", "wgain", "wzero", "zero"] lin_names = ["concentrations" + str(i) for i in range(self.npeaks)] names = lin_names + nonlin_names - if is_concat: + if is_combined_model: model.validate_shared_attributes() self.assertEqual(model.nshared_parameters, self.npeaks) self.assertEqual(model.nshared_linear_parameters, self.npeaks) @@ -192,11 +280,11 @@ def _validate_model(self, model, is_concat): self.assertEqual(model.parameter_names, names) self.assertEqual(model.linear_parameter_names, lin_names) - if not is_concat: + if not is_combined_model: for linear in [not model.linear, model.linear]: model.linear = linear for param_name, calc, numerical in model.compare_derivatives(): - err_msg = "[linear={}] Analytical and numerical derivative of {} are not equal".format( + err_msg = "[only_linear={}] Analytical and numerical derivative of {} are not equal".format( linear, repr(param_name) ) numpy.testing.assert_allclose( @@ -206,169 +294,3 @@ def _validate_model(self, model, is_concat): numpy.testing.assert_array_equal(keep_parameters, model.parameters) numpy.testing.assert_array_equal(keep_linear_parameters, model.linear_parameters) - @with_model(1) - def testLinearFit(self): - self._testLinearFit() - - @with_model(8) - def testLinearFitConcat(self): - self._testLinearFit() - - def _testLinearFit(self): - self.fitmodel.linear = True - expected = self.fitmodel.linear_parameters.copy() - self.modify_random(only_linear=True) - - result = self.fitmodel.fit() - try: - self.assert_result(result, expected) - except Exception: - breakpoint() - raise - self.assertTrue( - not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - ) - self.assertTrue(not numpy.allclose(self.fitmodel.linear_parameters, expected)) - - self.fitmodel.use_fit_result(result) - numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - numpy.testing.assert_allclose(self.fitmodel.linear_parameters, expected) - - @with_model(1) - def testNonLinearFit(self): - self._testNonLinearFit() - - @with_model(8) - def testNonLinearFitConcat(self): - self._testNonLinearFit() - - def _testNonLinearFit(self): - self.fitmodel.linear = False - expected1 = self.fitmodel.parameters.copy() - expected2 = self.fitmodel.linear_parameters.copy() - self.modify_random(only_linear=False) - - # from PyMca5.PyMcaMisc.ProfilingUtils import profile - # filename = "testNonLinearFit{}.pyprof".format(self.nmodels) - # with profile(memory=False, filename=filename): - result = self.fitmodel.fit(full_output=True) - - # TODO: non-linear parameters not precise - # self.assert_result(result, expected1) - self.assertTrue( - not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - ) - self.assertTrue(not numpy.allclose(self.fitmodel.parameters, expected1)) - self.assertTrue(not numpy.allclose(self.fitmodel.linear_parameters, expected2)) - - if False: - import matplotlib.pyplot as plt - - plt.plot(self.fitmodel.ydata) - plt.plot(self.fitmodel.yfullmodel) - plt.show() - - self.fitmodel.use_fit_result(result) - if self.is_concat: - self.fitmodel.share_attributes() - - # TODO: non-linear parameters not precise - # numpy.testing.assert_allclose(self.fitmodel.parameters, expected1) - if True: - import matplotlib.pyplot as plt - - plt.plot(self.fitmodel.ydata) - plt.plot(self.fitmodel.yfullmodel) - plt.show() - numpy.testing.assert_allclose( - self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 - ) - numpy.testing.assert_allclose( - self.fitmodel.linear_parameters, expected2, rtol=1e-3 - ) - - def assert_result(self, result, expected): - p = numpy.asarray(result["parameters"]) - pstd = numpy.asarray(result["uncertainties"]) - ll = p - 3 * pstd - ul = p + 3 * pstd - self.assertTrue(all((expected >= ll) & (expected <= ul))) - - @with_model(8) - def testParameterIndex(self): - # Test parameter index conversion from concatenated model to single model - nmodels = self.fitmodel.nmodels - npeaks = self.npeaks - for linear in [False, True]: - self.fitmodel.linear = linear - if linear: - nshapeparams = 0 - else: - nshapeparams = self.nshapeparams - imodels = [] - iparams = [] - for param_idx, param_name in enumerate(self.fitmodel.parameter_names): - lst = list(self.fitmodel._parameter_model_index(param_idx)) - # imodel: model indices - # iparam: index of the parameter in the corresponing models - if lst: - imodel, iparam = list(zip(*lst)) - else: - imodel, iparam = tuple(), tuple() - if "concentrations" in param_name: - offset = 0 - # Shared parameters - self.assertEqual(imodel, tuple(range(nmodels))) - self.assertEqual(iparam, (param_idx - offset,) * nmodels) - else: - # Non-shared peak shape parameters - offset = npeaks - imodels.extend(imodel) - iparams.extend(i - offset for i in iparam) - self.assertEqual(len(imodels), nshapeparams * nmodels) - expected = numpy.repeat(list(range(nmodels)), nshapeparams) - self.assertEqual(imodels, expected.tolist()) - expected = numpy.tile(list(range(nshapeparams)), nmodels) - self.assertEqual(iparams, expected.tolist()) - - @with_model(8) - def testChannelIndex(self): - # Test model index in concatenated - strides = [2, 3, 100, 1000, 1100, 1200, 3000] - for stride in strides: - x = self.fitmodel.xdata - x2 = x[::stride] - access_cnt = numpy.zeros(len(x2), dtype=int) - vstride = stride - if stride < 1000: - vstride = None - for idx in self.fitmodel._generate_model_data_slices( - len(x2), stride=vstride - ): - chunk = x2[idx] - access_cnt[idx] += 1 - self.assertTrue(all(numpy.diff(chunk) == stride)) - self.assertTrue(all(access_cnt == 1)) - - -def getSuite(auto=True): - testSuite = unittest.TestSuite() - if auto: - testSuite.addTest(unittest.TestLoader().loadTestsFromTestCase(testFitModel)) - else: - # use a predefined order - testSuite.addTest(testFitModel("testParameterIndex")) - testSuite.addTest(testFitModel("testChannelIndex")) - testSuite.addTest(testFitModel("testLinearFit")) - testSuite.addTest(testFitModel("testNonLinearFit")) - testSuite.addTest(testFitModel("testLinearFitConcat")) - testSuite.addTest(testFitModel("testNonLinearFitConcat")) - return testSuite - - -def test(auto=False): - unittest.TextTestRunner(verbosity=2).run(getSuite(auto=auto)) - - -if __name__ == "__main__": - test() diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 5e8b44210..214d5fa55 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -362,13 +362,13 @@ def _parameterize_linear_test(self): for local_linear, global_linear in [[True, False], [None, True]]: with self.subTest(local_linear=local_linear, global_linear=global_linear): self.concat_model.linear = global_linear - yield {"linear": local_linear} + yield {"only_linear": local_linear} def _parameterize_nonlinear_test(self): for local_linear, global_linear in [[False, True], [None, False]]: with self.subTest(local_linear=local_linear, global_linear=global_linear): self.concat_model.linear = global_linear - yield {"linear": local_linear} + yield {"only_linear": local_linear} @contextmanager def _unlink_var1_lin(self): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index af7772b53..efe6a42de 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -244,7 +244,7 @@ def derivative_fitmodel(self, param_idx, xdata=None): return y -class SimpleConcatModel(LeastSquaresCombinedFitModel): +class SimpleCombinedModel(LeastSquaresCombinedFitModel): def __init__(self, ndetectors=1): models = {f"detector{i}":SimpleModel() for i in range(ndetectors)} super().__init__(models) From cfb2d4b3e2bd91e66c797965403d726a0db50142 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 14:07:39 +0200 Subject: [PATCH 51/74] fixup --- PyMca5/tests/FitSimpleModelTest.py | 65 ++++++++++++++++-------------- PyMca5/tests/SimpleModel.py | 13 +++--- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 04aecae20..7d56ded1f 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -32,24 +32,12 @@ __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" import unittest +from contextlib import contextmanager import numpy from PyMca5.tests.SimpleModel import SimpleModel from PyMca5.tests.SimpleModel import SimpleCombinedModel -def with_model(nmodels): - def inner1(method): - def inner2(self, *args, **kw): - self.create_model(nmodels=nmodels) - result = method(self, *args, **kw) - self.validate_model() - return result - - return inner2 - - return inner1 - - class testFitModel(unittest.TestCase): def setUp(self): self.random_state = numpy.random.RandomState(seed=100) @@ -76,7 +64,9 @@ def testLinearFit(self): def testNonLinearFit(self): with self._fit_model_subtests(): self.fitmodel.linear = False - expected_nonlin = self.fitmodel.get_parameter_values(only_linear=False).copy() + expected_nonlin = self.fitmodel.get_parameter_values( + only_linear=False + ).copy() expected_lin = self.fitmodel.get_parameter_values(only_linear=True).copy() self.modify_random(only_linear=False) @@ -128,15 +118,16 @@ def _assert_fit_result(self, result, expected): ul = p + 3 * pstd self.assertTrue(all((expected >= ll) & (expected <= ul))) + @contextmanager def _fit_model_subtests(self): for nmodels in (1, 8): with self.subTest(nmodels=nmodels): - self.create_model(nmodels=nmodels) + self._create_model(nmodels=nmodels) self.validate_model() yield self.validate_model() - def create_model(self, nmodels): + def _create_model(self, nmodels): self.nmodels = nmodels self.is_combined_model = nmodels != 1 if nmodels == 1: @@ -144,18 +135,23 @@ def create_model(self, nmodels): else: self.fitmodel = SimpleCombinedModel(ndetectors=nmodels) self.assertTrue(not self.fitmodel.linear) + self.init_random() + ydata = self.fitmodel.yfullmodel.copy() self.fitmodel.ydata = ydata numpy.testing.assert_array_equal(self.fitmodel.ydata, ydata) numpy.testing.assert_array_equal(self.fitmodel.yfullmodel, ydata) - numpy.testing.assert_allclose(self.fitmodel.yfitmodel, ydata - 10, atol=1e-12) + numpy.testing.assert_allclose( + self.fitmodel.yfitmodel, ydata - self.background, atol=1e-12 + ) def init_random(self, **kw): - self.npeaks = 10 # concentrations + self.npeaks = 13 # concentrations self.nshapeparams = 4 # zero, gain, wzero, wgain self.nchannels = 2048 self.border = 0.1 # peak positions not within this border fraction + self.background = 10 if self.is_combined_model: for model in self.fitmodel.models: self._init_random(model) @@ -166,7 +162,7 @@ def _init_random(self, model): nchannels = self.nchannels model.xdata_raw = numpy.arange(nchannels) model.ydata_raw = numpy.full(nchannels, numpy.nan) - model.ybkg = 10 + model.ybkg = self.background model.xmin = self.random_state.randint(low=0, high=10) model.xmax = self.random_state.randint(low=nchannels - 10, high=nchannels) model.zero = self.random_state.uniform(low=1, high=1.5) @@ -231,16 +227,24 @@ def _validate_model(self, model, is_combined_model): keep_linear_parameters = model.get_parameter_values(only_linear=True).copy() if not is_combined_model: - # Alphabetic order + names = model.get_parameter_group_names(only_linear=False) expected = ["concentrations", "gain", "wgain", "wzero", "zero"] - self.assertEqual(model.parameter_group_names, expected) + self.assertEqual(names, expected) + names = model.get_parameter_group_names(only_linear=True) expected = ["concentrations"] - self.assertEqual(model.linear_parameter_group_names, expected) - self.assertTrue(not model._excluded_parameters) - self.assertTrue(not model._included_parameters) - self.assertEqual(model.ndata, len(model.xdata)) - self.assertEqual(model.nparameters, len(model.parameters)) - self.assertEqual(model.nlinear_parameters, len(model.linear_parameters)) + self.assertEqual(names, expected) + + n = model.ndata + nexpected = len(model.xdata) + self.assertEqual(n, nexpected) + + n = model.get_n_parameters(only_linear=False) + nexpected = len(model.get_parameter_values(only_linear=False)) + self.assertEqual(n, nexpected) + + n = model.get_n_parameters(only_linear=True) + nexpected = len(model.get_parameter_values(only_linear=True)) + self.assertEqual(n, nexpected) arr1 = model.evaluate_fullmodel() arr2 = model.evaluate_linear_fullmodel() @@ -252,12 +256,10 @@ def _validate_model(self, model, is_combined_model): arr2 = model.evaluate_linear_fitmodel() arr3 = model.yfitmodel arr4 = sum(model.linear_decomposition_fitmodel()) - numpy.testing.assert_allclose(arr1, arr2) numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) - # Alphabetic order nonlin_names = ["gain", "wgain", "wzero", "zero"] lin_names = ["concentrations" + str(i) for i in range(self.npeaks)] names = lin_names + nonlin_names @@ -292,5 +294,6 @@ def _validate_model(self, model, is_combined_model): ) numpy.testing.assert_array_equal(keep_parameters, model.parameters) - numpy.testing.assert_array_equal(keep_linear_parameters, model.linear_parameters) - + numpy.testing.assert_array_equal( + keep_linear_parameters, model.linear_parameters + ) diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index efe6a42de..75750d109 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -31,7 +31,6 @@ __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import numpy from PyMca5.PyMcaMath.fitting import SpecfitFuns from PyMca5.PyMcaMath.fitting.model import parameter_group @@ -41,7 +40,7 @@ class SimpleModel(LeastSquaresFitModel): - """Model MCA data using a fixed list of peak positions and efficiencies""" + """Simplfied MCA model a fixed list of peak positions and efficiencies""" SIGMA_TO_FWHM = 2 * numpy.sqrt(2 * numpy.log(2)) @@ -195,8 +194,8 @@ def npeaks(self): def evaluate_fitmodel(self, xdata=None): """Evaluate model - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ if xdata is None: xdata = self.xdata @@ -208,8 +207,8 @@ def derivative_fitmodel(self, param_idx, xdata=None): """Derivate to a specific parameter_group :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ if xdata is None: xdata = self.xdata @@ -246,6 +245,6 @@ def derivative_fitmodel(self, param_idx, xdata=None): class SimpleCombinedModel(LeastSquaresCombinedFitModel): def __init__(self, ndetectors=1): - models = {f"detector{i}":SimpleModel() for i in range(ndetectors)} + models = {f"detector{i}": SimpleModel() for i in range(ndetectors)} super().__init__(models) self._enable_property_link("concentrations", "positions") From 80162f82eb00f533326914f3aaeb775fa05d2631 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 14:14:48 +0200 Subject: [PATCH 52/74] fixup --- .../PyMcaMath/fitting/model/ParameterModel.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 39f919a23..31dd200a3 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -83,7 +83,7 @@ class ParameterGroupId: property_name: str = field(compare=False, hash=False) instance_key: Any = field(compare=False, hash=False) - def _iter_parameter_names(self): + def parameter_names(self): if self.count > 1: for i in range(self.count): yield self.name + str(i) @@ -115,10 +115,10 @@ def linear_context(self, linear): finally: self.linear = keep - def _property_cache_key(self, linear=None, **paramtype): - if linear is None: - linear = self.linear - return linear + def _property_cache_key(self, only_linear=None, **paramtype): + if only_linear is None: + only_linear = self.linear + return only_linear def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) @@ -131,7 +131,7 @@ def get_parameter_names(self, **paramtype): def _iter_parameter_names(self, **paramtype): for group in self._iter_parameter_groups(**paramtype): - yield from group._iter_parameter_names() + yield from group.parameter_names() def get_n_parameters(self, **paramtype): return sum(group.count for group in self._iter_parameter_groups(**paramtype)) @@ -147,9 +147,18 @@ def get_parameter_constraints(self, **paramtype): :returns array: nparams x 3 """ return numpy.vstack( - group.constraints for group in self._iter_parameter_groups(**paramtype) + self._normalize_constraints(group.constraints) for group in self._iter_parameter_groups(**paramtype) ) + @staticmethod + def _normalize_constraints(constraints): + constraints = numpy.atleast_1d(constraints) + if constraints.ndim not in (1, 2): + raise ValueError("Parameter group constraints must be of shape (3,) or (nparams, 3)") + if constraints.shape[-1] != 3: + raise ValueError("Parameter group constraints must be of shape (3,) or (nparams, 3)") + return constraints.tolist() + def get_parameter_group_value(self, group, **paramtype): return self._get_property_value(group, **paramtype) @@ -163,7 +172,8 @@ def get_parameter_group_names(self, **paramtype): return tuple(group.name for group in self._iter_parameter_groups(**paramtype)) def _iter_parameter_groups(self, **paramtype): - """ + """This will only yield the groups with count > 0. + :yields ParameterGroupId: """ yield from self._iter_cached_property_names(**paramtype) @@ -177,7 +187,7 @@ def _group_from_parameter_index(self, param_idx, **paramtype): class ParameterModel(ParameterModelBase, LinkedModel): """Model that implements fit parameters""" - def _instance_cached_property_names(self, linear=None, linked=None, tracker=None): + def _instance_cached_property_names(self, only_linear=None, linked=None, tracker=None): """ :yields ParameterGroupId: """ @@ -201,9 +211,9 @@ def _instance_cached_property_names(self, linear=None, linked=None, tracker=None continue group_is_linear = isinstance(prop, linear_parameter_group) - if linear is None: - linear = self.linear - if linear: + if only_linear is None: + only_linear = self.linear + if only_linear: if not group_is_linear: continue From 4a05afe1c164db7d37ce408fdb006953b5ae352a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 30 Jun 2021 14:15:18 +0200 Subject: [PATCH 53/74] fixup --- PyMca5/PyMcaMath/fitting/model/ParameterModel.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 31dd200a3..1e092ec2d 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -147,16 +147,21 @@ def get_parameter_constraints(self, **paramtype): :returns array: nparams x 3 """ return numpy.vstack( - self._normalize_constraints(group.constraints) for group in self._iter_parameter_groups(**paramtype) + self._normalize_constraints(group.constraints) + for group in self._iter_parameter_groups(**paramtype) ) @staticmethod def _normalize_constraints(constraints): constraints = numpy.atleast_1d(constraints) if constraints.ndim not in (1, 2): - raise ValueError("Parameter group constraints must be of shape (3,) or (nparams, 3)") + raise ValueError( + "Parameter group constraints must be of shape (3,) or (nparams, 3)" + ) if constraints.shape[-1] != 3: - raise ValueError("Parameter group constraints must be of shape (3,) or (nparams, 3)") + raise ValueError( + "Parameter group constraints must be of shape (3,) or (nparams, 3)" + ) return constraints.tolist() def get_parameter_group_value(self, group, **paramtype): @@ -187,7 +192,9 @@ def _group_from_parameter_index(self, param_idx, **paramtype): class ParameterModel(ParameterModelBase, LinkedModel): """Model that implements fit parameters""" - def _instance_cached_property_names(self, only_linear=None, linked=None, tracker=None): + def _instance_cached_property_names( + self, only_linear=None, linked=None, tracker=None + ): """ :yields ParameterGroupId: """ From fce63f9b8e217f8d056e7d998c07eaa6dafd7671 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 6 Jul 2021 21:57:32 +0200 Subject: [PATCH 54/74] fixup --- .../PyMcaMath/fitting/model/CachingModel.py | 166 ++++++++++-------- .../fitting/model/LeastSquaresFitModel.py | 76 ++++---- PyMca5/PyMcaMath/fitting/model/LinkedModel.py | 5 +- .../PyMcaMath/fitting/model/ParameterModel.py | 75 +++++--- .../fitting/model/PolynomialModels.py | 2 +- .../PyMcaMath/fitting/model/PropertyUtils.py | 47 ++++- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 2 +- PyMca5/tests/CachingModelTest.py | 4 +- PyMca5/tests/FitSimpleModelTest.py | 103 ++++++----- PyMca5/tests/SimpleModel.py | 9 +- 10 files changed, 305 insertions(+), 184 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index 34f1cab15..93417238c 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -14,9 +14,9 @@ def _create_empty_property_values_cache(self, key, **cacheoptions): # By default the property cache is a dictionary return dict() - def _property_cache_index(self, name, **cacheoptions): - # By default the property cache index is its name - return name + def _property_index_from_id(self, propid, **cacheoptions): + # By default the property cache index is its propid + return propid def _property_cache_key(self, **cacheoptions): # By default we only manage 1 cache (None) @@ -80,22 +80,24 @@ class cached_property(wrapped_property): """ def _wrap_getter(self, fget): - fget = super()._wrap_getter(fget) - @functools.wraps(fget) - def wrapper(oself): - return oself._cached_property_fget(fget) + def wrapper(oself, nocache=False): + if nocache: + return fget(oself) + else: + return oself._cached_property_fget(fget) - return wrapper + return super()._wrap_getter(wrapper) def _wrap_setter(self, fset): - fset = super()._wrap_setter(fset) - @functools.wraps(fset) - def wrapper(oself, value): - return oself._cached_property_fset(fset, value) + def wrapper(oself, value, nocache=False): + if nocache: + return fset(oself, value) + else: + return oself._cached_property_fset(fset, value) - return wrapper + return super()._wrap_setter(wrapper) class CachedPropertiesModel(CachingModel): @@ -116,19 +118,21 @@ def _cached_property_names(cls): """All property names for this class""" return cls._CACHED_PROPERTIES - def _instance_cached_property_names(self, **cacheoptions): - """All property names for this instance and the provided options""" + def _instance_cached_property_ids(self, **cacheoptions): + """All property id's for this instance and the provided cache options""" return self._cached_property_names() - def _iter_cached_property_names(self, **cacheoptions): - """To be used when iterating over all property names + def _iter_cached_property_ids(self, **cacheoptions): + """To be used when iterating over all property id's of this instance. """ - name_to_index = self._get_property_indices_cache(**cacheoptions) - if name_to_index is None: - yield from self._instance_cached_property_names(**cacheoptions) + propid_to_index = self._get_property_mapping_cache( + "_propid_to_index", **cacheoptions + ) + if propid_to_index is None: + yield from self._instance_cached_property_ids(**cacheoptions) else: - yield from name_to_index.keys() + yield from propid_to_index.keys() @contextmanager def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): @@ -138,13 +142,13 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption yield values_cache return - with self._cachingContext("_cached_property_indices"): + with self._cachingContext("_propid"): if start_cache is None: values_cache = self._create_start_property_values_cache(**cacheoptions) else: values_cache = start_cache - with self._cachingContext("_instance_cached_property_names"): + with self._cachingContext("_property_values"): self._set_property_values_cache(values_cache, **cacheoptions) yield values_cache @@ -152,14 +156,14 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption self._persist_property_values(values_cache, **cacheoptions) def _get_property_values_cache(self, **cacheoptions): - caches = self._getCache("_instance_cached_property_names") + caches = self._getCache("_property_values") if caches is None: return None key = self._cache_manager._property_cache_key(**cacheoptions) return caches.get(key, None) def _set_property_values_cache(self, values_cache, **cacheoptions): - caches = self._getCache("_instance_cached_property_names") + caches = self._getCache("_property_values") if caches is None: return False key = self._cache_manager._property_cache_key(**cacheoptions) @@ -173,9 +177,9 @@ def _create_start_property_values_cache(self, **cacheoptions): values_cache = _cache_manager._create_empty_property_values_cache( key, **cacheoptions ) - for name in self._iter_cached_property_names(**cacheoptions): - index = self._get_property_cache_index(name, **cacheoptions) - values_cache[index] = self._get_noncached_property_value(name) + for propid in self._iter_cached_property_ids(**cacheoptions): + index = self._propid_to_index(propid, **cacheoptions) + values_cache[index] = self._get_noncached_property_value(propid) return values_cache def _get_property_values(self, **cacheoptions): @@ -189,68 +193,92 @@ def _set_property_values(self, values, **cacheoptions): """Set the cache when enabled, set instance property values when disabled""" success = self._set_property_values_cache(values, **cacheoptions) if not success: - self._persist_property_values(values) + self._persist_property_values(values, **cacheoptions) def _persist_property_values(self, values, **cacheoptions): - for name in self._iter_cached_property_names(**cacheoptions): - index = self._get_property_cache_index(name, **cacheoptions) - self._set_noncached_property_value(name, values[index]) + for propid in self._iter_cached_property_ids(**cacheoptions): + index = self._propid_to_index(propid, **cacheoptions) + self._set_noncached_property_value(propid, values[index]) - def _get_property_value(self, name, **cacheoptions): + def _get_property_value(self, propid, **cacheoptions): """Get the value from the cache or from the property""" values_cache = self._get_property_values_cache(**cacheoptions) if values_cache is None: - return self._get_noncached_property_value(name) - index = self._get_property_cache_index(name, **cacheoptions) + return self._get_noncached_property_value(propid) + index = self._propid_to_index(propid, **cacheoptions) return values_cache[index] - def _cached_property_fget(self, fget): - """Same as _get_property_value but well have the property object - instead of the name - """ - values_cache = self._get_property_values_cache() - if values_cache is None: - return fget(self) - index = self._get_property_cache_index(fget.__name__) - return values_cache[index] + def _get_noncached_property_value(self, propid): + name = self._property_name_from_id(propid) + return getattr(self, name) - def _set_property_value(self, name, value, **cacheoptions): + def _set_property_value(self, propid, value, **cacheoptions): """Set the value in the cache or the property""" values_cache = self._get_property_values_cache(**cacheoptions) if values_cache is None: - return self._set_noncached_property_value(name, value) - index = self._get_property_cache_index(name, **cacheoptions) + return self._set_noncached_property_value(propid, value) + index = self._propid_to_index(propid, **cacheoptions) values_cache[index] = value + def _set_noncached_property_value(self, propid, value): + name = self._property_name_from_id(propid) + setattr(self, name, value) + + def _cached_property_fget(self, fget): + """Same as _get_property_value but we have the property object + instead of the propid + """ + values_cache = self._get_property_values_cache() + propid = self._name_to_propid(fget.__name__) + if values_cache is None or propid is None: + return fget(self) + index = self._propid_to_index(propid) + return values_cache[index] + def _cached_property_fset(self, fset, value): - """Same as _set_property_value but well have the property object - instead of the name + """Same as _set_property_value but we have the property object + instead of the propid """ values_cache = self._get_property_values_cache() - if values_cache is None: + propid = self._name_to_propid(fset.__name__) + if values_cache is None or propid is None: return fset(self, value) - index = self._get_property_cache_index(fset.__name__) + index = self._propid_to_index(propid) values_cache[index] = value - def _get_property_cache_index(self, name, **cacheoptions): - name_to_index = self._get_property_indices_cache(**cacheoptions) - if name_to_index is None: - return self._cache_manager._property_cache_index(name, **cacheoptions) - if name in name_to_index: - return self.name_to_index[name] - index = self._cache_manager._property_cache_index(name, **cacheoptions) - self.name_to_index[name] = index - return index - - def _get_noncached_property_value(self, name): - return getattr(self, name) - - def _set_noncached_property_value(self, name, value): - setattr(self, name, value) - - def _get_property_indices_cache(self, **cacheoptions): - caches = self._getCache("_cached_property_indices") + def _get_property_mapping_cache(self, *subnames, **cacheoptions): + caches = self._getCache("_propid", *subnames) if caches is None: return None key = self._cache_manager._property_cache_key(**cacheoptions) return caches.get(key, None) + + def _propid_to_index(self, propid, **cacheoptions): + propid_to_index = self._get_property_mapping_cache( + "_propid_to_index", **cacheoptions + ) + if propid_to_index is None: + return self._cache_manager._property_index_from_id(propid, **cacheoptions) + if propid in propid_to_index: + return propid_to_index[propid] + index = self._cache_manager._property_index_from_id(propid, **cacheoptions) + propid_to_index[propid] = index + return index + + def _name_to_propid(self, property_name, **cacheoptions): + name_to_propid = self._get_property_mapping_cache( + "_name_to_propid", **cacheoptions + ) + if name_to_propid is None: + return self._property_id_from_name(property_name) + if property_name in name_to_propid: + return name_to_propid[property_name] + propid = self._property_id_from_name(property_name) + name_to_propid[property_name] = propid + return propid + + def _property_id_from_name(self, property_name): + return property_name + + def _property_name_from_id(self, propid): + return propid diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 038e8c732..765d4e558 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -45,14 +45,15 @@ def evaluate_fitmodel(self, xdata=None): """ raise NotImplementedError - def derivative_fitmodel(self, param_idx, xdata=None, linear=None): + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter of the fit model. Only required when you want to implement analytical derivatives of the fit model. Numerical derivatives are used by default. - Note that the numerical derivatives for non-linear fitting - are approximations. They are exact for linear fitting. + Note that the numerical derivatives for non-linear parameters + are approximations. They are exact with arithmetic precission + for linear parameters. :param int param_idx: :param array xdata: shape (ndata,) @@ -119,7 +120,7 @@ def linear_decomposition_fitmodel(self, xdata=None): :returns array: nparams x ndata """ derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - parameters = self.get_parameter_values(linear=True) + parameters = self.get_parameter_values(only_linear=True) return parameters[:, numpy.newaxis] * derivatives @property @@ -225,7 +226,7 @@ def niter_non_leastsquares(self): @contextmanager def __linear_fit_context(self): with ExitStack() as stack: - ctx = self._linear_context(True) + ctx = self.linear_context(True) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) @@ -236,7 +237,7 @@ def __linear_fit_context(self): @contextmanager def __nonlinear_fit_context(self): with ExitStack() as stack: - ctx = self._linear_context(False) + ctx = self.linear_context(False) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) @@ -273,13 +274,13 @@ def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): :returns array: shape (ndata,) """ self.set_parameter_values(parameters) - return self.derivative_fitmodel(param_idx, xdata=xdata, linear=False) + return self.derivative_fitmodel(param_idx, xdata=xdata) def use_fit_result(self, result): """ :param dict result: """ - self.set_parameter_values(result["parameters"], linear=result["linear"]) + self.set_parameter_values(result["parameters"], only_linear=result["linear"]) @contextmanager def use_fit_result_context(self, result): @@ -287,7 +288,7 @@ def use_fit_result_context(self, result): :param dict result: """ - with self._linear_context(result["linear"]): + with self.linear_context(result["linear"]): with self._propertyCachingContext(): self.use_fit_result(result) yield @@ -375,7 +376,7 @@ def evaluate_linear_fitmodel(self, xdata=None): :returns array: shape (ndata,) """ derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - parameters = self.get_parameter_values(linear=True) + parameters = self.get_parameter_values(only_linear=True) return parameters.dot(derivatives) def linear_derivatives_fitmodel(self, xdata=None): @@ -384,57 +385,64 @@ def linear_derivatives_fitmodel(self, xdata=None): :param array xdata: shape (ndata,) :returns array: shape (nparams, ndata) """ - nparams = self.get_n_parameters(linear=True) + nparams = self.get_n_parameters(only_linear=True) return numpy.array( [ - self.derivative_fitmodel(i, xdata=xdata, linear=True) + self.derivative_fitmodel(i, xdata=xdata, only_linear=True) for i in range(nparams) ] ) - def derivative_fitmodel(self, param_idx, xdata=None, linear=None): + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: shape (ndata,) :returns array: shape (ndata,) """ - return self.numerical_derivative_fitmodel(param_idx, xdata=xdata, linear=linear) + return self.numerical_derivative_fitmodel(param_idx, xdata=xdata, **paramtype) - def numerical_derivative_fitmodel(self, param_idx, xdata=None, linear=None): + def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: shape (ndata,) :returns array: shape (ndata,) """ - if linear is None: - linear = self.linear - parameters = self.get_parameter_values(linear=linear) + group = self._group_from_parameter_index(param_idx, **paramtype) + param_is_linear = group.linear + parameters = self.get_parameter_values(**paramtype) try: - if linear: + if param_is_linear: return self._numerical_derivative_linear_param( - parameters, param_idx, xdata=xdata + parameters, param_idx, xdata=xdata, **paramtype ) else: return self._numerical_derivative_nonlinear_param( - parameters, param_idx, xdata=xdata + parameters, param_idx, xdata=xdata, **paramtype ) finally: - self.set_parameter_values(parameters, linear=linear) + self.set_parameter_values(parameters, **paramtype) - def _numerical_derivative_linear_param(self, parameters, param_idx, xdata=None): - """The numerical derivative to a linear parameter is exact so - far as the calculation of the fit model itself is exact. + def _numerical_derivative_linear_param( + self, parameters, param_idx, xdata=None, **paramtype + ): + """The numerical derivative to a linear parameter is exact + within arithmetic precision. """ # y(x) = p0*f0(x) + ... + pi*fi(x) + ... # dy/dpi(x) = fi(x) - parameters = numpy.zeros_like(parameters) + parameters = parameters.copy() + for group in self._iter_parameter_groups(**paramtype): + if group.linear: + parameters[group.index] = 0 parameters[param_idx] = 1 - self.set_parameter_values(parameters) + self.set_parameter_values(parameters, **paramtype) return self.evaluate_fitmodel(xdata=xdata) - def _numerical_derivative_nonlinear_param(self, parameters, param_idx, xdata=None): + def _numerical_derivative_nonlinear_param( + self, parameters, param_idx, xdata=None, **paramtype + ): """The numerical derivative to a non-linear parameter is an approximation""" # Choose delta to be a small fraction of the parameter value but not too small, # otherwise the derivative is zero. @@ -447,25 +455,25 @@ def _numerical_derivative_nonlinear_param(self, parameters, param_idx, xdata=Non parameters = parameters.copy() parameters[param_idx] = p0 + delta - self.set_parameter_values(parameters) + self.set_parameter_values(parameters, **paramtype) f1 = self.evaluate_fitmodel(xdata=xdata) parameters[param_idx] = p0 - delta - self.set_parameter_values(parameters) + self.set_parameter_values(parameters, **paramtype) f2 = self.evaluate_fitmodel(xdata=xdata) return (f1 - f2) / (2.0 * delta) - def compare_derivatives(self, xdata=None, linear=None): + def compare_derivatives(self, xdata=None, **paramtype): """Compare analytical and numerical derivatives. Useful to validate the user defined `derivative_fitmodel`. :yields str, array, array: parameter name, analytical, numerical """ - for param_idx, name in enumerate(self.fit_parameter_names): - ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata, linear=linear) + for param_idx, name in enumerate(self.get_parameter_names(**paramtype)): + ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata, **paramtype) ynumderiv = self.numerical_derivative_fitmodel( - param_idx, xdata=xdata, linear=linear + param_idx, xdata=xdata, **paramtype ) yield name, ycalderiv, ynumderiv diff --git a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py index be947149d..bd4d60936 100644 --- a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py @@ -10,12 +10,11 @@ class linked_property(wrapped_property): """ def __init__(self, *args, **kw): - super().__init__(*args, **kw) self.propagate = False + super().__init__(*args, **kw) def _wrap_setter(self, fset): propname = fset.__name__ - fset = super()._wrap_setter(fset) @functools.wraps(fset) def wrapper(oself, value): @@ -27,7 +26,7 @@ def wrapper(oself, value): ): setattr(instance, propname, value) - return wrapper + return super()._wrap_setter(wrapper) def linked_contextmanager(method): diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 1e092ec2d..5fbc4d785 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -38,9 +38,9 @@ def myparam(self): """ def __init__(self, *args, **kw): - super().__init__(*args, **kw) self.fcount = self._fcount_default() self.fconstraints = self._fconstraints_default() + super().__init__(*args, **kw) def counter(self, fcount): self.fcount = fcount @@ -52,8 +52,9 @@ def constraints(self, fconstraints): def _fcount_default(self): def fcount(oself): + values = self.fget(oself, nocache=True) try: - return len(self.fget(oself)) + return len(values) except TypeError: return 1 @@ -73,15 +74,15 @@ class linear_parameter_group(parameter_group): @dataclass(frozen=True, eq=True) class ParameterGroupId: name: str + property_name: str = field(compare=False, hash=False) linear: bool = field(compare=False, hash=False) linked: bool = field(compare=False, hash=False) count: int = field(compare=False, hash=False) - constraints: Any = field(compare=False, hash=False) start_index: int = field(compare=False, hash=False) stop_index: int = field(compare=False, hash=False) index: Any = field(compare=False, hash=False) - property_name: str = field(compare=False, hash=False) instance_key: Any = field(compare=False, hash=False) + constraints: Any = field(compare=False, hash=False, repr=False) def parameter_names(self): if self.count > 1: @@ -90,6 +91,14 @@ def parameter_names(self): elif self.count == 1: yield self.name + def contains_parameter_index(self, param_idx): + return self.start_index <= param_idx < self.stop_index + + def parameter_index_in_group(self, param_idx): + if self.contains_parameter_index(param_idx): + return param_idx - self.start_index + return None + class ParameterModelBase(CachedPropertiesModel): """Interface for all models that manage fit parameters""" @@ -98,6 +107,9 @@ def __init__(self, *args, **kw): super().__init__(*args, **kw) self._linear = False + for prop in self._cached_property_names(): + prop.count + @property def linear(self): return self._linear @@ -123,9 +135,12 @@ def _property_cache_key(self, only_linear=None, **paramtype): def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) - def _property_cache_index(self, group, **paramtype): + def _property_index_from_id(self, group, **cacheoptions): return group.index + def _property_name_from_id(self, group): + return group.property_name + def get_parameter_names(self, **paramtype): return tuple(self._iter_parameter_names(**paramtype)) @@ -147,8 +162,10 @@ def get_parameter_constraints(self, **paramtype): :returns array: nparams x 3 """ return numpy.vstack( - self._normalize_constraints(group.constraints) - for group in self._iter_parameter_groups(**paramtype) + tuple( + self._normalize_constraints(group.constraints()) + for group in self._iter_parameter_groups(**paramtype) + ) ) @staticmethod @@ -173,6 +190,12 @@ def set_parameter_group_value(self, group, value, **paramtype): def get_parameter_groups(self, **paramtype): return tuple(self._iter_parameter_groups(**paramtype)) + def _property_id_from_name(self, property_name): + for group in self._iter_parameter_groups(): + if group.property_name == property_name: + return group + return None + def get_parameter_group_names(self, **paramtype): return tuple(group.name for group in self._iter_parameter_groups(**paramtype)) @@ -181,7 +204,7 @@ def _iter_parameter_groups(self, **paramtype): :yields ParameterGroupId: """ - yield from self._iter_cached_property_names(**paramtype) + yield from self._iter_cached_property_ids(**paramtype) def _group_from_parameter_index(self, param_idx, **paramtype): for group in self._iter_parameter_groups(**paramtype): @@ -192,10 +215,23 @@ def _group_from_parameter_index(self, param_idx, **paramtype): class ParameterModel(ParameterModelBase, LinkedModel): """Model that implements fit parameters""" - def _instance_cached_property_names( + def _iter_parameter_group_properties(self): + cls = type(self) + for property_name in self._cached_property_names(): + prop = getattr(cls, property_name) + if not isinstance(prop, parameter_group): + raise TypeError( + "Currently only 'parameter_group' properties support caching" + ) + yield property_name, prop + + def _instance_cached_property_ids( self, only_linear=None, linked=None, tracker=None ): """ + :param only_linear bool: all parameters (linear and non-linear) or only linear parameters + :param linked bool: linked parameters or unlinked parameters + :param tracker _IterGroupTracker: :yields ParameterGroupId: """ count = None @@ -204,14 +240,7 @@ def _instance_cached_property_names( start_index = 0 else: start_index = tracker.start_index - cls = type(self) - for property_name in self._cached_property_names(): - prop = getattr(cls, property_name) - if not isinstance(prop, parameter_group): - raise TypeError( - "Currently only 'parameter_group' properties support caching" - ) - + for property_name, prop in self._iter_parameter_group_properties(): group_is_linked = prop.propagate if linked is not None: if group_is_linked is not linked: @@ -220,9 +249,8 @@ def _instance_cached_property_names( group_is_linear = isinstance(prop, linear_parameter_group) if only_linear is None: only_linear = self.linear - if only_linear: - if not group_is_linear: - continue + if only_linear and not group_is_linear: + continue count = prop.fcount(self) if not count: @@ -236,10 +264,11 @@ def _instance_cached_property_names( else: index = None - constraints = prop.fconstraints(self) + def constraints(): + return prop.fconstraints(self) instance_key = self._linked_instance_to_key - if group_is_linked: + if group_is_linked or instance_key is None: name = property_name else: name = f"{instance_key}:{property_name}" @@ -324,7 +353,7 @@ def linear(self): def linear(self, value): self._set_linked_property_value("linear", value) - def _instance_cached_property_names(self, **paramtype): + def _instance_cached_property_ids(self, **paramtype): """ :yields ParameterGroupId: """ diff --git a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py index a847381da..5a5bd3d92 100644 --- a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py @@ -127,7 +127,7 @@ def evaluate_fitmodel(self, xdata=None): y += coeff[i] * (xdata ** i) return y - def derivative_fitmodel(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter :param int param_idx: diff --git a/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py b/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py index 240fa60c4..00778c2de 100644 --- a/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py +++ b/PyMca5/PyMcaMath/fitting/model/PropertyUtils.py @@ -1,4 +1,4 @@ -class wrapped_property(property): +class _wrapped_property(property): """Property that prepares fget, fset and fdel wrappers for derived property classes. """ @@ -38,7 +38,7 @@ def setter(self, fset): def deleter(self, fdel): """Decorator to change fdel after property instantiation""" if fdel is not None: - fget = self._wrap_deleter(fdel) + fdel = self._wrap_deleter(fdel) return super().deleter(fdel) def _wrap_getter(self, fget): @@ -52,3 +52,46 @@ def _wrap_setter(self, fset): def _wrap_deleter(self, fdel): """Intended for derived property classes""" return fdel + + +class wrapped_property: + """Can be used like python's builtin property but with + getter and setter hooks for derived classes. + """ + + def __init__(self, fget): + self.fget = self._wrap_getter(fget) + self.fset = None + self.attrname = None + + def __set_name__(self, owner, name): + if self.attrname is None: + self.attrname = name + else: + raise TypeError(f"Cannot use the same {type(self).__name__} twice") + + def __get__(self, instance, owner=None): + if instance is None: + return self + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError( + f"{type(instance).__name__}.{self.attrname} has no setter" + ) + if instance is None: + return self + return self.fset(instance, value) + + def setter(self, fset): + self.fset = self._wrap_setter(fset) + return self + + def _wrap_getter(self, fget): + """Intended for derived property classes""" + return fget + + def _wrap_setter(self, fset): + """Intended for derived property classes""" + return fset diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 21e3d34ee..c7d6cd243 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -2027,7 +2027,7 @@ def _keep_parameters(self): finally: self.parameters = keep - def derivative_fitmodel(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter :param int param_idx: diff --git a/PyMca5/tests/CachingModelTest.py b/PyMca5/tests/CachingModelTest.py index 5a5bfefd0..44f14ea13 100644 --- a/PyMca5/tests/CachingModelTest.py +++ b/PyMca5/tests/CachingModelTest.py @@ -37,7 +37,7 @@ def var2(self, value): self.set_counter["var2"] += 1 self._cfg["var2"] = value - def _property_cache_index(self, name): + def _property_index_from_id(self, name): if name == "var1": return 0 else: @@ -48,7 +48,7 @@ def _create_empty_property_values_cache(self, key, **_): class ExternalCached(CachingModel): - def _property_cache_index(self, name): + def _property_index_from_id(self, name): if name == "var1": return 2 elif name == "var2": diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 7d56ded1f..b191a93cd 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -48,7 +48,8 @@ def testLinearFit(self): expected = self.fitmodel.get_parameter_values().copy() self.modify_random(only_linear=True) - result = self.fitmodel.fit() + with self._profile("test"): + result = self.fitmodel.fit() self._assert_fit_result(result, expected) self.assertTrue( not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) @@ -70,9 +71,6 @@ def testNonLinearFit(self): expected_lin = self.fitmodel.get_parameter_values(only_linear=True).copy() self.modify_random(only_linear=False) - # from PyMca5.PyMcaMisc.ProfilingUtils import profile - # filename = "testNonLinearFit{}.pyprof".format(self.nmodels) - # with profile(memory=False, filename=filename): result = self.fitmodel.fit(full_output=True) # TODO: non-linear parameters not precise @@ -85,25 +83,10 @@ def testNonLinearFit(self): parameters = self.fitmodel.get_parameter_values(only_linear=True) self.assertTrue(not numpy.allclose(parameters, expected_lin)) - if False: - import matplotlib.pyplot as plt - - plt.plot(self.fitmodel.ydata) - plt.plot(self.fitmodel.yfullmodel) - plt.show() - self.fitmodel.use_fit_result(result) - if self.is_combined_model: - self.fitmodel.share_attributes() # TODO: non-linear parameters not precise # numpy.testing.assert_allclose(self.fitmodel.parameters, expected_nonlin) - if False: - import matplotlib.pyplot as plt - - plt.plot(self.fitmodel.ydata) - plt.plot(self.fitmodel.yfullmodel) - plt.show() numpy.testing.assert_allclose( self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 ) @@ -120,7 +103,7 @@ def _assert_fit_result(self, result, expected): @contextmanager def _fit_model_subtests(self): - for nmodels in (1, 8): + for nmodels in (1,): with self.subTest(nmodels=nmodels): self._create_model(nmodels=nmodels) self.validate_model() @@ -205,17 +188,31 @@ def _modify_random(self, only_linear=False): plin = self.fitmodel.get_parameter_values(only_linear=True) for group in self.fitmodel.get_parameter_groups(only_linear=False): + current = p[group.index] + expected = porg[group.index] if only_linear and not group.linear: - self.assertEqual(p[group.index], porg[group.index], msg=group.name) + if group.count == 1: + self.assertEqual(current, expected, msg=group.name) + else: + self.assertTrue(all(current == expected), msg=group.name) else: - self.assertNotEqual(p[group.index], porg[group.index], msg=group.name) + if group.count == 1: + self.assertNotEqual(current, expected, msg=group.name) + else: + self.assertFalse(all(current == expected), msg=group.name) for group in self.fitmodel.get_parameter_groups(only_linear=True): - self.assertNotEqual(plin[group.index], plinorg[group.index], msg=group.name) + current = plin[group.index] + expected = plinorg[group.index] + if group.count == 1: + self.assertNotEqual(current, expected, msg=group.name) + else: + self.assertFalse(all(current == expected), msg=group.name) return p def validate_model(self): + return self._validate_model(self.fitmodel, self.is_combined_model) if self.is_combined_model: for model in self.fitmodel.models: @@ -228,11 +225,11 @@ def _validate_model(self, model, is_combined_model): if not is_combined_model: names = model.get_parameter_group_names(only_linear=False) - expected = ["concentrations", "gain", "wgain", "wzero", "zero"] - self.assertEqual(names, expected) + expected = {"concentrations", "zero", "gain", "wzero", "wgain"} + self.assertEqual(set(names), expected) names = model.get_parameter_group_names(only_linear=True) - expected = ["concentrations"] - self.assertEqual(names, expected) + expected = {"concentrations"} + self.assertEqual(set(names), expected) n = model.ndata nexpected = len(model.xdata) @@ -260,27 +257,26 @@ def _validate_model(self, model, is_combined_model): numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) - nonlin_names = ["gain", "wgain", "wzero", "zero"] - lin_names = ["concentrations" + str(i) for i in range(self.npeaks)] - names = lin_names + nonlin_names + nonlin_names = {"gain", "wgain", "wzero", "zero"} + lin_names = {"concentrations" + str(i) for i in range(self.npeaks)} + names = lin_names | nonlin_names if is_combined_model: - model.validate_shared_attributes() - self.assertEqual(model.nshared_parameters, self.npeaks) - self.assertEqual(model.nshared_linear_parameters, self.npeaks) nmodels = model.nmodels names = lin_names + [ name + str(i) for i in range(nmodels) for name in nonlin_names ] - n = self.npeaks + self.nshapeparams * nmodels - self.assertEqual(model.nparameters, n) - self.assertEqual(model.nlinear_parameters, self.npeaks) - self.assertEqual(model.parameter_names, names) - self.assertEqual(model.linear_parameter_names, lin_names) + nexpected = self.npeaks + self.nshapeparams * nmodels else: - self.assertEqual(model.nparameters, self.npeaks + self.nshapeparams) - self.assertEqual(model.nlinear_parameters, self.npeaks) - self.assertEqual(model.parameter_names, names) - self.assertEqual(model.linear_parameter_names, lin_names) + nexpected = self.npeaks + self.nshapeparams + + n = model.get_n_parameters(only_linear=False) + self.assertEqual(n, nexpected) + n = model.get_n_parameters(only_linear=True) + self.assertEqual(n, self.npeaks) + mnames = model.get_parameter_names(only_linear=False) + self.assertEqual(set(mnames), names) + mnames = model.get_parameter_names(only_linear=True) + self.assertEqual(set(mnames), lin_names) if not is_combined_model: for linear in [not model.linear, model.linear]: @@ -293,7 +289,22 @@ def _validate_model(self, model, is_combined_model): calc, numerical, err_msg=err_msg, rtol=1e-3, atol=1e-6 ) - numpy.testing.assert_array_equal(keep_parameters, model.parameters) - numpy.testing.assert_array_equal( - keep_linear_parameters, model.linear_parameters - ) + parameters = model.get_parameter_values(only_linear=False) + numpy.testing.assert_array_equal(keep_parameters, parameters) + parameters = model.get_parameter_values(only_linear=True) + numpy.testing.assert_array_equal(keep_linear_parameters, parameters) + + def _vis(self, a, b): + import matplotlib.pyplot as plt + + plt.plot(a) + plt.plot(b) + plt.show() + + @contextmanager + def _profile(self, name): + from PyMca5.PyMcaMisc.ProfilingUtils import profile + + filename = f"{name}.pyprof" + with profile(memory=False, filename=filename): + yield diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 75750d109..0ad3307bb 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -37,6 +37,7 @@ from PyMca5.PyMcaMath.fitting.model import linear_parameter_group from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel from PyMca5.PyMcaMath.fitting.model import LeastSquaresCombinedFitModel +from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property class SimpleModel(LeastSquaresFitModel): @@ -104,7 +105,7 @@ def efficiency(self, value): arr = self.config["matrix"]["efficiency"] self.config["matrix"]["efficiency"] = value - @property + @linked_property def positions(self): return self.config["matrix"]["positions"] @@ -203,7 +204,7 @@ def evaluate_fitmodel(self, xdata=None): p = list(zip(self.areas, self.positions, self.fwhms)) return SpecfitFuns.agauss(p, x) - def derivative_fitmodel(self, param_idx, xdata=None): + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter_group :param int param_idx: @@ -214,8 +215,10 @@ def derivative_fitmodel(self, param_idx, xdata=None): xdata = self.xdata x = self.zero + self.gain * xdata - name, i = self._parameter_name_from_index(param_idx) + group = self._group_from_parameter_index(param_idx, **paramtype) + name = group.property_name if name == "concentrations": + i = group.parameter_index_in_group(param_idx) p = self.positions[i] a = self.efficiency[i] w = self.wzero + self.wgain * p From 8279d4bc3970b90464f2d6623107095b831eb189 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 6 Jul 2021 22:19:32 +0200 Subject: [PATCH 55/74] fixup --- .../PyMcaMath/fitting/model/ParameterModel.py | 23 ++++++++----------- PyMca5/tests/FitSimpleModelTest.py | 3 +-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 5fbc4d785..763b7a895 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -82,7 +82,7 @@ class ParameterGroupId: stop_index: int = field(compare=False, hash=False) index: Any = field(compare=False, hash=False) instance_key: Any = field(compare=False, hash=False) - constraints: Any = field(compare=False, hash=False, repr=False) + get_constraints: Any = field(compare=False, hash=False, repr=False) def parameter_names(self): if self.count > 1: @@ -103,20 +103,13 @@ def parameter_index_in_group(self, param_idx): class ParameterModelBase(CachedPropertiesModel): """Interface for all models that manage fit parameters""" - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self._linear = False - - for prop in self._cached_property_names(): - prop.count - @property def linear(self): - return self._linear + raise NotImplementedError @linear.setter def linear(self, value): - self._linear = value + raise NotImplementedError @contextmanager def linear_context(self, linear): @@ -163,7 +156,7 @@ def get_parameter_constraints(self, **paramtype): """ return numpy.vstack( tuple( - self._normalize_constraints(group.constraints()) + self._normalize_constraints(group.get_constraints()) for group in self._iter_parameter_groups(**paramtype) ) ) @@ -215,6 +208,10 @@ def _group_from_parameter_index(self, param_idx, **paramtype): class ParameterModel(ParameterModelBase, LinkedModel): """Model that implements fit parameters""" + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self._linear= False + def _iter_parameter_group_properties(self): cls = type(self) for property_name in self._cached_property_names(): @@ -264,7 +261,7 @@ def _instance_cached_property_ids( else: index = None - def constraints(): + def get_constraints(): return prop.fconstraints(self) instance_key = self._linked_instance_to_key @@ -283,7 +280,7 @@ def constraints(): start_index=start_index, stop_index=stop_index, index=index, - constraints=constraints, + get_constraints=get_constraints, ) if tracker is None: yield group diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index b191a93cd..c35961d28 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -84,9 +84,9 @@ def testNonLinearFit(self): self.assertTrue(not numpy.allclose(parameters, expected_lin)) self.fitmodel.use_fit_result(result) - # TODO: non-linear parameters not precise # numpy.testing.assert_allclose(self.fitmodel.parameters, expected_nonlin) + self._vis(self.fitmodel.ydata, self.fitmodel.yfullmodel) numpy.testing.assert_allclose( self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 ) @@ -212,7 +212,6 @@ def _modify_random(self, only_linear=False): return p def validate_model(self): - return self._validate_model(self.fitmodel, self.is_combined_model) if self.is_combined_model: for model in self.fitmodel.models: From 5fbb37cfc4e687733d247d6a39759da80bbffbf8 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 7 Jul 2021 09:49:19 +0200 Subject: [PATCH 56/74] fixup --- PyMca5/tests/FitSimpleModelTest.py | 106 ++++++++++++++--------------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index c35961d28..dc7df3711 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -44,55 +44,47 @@ def setUp(self): def testLinearFit(self): with self._fit_model_subtests(): - self.fitmodel.linear = True - expected = self.fitmodel.get_parameter_values().copy() - self.modify_random(only_linear=True) - - with self._profile("test"): - result = self.fitmodel.fit() - self._assert_fit_result(result, expected) - self.assertTrue( - not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - ) - parameters = self.fitmodel.get_parameter_values(only_linear=True) - self.assertTrue(not numpy.allclose(parameters, expected)) - - self.fitmodel.use_fit_result(result) - numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - parameters = self.fitmodel.get_parameter_values(only_linear=True) - numpy.testing.assert_allclose(parameters, expected) + self._test_fit(True) def testNonLinearFit(self): with self._fit_model_subtests(): - self.fitmodel.linear = False - expected_nonlin = self.fitmodel.get_parameter_values( - only_linear=False - ).copy() - expected_lin = self.fitmodel.get_parameter_values(only_linear=True).copy() - self.modify_random(only_linear=False) - - result = self.fitmodel.fit(full_output=True) - - # TODO: non-linear parameters not precise - # self._assert_fit_result(result, expected_nonlin) - self.assertTrue( - not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) - ) - parameters = self.fitmodel.get_parameter_values(only_linear=False) - self.assertTrue(not numpy.allclose(parameters, expected_nonlin)) - parameters = self.fitmodel.get_parameter_values(only_linear=True) - self.assertTrue(not numpy.allclose(parameters, expected_lin)) - - self.fitmodel.use_fit_result(result) - # TODO: non-linear parameters not precise - # numpy.testing.assert_allclose(self.fitmodel.parameters, expected_nonlin) - self._vis(self.fitmodel.ydata, self.fitmodel.yfullmodel) - numpy.testing.assert_allclose( - self.fitmodel.ydata, self.fitmodel.yfullmodel, rtol=1e-3 - ) - numpy.testing.assert_allclose( - self.fitmodel.linear_parameters, expected_lin, rtol=1e-3 - ) + self._test_fit(False) + + def _test_fit(self, linear): + self.fitmodel.linear = linear + refined_params = self.fitmodel.get_parameter_values(only_linear=False).copy() + lin_refined_params = self.fitmodel.get_parameter_values(only_linear=True).copy() + self.modify_random(only_linear=linear) + + before = self.fitmodel.get_parameter_values(only_linear=False) + lin_before = self.fitmodel.get_parameter_values(only_linear=True) + + result = self.fitmodel.fit(full_output=True) + + after = self.fitmodel.get_parameter_values(only_linear=False) + lin_after = self.fitmodel.get_parameter_values(only_linear=True) + numpy.testing.assert_array_equal(before, after) + numpy.testing.assert_array_equal(lin_before, lin_after) + + self._assert_model_not_refined(refined_params, lin_refined_params) + self.fitmodel.use_fit_result(result) + self._assert_model_refined(refined_params, lin_refined_params) + + def _assert_model_not_refined(self, refined_params, lin_refined_params): + self.assertTrue( + not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + ) + parameters = self.fitmodel.get_parameter_values(only_linear=False) + self.assertTrue(not numpy.allclose(parameters, refined_params)) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + self.assertTrue(not numpy.allclose(parameters, lin_refined_params)) + + def _assert_model_refined(self, refined_params, lin_refined_params): + numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + parameters = self.fitmodel.get_parameter_values(only_linear=False) + numpy.testing.assert_allclose(parameters, refined_params) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + numpy.testing.assert_allclose(parameters, lin_refined_params) def _assert_fit_result(self, result, expected): p = numpy.asarray(result["parameters"]) @@ -172,20 +164,22 @@ def modify_random(self, only_linear=False): def _modify_random(self, only_linear=False): porg = self.fitmodel.get_parameter_values(only_linear=False).copy() plinorg = self.fitmodel.get_parameter_values(only_linear=True).copy() - if only_linear: - plin = self.fitmodel.get_parameter_values(only_linear=True) - plin *= self.random_state.uniform(0.5, 0.8, len(plin)) - self.fitmodel.set_parameter_values(plin, only_linear=True) - parameters = self.fitmodel.get_parameter_values(only_linear=True) - numpy.testing.assert_array_equal(parameters, plin) - p = self.fitmodel.get_parameter_values(only_linear=False) - else: - p = self.fitmodel.get_parameter_values(only_linear=False) + + if not only_linear: + p = porg.copy() p *= self.random_state.uniform(0.95, 1, len(p)) self.fitmodel.set_parameter_values(p, only_linear=False) parameters = self.fitmodel.get_parameter_values(only_linear=False) numpy.testing.assert_array_equal(parameters, p) - plin = self.fitmodel.get_parameter_values(only_linear=True) + + plin = plinorg.copy() + plin *= self.random_state.uniform(0.5, 0.8, len(plin)) + self.fitmodel.set_parameter_values(plin, only_linear=True) + parameters = self.fitmodel.get_parameter_values(only_linear=True) + numpy.testing.assert_array_equal(parameters, plin) + + if only_linear: + p = self.fitmodel.get_parameter_values(only_linear=False) for group in self.fitmodel.get_parameter_groups(only_linear=False): current = p[group.index] From 57738c5ae63f3156ebc99f0127bb5f0c2ebc816f Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 7 Jul 2021 09:54:22 +0200 Subject: [PATCH 57/74] fixup --- PyMca5/tests/FitSimpleModelTest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index dc7df3711..07e16a765 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -279,7 +279,7 @@ def _validate_model(self, model, is_combined_model): linear, repr(param_name) ) numpy.testing.assert_allclose( - calc, numerical, err_msg=err_msg, rtol=1e-3, atol=1e-6 + calc, numerical, err_msg=err_msg, rtol=1e-4 ) parameters = model.get_parameter_values(only_linear=False) @@ -287,7 +287,7 @@ def _validate_model(self, model, is_combined_model): parameters = model.get_parameter_values(only_linear=True) numpy.testing.assert_array_equal(keep_linear_parameters, parameters) - def _vis(self, a, b): + def _vis_compare(self, a, b): import matplotlib.pyplot as plt plt.plot(a) From 8370f7e50ec021df0f842d280878ee7cdedd35a6 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 7 Jul 2021 12:51:36 +0200 Subject: [PATCH 58/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 14 ++- PyMca5/tests/FitSimpleModelTest.py | 105 +++++++++++------- PyMca5/tests/SimpleModel.py | 2 +- 3 files changed, 77 insertions(+), 44 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 765d4e558..f7c4d3983 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -31,7 +31,7 @@ def ydata(self, value): @property def ystd(self): - raise NotImplementedError + return None @ystd.setter def ystd(self, value): @@ -343,12 +343,15 @@ def yfitstd(self): return self._ystd_full_to_fit(self.ystd) def _y_full_to_fit(self, y, xdata=None): + """Convert data from full model to fit model""" return y def _ystd_full_to_fit(self, ystd, xdata=None): + """Convert standard deviation from full model to fit model""" return ystd def _y_fit_to_full(self, y, xdata=None): + """Convert data from fit model to full model""" return y def evaluate_fullmodel(self, xdata=None): @@ -552,11 +555,16 @@ def evaluate_linear_fitmodel(self, xdata=None): def _get_concatenated_data(self, attr): """ :param str attr: - :returns array: + :returns array or None: """ if self.nmodels == 0: return None - return numpy.concatenate([getattr(model, attr) for model in self.models]) + data = [getattr(model, attr) for model in self.models] + isnone = [d is None for d in data] + if any(isnone): + assert all(isnone) + return None + return numpy.concatenate(data) def _set_concatenated_data(self, attr, values): """ diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 07e16a765..33e4990a3 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -54,7 +54,7 @@ def _test_fit(self, linear): self.fitmodel.linear = linear refined_params = self.fitmodel.get_parameter_values(only_linear=False).copy() lin_refined_params = self.fitmodel.get_parameter_values(only_linear=True).copy() - self.modify_random(only_linear=linear) + self._modify_random(only_linear=linear) before = self.fitmodel.get_parameter_values(only_linear=False) lin_before = self.fitmodel.get_parameter_values(only_linear=True) @@ -95,12 +95,12 @@ def _assert_fit_result(self, result, expected): @contextmanager def _fit_model_subtests(self): - for nmodels in (1,): + for nmodels in (2,): with self.subTest(nmodels=nmodels): self._create_model(nmodels=nmodels) - self.validate_model() + self._validate_model() yield - self.validate_model() + self._validate_model() def _create_model(self, nmodels): self.nmodels = nmodels @@ -111,7 +111,7 @@ def _create_model(self, nmodels): self.fitmodel = SimpleCombinedModel(ndetectors=nmodels) self.assertTrue(not self.fitmodel.linear) - self.init_random() + self._init_random() ydata = self.fitmodel.yfullmodel.copy() self.fitmodel.ydata = ydata @@ -121,7 +121,7 @@ def _create_model(self, nmodels): self.fitmodel.yfitmodel, ydata - self.background, atol=1e-12 ) - def init_random(self, **kw): + def _init_random(self, **kw): self.npeaks = 13 # concentrations self.nshapeparams = 4 # zero, gain, wzero, wgain self.nchannels = 2048 @@ -129,11 +129,11 @@ def init_random(self, **kw): self.background = 10 if self.is_combined_model: for model in self.fitmodel.models: - self._init_random(model) + self._init_random_model(model) else: - self._init_random(self.fitmodel) + self._init_random_model(self.fitmodel) - def _init_random(self, model): + def _init_random_model(self, model): nchannels = self.nchannels model.xdata_raw = numpy.arange(nchannels) model.ydata_raw = numpy.full(nchannels, numpy.nan) @@ -157,20 +157,20 @@ def _init_random(self, model): model.concentrations = self.random_state.uniform(low=0.5, high=1, size=npeaks) model.efficiency = self.random_state.uniform(low=5000, high=6000, size=npeaks) - def modify_random(self, only_linear=False): - self._modify_random(only_linear=only_linear) - self.validate_model() - def _modify_random(self, only_linear=False): - porg = self.fitmodel.get_parameter_values(only_linear=False).copy() + self._modify_random_model(only_linear=only_linear) + self._validate_model() + + def _modify_random_model(self, only_linear=False): + pallorg = self.fitmodel.get_parameter_values(only_linear=False).copy() plinorg = self.fitmodel.get_parameter_values(only_linear=True).copy() if not only_linear: - p = porg.copy() - p *= self.random_state.uniform(0.95, 1, len(p)) - self.fitmodel.set_parameter_values(p, only_linear=False) + pall = pallorg.copy() + pall *= self.random_state.uniform(0.95, 1, len(pall)) + self.fitmodel.set_parameter_values(pall, only_linear=False) parameters = self.fitmodel.get_parameter_values(only_linear=False) - numpy.testing.assert_array_equal(parameters, p) + numpy.testing.assert_array_equal(parameters, pall) plin = plinorg.copy() plin *= self.random_state.uniform(0.5, 0.8, len(plin)) @@ -179,11 +179,11 @@ def _modify_random(self, only_linear=False): numpy.testing.assert_array_equal(parameters, plin) if only_linear: - p = self.fitmodel.get_parameter_values(only_linear=False) + pall = self.fitmodel.get_parameter_values(only_linear=False) for group in self.fitmodel.get_parameter_groups(only_linear=False): - current = p[group.index] - expected = porg[group.index] + current = pall[group.index] + expected = pallorg[group.index] if only_linear and not group.linear: if group.count == 1: self.assertEqual(current, expected, msg=group.name) @@ -203,26 +203,52 @@ def _modify_random(self, only_linear=False): else: self.assertFalse(all(current == expected), msg=group.name) - return p + return pall - def validate_model(self): - self._validate_model(self.fitmodel, self.is_combined_model) + def _validate_model(self): + self._validate_submodel(self.fitmodel, self.is_combined_model) if self.is_combined_model: for model in self.fitmodel.models: - self._validate_model(model, False) - self._validate_model(self.fitmodel, self.is_combined_model) + self._validate_submodel(model, False) + self._validate_submodel(self.fitmodel, self.is_combined_model) - def _validate_model(self, model, is_combined_model): + def _validate_submodel(self, model, is_combined_model): keep_parameters = model.get_parameter_values(only_linear=False).copy() keep_linear_parameters = model.get_parameter_values(only_linear=True).copy() - if not is_combined_model: - names = model.get_parameter_group_names(only_linear=False) - expected = {"concentrations", "zero", "gain", "wzero", "wgain"} - self.assertEqual(set(names), expected) - names = model.get_parameter_group_names(only_linear=True) - expected = {"concentrations"} - self.assertEqual(set(names), expected) + lin_expected = {"concentrations"} + expected = {"concentrations"} + if is_combined_model: + for i in range(self.nmodels): + expected |= { + f"detector{i}:zero", + f"detector{i}:gain", + f"detector{i}:wzero", + f"detector{i}:wgain", + } + else: + expected |= {"zero", "gain", "wzero", "wgain"} + names = model.get_parameter_group_names(only_linear=False) + self.assertEqual(set(names), expected) + names = model.get_parameter_group_names(only_linear=True) + self.assertEqual(set(names), lin_expected) + + lin_expected = {f"concentrations{i}" for i in range(self.npeaks)} + expected = {f"concentrations{i}" for i in range(self.npeaks)} + if is_combined_model: + for i in range(self.nmodels): + expected |= { + f"detector{i}:zero", + f"detector{i}:gain", + f"detector{i}:wzero", + f"detector{i}:wgain", + } + else: + expected |= {"zero", "gain", "wzero", "wgain"} + names = model.get_parameter_names(only_linear=False) + self.assertEqual(set(names), expected) + names = model.get_parameter_names(only_linear=True) + self.assertEqual(set(names), lin_expected) n = model.ndata nexpected = len(model.xdata) @@ -250,13 +276,13 @@ def _validate_model(self, model, is_combined_model): numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) - nonlin_names = {"gain", "wgain", "wzero", "zero"} + all_names = {"gain", "wgain", "wzero", "zero"} lin_names = {"concentrations" + str(i) for i in range(self.npeaks)} - names = lin_names | nonlin_names + names = lin_names | all_names if is_combined_model: nmodels = model.nmodels names = lin_names + [ - name + str(i) for i in range(nmodels) for name in nonlin_names + name + str(i) for i in range(nmodels) for name in all_names ] nexpected = self.npeaks + self.nshapeparams * nmodels else: @@ -271,9 +297,8 @@ def _validate_model(self, model, is_combined_model): mnames = model.get_parameter_names(only_linear=True) self.assertEqual(set(mnames), lin_names) - if not is_combined_model: - for linear in [not model.linear, model.linear]: - model.linear = linear + for linear in (True, False): + with model.linear_context(linear): for param_name, calc, numerical in model.compare_derivatives(): err_msg = "[only_linear={}] Analytical and numerical derivative of {} are not equal".format( linear, repr(param_name) diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 0ad3307bb..1f6e6d649 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -129,7 +129,7 @@ def concentrations(self): def concentrations(self, value): self.config["matrix"]["concentrations"] = value - @property + @linked_property def linear(self): return self.config["fit"]["linear"] From 2a7a21e7ae2a4865257aa8e51389ae55d8b8f60a Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 22 Jul 2021 17:52:28 +0200 Subject: [PATCH 59/74] fixup --- .../PyMcaMath/fitting/model/CachingModel.py | 3 + .../fitting/model/LeastSquaresFitModel.py | 151 ++++++++++++++---- .../PyMcaMath/fitting/model/ParameterModel.py | 26 ++- PyMca5/tests/FitSimpleModelTest.py | 62 +++---- 4 files changed, 165 insertions(+), 77 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index 93417238c..1b06a5f47 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -155,6 +155,9 @@ def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoption if persist: self._persist_property_values(values_cache, **cacheoptions) + def _in_property_caching_context(self): + return self._getCache("_property_values") is not None + def _get_property_values_cache(self, **cacheoptions): caches = self._getCache("_property_values") if caches is None: diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index f7c4d3983..850cdad1f 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -101,18 +101,48 @@ def evaluate_linear_fitmodel(self, xdata=None): """Evaluate the fit model. :param array xdata: shape (ndata,) - :returns array: n x ndata + :returns array: shape (ndata,) """ - raise NotImplementedError + derivatives = self.linear_derivatives_fitmodel(xdata=xdata) + parameters = self.get_parameter_values(only_linear=True) + return parameters.dot(derivatives) def linear_derivatives_fitmodel(self, xdata=None): """Derivates to all linear parameters :param array xdata: shape (ndata,) - :returns array: nparams x ndata + :returns array: shape (nparams, ndata) + """ + nparams = self.get_n_parameters(only_linear=True) + return numpy.array( + [ + self.derivative_fitmodel(i, xdata=xdata, only_linear=True) + for i in range(nparams) + ] + ) + + def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ raise NotImplementedError + def compare_derivatives(self, xdata=None, **paramtype): + """Compare analytical and numerical derivatives. Useful to + validate the user defined `derivative_fitmodel`. + + :yields str, array, array: parameter name, analytical, numerical + """ + for param_idx, name in enumerate(self.get_parameter_names(**paramtype)): + ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata, **paramtype) + ynumderiv = self.numerical_derivative_fitmodel( + param_idx, xdata=xdata, **paramtype + ) + yield name, ycalderiv, ynumderiv + def linear_decomposition_fitmodel(self, xdata=None): """Linear decomposition of the fit model. @@ -372,30 +402,6 @@ def evaluate_linear_fullmodel(self, xdata=None): y = self.evaluate_linear_fitmodel(xdata=xdata) return self._y_fit_to_full(y, xdata=xdata) - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. - - :param array xdata: shape (ndata,) - :returns array: shape (ndata,) - """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - parameters = self.get_parameter_values(only_linear=True) - return parameters.dot(derivatives) - - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: shape (ndata,) - :returns array: shape (nparams, ndata) - """ - nparams = self.get_n_parameters(only_linear=True) - return numpy.array( - [ - self.derivative_fitmodel(i, xdata=xdata, only_linear=True) - for i in range(nparams) - ] - ) - def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): """Derivate to a specific parameter of the fit model. @@ -544,13 +550,27 @@ def evaluate_fitmodel(self, xdata=None): """ return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) - def evaluate_linear_fitmodel(self, xdata=None): - """Evaluate the fit model. + def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): + """Derivate to a specific parameter of the fit model. - :param array xdata: shape (ndata,) or (nmodels, ndatai) - :returns array: shape (ndata,) or (sum(ndatai),) + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + return self._get_concatenated_derivative( + "derivative_fitmodel", param_idx, xdata=xdata, **paramtype + ) + + def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ - return self._concatenate_evaluation("evaluate_linear_fitmodel", xdata=xdata) + return self._get_concatenated_derivative( + "numerical_derivative_fitmodel", param_idx, xdata=xdata, **paramtype + ) def _get_concatenated_data(self, attr): """ @@ -588,6 +608,43 @@ def _concatenate_evaluation(self, funcname, xdata=None): ret[idx] = func(xdata=xdata) return ret + def _get_concatenated_derivative( + self, funcname, param_idx, xdata=None, **paramtype + ): + """Derivate to a specific parameter of the fit model. + + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + if xdata is None: + n = self.ndata + else: + # Note: RestreinedLeastSquaresFit strides the data on the first iteration + n = len(xdata) + ret = numpy.zeros(n) + group = self._group_from_parameter_index(param_idx, **paramtype) + model = self._linked_key_to_instance(group.instance_key) + + param_idx0 = param_idx + cached = self._in_property_caching_context() + if not cached: + parameter_index_in_group = group.parameter_index_in_group(param_idx0) + for modeli, xdata, idx in self._iter_model_data_slices(xdata, strided=True): + if not group.linked and modeli is not model: + continue + if cached: + param_idx = param_idx0 + else: + groupi = modeli._group_from_parameter_name( + group.property_name, **paramtype + ) + param_idx = groupi.start_index + parameter_index_in_group + func = getattr(modeli, funcname) + ret[idx] = func(param_idx, xdata=xdata, **paramtype) + + return ret + def _get_ndata(self, data): """ :param array data: shape (ndata,) or (nmodels, ndatai) @@ -602,9 +659,10 @@ def _get_ndata(self, data): raise ValueError(f"Expected {self.nmodels} data arrays") return sum(len(x) for x in data) - def _iter_model_data_slices(self, data): + def _iter_model_data_slices(self, data, strided=False): """ :param array data: shape (ndata,) or (nmodels, ndatai) + :param bool strided: :yields tuple: model, datai, slice """ i = 0 @@ -619,6 +677,10 @@ def _iter_model_data_slices(self, data): idx = slice(i, i + n) yield model, data[idx], idx i += n + elif strided: + indices = self.__model_indices_after_slicing(data) + for model, idx in zip(self.models, indices): + yield model, data[idx], idx else: if len(data) != self.nmodels: raise ValueError(f"Expected {self.nmodels} data arrays") @@ -626,3 +688,26 @@ def _iter_model_data_slices(self, data): n = len(xdata_model) yield model, xdata_model, slice(i, i + n) i += n + + def __model_indices_after_slicing(self, data): + ndata_models = [model.ndata for model in self.models] + stride, remain = divmod(sum(ndata_models), len(data)) + stride += remain > 0 + start = 0 + offset = 0 + i = 0 + for n in ndata_models: + # Index of model in concatenated xdata due to slicing + stop = start + n + lst = list(range(start + offset, stop, stride)) + nlst = len(lst) + # Index of model in concatenated xdata after slicing + idx = slice(i, i + nlst) + i += nlst + # Prepare for next model + if lst: + offset = lst[-1] + stride - stop + else: + offset -= n + start = stop + yield idx \ No newline at end of file diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 763b7a895..c90f9a73b 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -144,11 +144,25 @@ def _iter_parameter_names(self, **paramtype): def get_n_parameters(self, **paramtype): return sum(group.count for group in self._iter_parameter_groups(**paramtype)) + @contextmanager + def __parameter_values_context(self): + not_cached = not self._in_property_caching_context() + if not_cached: + keep = self._cache_manager + self._cache_manager = self + try: + yield + finally: + if not_cached: + self._cache_manager = keep + def get_parameter_values(self, **paramtype): - return self._get_property_values(**paramtype) + with self.__parameter_values_context(): + return self._get_property_values(**paramtype) def set_parameter_values(self, values, **paramtype): - self._set_property_values(values, **paramtype) + with self.__parameter_values_context(): + self._set_property_values(values, **paramtype) def get_parameter_constraints(self, **paramtype): """ @@ -204,13 +218,18 @@ def _group_from_parameter_index(self, param_idx, **paramtype): if group.start_index <= param_idx < group.stop_index: return group + def _group_from_parameter_name(self, prop_name, **paramtype): + for group in self._iter_parameter_groups(**paramtype): + if group.property_name == prop_name: + return group + class ParameterModel(ParameterModelBase, LinkedModel): """Model that implements fit parameters""" def __init__(self, *args, **kw): super().__init__(*args, **kw) - self._linear= False + self._linear = False def _iter_parameter_group_properties(self): cls = type(self) @@ -356,7 +375,6 @@ def _instance_cached_property_ids(self, **paramtype): """ # Shared parameters tracker = _IterGroupTracker() - start_index = 0 for model in self.models: yield from model._iter_parameter_groups( linked=True, tracker=tracker, **paramtype diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 33e4990a3..659636a90 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -206,47 +206,40 @@ def _modify_random_model(self, only_linear=False): return pall def _validate_model(self): - self._validate_submodel(self.fitmodel, self.is_combined_model) + self._validate_submodel(self.fitmodel) if self.is_combined_model: - for model in self.fitmodel.models: - self._validate_submodel(model, False) - self._validate_submodel(self.fitmodel, self.is_combined_model) + for model_idx, model in enumerate(self.fitmodel.models): + self._validate_submodel(model, model_idx) + self._validate_submodel(self.fitmodel) - def _validate_submodel(self, model, is_combined_model): + def _validate_submodel(self, model, model_idx=None): + is_combined_model = self.is_combined_model and model_idx is None keep_parameters = model.get_parameter_values(only_linear=False).copy() keep_linear_parameters = model.get_parameter_values(only_linear=True).copy() - lin_expected = {"concentrations"} - expected = {"concentrations"} - if is_combined_model: - for i in range(self.nmodels): - expected |= { - f"detector{i}:zero", - f"detector{i}:gain", - f"detector{i}:wzero", - f"detector{i}:wgain", + nonlin_expected = {"gain", "wgain", "wzero", "zero"} + if self.is_combined_model: + if is_combined_model: + nonlin_expected = { + f"detector{model_idx}:{name}" + for model_idx in range(self.nmodels) + for name in nonlin_expected } - else: - expected |= {"zero", "gain", "wzero", "wgain"} + else: + nonlin_expected = { + f"detector{model_idx}:{name}" for name in nonlin_expected + } + lin_expected = {"concentrations"} + all_expected = lin_expected | nonlin_expected names = model.get_parameter_group_names(only_linear=False) - self.assertEqual(set(names), expected) + self.assertEqual(set(names), all_expected) names = model.get_parameter_group_names(only_linear=True) self.assertEqual(set(names), lin_expected) lin_expected = {f"concentrations{i}" for i in range(self.npeaks)} - expected = {f"concentrations{i}" for i in range(self.npeaks)} - if is_combined_model: - for i in range(self.nmodels): - expected |= { - f"detector{i}:zero", - f"detector{i}:gain", - f"detector{i}:wzero", - f"detector{i}:wgain", - } - else: - expected |= {"zero", "gain", "wzero", "wgain"} + all_expected = lin_expected | nonlin_expected names = model.get_parameter_names(only_linear=False) - self.assertEqual(set(names), expected) + self.assertEqual(set(names), all_expected) names = model.get_parameter_names(only_linear=True) self.assertEqual(set(names), lin_expected) @@ -276,26 +269,15 @@ def _validate_submodel(self, model, is_combined_model): numpy.testing.assert_allclose(arr1, arr3) numpy.testing.assert_allclose(arr1, arr4) - all_names = {"gain", "wgain", "wzero", "zero"} - lin_names = {"concentrations" + str(i) for i in range(self.npeaks)} - names = lin_names | all_names if is_combined_model: nmodels = model.nmodels - names = lin_names + [ - name + str(i) for i in range(nmodels) for name in all_names - ] nexpected = self.npeaks + self.nshapeparams * nmodels else: nexpected = self.npeaks + self.nshapeparams - n = model.get_n_parameters(only_linear=False) self.assertEqual(n, nexpected) n = model.get_n_parameters(only_linear=True) self.assertEqual(n, self.npeaks) - mnames = model.get_parameter_names(only_linear=False) - self.assertEqual(set(mnames), names) - mnames = model.get_parameter_names(only_linear=True) - self.assertEqual(set(mnames), lin_names) for linear in (True, False): with model.linear_context(linear): From cfce49fa7391f494231a3b2dfc39a7b281b22257 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 28 Jul 2021 10:53:25 +0200 Subject: [PATCH 60/74] fixup --- PyMca5/tests/ParameterModelTest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 214d5fa55..5fe370373 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -296,7 +296,7 @@ def test_linear_parameter_contraints(self): def test_get_parameter_values(self): for cacheoptions in self._parameterize_nonlinear_test(): values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 0, 0]) + self.assertEqual(values.tolist(), [11, 11, 12]) values = self.concat_model[-1].get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) @@ -328,7 +328,7 @@ def test_get_parameter_values_in_caching_context(self): def test_get_linear_parameter_values(self): for cacheoptions in self._parameterize_linear_test(): values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 0]) + self.assertEqual(values.tolist(), [11, 11]) values = self.concat_model[-1].get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 41]) From 39ebb49d354b5ddf718b4434746b52b72cb1b70c Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 28 Jul 2021 15:53:54 +0200 Subject: [PATCH 61/74] fixup --- .../fitting/model/CachingLinkedModel.py | 49 ++++ .../PyMcaMath/fitting/model/CachingModel.py | 38 +-- .../fitting/model/LeastSquaresFitModel.py | 47 ++-- PyMca5/PyMcaMath/fitting/model/LinkedModel.py | 4 + .../PyMcaMath/fitting/model/ParameterModel.py | 38 ++- PyMca5/tests/ParameterModelTest.py | 228 +++++++++--------- 6 files changed, 227 insertions(+), 177 deletions(-) create mode 100644 PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py diff --git a/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py new file mode 100644 index 000000000..29dda94e9 --- /dev/null +++ b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py @@ -0,0 +1,49 @@ +import numpy +from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel + + +class CachedPropertiesLinkModel(CachedPropertiesModel, LinkedModel): + def _iter_cached_property_ids(self, **cacheoptions): + if self.is_linked and self._in_property_caching_context(): + # Yield only the property id's that belong to this model + names = self._cached_property_names() + for propid in super()._iter_cached_property_ids(**cacheoptions): + if propid.property_name in names: + yield propid + else: + yield from super()._iter_cached_property_ids(**cacheoptions) + + def _get_property_values(self, **cacheoptions): + values = super()._get_property_values(**cacheoptions) + return self.__extract_values(values, **cacheoptions) + + def _set_property_values(self, values, **cacheoptions): + values = self.__insert_values(values, **cacheoptions) + super()._set_property_values(values, **cacheoptions) + + def __extract_values(self, values, **cacheoptions): + if not self.is_linked: + return values + # Only the values that belong to this model + data = list() + for propid in self._iter_cached_property_ids(**cacheoptions): + index = self._propid_to_index(propid, **cacheoptions) + data.append(numpy.atleast_1d(values[index]).tolist()) + + return numpy.concatenate(data) + + def __insert_values(self, values, **cacheoptions): + if not self.is_linked: + return values + gvalues = super()._get_property_values(**cacheoptions) + i = 0 + for propid in self._iter_cached_property_ids(**cacheoptions): + index = self._propid_to_index(propid, **cacheoptions) + try: + n = len(gvalues[index]) + except TypeError: + n = 1 + gvalues[index] = values[i : i + n] + i += n + return gvalues diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index 1b06a5f47..b94a0b4d7 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -122,18 +122,6 @@ def _instance_cached_property_ids(self, **cacheoptions): """All property id's for this instance and the provided cache options""" return self._cached_property_names() - def _iter_cached_property_ids(self, **cacheoptions): - """To be used when iterating over all property id's - of this instance. - """ - propid_to_index = self._get_property_mapping_cache( - "_propid_to_index", **cacheoptions - ) - if propid_to_index is None: - yield from self._instance_cached_property_ids(**cacheoptions) - else: - yield from propid_to_index.keys() - @contextmanager def _propertyCachingContext(self, persist=False, start_cache=None, **cacheoptions): values_cache = self._get_property_values_cache(**cacheoptions) @@ -249,16 +237,30 @@ def _cached_property_fset(self, fset, value): index = self._propid_to_index(propid) values_cache[index] = value - def _get_property_mapping_cache(self, *subnames, **cacheoptions): + def _get_from_propid_cache(self, *subnames, dtype=dict, **cacheoptions): caches = self._getCache("_propid", *subnames) if caches is None: return None key = self._cache_manager._property_cache_key(**cacheoptions) - return caches.get(key, None) + return caches.setdefault(key, dtype()) + + def _iter_cached_property_ids(self, **cacheoptions): + """To be used when iterating over all property id's + of this instance. + """ + propid_list = self._get_from_propid_cache( + "_propid_list", dtype=list, **cacheoptions + ) + if propid_list is None: + yield from self._instance_cached_property_ids(**cacheoptions) + return + if not propid_list: + propid_list.extend(self._instance_cached_property_ids(**cacheoptions)) + yield from propid_list def _propid_to_index(self, propid, **cacheoptions): - propid_to_index = self._get_property_mapping_cache( - "_propid_to_index", **cacheoptions + propid_to_index = self._get_from_propid_cache( + "_propid_to_index", dtype=dict, **cacheoptions ) if propid_to_index is None: return self._cache_manager._property_index_from_id(propid, **cacheoptions) @@ -269,8 +271,8 @@ def _propid_to_index(self, propid, **cacheoptions): return index def _name_to_propid(self, property_name, **cacheoptions): - name_to_propid = self._get_property_mapping_cache( - "_name_to_propid", **cacheoptions + name_to_propid = self._get_from_propid_cache( + "_name_to_propid", dtype=dict, **cacheoptions ) if name_to_propid is None: return self._property_id_from_name(property_name) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 850cdad1f..1544185ee 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -293,7 +293,7 @@ def _gefit_evaluate_fitmodel(self, parameters, xdata): :returns array: shape (ndata,) """ self.set_parameter_values(parameters) - return self.evaluate_fitmodel(xdata=xdata) + return self.evaluate_fitmodel(xdata=xdata, strided=True) def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): """Update parameters and return derivate to a specific parameter @@ -304,7 +304,7 @@ def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): :returns array: shape (ndata,) """ self.set_parameter_values(parameters) - return self.derivative_fitmodel(param_idx, xdata=xdata) + return self.derivative_fitmodel(param_idx, xdata=xdata, strided=True) def use_fit_result(self, result): """ @@ -542,23 +542,27 @@ def evaluate_linear_fullmodel(self, xdata=None): """ return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) - def evaluate_fitmodel(self, xdata=None): + def evaluate_fitmodel(self, xdata=None, strided=False): """Evaluate the fit model. :param array xdata: shape (ndata,) or (nmodels, ndatai) + :param bool strided: :returns array: shape (ndata,) or (sum(ndatai),) """ - return self._concatenate_evaluation("evaluate_fitmodel", xdata=xdata) + return self._concatenate_evaluation( + "evaluate_fitmodel", xdata=xdata, strided=strided + ) - def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): + def derivative_fitmodel(self, param_idx, xdata=None, strided=False, **paramtype): """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: shape (ndata,) + :param bool strided: :returns array: shape (ndata,) """ return self._get_concatenated_derivative( - "derivative_fitmodel", param_idx, xdata=xdata, **paramtype + "derivative_fitmodel", param_idx, xdata=xdata, strided=strided, **paramtype ) def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): @@ -596,41 +600,40 @@ def _set_concatenated_data(self, attr, values): for model, values, _ in self._iter_model_data_slices(values): setattr(model, attr, values) - def _concatenate_evaluation(self, funcname, xdata=None): + def _concatenate_evaluation(self, funcname, xdata=None, strided=False): """Evaluate model :param array xdata: shape (ndata,) or (nmodels, ndatai) + :param bool strided: :returns array: shape (ndata,) or (sum(ndatai),) """ - ret = numpy.empty(self._get_ndata(xdata)) - for model, xdata, idx in self._iter_model_data_slices(xdata): + ret = numpy.empty(self._get_ndata(xdata, strided=strided)) + for model, xdata, idx in self._iter_model_data_slices(xdata, strided=strided): func = getattr(model, funcname) ret[idx] = func(xdata=xdata) return ret def _get_concatenated_derivative( - self, funcname, param_idx, xdata=None, **paramtype + self, funcname, param_idx, xdata=None, strided=False, **paramtype ): """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: shape (ndata,) + :param bool strided: :returns array: shape (ndata,) """ - if xdata is None: - n = self.ndata - else: - # Note: RestreinedLeastSquaresFit strides the data on the first iteration - n = len(xdata) + n = self._get_ndata(xdata, strided=strided) ret = numpy.zeros(n) group = self._group_from_parameter_index(param_idx, **paramtype) model = self._linked_key_to_instance(group.instance_key) param_idx0 = param_idx cached = self._in_property_caching_context() + cached = False if not cached: parameter_index_in_group = group.parameter_index_in_group(param_idx0) - for modeli, xdata, idx in self._iter_model_data_slices(xdata, strided=True): + for modeli, xdata, idx in self._iter_model_data_slices(xdata, strided=strided): if not group.linked and modeli is not model: continue if cached: @@ -641,17 +644,25 @@ def _get_concatenated_derivative( ) param_idx = groupi.start_index + parameter_index_in_group func = getattr(modeli, funcname) + + tmpgroup = modeli._group_from_parameter_index(param_idx, **paramtype) + if tmpgroup is None: + breakpoint() ret[idx] = func(param_idx, xdata=xdata, **paramtype) return ret - def _get_ndata(self, data): + def _get_ndata(self, data, strided=False): """ :param array data: shape (ndata,) or (nmodels, ndatai) + :param bool strided: + :returns int: """ ndata = self.ndata if data is None: return ndata + elif strided: + return len(data) elif len(data) == ndata: return ndata else: @@ -710,4 +721,4 @@ def __model_indices_after_slicing(self, data): else: offset -= n start = stop - yield idx \ No newline at end of file + yield idx diff --git a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py index bd4d60936..9014fc753 100644 --- a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py @@ -93,6 +93,10 @@ def _enable_property_link(cls, *prop_names): if prop is not None: prop.propagate = True + @property + def is_linked(self): + return self._link_manager is not None + @property def _link_manager(self): return self.__link_manager diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index c90f9a73b..28c8e34c6 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from contextlib import contextmanager import numpy -from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModel +from PyMca5.PyMcaMath.fitting.model.CachingLinkedModel import CachedPropertiesLinkModel from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModelManager from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel @@ -144,25 +144,11 @@ def _iter_parameter_names(self, **paramtype): def get_n_parameters(self, **paramtype): return sum(group.count for group in self._iter_parameter_groups(**paramtype)) - @contextmanager - def __parameter_values_context(self): - not_cached = not self._in_property_caching_context() - if not_cached: - keep = self._cache_manager - self._cache_manager = self - try: - yield - finally: - if not_cached: - self._cache_manager = keep - def get_parameter_values(self, **paramtype): - with self.__parameter_values_context(): - return self._get_property_values(**paramtype) + return self._get_property_values(**paramtype) def set_parameter_values(self, values, **paramtype): - with self.__parameter_values_context(): - self._set_property_values(values, **paramtype) + self._set_property_values(values, **paramtype) def get_parameter_constraints(self, **paramtype): """ @@ -224,14 +210,14 @@ def _group_from_parameter_name(self, prop_name, **paramtype): return group -class ParameterModel(ParameterModelBase, LinkedModel): +class ParameterModel(CachedPropertiesLinkModel, ParameterModelBase): """Model that implements fit parameters""" def __init__(self, *args, **kw): super().__init__(*args, **kw) self._linear = False - def _iter_parameter_group_properties(self): + def __iter_parameter_group_properties(self): cls = type(self) for property_name in self._cached_property_names(): prop = getattr(cls, property_name) @@ -256,7 +242,7 @@ def _instance_cached_property_ids( start_index = 0 else: start_index = tracker.start_index - for property_name, prop in self._iter_parameter_group_properties(): + for property_name, prop in self.__iter_parameter_group_properties(): group_is_linked = prop.propagate if linked is not None: if group_is_linked is not linked: @@ -299,7 +285,7 @@ def get_constraints(): start_index=start_index, stop_index=stop_index, index=index, - get_constraints=get_constraints, + get_constraints=self.__constraints_getter(prop), ) if tracker is None: yield group @@ -308,6 +294,12 @@ def get_constraints(): yield group start_index = tracker.start_index + def __constraints_getter(self, prop): + def get_constraints(): + return prop.fconstraints(self) + + return get_constraints + @linked_property def linear(self): return self._linear @@ -376,12 +368,12 @@ def _instance_cached_property_ids(self, **paramtype): # Shared parameters tracker = _IterGroupTracker() for model in self.models: - yield from model._iter_parameter_groups( + yield from model._instance_cached_property_ids( linked=True, tracker=tracker, **paramtype ) # Non-shared parameters for model in self.models: - yield from model._iter_parameter_groups( + yield from model._instance_cached_property_ids( linked=False, tracker=tracker, **paramtype ) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 5fe370373..76943368c 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -121,18 +121,19 @@ def test_parameter_group_names(self): ) self.assertEqual(set(names), set(expected)) - with self._unlink_var1_lin(): - names = self.concat_model.get_parameter_group_names(**cacheoptions) - expected = ( - "model0:var1_lin", - "model1:var1_lin", - "model2:var1_lin", - "model3:var1_lin", - "var1_nonlin", - "var2_lin", - "var2_nonlin", - ) - self.assertEqual(set(names), set(expected)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ( + "model0:var1_lin", + "model1:var1_lin", + "model2:var1_lin", + "model3:var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) def test_linear_parameter_group_names(self): for cacheoptions in self._parameterize_linear_test(): @@ -148,16 +149,17 @@ def test_linear_parameter_group_names(self): expected = ("var1_lin", "var2_lin") self.assertEqual(set(names), set(expected)) - with self._unlink_var1_lin(): - names = self.concat_model.get_parameter_group_names(**cacheoptions) - expected = ( - "model0:var1_lin", - "model1:var1_lin", - "model2:var1_lin", - "model3:var1_lin", - "var2_lin", - ) - self.assertEqual(set(names), set(expected)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + names = self.concat_model.get_parameter_group_names(**cacheoptions) + expected = ( + "model0:var1_lin", + "model1:var1_lin", + "model2:var1_lin", + "model3:var1_lin", + "var2_lin", + ) + self.assertEqual(set(names), set(expected)) def test_parameter_names(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -185,22 +187,23 @@ def test_parameter_names(self): ) self.assertEqual(set(names), set(expected)) - with self._unlink_var1_lin(): - names = self.concat_model.get_parameter_names(**cacheoptions) - expected = ( - "model0:var1_lin0", - "model0:var1_lin1", - "model1:var1_lin0", - "model1:var1_lin1", - "model2:var1_lin0", - "model2:var1_lin1", - "model3:var1_lin0", - "model3:var1_lin1", - "var1_nonlin", - "var2_lin", - "var2_nonlin", - ) - self.assertEqual(set(names), set(expected)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model3:var1_lin0", + "model3:var1_lin1", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + ) + self.assertEqual(set(names), set(expected)) def test_linear_parameter_names(self): for cacheoptions in self._parameterize_linear_test(): @@ -216,20 +219,21 @@ def test_linear_parameter_names(self): expected = ("var1_lin0", "var1_lin1", "var2_lin") self.assertEqual(set(names), set(expected)) - with self._unlink_var1_lin(): - names = self.concat_model.get_parameter_names(**cacheoptions) - expected = ( - "model0:var1_lin0", - "model0:var1_lin1", - "model1:var1_lin0", - "model1:var1_lin1", - "model2:var1_lin0", - "model2:var1_lin1", - "model3:var1_lin0", - "model3:var1_lin1", - "var2_lin", - ) - self.assertEqual(set(names), set(expected)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + names = self.concat_model.get_parameter_names(**cacheoptions) + expected = ( + "model0:var1_lin0", + "model0:var1_lin1", + "model1:var1_lin0", + "model1:var1_lin1", + "model2:var1_lin0", + "model2:var1_lin1", + "model3:var1_lin0", + "model3:var1_lin1", + "var2_lin", + ) + self.assertEqual(set(names), set(expected)) def test_n_parameter(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -242,9 +246,10 @@ def test_n_parameter(self): n = self.concat_model.get_n_parameters(**cacheoptions) self.assertEqual(n, 5) - with self._unlink_var1_lin(): - n = self.concat_model.get_n_parameters(**cacheoptions) - self.assertEqual(n, 11) + with self._unlink_var1_lin() as unlinked: + if unlinked: + n = self.concat_model.get_n_parameters(**cacheoptions) + self.assertEqual(n, 11) def test_n_linear_parameter(self): for cacheoptions in self._parameterize_linear_test(): @@ -257,10 +262,10 @@ def test_n_linear_parameter(self): n = self.concat_model.get_n_parameters(**cacheoptions) self.assertEqual(n, 3) - with self._unlink_var1_lin(): - n = self.concat_model.get_n_parameters(**cacheoptions) - self.concat_model._enable_property_link("var1_lin") - self.assertEqual(n, 9) + with self._unlink_var1_lin() as unlinked: + if unlinked: + n = self.concat_model.get_n_parameters(**cacheoptions) + self.assertEqual(n, 9) def test_parameter_constraints(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -273,14 +278,20 @@ def test_parameter_constraints(self): arr = self.concat_model.get_parameter_constraints(**cacheoptions) self.assertEqual(arr.shape, (5, 3)) - with self._unlink_var1_lin(): - arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (11, 3)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (11, 3)) def test_linear_parameter_contraints(self): for cacheoptions in self._parameterize_linear_test(): + if not self._cached: + continue arr = self.concat_model[0].get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (2, 3)) + try: + self.assertEqual(arr.shape, (2, 3)) + except Exception: + breakpoint() arr = self.concat_model[-1].get_parameter_constraints(**cacheoptions) self.assertEqual(arr.shape, (3, 3)) @@ -288,10 +299,10 @@ def test_linear_parameter_contraints(self): arr = self.concat_model.get_parameter_constraints(**cacheoptions) self.assertEqual(arr.shape, (3, 3)) - with self._unlink_var1_lin(): - arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.concat_model._enable_property_link("var1_lin") - self.assertEqual(arr.shape, (9, 3)) + with self._unlink_var1_lin() as unlinked: + if unlinked: + arr = self.concat_model.get_parameter_constraints(**cacheoptions) + self.assertEqual(arr.shape, (9, 3)) def test_get_parameter_values(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -304,24 +315,8 @@ def test_get_parameter_values(self): values = self.concat_model.get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - with self._unlink_var1_lin(): - values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) - - def test_get_parameter_values_in_caching_context(self): - for cacheoptions in self._parameterize_nonlinear_test(): - with self.concat_model._propertyCachingContext(**cacheoptions): - values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - - values = self.concat_model[-1].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - - values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) - - with self._unlink_var1_lin(): - with self.concat_model._propertyCachingContext(**cacheoptions): + with self._unlink_var1_lin() as unlinked: + if unlinked: values = self.concat_model.get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) @@ -336,44 +331,41 @@ def test_get_linear_parameter_values(self): values = self.concat_model.get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [11, 11, 41]) - with self._unlink_var1_lin(): - values = self.concat_model.get_parameter_values(**cacheoptions) - self.concat_model._enable_property_link("var1_lin") - self.assertEqual(values.tolist(), [41] + [11] * 8) - - def test_get_linear_parameter_values_in_caching_context(self): - for cacheoptions in self._parameterize_linear_test(): - with self.concat_model._propertyCachingContext(**cacheoptions): - values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 41]) - - values = self.concat_model[-1].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 41]) - - values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 41]) - - with self._unlink_var1_lin(): - with self.concat_model._propertyCachingContext(**cacheoptions): + with self._unlink_var1_lin() as unlinked: + if unlinked: values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [41] + [11] * 8) + self.assertEqual(values.tolist(), [41] + [11] * 8) def _parameterize_linear_test(self): - for local_linear, global_linear in [[True, False], [None, True]]: - with self.subTest(local_linear=local_linear, global_linear=global_linear): - self.concat_model.linear = global_linear - yield {"only_linear": local_linear} + yield from self._parameterize_tests([[True, False], [None, True]]) def _parameterize_nonlinear_test(self): - for local_linear, global_linear in [[False, True], [None, False]]: - with self.subTest(local_linear=local_linear, global_linear=global_linear): - self.concat_model.linear = global_linear - yield {"only_linear": local_linear} + yield from self._parameterize_tests([[False, True], [None, False]]) + + def _parameterize_tests(self, linear_options): + for local_linear, global_linear in linear_options: + for cached in [True, False]: + with self.subTest( + local_linear=local_linear, + global_linear=global_linear, + cached=cached, + ): + self.concat_model.linear = global_linear + cacheoptions = {"only_linear": local_linear} + self._cached = cached + if cached: + with self.concat_model._propertyCachingContext(**cacheoptions): + yield cacheoptions + else: + yield cacheoptions @contextmanager def _unlink_var1_lin(self): - self.concat_model._disable_property_link("var1_lin") - try: - yield - finally: - self.concat_model._enable_property_link("var1_lin") + if self._cached: + yield False + else: + self.concat_model._disable_property_link("var1_lin") + try: + yield True + finally: + self.concat_model._enable_property_link("var1_lin") From a9920b8135b104ca64da87cdc85a57fc495aa2a3 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 28 Jul 2021 16:25:13 +0200 Subject: [PATCH 62/74] fixup --- .../fitting/model/CachingLinkedModel.py | 5 +- PyMca5/tests/ParameterModelTest.py | 63 +++++++++++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py index 29dda94e9..36ddc8d4a 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py @@ -44,6 +44,9 @@ def __insert_values(self, values, **cacheoptions): n = len(gvalues[index]) except TypeError: n = 1 - gvalues[index] = values[i : i + n] + if n == 1: + gvalues[index] = values[i] + else: + gvalues[index] = values[i : i + n] i += n return gvalues diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 76943368c..b16eef788 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -1,4 +1,5 @@ import unittest +import numpy from contextlib import contextmanager from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModel from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelManager @@ -320,21 +321,51 @@ def test_get_parameter_values(self): values = self.concat_model.get_parameter_values(**cacheoptions) self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) - def test_get_linear_parameter_values(self): - for cacheoptions in self._parameterize_linear_test(): - values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11]) - - values = self.concat_model[-1].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 41]) - - values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 41]) + def test_parameter_values(self): + for cacheoptions in self._parameterize_nonlinear_test(): + self._assert_set_get_parameter_values( + self.concat_model[0], [11, 11, 12], **cacheoptions + ) + self._assert_set_get_parameter_values( + self.concat_model[-1], [11, 11, 12, 41, 42], **cacheoptions + ) + self._assert_set_get_parameter_values( + self.concat_model, [11, 11, 12, 41, 42], **cacheoptions + ) + with self._unlink_var1_lin() as unlinked: + if unlinked: + self._assert_set_get_parameter_values( + self.concat_model, [12, 41, 42] + [11] * 8, **cacheoptions + ) + def test_linear_parameter_values(self): + for cacheoptions in self._parameterize_linear_test(): + self._assert_set_get_parameter_values( + self.concat_model[0], [11, 11], **cacheoptions + ) + self._assert_set_get_parameter_values( + self.concat_model[-1], [11, 11, 41], **cacheoptions + ) + self._assert_set_get_parameter_values( + self.concat_model, [11, 11, 41], **cacheoptions + ) with self._unlink_var1_lin() as unlinked: if unlinked: - values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [41] + [11] * 8) + self._assert_set_get_parameter_values( + self.concat_model, [41] + [11] * 8, **cacheoptions + ) + + def _assert_set_get_parameter_values(self, model, expected, **cacheoptions): + self._assert_get_parameter_values(model, expected, **cacheoptions) + with self._protect_parameter_values(**cacheoptions): + expected2 = list(range(len(expected))) + model.set_parameter_values(numpy.array(expected2), **cacheoptions) + self._assert_get_parameter_values(model, expected2, **cacheoptions) + self._assert_get_parameter_values(model, expected, **cacheoptions) + + def _assert_get_parameter_values(self, model, expected, **cacheoptions): + values = model.get_parameter_values(**cacheoptions) + self.assertEqual(values.tolist(), expected) def _parameterize_linear_test(self): yield from self._parameterize_tests([[True, False], [None, True]]) @@ -369,3 +400,11 @@ def _unlink_var1_lin(self): yield True finally: self.concat_model._enable_property_link("var1_lin") + + @contextmanager + def _protect_parameter_values(self, **cacheoptions): + values = self.concat_model.get_parameter_values(**cacheoptions).copy() + try: + yield + finally: + self.concat_model.set_parameter_values(values, **cacheoptions) From b02f08951fee21d00ca39c5b1d1f7c5cd09ad8e0 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 28 Jul 2021 16:58:25 +0200 Subject: [PATCH 63/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 38 ++++++++++++++----- PyMca5/tests/FitSimpleModelTest.py | 24 ++++++++---- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 1544185ee..098c168c9 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -293,7 +293,7 @@ def _gefit_evaluate_fitmodel(self, parameters, xdata): :returns array: shape (ndata,) """ self.set_parameter_values(parameters) - return self.evaluate_fitmodel(xdata=xdata, strided=True) + return self.evaluate_fitmodel(xdata=xdata) def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): """Update parameters and return derivate to a specific parameter @@ -304,7 +304,7 @@ def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): :returns array: shape (ndata,) """ self.set_parameter_values(parameters) - return self.derivative_fitmodel(param_idx, xdata=xdata, strided=True) + return self.derivative_fitmodel(param_idx, xdata=xdata) def use_fit_result(self, result): """ @@ -542,27 +542,27 @@ def evaluate_linear_fullmodel(self, xdata=None): """ return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) - def evaluate_fitmodel(self, xdata=None, strided=False): + def evaluate_fitmodel(self, xdata=None, _strided=False): """Evaluate the fit model. :param array xdata: shape (ndata,) or (nmodels, ndatai) - :param bool strided: + :param bool _strided: :returns array: shape (ndata,) or (sum(ndatai),) """ return self._concatenate_evaluation( - "evaluate_fitmodel", xdata=xdata, strided=strided + "evaluate_fitmodel", xdata=xdata, strided=_strided ) - def derivative_fitmodel(self, param_idx, xdata=None, strided=False, **paramtype): + def derivative_fitmodel(self, param_idx, xdata=None, _strided=False, **paramtype): """Derivate to a specific parameter of the fit model. :param int param_idx: :param array xdata: shape (ndata,) - :param bool strided: + :param bool _strided: :returns array: shape (ndata,) """ return self._get_concatenated_derivative( - "derivative_fitmodel", param_idx, xdata=xdata, strided=strided, **paramtype + "derivative_fitmodel", param_idx, xdata=xdata, strided=_strided, **paramtype ) def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): @@ -576,6 +576,27 @@ def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): "numerical_derivative_fitmodel", param_idx, xdata=xdata, **paramtype ) + def _gefit_evaluate_fitmodel(self, parameters, xdata): + """Update parameters and evaluate model + + :param array parameters: shape (nparams,) + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + self.set_parameter_values(parameters) + return self.evaluate_fitmodel(xdata=xdata, _strided=True) + + def _gefit_derivative_fitmodel(self, parameters, param_idx, xdata): + """Update parameters and return derivate to a specific parameter + + :param array parameters: shape (nparams,) + :param int param_idx: + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) + """ + self.set_parameter_values(parameters) + return self.derivative_fitmodel(param_idx, xdata=xdata, _strided=True) + def _get_concatenated_data(self, attr): """ :param str attr: @@ -630,7 +651,6 @@ def _get_concatenated_derivative( param_idx0 = param_idx cached = self._in_property_caching_context() - cached = False if not cached: parameter_index_in_group = group.parameter_index_in_group(param_idx0) for modeli, xdata, idx in self._iter_model_data_slices(xdata, strided=strided): diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 659636a90..9ccd4874a 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -59,16 +59,26 @@ def _test_fit(self, linear): before = self.fitmodel.get_parameter_values(only_linear=False) lin_before = self.fitmodel.get_parameter_values(only_linear=True) + #with self._profile("test"): result = self.fitmodel.fit(full_output=True) + # Verify the expected fit parameters + rtol = 1e-3 + if result["linear"]: + numpy.testing.assert_allclose(result["parameters"], lin_refined_params, rtol=rtol) + else: + numpy.testing.assert_allclose(result["parameters"], refined_params, rtol=rtol) + + # Check that the model has not been affected after = self.fitmodel.get_parameter_values(only_linear=False) lin_after = self.fitmodel.get_parameter_values(only_linear=True) numpy.testing.assert_array_equal(before, after) numpy.testing.assert_array_equal(lin_before, lin_after) + # Modify the fit model self._assert_model_not_refined(refined_params, lin_refined_params) self.fitmodel.use_fit_result(result) - self._assert_model_refined(refined_params, lin_refined_params) + self._assert_model_refined(refined_params, lin_refined_params, rtol=rtol) def _assert_model_not_refined(self, refined_params, lin_refined_params): self.assertTrue( @@ -79,12 +89,12 @@ def _assert_model_not_refined(self, refined_params, lin_refined_params): parameters = self.fitmodel.get_parameter_values(only_linear=True) self.assertTrue(not numpy.allclose(parameters, lin_refined_params)) - def _assert_model_refined(self, refined_params, lin_refined_params): - numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) + def _assert_model_refined(self, refined_params, lin_refined_params, rtol=1e-7): parameters = self.fitmodel.get_parameter_values(only_linear=False) - numpy.testing.assert_allclose(parameters, refined_params) + numpy.testing.assert_allclose(parameters, refined_params, rtol=rtol) parameters = self.fitmodel.get_parameter_values(only_linear=True) - numpy.testing.assert_allclose(parameters, lin_refined_params) + numpy.testing.assert_allclose(parameters, lin_refined_params, rtol=rtol) + numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) def _assert_fit_result(self, result, expected): p = numpy.asarray(result["parameters"]) @@ -95,7 +105,7 @@ def _assert_fit_result(self, result, expected): @contextmanager def _fit_model_subtests(self): - for nmodels in (2,): + for nmodels in (8,): with self.subTest(nmodels=nmodels): self._create_model(nmodels=nmodels) self._validate_model() @@ -286,7 +296,7 @@ def _validate_submodel(self, model, model_idx=None): linear, repr(param_name) ) numpy.testing.assert_allclose( - calc, numerical, err_msg=err_msg, rtol=1e-4 + calc, numerical, err_msg=err_msg, rtol=1e-3 ) parameters = model.get_parameter_values(only_linear=False) From fb449cadf3d8014861c44d02287d127fcf2e01ff Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 10 Aug 2021 12:14:04 +0200 Subject: [PATCH 64/74] fixup --- .../PyMcaMath/fitting/model/CachingModel.py | 5 + .../PyMcaMath/fitting/model/ParameterModel.py | 16 +- PyMca5/tests/ParameterModelTest.py | 315 ++++++++++++++---- 3 files changed, 263 insertions(+), 73 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index b94a0b4d7..90d72fc49 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -238,6 +238,9 @@ def _cached_property_fset(self, fset, value): values_cache[index] = value def _get_from_propid_cache(self, *subnames, dtype=dict, **cacheoptions): + """Returns the cache when propid caching is enabled, potentially initialized + by instantiating `dtype`. Returns `None` when propid caching is disabled. + """ caches = self._getCache("_propid", *subnames) if caches is None: return None @@ -271,6 +274,8 @@ def _propid_to_index(self, propid, **cacheoptions): return index def _name_to_propid(self, property_name, **cacheoptions): + return self._property_id_from_name(property_name) + # TODO: duplicate names for linked models name_to_propid = self._get_from_propid_cache( "_name_to_propid", dtype=dict, **cacheoptions ) diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 28c8e34c6..1b67c7e14 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -184,10 +184,7 @@ def get_parameter_groups(self, **paramtype): return tuple(self._iter_parameter_groups(**paramtype)) def _property_id_from_name(self, property_name): - for group in self._iter_parameter_groups(): - if group.property_name == property_name: - return group - return None + return self._group_from_parameter_name(property_name) def get_parameter_group_names(self, **paramtype): return tuple(group.name for group in self._iter_parameter_groups(**paramtype)) @@ -204,10 +201,11 @@ def _group_from_parameter_index(self, param_idx, **paramtype): if group.start_index <= param_idx < group.stop_index: return group - def _group_from_parameter_name(self, prop_name, **paramtype): + def _group_from_parameter_name(self, property_name, **paramtype): for group in self._iter_parameter_groups(**paramtype): - if group.property_name == prop_name: + if group.property_name == property_name: return group + return None class ParameterModel(CachedPropertiesLinkModel, ParameterModelBase): @@ -217,6 +215,12 @@ def __init__(self, *args, **kw): super().__init__(*args, **kw) self._linear = False + def _iter_cached_property_ids(self, **paramtype): + instance_key = self._linked_instance_to_key + for propid in super()._iter_cached_property_ids(**paramtype): + if propid.linked or propid.instance_key == instance_key: + yield propid + def __iter_parameter_group_properties(self): cls = type(self) for property_name in self._cached_property_names(): diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index b16eef788..0e9ec84ec 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -32,6 +32,22 @@ def var1_lin(self, value): def var1_lin(self): return 2 + @parameter_group + def var3_nonlin(self): + return self._cfg["var3_nonlin"] + + @var3_nonlin.setter + def var3_nonlin(self, value): + self._cfg["var3_nonlin"] = value + + @linear_parameter_group + def var3_lin(self): + return self._cfg["var3_lin"] + + @var3_lin.setter + def var3_lin(self, value): + self._cfg["var3_lin"] = value + def __getitem__(self, index): return self._linked_instance_mapping[index] @@ -56,19 +72,33 @@ def var2_lin(self, value): class ConcatModel(ParameterModelManager): def __init__(self): - cfg1a = {"var1_lin": [11, 11], "var1_nonlin": 12} - cfg1b = {"var1_lin": [21, 21], "var1_nonlin": 22} + cfg1a = { + "var1_lin": [11, 11], + "var1_nonlin": 12, + "var3_lin": 101, + "var3_nonlin": 102, + } + cfg1b = { + "var1_lin": [21, 21], + "var1_nonlin": 22, + "var3_lin": 201, + "var3_nonlin": 202, + } cfg2a = { "var1_lin": [31, 31], "var1_nonlin": 32, "var2_lin": 41, "var2_nonlin": 42, + "var3_lin": 301, + "var3_nonlin": 302, } cfg2b = { "var1_lin": [51, 51], "var1_nonlin": 12, "var2_lin": 61, "var2_nonlin": 62, + "var3_lin": 401, + "var3_nonlin": 402, } models = { "model0": Model1(cfg1a), @@ -106,12 +136,24 @@ def test_linear_context(self): def test_parameter_group_names(self): for cacheoptions in self._parameterize_nonlinear_test(): names = self.concat_model[0].get_parameter_group_names(**cacheoptions) - expected = ("var1_lin", "var1_nonlin") - self.assertEqual(set(names), set(expected)) + expected = ( + "var1_lin", + "var1_nonlin", + "model0:var3_lin", + "model0:var3_nonlin", + ) + self.assertEqual(tuple(names), expected) names = self.concat_model[-1].get_parameter_group_names(**cacheoptions) - expected = ("var1_lin", "var1_nonlin", "var2_lin", "var2_nonlin") - self.assertEqual(set(names), set(expected)) + expected = ( + "var1_lin", + "var1_nonlin", + "var2_lin", + "var2_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", + ) + self.assertEqual(tuple(names), expected) names = self.concat_model.get_parameter_group_names(**cacheoptions) expected = ( @@ -119,54 +161,87 @@ def test_parameter_group_names(self): "var1_nonlin", "var2_lin", "var2_nonlin", + "model0:var3_lin", + "model0:var3_nonlin", + "model1:var3_lin", + "model1:var3_nonlin", + "model2:var3_lin", + "model2:var3_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) with self._unlink_var1_lin() as unlinked: if unlinked: names = self.concat_model.get_parameter_group_names(**cacheoptions) expected = ( + "var1_nonlin", + "var2_lin", + "var2_nonlin", "model0:var1_lin", + "model0:var3_lin", + "model0:var3_nonlin", "model1:var1_lin", + "model1:var3_lin", + "model1:var3_nonlin", "model2:var1_lin", + "model2:var3_lin", + "model2:var3_nonlin", "model3:var1_lin", - "var1_nonlin", - "var2_lin", - "var2_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) def test_linear_parameter_group_names(self): for cacheoptions in self._parameterize_linear_test(): names = self.concat_model[0].get_parameter_group_names(**cacheoptions) - expected = ("var1_lin",) - self.assertEqual(set(names), set(expected)) + expected = ("var1_lin", "model0:var3_lin") + self.assertEqual(tuple(names), expected) names = self.concat_model[-1].get_parameter_group_names(**cacheoptions) - expected = ("var1_lin", "var2_lin") - self.assertEqual(set(names), set(expected)) + expected = ("var1_lin", "var2_lin", "model3:var3_lin") + self.assertEqual(tuple(names), expected) names = self.concat_model.get_parameter_group_names(**cacheoptions) - expected = ("var1_lin", "var2_lin") - self.assertEqual(set(names), set(expected)) + expected = ( + "var1_lin", + "var2_lin", + "model0:var3_lin", + "model1:var3_lin", + "model2:var3_lin", + "model3:var3_lin", + ) + self.assertEqual(tuple(names), expected) with self._unlink_var1_lin() as unlinked: if unlinked: names = self.concat_model.get_parameter_group_names(**cacheoptions) expected = ( + "var2_lin", "model0:var1_lin", + "model0:var3_lin", "model1:var1_lin", + "model1:var3_lin", "model2:var1_lin", + "model2:var3_lin", "model3:var1_lin", - "var2_lin", + "model3:var3_lin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) def test_parameter_names(self): for cacheoptions in self._parameterize_nonlinear_test(): names = self.concat_model[0].get_parameter_names(**cacheoptions) - expected = ("var1_lin0", "var1_lin1", "var1_nonlin") - self.assertEqual(set(names), set(expected)) + expected = ( + "var1_lin0", + "var1_lin1", + "var1_nonlin", + "model0:var3_lin", + "model0:var3_nonlin", + ) + self.assertEqual(tuple(names), expected) names = self.concat_model[-1].get_parameter_names(**cacheoptions) expected = ( @@ -175,8 +250,10 @@ def test_parameter_names(self): "var1_nonlin", "var2_lin", "var2_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) names = self.concat_model.get_parameter_names(**cacheoptions) expected = ( @@ -185,176 +262,274 @@ def test_parameter_names(self): "var1_nonlin", "var2_lin", "var2_nonlin", + "model0:var3_lin", + "model0:var3_nonlin", + "model1:var3_lin", + "model1:var3_nonlin", + "model2:var3_lin", + "model2:var3_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) with self._unlink_var1_lin() as unlinked: if unlinked: names = self.concat_model.get_parameter_names(**cacheoptions) expected = ( + "var1_nonlin", + "var2_lin", + "var2_nonlin", "model0:var1_lin0", "model0:var1_lin1", + "model0:var3_lin", + "model0:var3_nonlin", "model1:var1_lin0", "model1:var1_lin1", + "model1:var3_lin", + "model1:var3_nonlin", "model2:var1_lin0", "model2:var1_lin1", + "model2:var3_lin", + "model2:var3_nonlin", "model3:var1_lin0", "model3:var1_lin1", - "var1_nonlin", - "var2_lin", - "var2_nonlin", + "model3:var3_lin", + "model3:var3_nonlin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) def test_linear_parameter_names(self): for cacheoptions in self._parameterize_linear_test(): names = self.concat_model[0].get_parameter_names(**cacheoptions) - expected = ("var1_lin0", "var1_lin1") - self.assertEqual(set(names), set(expected)) + expected = ("var1_lin0", "var1_lin1", "model0:var3_lin") + self.assertEqual(tuple(names), expected) names = self.concat_model[-1].get_parameter_names(**cacheoptions) - expected = ("var1_lin0", "var1_lin1", "var2_lin") - self.assertEqual(set(names), set(expected)) + expected = ("var1_lin0", "var1_lin1", "var2_lin", "model3:var3_lin") + self.assertEqual(tuple(names), expected) names = self.concat_model.get_parameter_names(**cacheoptions) - expected = ("var1_lin0", "var1_lin1", "var2_lin") - self.assertEqual(set(names), set(expected)) + expected = ( + "var1_lin0", + "var1_lin1", + "var2_lin", + "model0:var3_lin", + "model1:var3_lin", + "model2:var3_lin", + "model3:var3_lin", + ) + self.assertEqual(tuple(names), expected) with self._unlink_var1_lin() as unlinked: if unlinked: names = self.concat_model.get_parameter_names(**cacheoptions) expected = ( + "var2_lin", "model0:var1_lin0", "model0:var1_lin1", + "model0:var3_lin", "model1:var1_lin0", "model1:var1_lin1", + "model1:var3_lin", "model2:var1_lin0", "model2:var1_lin1", + "model2:var3_lin", "model3:var1_lin0", "model3:var1_lin1", - "var2_lin", + "model3:var3_lin", ) - self.assertEqual(set(names), set(expected)) + self.assertEqual(tuple(names), expected) def test_n_parameter(self): for cacheoptions in self._parameterize_nonlinear_test(): n = self.concat_model[0].get_n_parameters(**cacheoptions) - self.assertEqual(n, 3) + self.assertEqual(n, 5) n = self.concat_model[-1].get_n_parameters(**cacheoptions) - self.assertEqual(n, 5) + self.assertEqual(n, 7) n = self.concat_model.get_n_parameters(**cacheoptions) - self.assertEqual(n, 5) + self.assertEqual(n, 13) with self._unlink_var1_lin() as unlinked: if unlinked: n = self.concat_model.get_n_parameters(**cacheoptions) - self.assertEqual(n, 11) + self.assertEqual(n, 19) def test_n_linear_parameter(self): for cacheoptions in self._parameterize_linear_test(): n = self.concat_model[0].get_n_parameters(**cacheoptions) - self.assertEqual(n, 2) + self.assertEqual(n, 3) n = self.concat_model[-1].get_n_parameters(**cacheoptions) - self.assertEqual(n, 3) + self.assertEqual(n, 4) n = self.concat_model.get_n_parameters(**cacheoptions) - self.assertEqual(n, 3) + self.assertEqual(n, 7) with self._unlink_var1_lin() as unlinked: if unlinked: n = self.concat_model.get_n_parameters(**cacheoptions) - self.assertEqual(n, 9) + self.assertEqual(n, 13) def test_parameter_constraints(self): for cacheoptions in self._parameterize_nonlinear_test(): arr = self.concat_model[0].get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (3, 3)) + self.assertEqual(arr.shape, (5, 3)) arr = self.concat_model[-1].get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (5, 3)) + self.assertEqual(arr.shape, (7, 3)) arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (5, 3)) + self.assertEqual(arr.shape, (13, 3)) with self._unlink_var1_lin() as unlinked: if unlinked: arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (11, 3)) + self.assertEqual(arr.shape, (19, 3)) def test_linear_parameter_contraints(self): for cacheoptions in self._parameterize_linear_test(): if not self._cached: continue arr = self.concat_model[0].get_parameter_constraints(**cacheoptions) - try: - self.assertEqual(arr.shape, (2, 3)) - except Exception: - breakpoint() + self.assertEqual(arr.shape, (3, 3)) arr = self.concat_model[-1].get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (3, 3)) + self.assertEqual(arr.shape, (4, 3)) arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (3, 3)) + self.assertEqual(arr.shape, (7, 3)) with self._unlink_var1_lin() as unlinked: if unlinked: arr = self.concat_model.get_parameter_constraints(**cacheoptions) - self.assertEqual(arr.shape, (9, 3)) + self.assertEqual(arr.shape, (13, 3)) def test_get_parameter_values(self): for cacheoptions in self._parameterize_nonlinear_test(): values = self.concat_model[0].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12]) + self.assertEqual(values.tolist(), [11, 11, 12, 101, 102]) values = self.concat_model[-1].get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + self.assertEqual(values.tolist(), [11, 11, 12, 41, 42, 401, 402]) values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [11, 11, 12, 41, 42]) + expected = [11, 11, 12, 41, 42, 101, 102, 201, 202, 301, 302, 401, 402] + self.assertEqual(values.tolist(), expected) with self._unlink_var1_lin() as unlinked: if unlinked: values = self.concat_model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), [12, 41, 42] + [11] * 8) + expected = [ + 12, + 41, + 42, + 11, + 11, + 101, + 102, + 11, + 11, + 201, + 202, + 11, + 11, + 301, + 302, + 11, + 11, + 401, + 402, + ] + self.assertEqual(values.tolist(), expected) def test_parameter_values(self): for cacheoptions in self._parameterize_nonlinear_test(): self._assert_set_get_parameter_values( - self.concat_model[0], [11, 11, 12], **cacheoptions + self.concat_model[0], [11, 11, 12, 101, 102], **cacheoptions ) self._assert_set_get_parameter_values( - self.concat_model[-1], [11, 11, 12, 41, 42], **cacheoptions + self.concat_model[-1], [11, 11, 12, 41, 42, 401, 402], **cacheoptions ) + expected = [11, 11, 12, 41, 42, 101, 102, 201, 202, 301, 302, 401, 402] self._assert_set_get_parameter_values( - self.concat_model, [11, 11, 12, 41, 42], **cacheoptions + self.concat_model, expected, **cacheoptions ) with self._unlink_var1_lin() as unlinked: if unlinked: + expected = [ + 12, + 41, + 42, + 11, + 11, + 101, + 102, + 11, + 11, + 201, + 202, + 11, + 11, + 301, + 302, + 11, + 11, + 401, + 402, + ] self._assert_set_get_parameter_values( - self.concat_model, [12, 41, 42] + [11] * 8, **cacheoptions + self.concat_model, expected, **cacheoptions ) def test_linear_parameter_values(self): for cacheoptions in self._parameterize_linear_test(): self._assert_set_get_parameter_values( - self.concat_model[0], [11, 11], **cacheoptions + self.concat_model[0], [11, 11, 101], **cacheoptions ) self._assert_set_get_parameter_values( - self.concat_model[-1], [11, 11, 41], **cacheoptions + self.concat_model[-1], [11, 11, 41, 401], **cacheoptions ) self._assert_set_get_parameter_values( - self.concat_model, [11, 11, 41], **cacheoptions + self.concat_model, [11, 11, 41, 101, 201, 301, 401], **cacheoptions ) with self._unlink_var1_lin() as unlinked: if unlinked: + expected = [ + 41, + 11, + 11, + 101, + 11, + 11, + 201, + 11, + 11, + 301, + 11, + 11, + 401, + ] self._assert_set_get_parameter_values( - self.concat_model, [41] + [11] * 8, **cacheoptions + self.concat_model, expected, **cacheoptions ) + def test_parameter_property_values(self): + for cacheoptions in self._parameterize_nonlinear_test(): + self._assertValuesEqual(self.concat_model[0].var1_lin, [11, 11]) + self._assertValuesEqual(self.concat_model[0].var1_nonlin, 12) + self._assertValuesEqual(self.concat_model[0].var3_lin, 101) + self._assertValuesEqual(self.concat_model[0].var3_nonlin, 102) + + self._assertValuesEqual(self.concat_model[-1].var1_lin, [11, 11]) + self._assertValuesEqual(self.concat_model[-1].var1_nonlin, 12) + self._assertValuesEqual(self.concat_model[-1].var2_lin, 41) + self._assertValuesEqual(self.concat_model[-1].var2_nonlin, 42) + self._assertValuesEqual(self.concat_model[-1].var3_lin, 401) + self._assertValuesEqual(self.concat_model[-1].var3_nonlin, 402) + def _assert_set_get_parameter_values(self, model, expected, **cacheoptions): self._assert_get_parameter_values(model, expected, **cacheoptions) with self._protect_parameter_values(**cacheoptions): @@ -365,7 +540,13 @@ def _assert_set_get_parameter_values(self, model, expected, **cacheoptions): def _assert_get_parameter_values(self, model, expected, **cacheoptions): values = model.get_parameter_values(**cacheoptions) - self.assertEqual(values.tolist(), expected) + self._assertValuesEqual(values, expected) + + def _assertValuesEqual(self, values, expected): + if isinstance(expected, list): + self.assertEqual(numpy.asarray(values).tolist(), expected) + else: + self.assertEqual(values, expected) def _parameterize_linear_test(self): yield from self._parameterize_tests([[True, False], [None, True]]) From 6df8a529f54512876a9ed2ef6a75c997a6848e08 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 10 Aug 2021 14:22:05 +0200 Subject: [PATCH 65/74] fixup --- PyMca5/tests/FitPolModelTest.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 7f7f228c4..7f2d192c4 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -52,11 +52,12 @@ def testLinearPol(self): ncoeff = degree + 1 expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected - self.assertEqual(model.parameter_group_names, ["fitmodel_coefficients"]) - self.assertEqual( - model.linear_parameter_group_names, ["fitmodel_coefficients"] - ) - numpy.testing.assert_array_equal(model.parameters, expected) + actual = model.get_parameter_values() + numpy.testing.assert_array_equal(actual, expected) + + names = model.get_parameter_group_names() + expected_names = "fitmodel_coefficients", + self.assertEqual(names, expected_names) fitmodel.ydata = model.yfullmodel numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) @@ -84,7 +85,8 @@ def testExpPol(self): expected = self.random_state.uniform(low=-5, high=5, size=ncoeff) model.coefficients = expected expected[0] = numpy.log(expected[0]) - numpy.testing.assert_array_equal(model.parameters, expected) + actual = model.get_parameter_values() + numpy.testing.assert_array_equal(actual, expected) fitmodel.ydata = model.yfullmodel numpy.testing.assert_array_equal(fitmodel.ydata, model.yfullmodel) From ce40b45bdfd9ec569e1a9ec7729eae545e916f94 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 10 Aug 2021 15:25:31 +0200 Subject: [PATCH 66/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 28 ++++---- .../PyMcaMath/fitting/model/ParameterModel.py | 1 + PyMca5/tests/FitSimpleModelTest.py | 70 ++++++++++++------- 3 files changed, 59 insertions(+), 40 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 098c168c9..9bb0ec482 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -1,5 +1,6 @@ from contextlib import contextmanager, ExitStack import numpy + from PyMca5.PyMcaMath.linalg import lstsq from PyMca5.PyMcaMath.fitting import Gefit @@ -325,22 +326,22 @@ def use_fit_result_context(self, result): class LeastSquaresFitModel(LeastSquaresFitModelBase, ParameterModel): - """A least-squares parameter model which implement the fit model + """A least-squares parameter model Derived classes: * implement the LeastSquaresFitModelInterface. - * add parameter like a python property by using the `parameter_group` or + * add fit parameters as python properties by using the `parameter_group` or `linear_parameter_group` decorators instead of the `property` decorator. There is a "fit model" and a "full model". The full model describes the data, the fit model describes the pre-processed data (for example smoothed, - numerical back subtracted, ...). By default the full model and the fit model + background subtracted, ...). By default the full model and the fit model are identical. - Fitting is done with linear least-squares optimization (not iterative) + Fitting is done with linear least-squares optimization (non iterative) or non-linear least-squares optimization (iterative). An outer loop of - non-least-squares optimization can be enabled (iterative). + non-least-squares optimization can be enabled for either (iterative). Example: @@ -647,14 +648,16 @@ def _get_concatenated_derivative( n = self._get_ndata(xdata, strided=strided) ret = numpy.zeros(n) group = self._group_from_parameter_index(param_idx, **paramtype) - model = self._linked_key_to_instance(group.instance_key) param_idx0 = param_idx cached = self._in_property_caching_context() if not cached: - parameter_index_in_group = group.parameter_index_in_group(param_idx0) + param_group_idx = group.parameter_index_in_group(param_idx0) for modeli, xdata, idx in self._iter_model_data_slices(xdata, strided=strided): - if not group.linked and modeli is not model: + if ( + not group.linked + and modeli._linked_instance_to_key != group.instance_key + ): continue if cached: param_idx = param_idx0 @@ -662,14 +665,9 @@ def _get_concatenated_derivative( groupi = modeli._group_from_parameter_name( group.property_name, **paramtype ) - param_idx = groupi.start_index + parameter_index_in_group - func = getattr(modeli, funcname) - - tmpgroup = modeli._group_from_parameter_index(param_idx, **paramtype) - if tmpgroup is None: - breakpoint() + param_idx = groupi.start_index + param_group_idx + func = getattr(modeli, funcname, funcname) ret[idx] = func(param_idx, xdata=xdata, **paramtype) - return ret def _get_ndata(self, data, strided=False): diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 1b67c7e14..6ba1995f1 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -200,6 +200,7 @@ def _group_from_parameter_index(self, param_idx, **paramtype): for group in self._iter_parameter_groups(**paramtype): if group.start_index <= param_idx < group.stop_index: return group + return None def _group_from_parameter_name(self, property_name, **paramtype): for group in self._iter_parameter_groups(**paramtype): diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 9ccd4874a..6083c650f 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -43,11 +43,11 @@ def setUp(self): self.random_state = numpy.random.RandomState(seed=100) def testLinearFit(self): - with self._fit_model_subtests(): + for _ in self._fit_model_subtests(): self._test_fit(True) def testNonLinearFit(self): - with self._fit_model_subtests(): + for _ in self._fit_model_subtests(): self._test_fit(False) def _test_fit(self, linear): @@ -59,15 +59,19 @@ def _test_fit(self, linear): before = self.fitmodel.get_parameter_values(only_linear=False) lin_before = self.fitmodel.get_parameter_values(only_linear=True) - #with self._profile("test"): + # with self._profile("test"): result = self.fitmodel.fit(full_output=True) # Verify the expected fit parameters rtol = 1e-3 if result["linear"]: - numpy.testing.assert_allclose(result["parameters"], lin_refined_params, rtol=rtol) + numpy.testing.assert_allclose( + result["parameters"], lin_refined_params, rtol=rtol + ) else: - numpy.testing.assert_allclose(result["parameters"], refined_params, rtol=rtol) + numpy.testing.assert_allclose( + result["parameters"], refined_params, rtol=rtol + ) # Check that the model has not been affected after = self.fitmodel.get_parameter_values(only_linear=False) @@ -103,14 +107,13 @@ def _assert_fit_result(self, result, expected): ul = p + 3 * pstd self.assertTrue(all((expected >= ll) & (expected <= ul))) - @contextmanager def _fit_model_subtests(self): - for nmodels in (8,): + for nmodels in [1, 4]: with self.subTest(nmodels=nmodels): self._create_model(nmodels=nmodels) - self._validate_model() + self._validate_models() yield - self._validate_model() + self._validate_models() def _create_model(self, nmodels): self.nmodels = nmodels @@ -169,7 +172,7 @@ def _init_random_model(self, model): def _modify_random(self, only_linear=False): self._modify_random_model(only_linear=only_linear) - self._validate_model() + self._validate_models() def _modify_random_model(self, only_linear=False): pallorg = self.fitmodel.get_parameter_values(only_linear=False).copy() @@ -215,17 +218,17 @@ def _modify_random_model(self, only_linear=False): return pall - def _validate_model(self): - self._validate_submodel(self.fitmodel) + def _validate_models(self): + self._validate_model(self.fitmodel) if self.is_combined_model: for model_idx, model in enumerate(self.fitmodel.models): - self._validate_submodel(model, model_idx) - self._validate_submodel(self.fitmodel) + self._validate_model(model, model_idx) + self._validate_model(self.fitmodel) - def _validate_submodel(self, model, model_idx=None): + def _validate_model(self, model, model_idx=None): is_combined_model = self.is_combined_model and model_idx is None - keep_parameters = model.get_parameter_values(only_linear=False).copy() - keep_linear_parameters = model.get_parameter_values(only_linear=True).copy() + original_parameters = model.get_parameter_values(only_linear=False).copy() + original_linear_parameters = model.get_parameter_values(only_linear=True).copy() nonlin_expected = {"gain", "wgain", "wzero", "zero"} if self.is_combined_model: @@ -257,12 +260,12 @@ def _validate_submodel(self, model, model_idx=None): nexpected = len(model.xdata) self.assertEqual(n, nexpected) - n = model.get_n_parameters(only_linear=False) nexpected = len(model.get_parameter_values(only_linear=False)) + n = model.get_n_parameters(only_linear=False) self.assertEqual(n, nexpected) - n = model.get_n_parameters(only_linear=True) nexpected = len(model.get_parameter_values(only_linear=True)) + n = model.get_n_parameters(only_linear=True) self.assertEqual(n, nexpected) arr1 = model.evaluate_fullmodel() @@ -289,20 +292,37 @@ def _validate_submodel(self, model, model_idx=None): n = model.get_n_parameters(only_linear=True) self.assertEqual(n, self.npeaks) + rtol = 1e-3 for linear in (True, False): with model.linear_context(linear): - for param_name, calc, numerical in model.compare_derivatives(): - err_msg = "[only_linear={}] Analytical and numerical derivative of {} are not equal".format( - linear, repr(param_name) + noncached = list(model.compare_derivatives()) + cached = list(model.compare_derivatives()) + err_fmt = "[only_linear={}] {{}} and {{}} derivative of '{{}}' (model: {}) are not equal".format( + linear, model_idx + ) + for deriv, deriv_cached in zip(noncached, cached): + param_name, calc, numerical = deriv + param_name_cached, calc_cached, numerical_cached = deriv_cached + err_msg = err_fmt.format("analytical", "numerical", param_name) + self.assertEqual(param_name, param_name_cached) + numpy.testing.assert_allclose( + calc, numerical, err_msg=err_msg, rtol=rtol + ) + err_msg = err_fmt.format( + "cached analytical", "numerical", param_name_cached + ) + numpy.testing.assert_allclose( + calc_cached, numerical_cached, err_msg=err_msg, rtol=rtol ) + err_msg = err_fmt.format("cached", "non-cached ", param_name) numpy.testing.assert_allclose( - calc, numerical, err_msg=err_msg, rtol=1e-3 + calc, calc_cached, err_msg=err_msg, rtol=rtol ) parameters = model.get_parameter_values(only_linear=False) - numpy.testing.assert_array_equal(keep_parameters, parameters) + numpy.testing.assert_array_equal(original_parameters, parameters) parameters = model.get_parameter_values(only_linear=True) - numpy.testing.assert_array_equal(keep_linear_parameters, parameters) + numpy.testing.assert_array_equal(original_linear_parameters, parameters) def _vis_compare(self, a, b): import matplotlib.pyplot as plt From a1ccee85c6bfe86e0bc9ee519d51a2c60fd99803 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 13 Aug 2021 10:54:46 +0200 Subject: [PATCH 67/74] fixup --- .../fitting/model/CachingLinkedModel.py | 5 ++- .../PyMcaMath/fitting/model/CachingModel.py | 3 +- .../fitting/model/LeastSquaresFitModel.py | 22 +++++------ PyMca5/PyMcaMath/fitting/model/LinkedModel.py | 3 +- .../PyMcaMath/fitting/model/ParameterModel.py | 11 +++--- .../fitting/model/PolynomialModels.py | 39 ++----------------- 6 files changed, 27 insertions(+), 56 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py index 36ddc8d4a..0af8b59eb 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingLinkedModel.py @@ -1,6 +1,7 @@ import numpy -from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModel -from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel + +from .LinkedModel import LinkedModel +from .CachingModel import CachedPropertiesModel class CachedPropertiesLinkModel(CachedPropertiesModel, LinkedModel): diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index 90d72fc49..c33a989fe 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -1,6 +1,7 @@ import functools from contextlib import contextmanager -from PyMca5.PyMcaMath.fitting.model.PropertyUtils import wrapped_property + +from .PropertyUtils import wrapped_property class CacheManager: diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 9bb0ec482..61bd423df 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -4,9 +4,9 @@ from PyMca5.PyMcaMath.linalg import lstsq from PyMca5.PyMcaMath.fitting import Gefit -from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelBase -from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModel -from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelManager +from .ParameterModel import ParameterModelBase +from .ParameterModel import ParameterModel +from .ParameterModel import ParameterModelManager class LeastSquaresFitModelInterface: @@ -179,7 +179,7 @@ def linear_fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - with self.__linear_fit_context(): + with self._linear_fit_context(): b = self.yfitdata for i in range(max(self.niter_non_leastsquares, 1)): A = self.linear_derivatives_fitmodel() @@ -204,7 +204,7 @@ def nonlinear_fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - with self.__nonlinear_fit_context(): + with self._nonlinear_fit_context(): constraints = self.get_parameter_constraints().T xdata = self.xdata ydata = self.yfitdata @@ -255,34 +255,34 @@ def niter_non_leastsquares(self): return 0 @contextmanager - def __linear_fit_context(self): + def _linear_fit_context(self): with ExitStack() as stack: ctx = self.linear_context(True) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) - ctx = self._linear_fit_context() + ctx = self._custom_linear_fit_context() stack.enter_context(ctx) yield @contextmanager - def __nonlinear_fit_context(self): + def _nonlinear_fit_context(self): with ExitStack() as stack: ctx = self.linear_context(False) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) - ctx = self._nonlinear_fit_context() + ctx = self._custom_nonlinear_fit_context() stack.enter_context(ctx) yield @contextmanager - def _linear_fit_context(self): + def _custom_linear_fit_context(self): """To allow derived classes to add context""" yield @contextmanager - def _nonlinear_fit_context(self): + def _custom_nonlinear_fit_context(self): """To allow derived classes to add context""" yield diff --git a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py index 9014fc753..32cccc2a6 100644 --- a/PyMca5/PyMcaMath/fitting/model/LinkedModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LinkedModel.py @@ -1,7 +1,8 @@ import functools from contextlib import ExitStack, contextmanager from collections.abc import Mapping -from PyMca5.PyMcaMath.fitting.model.PropertyUtils import wrapped_property + +from .PropertyUtils import wrapped_property class linked_property(wrapped_property): diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 6ba1995f1..3197aa668 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -2,11 +2,12 @@ from dataclasses import dataclass, field from contextlib import contextmanager import numpy -from PyMca5.PyMcaMath.fitting.model.CachingLinkedModel import CachedPropertiesLinkModel -from PyMca5.PyMcaMath.fitting.model.LinkedModel import LinkedModelManager -from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property -from PyMca5.PyMcaMath.fitting.model.CachingModel import CachedPropertiesModel -from PyMca5.PyMcaMath.fitting.model.CachingModel import cached_property + +from .CachingLinkedModel import CachedPropertiesLinkModel +from .LinkedModel import LinkedModelManager +from .LinkedModel import linked_property +from .CachingModel import CachedPropertiesModel +from .CachingModel import cached_property class parameter_group(cached_property, linked_property): diff --git a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py index 5a5bd3d92..e7cc648da 100644 --- a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py @@ -1,40 +1,7 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import numpy -from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel -from PyMca5.PyMcaMath.fitting.model import parameter_group -from PyMca5.PyMcaMath.fitting.model import linear_parameter_group + +from .LeastSquaresFitModel import LeastSquaresFitModel +from .ParameterModel import linear_parameter_group class PolynomialModel(LeastSquaresFitModel): From 00b093544a6abe3910f968e923c6b4b2aab2ee82 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Fri, 13 Aug 2021 17:42:44 +0200 Subject: [PATCH 68/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 134 +++++++----- .../PyMcaMath/fitting/model/ParameterModel.py | 105 +++++---- .../fitting/model/PolynomialModels.py | 15 +- PyMca5/PyMcaMath/fitting/model/__init__.py | 4 +- PyMca5/tests/FitPolModelTest.py | 50 +---- PyMca5/tests/FitSimpleModelTest.py | 206 +++++++++--------- PyMca5/tests/ParameterModelTest.py | 68 ++++-- PyMca5/tests/SimpleModel.py | 61 ++---- 8 files changed, 318 insertions(+), 325 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index 61bd423df..f71a1a6cd 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -7,6 +7,7 @@ from .ParameterModel import ParameterModelBase from .ParameterModel import ParameterModel from .ParameterModel import ParameterModelManager +from .ParameterModel import ParameterType class LeastSquaresFitModelInterface: @@ -82,6 +83,22 @@ def yfitdata(self): def yfitstd(self): raise NotImplementedError + @property + def yfit_weighted_residuals(self): + return (self.yfitdata - self.yfitmodel) / self.yfitstd + + @property + def chi_squared(self): + return (self.yfit_weighted_residuals ** 2).sum() + + @property + def reduced_chi_squared(self): + return self.chi_squared / self.degrees_of_freedom + + @property + def degrees_of_freedom(self): + return self.ndata - self.get_n_parameters() + def evaluate_fullmodel(self, xdata=None): """Evaluate the full model. @@ -90,7 +107,7 @@ def evaluate_fullmodel(self, xdata=None): """ raise NotImplementedError - def evaluate_linear_fullmodel(self, xdata=None): + def evaluate_decomposed_fullmodel(self, xdata=None): """Evaluate the full model. :param array xdata: shape (ndata,) @@ -98,26 +115,30 @@ def evaluate_linear_fullmodel(self, xdata=None): """ raise NotImplementedError - def evaluate_linear_fitmodel(self, xdata=None): + def evaluate_decomposed_fitmodel(self, xdata=None): """Evaluate the fit model. :param array xdata: shape (ndata,) :returns array: shape (ndata,) """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - parameters = self.get_parameter_values(only_linear=True) + derivatives = self.independent_linear_derivatives_fitmodel(xdata=xdata) + parameters = self.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) return parameters.dot(derivatives) - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters + def independent_linear_derivatives_fitmodel(self, xdata=None): + """Derivates to all independent linear parameters :param array xdata: shape (ndata,) :returns array: shape (nparams, ndata) """ - nparams = self.get_n_parameters(only_linear=True) + nparams = self.get_n_parameters(parameter_type=ParameterType.independent_linear) return numpy.array( [ - self.derivative_fitmodel(i, xdata=xdata, only_linear=True) + self.derivative_fitmodel( + i, xdata=xdata, parameter_type=ParameterType.independent_linear + ) for i in range(nparams) ] ) @@ -150,8 +171,10 @@ def linear_decomposition_fitmodel(self, xdata=None): :param array xdata: shape (ndata,) :returns array: nparams x ndata """ - derivatives = self.linear_derivatives_fitmodel(xdata=xdata) - parameters = self.get_parameter_values(only_linear=True) + derivatives = self.independent_linear_derivatives_fitmodel(xdata=xdata) + parameters = self.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) return parameters[:, numpy.newaxis] * derivatives @property @@ -169,42 +192,45 @@ def fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - if self.linear: - return self.linear_fit(full_output=full_output) + if self.parameter_type == ParameterType.independent_linear: + return self.non_iterative_optimization(full_output=full_output) else: - return self.nonlinear_fit(full_output=full_output) + return self.iterative_optimization(full_output=full_output) - def linear_fit(self, full_output=False): + def non_iterative_optimization(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters :returns dict: """ - with self._linear_fit_context(): + with self._non_iterative_optimization_context(): b = self.yfitdata + sigma_b = self.yfitstd for i in range(max(self.niter_non_leastsquares, 1)): - A = self.linear_derivatives_fitmodel() + A = self.independent_linear_derivatives_fitmodel() result = lstsq( A.T, # ndata, nparams b.copy(), # ndata uncertainties=True, covariances=False, digested_output=True, + sigma_b=sigma_b, + weight=self.weightflag, ) if self.niter_non_leastsquares: self.set_parameter_values(result["parameters"]) self.non_leastsquares_increment() - result["linear"] = True - result["parameters"] = result["parameters"] - result["uncertainties"] = result["uncertainties"] - result.pop("svd") - return result + result["parameter_type"] = self.parameter_type + result["parameters"] = result["parameters"] + result["uncertainties"] = result["uncertainties"] + result.pop("svd") + return result - def nonlinear_fit(self, full_output=False): + def iterative_optimization(self, full_output=False): """ :param bool full_output: add statistics to fitted parameters :returns dict: """ - with self._nonlinear_fit_context(): + with self._iterative_optimization_context(): constraints = self.get_parameter_constraints().T xdata = self.xdata ydata = self.yfitdata @@ -223,20 +249,21 @@ def nonlinear_fit(self, full_output=False): weightflag=self.weightflag, deltachi=self.deltachi, fulloutput=full_output, + linear=self.only_linear_parameters, ) if self.niter_non_leastsquares: self.set_parameter_values(result[0]) self.non_leastsquares_increment() - ret = { - "linear": False, - "parameters": result[0], - "uncertainties": result[2], - "chi2_red": result[1], - } - if full_output: - ret["niter"] = result[3] - ret["lastdeltachi"] = result[4] - return ret + ret = { + "parameter_type": self.parameter_type, + "parameters": result[0], + "uncertainties": result[2], + "chi2_red": result[1], + } + if full_output: + ret["niter"] = result[3] + ret["lastdeltachi"] = result[4] + return ret @property def maxiter(self): @@ -255,34 +282,32 @@ def niter_non_leastsquares(self): return 0 @contextmanager - def _linear_fit_context(self): + def _non_iterative_optimization_context(self): with ExitStack() as stack: - ctx = self.linear_context(True) + ctx = self.parameter_type_context(ParameterType.independent_linear) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) - ctx = self._custom_linear_fit_context() + ctx = self._custom_non_iterative_optimization_context() stack.enter_context(ctx) yield @contextmanager - def _nonlinear_fit_context(self): + def _iterative_optimization_context(self): with ExitStack() as stack: - ctx = self.linear_context(False) - stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) - ctx = self._custom_nonlinear_fit_context() + ctx = self._custom_iterative_optimization_context() stack.enter_context(ctx) yield @contextmanager - def _custom_linear_fit_context(self): + def _custom_non_iterative_optimization_context(self): """To allow derived classes to add context""" yield @contextmanager - def _custom_nonlinear_fit_context(self): + def _custom_iterative_optimization_context(self): """To allow derived classes to add context""" yield @@ -311,7 +336,9 @@ def use_fit_result(self, result): """ :param dict result: """ - self.set_parameter_values(result["parameters"], only_linear=result["linear"]) + self.set_parameter_values( + result["parameters"], parameter_type=result["parameter_type"] + ) @contextmanager def use_fit_result_context(self, result): @@ -319,7 +346,7 @@ def use_fit_result_context(self, result): :param dict result: """ - with self.linear_context(result["linear"]): + with self.parameter_type_context(result["parameter_type"]): with self._propertyCachingContext(): self.use_fit_result(result) yield @@ -331,8 +358,8 @@ class LeastSquaresFitModel(LeastSquaresFitModelBase, ParameterModel): Derived classes: * implement the LeastSquaresFitModelInterface. - * add fit parameters as python properties by using the `parameter_group` or - `linear_parameter_group` decorators instead of the `property` decorator. + * add fit parameters as python properties by using the `nonlinear_parameter_group` or + `independent_linear_parameter_group` decorators instead of the `property` decorator. There is a "fit model" and a "full model". The full model describes the data, the fit model describes the pre-processed data (for example smoothed, @@ -394,13 +421,13 @@ def evaluate_fullmodel(self, xdata=None): y = self.evaluate_fitmodel(xdata=xdata) return self._y_fit_to_full(y, xdata=xdata) - def evaluate_linear_fullmodel(self, xdata=None): + def evaluate_decomposed_fullmodel(self, xdata=None): """Evaluate the full model. :param array xdata: shape (ndata,) :returns array: shape (ndata,) """ - y = self.evaluate_linear_fitmodel(xdata=xdata) + y = self.evaluate_decomposed_fitmodel(xdata=xdata) return self._y_fit_to_full(y, xdata=xdata) def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): @@ -420,10 +447,9 @@ def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): :returns array: shape (ndata,) """ group = self._group_from_parameter_index(param_idx, **paramtype) - param_is_linear = group.linear parameters = self.get_parameter_values(**paramtype) try: - if param_is_linear: + if group.is_linear: return self._numerical_derivative_linear_param( parameters, param_idx, xdata=xdata, **paramtype ) @@ -444,7 +470,7 @@ def _numerical_derivative_linear_param( # dy/dpi(x) = fi(x) parameters = parameters.copy() for group in self._iter_parameter_groups(**paramtype): - if group.linear: + if group.is_independent_linear: parameters[group.index] = 0 parameters[param_idx] = 1 self.set_parameter_values(parameters, **paramtype) @@ -535,13 +561,15 @@ def evaluate_fullmodel(self, xdata=None): """ return self._concatenate_evaluation("evaluate_fullmodel", xdata=xdata) - def evaluate_linear_fullmodel(self, xdata=None): + def evaluate_decomposed_fullmodel(self, xdata=None): """Evaluate the full model. :param array xdata: shape (ndata,) or (nmodels, ndatai) :returns array: shape (ndata,) or (sum(ndatai),) """ - return self._concatenate_evaluation("evaluate_linear_fullmodel", xdata=xdata) + return self._concatenate_evaluation( + "evaluate_decomposed_fullmodel", xdata=xdata + ) def evaluate_fitmodel(self, xdata=None, _strided=False): """Evaluate the fit model. diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 3197aa668..6eef07c05 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from contextlib import contextmanager import numpy +from enum import Enum from .CachingLinkedModel import CachedPropertiesLinkModel from .LinkedModel import LinkedModelManager @@ -10,7 +11,10 @@ from .CachingModel import cached_property -class parameter_group(cached_property, linked_property): +ParameterType = Enum("ParameterType", "non_linear dependent_linear independent_linear") + + +class _parameter_group(cached_property, linked_property): """Usage: .. highlight:: python @@ -21,7 +25,7 @@ class MyClass(Model): def __init__(self): self._myparam = 0. - @parameter_group + @nonlinear_parameter_group def myparam(self): return self._myparam @@ -38,6 +42,8 @@ def myparam(self): return 1, 0, 0 """ + TYPE = NotImplemented + def __init__(self, *args, **kw): self.fcount = self._fcount_default() self.fconstraints = self._fconstraints_default() @@ -68,15 +74,23 @@ def fconstraints(oself): return fconstraints -class linear_parameter_group(parameter_group): - pass +class nonlinear_parameter_group(_parameter_group): + TYPE = ParameterType.non_linear + + +class dependent_linear_parameter_group(_parameter_group): + TYPE = ParameterType.dependent_linear + + +class independent_linear_parameter_group(_parameter_group): + TYPE = ParameterType.independent_linear @dataclass(frozen=True, eq=True) class ParameterGroupId: name: str property_name: str = field(compare=False, hash=False) - linear: bool = field(compare=False, hash=False) + type: ParameterType = field(compare=False, hash=False) linked: bool = field(compare=False, hash=False) count: int = field(compare=False, hash=False) start_index: int = field(compare=False, hash=False) @@ -100,31 +114,46 @@ def parameter_index_in_group(self, param_idx): return param_idx - self.start_index return None + @property + def is_linear(self): + return self.type != ParameterType.non_linear + + @property + def is_independent_linear(self): + return self.type == ParameterType.independent_linear + class ParameterModelBase(CachedPropertiesModel): """Interface for all models that manage fit parameters""" @property - def linear(self): + def parameter_type(self): raise NotImplementedError - @linear.setter - def linear(self, value): + @parameter_type.setter + def parameter_type(self, value): raise NotImplementedError + @property + def only_linear_parameters(self): + return self.parameter_type in ( + ParameterType.dependent_linear, + ParameterType.independent_linear, + ) + @contextmanager - def linear_context(self, linear): - keep = self.linear - self.linear = linear + def parameter_type_context(self, value): + keep = self.parameter_type + self.parameter_type = value try: yield finally: - self.linear = keep + self.parameter_type = keep - def _property_cache_key(self, only_linear=None, **paramtype): - if only_linear is None: - only_linear = self.linear - return only_linear + def _property_cache_key(self, parameter_type=NotImplemented, **paramtype): + if parameter_type is NotImplemented: + parameter_type = self.parameter_type + return parameter_type def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) @@ -215,7 +244,7 @@ class ParameterModel(CachedPropertiesLinkModel, ParameterModelBase): def __init__(self, *args, **kw): super().__init__(*args, **kw) - self._linear = False + self._parameter_type = None def _iter_cached_property_ids(self, **paramtype): instance_key = self._linked_instance_to_key @@ -227,17 +256,17 @@ def __iter_parameter_group_properties(self): cls = type(self) for property_name in self._cached_property_names(): prop = getattr(cls, property_name) - if not isinstance(prop, parameter_group): + if not isinstance(prop, _parameter_group): raise TypeError( - "Currently only 'parameter_group' properties support caching" + "Currently only parameter _group properties support caching" ) yield property_name, prop def _instance_cached_property_ids( - self, only_linear=None, linked=None, tracker=None + self, parameter_type=NotImplemented, linked=None, tracker=None ): """ - :param only_linear bool: all parameters (linear and non-linear) or only linear parameters + :param parameter_type bool: only this parameter type :param linked bool: linked parameters or unlinked parameters :param tracker _IterGroupTracker: :yields ParameterGroupId: @@ -254,10 +283,9 @@ def _instance_cached_property_ids( if group_is_linked is not linked: continue - group_is_linear = isinstance(prop, linear_parameter_group) - if only_linear is None: - only_linear = self.linear - if only_linear and not group_is_linear: + if parameter_type is NotImplemented: + parameter_type = self.parameter_type + if parameter_type is not None and prop.TYPE != parameter_type: continue count = prop.fcount(self) @@ -272,9 +300,6 @@ def _instance_cached_property_ids( else: index = None - def get_constraints(): - return prop.fconstraints(self) - instance_key = self._linked_instance_to_key if group_is_linked or instance_key is None: name = property_name @@ -283,7 +308,7 @@ def get_constraints(): group = ParameterGroupId( name=name, - linear=group_is_linear, + type=prop.TYPE, linked=group_is_linked, property_name=property_name, instance_key=instance_key, @@ -307,12 +332,12 @@ def get_constraints(): return get_constraints @linked_property - def linear(self): - return self._linear + def parameter_type(self): + return self._parameter_type - @linear.setter - def linear(self, value): - self._linear = value + @parameter_type.setter + def parameter_type(self, value): + self._parameter_type = value def _get_noncached_property_value(self, group): return getattr(self, group.property_name) @@ -343,7 +368,7 @@ class ParameterModelManager(ParameterModelBase, LinkedModelManager): def __init__(self, *args, **kw): super().__init__(*args, **kw) - self._enable_property_link("linear") + self._enable_property_link("parameter_type") for model in self.models: model._cache_manager = self @@ -360,12 +385,12 @@ def nmodels(self): return len(self.model_mapping) @property - def linear(self): - return self._get_linked_property_value("linear") + def parameter_type(self): + return self._get_linked_property_value("parameter_type") - @linear.setter - def linear(self, value): - self._set_linked_property_value("linear", value) + @parameter_type.setter + def parameter_type(self, value): + self._set_linked_property_value("parameter_type", value) def _instance_cached_property_ids(self, **paramtype): """ diff --git a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py index e7cc648da..ad3202125 100644 --- a/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py +++ b/PyMca5/PyMcaMath/fitting/model/PolynomialModels.py @@ -1,7 +1,7 @@ import numpy from .LeastSquaresFitModel import LeastSquaresFitModel -from .ParameterModel import linear_parameter_group +from .ParameterModel import independent_linear_parameter_group class PolynomialModel(LeastSquaresFitModel): @@ -9,7 +9,6 @@ def __init__(self, degree=0, maxiter=100): self._xdata = None self._ydata = None self._mask = None - self._linear = True self.degree = degree self.maxiter = maxiter super().__init__() @@ -52,14 +51,6 @@ def ydata(self, values): def ystd(self): return None - @property - def linear(self): - return self._linear - - @linear.setter - def linear(self, value): - self._linear = value - @property def maxiter(self): return self._maxiter @@ -72,7 +63,7 @@ def maxiter(self, value): class LinearPolynomialModel(PolynomialModel): """y = c0 + c1*x + c2*x^2 + ...""" - @linear_parameter_group + @independent_linear_parameter_group def fitmodel_coefficients(self): return self.coefficients @@ -114,7 +105,7 @@ class ExponentialPolynomialModel(LinearPolynomialModel): yfit = log(y) = log(c1) + c1*x + c2*x^2 + ... """ - @linear_parameter_group + @independent_linear_parameter_group def fitmodel_coefficients(self): coefficients = self.coefficients.copy() coefficients[0] = numpy.log(coefficients[0]) diff --git a/PyMca5/PyMcaMath/fitting/model/__init__.py b/PyMca5/PyMcaMath/fitting/model/__init__.py index 5e54e6598..ff38fdcb1 100644 --- a/PyMca5/PyMcaMath/fitting/model/__init__.py +++ b/PyMca5/PyMcaMath/fitting/model/__init__.py @@ -2,8 +2,8 @@ """ from PyMca5.PyMcaMath.fitting.model.ParameterModel import ( - parameter_group, - linear_parameter_group, + nonlinear_parameter_group, + independent_linear_parameter_group, ) from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import ( LeastSquaresFitModel, diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 7f2d192c4..868b3483d 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -1,39 +1,7 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import unittest import numpy from PyMca5.PyMcaMath.fitting.model import PolynomialModels +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType class testFitPolModel(unittest.TestCase): @@ -56,7 +24,7 @@ def testLinearPol(self): numpy.testing.assert_array_equal(actual, expected) names = model.get_parameter_group_names() - expected_names = "fitmodel_coefficients", + expected_names = ("fitmodel_coefficients",) self.assertEqual(names, expected_names) fitmodel.ydata = model.yfullmodel @@ -64,9 +32,9 @@ def testLinearPol(self): numpy.testing.assert_array_equal(fitmodel.yfitdata, model.yfitmodel) numpy.testing.assert_array_equal(model.yfitmodel, model.yfullmodel) - for linear in [True, False]: - with self.subTest(degree=degree, linear=linear): - fitmodel.linear = linear + for parameter_type in [ParameterType.independent_linear, None]: + with self.subTest(degree=degree, parameter_type=parameter_type): + fitmodel.parameter_type = parameter_type fitmodel.coefficients = numpy.zeros_like(expected) self.assertEqual(fitmodel.degree, degree) result = fitmodel.fit()["parameters"] @@ -95,11 +63,11 @@ def testExpPol(self): model.yfitmodel, numpy.log(model.yfullmodel) ) - for linear in [True, False]: - with self.subTest(degree=degree, linear=linear): - fitmodel.linear = linear + for parameter_type in [ParameterType.independent_linear, None]: + with self.subTest(degree=degree, parameter_type=parameter_type): + fitmodel.parameter_type = parameter_type fitmodel.coefficients = numpy.zeros_like(expected) - if not linear: + if parameter_type is None: fitmodel.coefficients[0] = 0.1 self.assertEqual(fitmodel.degree, degree) result = fitmodel.fit()["parameters"] diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 6083c650f..703533236 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -1,41 +1,9 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import unittest from contextlib import contextmanager import numpy from PyMca5.tests.SimpleModel import SimpleModel from PyMca5.tests.SimpleModel import SimpleCombinedModel +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType class testFitModel(unittest.TestCase): @@ -44,27 +12,31 @@ def setUp(self): def testLinearFit(self): for _ in self._fit_model_subtests(): - self._test_fit(True) + self._test_fit(ParameterType.independent_linear) def testNonLinearFit(self): for _ in self._fit_model_subtests(): - self._test_fit(False) - - def _test_fit(self, linear): - self.fitmodel.linear = linear - refined_params = self.fitmodel.get_parameter_values(only_linear=False).copy() - lin_refined_params = self.fitmodel.get_parameter_values(only_linear=True).copy() - self._modify_random(only_linear=linear) - - before = self.fitmodel.get_parameter_values(only_linear=False) - lin_before = self.fitmodel.get_parameter_values(only_linear=True) + self._test_fit(None) + + def _test_fit(self, parameter_type): + self.fitmodel.parameter_type = parameter_type + refined_params = self.fitmodel.get_parameter_values(parameter_type=None).copy() + lin_refined_params = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ).copy() + self._modify_random(parameter_type=parameter_type) + + before = self.fitmodel.get_parameter_values(parameter_type=None) + lin_before = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) # with self._profile("test"): result = self.fitmodel.fit(full_output=True) # Verify the expected fit parameters rtol = 1e-3 - if result["linear"]: + if result["parameter_type"]: numpy.testing.assert_allclose( result["parameters"], lin_refined_params, rtol=rtol ) @@ -74,8 +46,10 @@ def _test_fit(self, linear): ) # Check that the model has not been affected - after = self.fitmodel.get_parameter_values(only_linear=False) - lin_after = self.fitmodel.get_parameter_values(only_linear=True) + after = self.fitmodel.get_parameter_values(parameter_type=None) + lin_after = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) numpy.testing.assert_array_equal(before, after) numpy.testing.assert_array_equal(lin_before, lin_after) @@ -88,15 +62,19 @@ def _assert_model_not_refined(self, refined_params, lin_refined_params): self.assertTrue( not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) ) - parameters = self.fitmodel.get_parameter_values(only_linear=False) + parameters = self.fitmodel.get_parameter_values(parameter_type=None) self.assertTrue(not numpy.allclose(parameters, refined_params)) - parameters = self.fitmodel.get_parameter_values(only_linear=True) + parameters = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) self.assertTrue(not numpy.allclose(parameters, lin_refined_params)) def _assert_model_refined(self, refined_params, lin_refined_params, rtol=1e-7): - parameters = self.fitmodel.get_parameter_values(only_linear=False) + parameters = self.fitmodel.get_parameter_values(parameter_type=None) numpy.testing.assert_allclose(parameters, refined_params, rtol=rtol) - parameters = self.fitmodel.get_parameter_values(only_linear=True) + parameters = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) numpy.testing.assert_allclose(parameters, lin_refined_params, rtol=rtol) numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) @@ -122,7 +100,7 @@ def _create_model(self, nmodels): self.fitmodel = SimpleModel() else: self.fitmodel = SimpleCombinedModel(ndetectors=nmodels) - self.assertTrue(not self.fitmodel.linear) + self.assertTrue(not self.fitmodel.parameter_type) self._init_random() @@ -170,53 +148,57 @@ def _init_random_model(self, model): model.concentrations = self.random_state.uniform(low=0.5, high=1, size=npeaks) model.efficiency = self.random_state.uniform(low=5000, high=6000, size=npeaks) - def _modify_random(self, only_linear=False): - self._modify_random_model(only_linear=only_linear) + def _modify_random(self, parameter_type): + self._modify_random_model(parameter_type) self._validate_models() - def _modify_random_model(self, only_linear=False): - pallorg = self.fitmodel.get_parameter_values(only_linear=False).copy() - plinorg = self.fitmodel.get_parameter_values(only_linear=True).copy() - - if not only_linear: - pall = pallorg.copy() - pall *= self.random_state.uniform(0.95, 1, len(pall)) - self.fitmodel.set_parameter_values(pall, only_linear=False) - parameters = self.fitmodel.get_parameter_values(only_linear=False) - numpy.testing.assert_array_equal(parameters, pall) + def _modify_random_model(self, parameter_type): + self.assertIn(parameter_type, (None, ParameterType.independent_linear)) + pnonlinorg = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.non_linear + ) + plinorg = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) + pallorg = self.fitmodel.get_parameter_values(parameter_type=None) - plin = plinorg.copy() - plin *= self.random_state.uniform(0.5, 0.8, len(plin)) - self.fitmodel.set_parameter_values(plin, only_linear=True) - parameters = self.fitmodel.get_parameter_values(only_linear=True) - numpy.testing.assert_array_equal(parameters, plin) + if parameter_type is None: + pmod = pnonlinorg.copy() + pmod *= self.random_state.uniform(0.95, 1, len(pmod)) + self.fitmodel.set_parameter_values( + pmod, parameter_type=ParameterType.non_linear + ) + parameters = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.non_linear + ) + numpy.testing.assert_array_equal(parameters, pmod) - if only_linear: - pall = self.fitmodel.get_parameter_values(only_linear=False) + pmod = plinorg.copy() + pmod *= self.random_state.uniform(0.5, 0.8, len(pmod)) + self.fitmodel.set_parameter_values( + pmod, parameter_type=ParameterType.independent_linear + ) + parameters = self.fitmodel.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) + numpy.testing.assert_array_equal(parameters, pmod) - for group in self.fitmodel.get_parameter_groups(only_linear=False): + pall = self.fitmodel.get_parameter_values(parameter_type=None) + for group in self.fitmodel.get_parameter_groups(parameter_type=None): current = pall[group.index] expected = pallorg[group.index] - if only_linear and not group.linear: - if group.count == 1: - self.assertEqual(current, expected, msg=group.name) - else: - self.assertTrue(all(current == expected), msg=group.name) - else: + if parameter_type is None or group.is_independent_linear: + # Values are expected to be modified if group.count == 1: self.assertNotEqual(current, expected, msg=group.name) else: self.assertFalse(all(current == expected), msg=group.name) - - for group in self.fitmodel.get_parameter_groups(only_linear=True): - current = plin[group.index] - expected = plinorg[group.index] - if group.count == 1: - self.assertNotEqual(current, expected, msg=group.name) else: - self.assertFalse(all(current == expected), msg=group.name) - - return pall + # Values are not expected to be modified + if group.count == 1: + self.assertEqual(current, expected, msg=group.name) + else: + self.assertTrue(all(current == expected), msg=group.name) def _validate_models(self): self._validate_model(self.fitmodel) @@ -227,8 +209,10 @@ def _validate_models(self): def _validate_model(self, model, model_idx=None): is_combined_model = self.is_combined_model and model_idx is None - original_parameters = model.get_parameter_values(only_linear=False).copy() - original_linear_parameters = model.get_parameter_values(only_linear=True).copy() + original_parameters = model.get_parameter_values(parameter_type=None).copy() + original_linear_parameters = model.get_parameter_values( + parameter_type=ParameterType.independent_linear + ).copy() nonlin_expected = {"gain", "wgain", "wzero", "zero"} if self.is_combined_model: @@ -244,38 +228,44 @@ def _validate_model(self, model, model_idx=None): } lin_expected = {"concentrations"} all_expected = lin_expected | nonlin_expected - names = model.get_parameter_group_names(only_linear=False) + names = model.get_parameter_group_names(parameter_type=None) self.assertEqual(set(names), all_expected) - names = model.get_parameter_group_names(only_linear=True) + names = model.get_parameter_group_names( + parameter_type=ParameterType.independent_linear + ) self.assertEqual(set(names), lin_expected) lin_expected = {f"concentrations{i}" for i in range(self.npeaks)} all_expected = lin_expected | nonlin_expected - names = model.get_parameter_names(only_linear=False) + names = model.get_parameter_names(parameter_type=None) self.assertEqual(set(names), all_expected) - names = model.get_parameter_names(only_linear=True) + names = model.get_parameter_names( + parameter_type=ParameterType.independent_linear + ) self.assertEqual(set(names), lin_expected) n = model.ndata nexpected = len(model.xdata) self.assertEqual(n, nexpected) - nexpected = len(model.get_parameter_values(only_linear=False)) - n = model.get_n_parameters(only_linear=False) + nexpected = len(model.get_parameter_values(parameter_type=None)) + n = model.get_n_parameters(parameter_type=None) self.assertEqual(n, nexpected) - nexpected = len(model.get_parameter_values(only_linear=True)) - n = model.get_n_parameters(only_linear=True) + nexpected = len( + model.get_parameter_values(parameter_type=ParameterType.independent_linear) + ) + n = model.get_n_parameters(parameter_type=ParameterType.independent_linear) self.assertEqual(n, nexpected) arr1 = model.evaluate_fullmodel() - arr2 = model.evaluate_linear_fullmodel() + arr2 = model.evaluate_decomposed_fullmodel() arr3 = model.yfullmodel numpy.testing.assert_allclose(arr1, arr2) numpy.testing.assert_allclose(arr1, arr3) arr1 = model.evaluate_fitmodel() - arr2 = model.evaluate_linear_fitmodel() + arr2 = model.evaluate_decomposed_fitmodel() arr3 = model.yfitmodel arr4 = sum(model.linear_decomposition_fitmodel()) numpy.testing.assert_allclose(arr1, arr2) @@ -287,18 +277,18 @@ def _validate_model(self, model, model_idx=None): nexpected = self.npeaks + self.nshapeparams * nmodels else: nexpected = self.npeaks + self.nshapeparams - n = model.get_n_parameters(only_linear=False) + n = model.get_n_parameters(parameter_type=None) self.assertEqual(n, nexpected) - n = model.get_n_parameters(only_linear=True) + n = model.get_n_parameters(parameter_type=ParameterType.independent_linear) self.assertEqual(n, self.npeaks) rtol = 1e-3 - for linear in (True, False): - with model.linear_context(linear): + for parameter_type in (ParameterType.independent_linear, None): + with model.parameter_type_context(parameter_type): noncached = list(model.compare_derivatives()) cached = list(model.compare_derivatives()) - err_fmt = "[only_linear={}] {{}} and {{}} derivative of '{{}}' (model: {}) are not equal".format( - linear, model_idx + err_fmt = "[parameter_type={}] {{}} and {{}} derivative of '{{}}' (model: {}) are not equal".format( + parameter_type, model_idx ) for deriv, deriv_cached in zip(noncached, cached): param_name, calc, numerical = deriv @@ -319,9 +309,11 @@ def _validate_model(self, model, model_idx=None): calc, calc_cached, err_msg=err_msg, rtol=rtol ) - parameters = model.get_parameter_values(only_linear=False) + parameters = model.get_parameter_values(parameter_type=None) numpy.testing.assert_array_equal(original_parameters, parameters) - parameters = model.get_parameter_values(only_linear=True) + parameters = model.get_parameter_values( + parameter_type=ParameterType.independent_linear + ) numpy.testing.assert_array_equal(original_linear_parameters, parameters) def _vis_compare(self, a, b): diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index 0e9ec84ec..da522f4c7 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -3,8 +3,11 @@ from contextlib import contextmanager from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModel from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterModelManager -from PyMca5.PyMcaMath.fitting.model.ParameterModel import parameter_group -from PyMca5.PyMcaMath.fitting.model.ParameterModel import linear_parameter_group +from PyMca5.PyMcaMath.fitting.model.ParameterModel import nonlinear_parameter_group +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ( + independent_linear_parameter_group, +) +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType class Model1(ParameterModel): @@ -12,7 +15,7 @@ def __init__(self, cfg): super().__init__() self._cfg = cfg - @parameter_group + @nonlinear_parameter_group def var1_nonlin(self): return self._cfg["var1_nonlin"] @@ -20,7 +23,7 @@ def var1_nonlin(self): def var1_nonlin(self, value): self._cfg["var1_nonlin"] = value - @linear_parameter_group + @independent_linear_parameter_group def var1_lin(self): return self._cfg["var1_lin"] @@ -32,7 +35,7 @@ def var1_lin(self, value): def var1_lin(self): return 2 - @parameter_group + @nonlinear_parameter_group def var3_nonlin(self): return self._cfg["var3_nonlin"] @@ -40,7 +43,7 @@ def var3_nonlin(self): def var3_nonlin(self, value): self._cfg["var3_nonlin"] = value - @linear_parameter_group + @independent_linear_parameter_group def var3_lin(self): return self._cfg["var3_lin"] @@ -53,7 +56,7 @@ def __getitem__(self, index): class Model2(Model1): - @parameter_group + @nonlinear_parameter_group def var2_nonlin(self): return self._cfg["var2_nonlin"] @@ -61,7 +64,7 @@ def var2_nonlin(self): def var2_nonlin(self, value): self._cfg["var2_nonlin"] = value - @linear_parameter_group + @independent_linear_parameter_group def var2_lin(self): return self._cfg["var2_lin"] @@ -120,18 +123,30 @@ def setUp(self): self.concat_model = ConcatModel() def test_instantiation(self): - self.assertFalse(self.concat_model.linear) + self.assertEqual(self.concat_model.parameter_type, None) self.assertEqual(self.concat_model.nmodels, 4) - def test_linear_context(self): - with self.concat_model.linear_context(True): - self.assertTrue(self.concat_model.linear) + def test_parameter_type(self): + for parameter_type in ParameterType: + for group in self.concat_model.get_parameter_groups( + parameter_type=parameter_type + ): + self.assertEqual(group.type, parameter_type) + types = set(group.type for group in self.concat_model.get_parameter_groups()) + expected = {ParameterType.independent_linear, ParameterType.non_linear} + self.assertEqual(types, expected) + + def test_parameter_type_context(self): + with self.concat_model.parameter_type_context(ParameterType.independent_linear): + self.assertEqual( + self.concat_model.parameter_type, ParameterType.independent_linear + ) for model in self.concat_model.models: - self.assertTrue(model.linear) + self.assertTrue(model.parameter_type, ParameterType.independent_linear) - self.assertFalse(self.concat_model.linear) + self.assertEqual(self.concat_model.parameter_type, None) for model in self.concat_model.models: - self.assertFalse(model.linear) + self.assertEqual(model.parameter_type, None) def test_parameter_group_names(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -194,7 +209,7 @@ def test_parameter_group_names(self): ) self.assertEqual(tuple(names), expected) - def test_linear_parameter_group_names(self): + def test_independent_linear_parameter_group_names(self): for cacheoptions in self._parameterize_linear_test(): names = self.concat_model[0].get_parameter_group_names(**cacheoptions) expected = ("var1_lin", "model0:var3_lin") @@ -549,21 +564,28 @@ def _assertValuesEqual(self, values, expected): self.assertEqual(values, expected) def _parameterize_linear_test(self): - yield from self._parameterize_tests([[True, False], [None, True]]) + yield from self._parameterize_tests( + [ + [ParameterType.independent_linear, None], + [NotImplemented, ParameterType.independent_linear], + ] + ) def _parameterize_nonlinear_test(self): - yield from self._parameterize_tests([[False, True], [None, False]]) + yield from self._parameterize_tests( + [[None, ParameterType.independent_linear], [NotImplemented, None]] + ) def _parameterize_tests(self, linear_options): - for local_linear, global_linear in linear_options: + for local_type, global_type in linear_options: for cached in [True, False]: with self.subTest( - local_linear=local_linear, - global_linear=global_linear, + local_type=local_type, + global_type=global_type, cached=cached, ): - self.concat_model.linear = global_linear - cacheoptions = {"only_linear": local_linear} + self.concat_model.parameter_type = global_type + cacheoptions = {"parameter_type": local_type} self._cached = cached if cached: with self.concat_model._propertyCachingContext(**cacheoptions): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 1f6e6d649..274ea5ee3 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -1,40 +1,7 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import numpy from PyMca5.PyMcaMath.fitting import SpecfitFuns -from PyMca5.PyMcaMath.fitting.model import parameter_group -from PyMca5.PyMcaMath.fitting.model import linear_parameter_group +from PyMca5.PyMcaMath.fitting.model import nonlinear_parameter_group +from PyMca5.PyMcaMath.fitting.model import independent_linear_parameter_group from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel from PyMca5.PyMcaMath.fitting.model import LeastSquaresCombinedFitModel from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property @@ -49,7 +16,7 @@ def __init__(self): self.config = { "detector": {"zero": 0.0, "gain": 1.0, "wzero": 0.0, "wgain": 1.0}, "matrix": {"positions": [], "concentrations": [], "efficiency": []}, - "fit": {"linear": False}, + "fit": {"parameter_type": None}, "xmin": 0.0, "xmax": 1.0, } @@ -64,7 +31,7 @@ def __str__(self): self.__class__, self.npeaks, self.zero, self.gain, self.wzero, self.wgain ) - @parameter_group + @nonlinear_parameter_group def zero(self): return self.config["detector"]["zero"] @@ -72,7 +39,7 @@ def zero(self): def zero(self, value): self.config["detector"]["zero"] = value - @parameter_group + @nonlinear_parameter_group def gain(self): return self.config["detector"]["gain"] @@ -80,7 +47,7 @@ def gain(self): def gain(self, value): self.config["detector"]["gain"] = value - @parameter_group + @nonlinear_parameter_group def wzero(self): return self.config["detector"]["wzero"] @@ -88,7 +55,7 @@ def wzero(self): def wzero(self, value): self.config["detector"]["wzero"] = value - @parameter_group + @nonlinear_parameter_group def wgain(self): return self.config["detector"]["wgain"] @@ -121,7 +88,7 @@ def fwhms(self): def areas(self): return self.efficiency * self.concentrations - @linear_parameter_group + @independent_linear_parameter_group def concentrations(self): return self.config["matrix"]["concentrations"] @@ -130,12 +97,12 @@ def concentrations(self, value): self.config["matrix"]["concentrations"] = value @linked_property - def linear(self): - return self.config["fit"]["linear"] + def parameter_type(self): + return self.config["fit"]["parameter_type"] - @linear.setter - def linear(self, value): - self.config["fit"]["linear"] = value + @parameter_type.setter + def parameter_type(self, value): + self.config["fit"]["parameter_type"] = value @property def idx_channels(self): @@ -205,7 +172,7 @@ def evaluate_fitmodel(self, xdata=None): return SpecfitFuns.agauss(p, x) def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): - """Derivate to a specific parameter_group + """Derivate to a specific nonlinear_parameter_group :param int param_idx: :param array xdata: shape (ndata,) From f637d8d2d91df5b76c5c9cb45c2fcb934b58db51 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 16 Aug 2021 16:55:57 +0200 Subject: [PATCH 69/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 56 +- .../PyMcaMath/fitting/model/ParameterModel.py | 86 +-- PyMca5/PyMcaMath/fitting/model/__init__.py | 1 + PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 573 +++++++++--------- PyMca5/tests/FitPolModelTest.py | 21 +- PyMca5/tests/FitSimpleModelTest.py | 119 ++-- PyMca5/tests/ParameterModelTest.py | 42 +- PyMca5/tests/SimpleModel.py | 13 +- PyMca5/tests/XrfTest.py | 108 ++-- 9 files changed, 556 insertions(+), 463 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index f71a1a6cd..bfa09dba9 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -123,7 +123,7 @@ def evaluate_decomposed_fitmodel(self, xdata=None): """ derivatives = self.independent_linear_derivatives_fitmodel(xdata=xdata) parameters = self.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) return parameters.dot(derivatives) @@ -133,11 +133,13 @@ def independent_linear_derivatives_fitmodel(self, xdata=None): :param array xdata: shape (ndata,) :returns array: shape (nparams, ndata) """ - nparams = self.get_n_parameters(parameter_type=ParameterType.independent_linear) + nparams = self.get_n_parameters( + parameter_types=ParameterType.independent_linear + ) return numpy.array( [ self.derivative_fitmodel( - i, xdata=xdata, parameter_type=ParameterType.independent_linear + i, xdata=xdata, parameter_types=ParameterType.independent_linear ) for i in range(nparams) ] @@ -173,7 +175,7 @@ def linear_decomposition_fitmodel(self, xdata=None): """ derivatives = self.independent_linear_derivatives_fitmodel(xdata=xdata) parameters = self.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) return parameters[:, numpy.newaxis] * derivatives @@ -192,7 +194,7 @@ def fit(self, full_output=False): :param bool full_output: add statistics to fitted parameters :returns dict: """ - if self.parameter_type == ParameterType.independent_linear: + if self.parameter_types == ParameterType.independent_linear: return self.non_iterative_optimization(full_output=full_output) else: return self.iterative_optimization(full_output=full_output) @@ -219,10 +221,14 @@ def non_iterative_optimization(self, full_output=False): if self.niter_non_leastsquares: self.set_parameter_values(result["parameters"]) self.non_leastsquares_increment() - result["parameter_type"] = self.parameter_type + result["parameter_types"] = self.parameter_types result["parameters"] = result["parameters"] result["uncertainties"] = result["uncertainties"] + result["chi2_red"] = numpy.nan result.pop("svd") + if full_output: + result["niter"] = 1 + result["lastdeltachi"] = numpy.nan return result def iterative_optimization(self, full_output=False): @@ -255,7 +261,7 @@ def iterative_optimization(self, full_output=False): self.set_parameter_values(result[0]) self.non_leastsquares_increment() ret = { - "parameter_type": self.parameter_type, + "parameter_types": self.parameter_types, "parameters": result[0], "uncertainties": result[2], "chi2_red": result[1], @@ -284,7 +290,7 @@ def niter_non_leastsquares(self): @contextmanager def _non_iterative_optimization_context(self): with ExitStack() as stack: - ctx = self.parameter_type_context(ParameterType.independent_linear) + ctx = self.parameter_types_context(ParameterType.independent_linear) stack.enter_context(ctx) ctx = self._propertyCachingContext() stack.enter_context(ctx) @@ -337,7 +343,7 @@ def use_fit_result(self, result): :param dict result: """ self.set_parameter_values( - result["parameters"], parameter_type=result["parameter_type"] + result["parameters"], parameter_types=result["parameter_types"] ) @contextmanager @@ -346,7 +352,7 @@ def use_fit_result_context(self, result): :param dict result: """ - with self.parameter_type_context(result["parameter_type"]): + with self.parameter_types_context(result["parameter_types"]): with self._propertyCachingContext(): self.use_fit_result(result) yield @@ -449,22 +455,22 @@ def numerical_derivative_fitmodel(self, param_idx, xdata=None, **paramtype): group = self._group_from_parameter_index(param_idx, **paramtype) parameters = self.get_parameter_values(**paramtype) try: - if group.is_linear: - return self._numerical_derivative_linear_param( + if group.is_independent_linear: + return self._numerical_derivative_independent_linear_param( parameters, param_idx, xdata=xdata, **paramtype ) else: - return self._numerical_derivative_nonlinear_param( + return self._numerical_derivative_param( parameters, param_idx, xdata=xdata, **paramtype ) finally: self.set_parameter_values(parameters, **paramtype) - def _numerical_derivative_linear_param( + def _numerical_derivative_independent_linear_param( self, parameters, param_idx, xdata=None, **paramtype ): - """The numerical derivative to a linear parameter is exact - within arithmetic precision. + """The numerical derivative to an independent linear parameter + is exact within arithmetic precision. """ # y(x) = p0*f0(x) + ... + pi*fi(x) + ... # dy/dpi(x) = fi(x) @@ -476,12 +482,14 @@ def _numerical_derivative_linear_param( self.set_parameter_values(parameters, **paramtype) return self.evaluate_fitmodel(xdata=xdata) - def _numerical_derivative_nonlinear_param( + def _numerical_derivative_param( self, parameters, param_idx, xdata=None, **paramtype ): - """The numerical derivative to a non-linear parameter is an approximation""" - # Choose delta to be a small fraction of the parameter value but not too small, - # otherwise the derivative is zero. + """The numerical derivative to a non-linear or dependent-linear + parameter is an approximation. + """ + # Choose delta to be a small fraction of the parameter value but + # not too small, otherwise the derivative is zero. p0 = parameters[param_idx] delta = p0 * 1e-5 if delta < 0: @@ -500,13 +508,17 @@ def _numerical_derivative_nonlinear_param( return (f1 - f2) / (2.0 * delta) - def compare_derivatives(self, xdata=None, **paramtype): + def compare_derivatives(self, xdata=None, exclude_fixed=False, **paramtype): """Compare analytical and numerical derivatives. Useful to validate the user defined `derivative_fitmodel`. :yields str, array, array: parameter name, analytical, numerical """ - for param_idx, name in enumerate(self.get_parameter_names(**paramtype)): + names = self.get_parameter_names(**paramtype) + constraints = self.get_parameter_constraints(**paramtype) + for param_idx, (name, constraint) in enumerate(zip(names, constraints)): + if exclude_fixed and constraint[0] == Gefit.CFIXED: + continue ycalderiv = self.derivative_fitmodel(param_idx, xdata=xdata, **paramtype) ynumderiv = self.numerical_derivative_fitmodel( param_idx, xdata=xdata, **paramtype diff --git a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py index 6eef07c05..802877518 100644 --- a/PyMca5/PyMcaMath/fitting/model/ParameterModel.py +++ b/PyMca5/PyMcaMath/fitting/model/ParameterModel.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from contextlib import contextmanager import numpy -from enum import Enum +from enum import Flag from .CachingLinkedModel import CachedPropertiesLinkModel from .LinkedModel import LinkedModelManager @@ -11,11 +11,23 @@ from .CachingModel import cached_property -ParameterType = Enum("ParameterType", "non_linear dependent_linear independent_linear") +ParameterType = Flag("ParameterType", "non_linear dependent_linear independent_linear") +LinearParameterTypes = ParameterType.dependent_linear | ParameterType.independent_linear +AllParameterTypes = ( + ParameterType.non_linear + | ParameterType.dependent_linear + | ParameterType.independent_linear +) class _parameter_group(cached_property, linked_property): - """Usage: + """Specify a getter and setter for a group of fit parameters. + The counter and constraints are optional. When the counter + returns zero, the variable is excluded from the fit parameters. + Use CFIXED in the constraints if the parameters should be + included but not optimized. + + Usage: .. highlight:: python .. code-block:: python @@ -127,33 +139,33 @@ class ParameterModelBase(CachedPropertiesModel): """Interface for all models that manage fit parameters""" @property - def parameter_type(self): + def parameter_types(self): raise NotImplementedError - @parameter_type.setter - def parameter_type(self, value): + @parameter_types.setter + def parameter_types(self, value): raise NotImplementedError @property def only_linear_parameters(self): - return self.parameter_type in ( - ParameterType.dependent_linear, - ParameterType.independent_linear, - ) + return not bool(self.parameter_types & ParameterType.non_linear) @contextmanager - def parameter_type_context(self, value): - keep = self.parameter_type - self.parameter_type = value + def parameter_types_context(self, value=None): + keep = self.parameter_types + if value is not None: + self.parameter_types = value try: yield finally: - self.parameter_type = keep + self.parameter_types = keep - def _property_cache_key(self, parameter_type=NotImplemented, **paramtype): - if parameter_type is NotImplemented: - parameter_type = self.parameter_type - return parameter_type + def _property_cache_key(self, parameter_types=None, **paramtype): + if parameter_types is None: + parameter_types = self.parameter_types + elif not isinstance(parameter_types, ParameterType): + raise TypeError(parameter_types, "must be None or ParameterType") + return parameter_types def _create_empty_property_values_cache(self, key, **paramtype): return numpy.zeros(self.get_n_parameters(**paramtype)) @@ -244,7 +256,7 @@ class ParameterModel(CachedPropertiesLinkModel, ParameterModelBase): def __init__(self, *args, **kw): super().__init__(*args, **kw) - self._parameter_type = None + self.__parameter_types = AllParameterTypes def _iter_cached_property_ids(self, **paramtype): instance_key = self._linked_instance_to_key @@ -263,14 +275,18 @@ def __iter_parameter_group_properties(self): yield property_name, prop def _instance_cached_property_ids( - self, parameter_type=NotImplemented, linked=None, tracker=None + self, parameter_types=None, linked=None, tracker=None ): """ - :param parameter_type bool: only this parameter type + :param parameter_types ParameterType: only these parameter types :param linked bool: linked parameters or unlinked parameters :param tracker _IterGroupTracker: :yields ParameterGroupId: """ + if parameter_types is None: + parameter_types = self.parameter_types + elif not isinstance(parameter_types, ParameterType): + raise TypeError(parameter_types, "must be of type ParameterType") count = None index = None if tracker is None: @@ -283,9 +299,7 @@ def _instance_cached_property_ids( if group_is_linked is not linked: continue - if parameter_type is NotImplemented: - parameter_type = self.parameter_type - if parameter_type is not None and prop.TYPE != parameter_type: + if not (parameter_types & prop.TYPE): continue count = prop.fcount(self) @@ -332,12 +346,14 @@ def get_constraints(): return get_constraints @linked_property - def parameter_type(self): - return self._parameter_type + def parameter_types(self): + return self.__parameter_types - @parameter_type.setter - def parameter_type(self, value): - self._parameter_type = value + @parameter_types.setter + def parameter_types(self, value): + if not isinstance(value, ParameterType): + raise TypeError(value, "must be None or ParameterType") + self.__parameter_types = value def _get_noncached_property_value(self, group): return getattr(self, group.property_name) @@ -368,7 +384,7 @@ class ParameterModelManager(ParameterModelBase, LinkedModelManager): def __init__(self, *args, **kw): super().__init__(*args, **kw) - self._enable_property_link("parameter_type") + self._enable_property_link("parameter_types") for model in self.models: model._cache_manager = self @@ -385,12 +401,12 @@ def nmodels(self): return len(self.model_mapping) @property - def parameter_type(self): - return self._get_linked_property_value("parameter_type") + def parameter_types(self): + return self._get_linked_property_value("parameter_types") - @parameter_type.setter - def parameter_type(self, value): - self._set_linked_property_value("parameter_type", value) + @parameter_types.setter + def parameter_types(self, value): + self._set_linked_property_value("parameter_types", value) def _instance_cached_property_ids(self, **paramtype): """ diff --git a/PyMca5/PyMcaMath/fitting/model/__init__.py b/PyMca5/PyMcaMath/fitting/model/__init__.py index ff38fdcb1..1d76e063b 100644 --- a/PyMca5/PyMcaMath/fitting/model/__init__.py +++ b/PyMca5/PyMcaMath/fitting/model/__init__.py @@ -3,6 +3,7 @@ from PyMca5.PyMcaMath.fitting.model.ParameterModel import ( nonlinear_parameter_group, + dependent_linear_parameter_group, independent_linear_parameter_group, ) from PyMca5.PyMcaMath.fitting.model.LeastSquaresFitModel import ( diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index c7d6cd243..fafcf309a 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -1,36 +1,3 @@ -# /*########################################################################## -# -# The PyMca X-Ray Fluorescence Toolkit -# -# Copyright (c) 2020 European Synchrotron Radiation Facility -# -# This file is part of the PyMca X-ray Fluorescence Toolkit developed at -# the ESRF by the Software group. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -#############################################################################*/ -__author__ = "Wout De Nolf" -__contact__ = "wout.de_nolf@esrf.eu" -__license__ = "MIT" -__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" - import os import sys import copy @@ -43,13 +10,16 @@ from PyMca5.PyMcaIO import ConfigDict from PyMca5.PyMcaMath.fitting import SpecfitFuns from PyMca5.PyMcaMath.fitting import Gefit -from PyMca5.PyMcaMath.fitting.Model import Model -from PyMca5.PyMcaMath.fitting.Model import ConcatModel -from PyMca5.PyMcaMath.fitting.Model import parameter -from PyMca5.PyMcaMath.fitting.Model import linear_parameter -from PyMca5.PyMcaMath.fitting.PolynomialModels import LinearPolynomialModel -from PyMca5.PyMcaMath.fitting.PolynomialModels import ExponentialPolynomialModel - +from PyMca5.PyMcaMath.fitting.model import nonlinear_parameter_group +from PyMca5.PyMcaMath.fitting.model import independent_linear_parameter_group +from PyMca5.PyMcaMath.fitting.model import dependent_linear_parameter_group +from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel +from PyMca5.PyMcaMath.fitting.model import LeastSquaresCombinedFitModel +from PyMca5.PyMcaMath.fitting.model.PolynomialModels import LinearPolynomialModel +from PyMca5.PyMcaMath.fitting.model.PolynomialModels import ExponentialPolynomialModel +from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType +from PyMca5.PyMcaMath.fitting.model.ParameterModel import AllParameterTypes from . import Elements from . import ConcentrationsTool @@ -93,7 +63,7 @@ def __init__(self, initdict=None, filelist=None, **kw): self._overwriteConfig(**kw) self.attflag = kw.get("attenuatorsflag", 1) self.configure() - super(McaTheoryConfigApi, self).__init__() + super().__init__() def _overwriteConfig(self, **kw): if "config" in kw: @@ -191,7 +161,7 @@ def _sourceLines(self): yield energy, weight, scatter and scatterflag def _scatterLines(self): - """Source lines that are included in the fir model + """Source lines that are included in the fit model :yields tuple: (energy, weight) """ @@ -368,10 +338,6 @@ def _emissionGroups(self): else: yield [Z, symb, peaks] - @property - def emissionGroupNames(self): - return [symb + " " + line for _, symb, line in sorted(self._emissionGroups())] - def _configureElementsModule(self): """Configure the globals in the Elements module""" for material, info in self.config["materials"].items(): @@ -470,8 +436,21 @@ def getStartingConfiguration(self): """ return self.configure() - def estimate(self): - warnings.warn("McaTheory.estimate is deprecated (does nothing)", FutureWarning) + def enableOptimizedLinearFit(self): + warnings.warn( + "McaTheory.enableOptimizedLinearFit is deprecated (does nothing)", + FutureWarning, + ) + + def disableOptimizedLinearFit(self): + warnings.warn( + "McaTheory.enableOptimizedLinearFit is deprecated (does nothing)", + FutureWarning, + ) + + @property + def codes(self): + return self.get_parameter_constraints() def specfitestimate(self, x, y, z, xscaling=1.0, yscaling=1.0): warnings.warn( @@ -497,7 +476,7 @@ def __init__(self, **kw): self._std = None self._lastDataCacheParams = None - super(McaTheoryDataApi, self).__init__(**kw) + super().__init__(**kw) @property def xdata(self): @@ -700,7 +679,7 @@ def __init__(self, **kw): self._continuumModel = None self._lastContinuumCacheParams = None - super(McaTheoryBackground, self).__init__(**kw) + super().__init__(**kw) @property def hasNumBkg(self): @@ -816,7 +795,7 @@ def continuum_name(self, name): self.config["fit"]["continuum_name"] = name def _initializeConfig(self): - super(McaTheoryBackground, self)._initializeConfig() + super()._initializeConfig() self.continuum = self.continuum # verify continuum name and index @property @@ -879,16 +858,16 @@ def continuum_coefficients(self): if model is None: return list() else: - return model.parameters + return model.get_parameter_values() @continuum_coefficients.setter def continuum_coefficients(self, values): model = self.continuumModel if model is not None: - model.parameters = values + model.set_parameter_values(values) -class McaTheory(McaTheoryBackground, McaTheoryLegacyApi, Model): +class McaTheory(McaTheoryBackground, McaTheoryLegacyApi, LeastSquaresFitModel): """Model for MCA data""" BAND_GAP = 0.00385 # keV, silicon @@ -909,7 +888,10 @@ def __init__(self, **kw): self._escapeLineGroups = [] self._lastAreasCacheParams = None - super(McaTheory, self).__init__(**kw) + # Misc + self._last_fit_result = None + + super().__init__(**kw) def useFisxEscape(self, flag=None, apply=True): """Make sure the model uses fisx to calculate the escape peaks @@ -931,7 +913,10 @@ def useFisxEscape(self, flag=None, apply=True): self.configure() def _initializeConfig(self): - super(McaTheory, self)._initializeConfig() + super()._initializeConfig() + self.linearfitflag = self.config["fit"][ + "linearfitflag" + ] # synchronize parameter_types self._preCalculateParameterIndependent() self._preCalculateParameterDependent() @@ -955,6 +940,8 @@ def _preCalculateLineGroups(self): # This is a filtered and normalized form of `_fluoRates` self._lineGroups = list(self._getEmissionLines()) self._lineGroups.extend(self._getScatterLines()) + self._lineGroupNames = list(self._getEmissionLineNames()) + self._lineGroupNames.extend(self._getScatterNames()) self._nLineGroups = len(self._lineGroups) self._lineGroupAreas = numpy.ones(self._nLineGroups) @@ -969,15 +956,15 @@ def _preCalculateLineGroups(self): ] def _peakProfileParams( - self, hypermet=None, selected_groups=None, normalized_peakgroups=False + self, hypermet=None, selected_groups=None, normalized_fit_parameters=False ): - """Raw parameters of emission/scatter/escape peaks. All parameters are - defined in the energy domain (X-axis is energy, not channels). + """Raw peak profile parameters of emission/scatter/escape peaks. + All parameters are defined in the energy domain. :param int hypermet: :param list or None selected_groups: all groups when `None` no groups when empty list (npeaks == 0) - :param bool normalized_peakgroups: + :param bool normalized_fit_parameters: :returns array: npeaks x npeakparams """ lineGroups = self._lineGroups @@ -991,7 +978,7 @@ def _peakProfileParams( linegroup_areas = [linegroup_areas[i] for i in selected_groups] if hypermet is None: hypermet = self._hypermet - if normalized_peakgroups: + if normalized_fit_parameters: linegroup_areas = numpy.ones(len(linegroup_areas)) npeaks = sum(len(group) for group in lineGroups) @@ -1026,18 +1013,27 @@ def _peakProfileParams( # Area parameters from channel to energy domain parameters[:, 0] *= self.gain - # FWHM + # FWHM in the energy domain parameters[:, 2] = self._peakFWHM(parameters[:, 1]) # Other peak shape parameters if hypermet: - shapeparams = [ - self.st_arearatio, - self.st_sloperatio, - self.lt_arearatio, - self.lt_sloperatio, - self.step_heightratio, - ] + if normalized_fit_parameters: + shapeparams = [ + 1, + self.st_sloperatio, + 1, + self.lt_sloperatio, + 1, + ] + else: + shapeparams = [ + self.st_arearatio, + self.st_sloperatio, + self.lt_arearatio, + self.lt_sloperatio, + self.step_heightratio, + ] else: shapeparams = [self.eta_factor] for i, param in enumerate(shapeparams, 3): @@ -1045,8 +1041,9 @@ def _peakProfileParams( return parameters - @linear_parameter + @independent_linear_parameter_group def linegroup_areas(self): + """line group areas in the channel domain""" self._refreshAreasCache() return self._lineGroupAreas @@ -1064,19 +1061,32 @@ def linegroup_areas(self): positive = Gefit.CPOSITIVE, 0, 0 return [positive if area else fixed for area in self.linegroup_areas] - def _refreshAreasCache(self): + @property + def linegroup_names(self): + return self._lineGroupNames + + def _refreshAreasCache(self, force=False): params = self._areasCacheParams - if self._lastAreasCacheParams == params: + if self._lastAreasCacheParams == params and not force: return # the cached data is still valid self._estimateLineGroupAreas() self._lastAreasCacheParams = params + import matplotlib.pyplot as plt + + plt.plot(self.yfitdata) + plt.plot(self.yfitmodel) + plt.show() + @property def _areasCacheParams(self): """Any change in these parameter will invalidate the cache""" - return id(self._dataCacheParams), id(self._lineGroupAreas) + zero = int(self.zero * 1000) + gain = int(self.gain * 1000) + return self._dataCacheParams + (zero, gain, id(self._lineGroupAreas)) def _estimateLineGroupAreas(self): + """Estimated line group areas in the channel domain""" if self.hasBackground: ydata = self.ydata - self.ybackground() else: @@ -1084,10 +1094,13 @@ def _estimateLineGroupAreas(self): xenergy = self.xenergy emin = xenergy.min() emax = xenergy.max() - factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM - gain = self.gain + factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM / self.gain + + import matplotlib.pyplot as plt - keep = list() + plt.figure() + plt.plot(xenergy, ydata) + zeroarea = max(ydata) * 1e-7 lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups @@ -1106,23 +1119,18 @@ def _estimateLineGroupAreas(self): if rate * escrate > selected_rate: selected_energy = peakenergy if selected_energy: - chan = (numpy.abs(xenergy - selected_energy)).argmin() + chan = numpy.abs(xenergy - selected_energy).argmin() height = ydata[chan] - keep.append((chan, height)) - fwhm = self._peakFWHM(selected_energy) + plt.plot([selected_energy, selected_energy], [0, height]) + fwhm = self._peakFWHM(selected_energy) # energy domain linegroup_areas[i] = height * fwhm * factor # Gaussian else: - linegroup_areas[i] = 0 # Fixed at zero - - if False: - print(linegroup_areas) - import matplotlib.pyplot as plt + linegroup_areas[i] = zeroarea + plt.title(f"NEW ESTIMATE {self.zero} {self.gain}") + plt.show() - plt.figure() - plt.plot(ydata) - x, y = zip(*keep) - plt.plot(x, y, "o") - plt.show() + def estimate(self): + self._refreshAreasCache(force=True) def _evaluatePeakProfiles( self, @@ -1130,7 +1138,7 @@ def _evaluatePeakProfiles( hypermet=None, fast=True, selected_groups=None, - normalized_peakgroups=False, + normalized_fit_parameters=False, ): """Summed peak profiles of emission/scatter/escape peaks @@ -1138,13 +1146,13 @@ def _evaluatePeakProfiles( :param int or None hypermet: :param bool fast: use lookup table for calculating exponentials :param bool selected_groups: - :param bool normalized_peakgroups: + :param bool normalized_fit_parameters: :returns array: same shape as x """ parameters = self._peakProfileParams( hypermet=hypermet, selected_groups=selected_groups, - normalized_peakgroups=normalized_peakgroups, + normalized_fit_parameters=normalized_fit_parameters, ) if parameters.size == 0: if xdata is None: @@ -1186,6 +1194,10 @@ def _getEmissionLines(self): for group in sorted(self._emissionGroups()): yield self._getGroupEmissionLines(*group) + def _getEmissionLineNames(self): + for _, symb, line in sorted(self._emissionGroups()): + yield symb + " " + line + def _getScatterLines(self): """Yields a list for scattering lines for each source line. @@ -1195,9 +1207,13 @@ def _getScatterLines(self): angleFactor = 1.0 - numpy.cos(scatteringAngle) for i, (en_elastic, _) in enumerate(self._scatterLines()): en_inelastic = en_elastic / (1.0 + (en_elastic / 511.0) * angleFactor) - name = "Scatter %03d" % i - yield [[en_elastic, 1.0, name]] - yield [[en_inelastic, 1.0, name]] + yield [[en_elastic, 1.0, "Scatter %03d" % i]] + yield [[en_inelastic, 1.0, "Scatter %03d" % i]] + + def _getScatterNames(self): + for i in range(self._nRayleighLines): + yield "elastic" + str(i) + yield "inelastic" + str(i) def _calcFluoRates(self): """Fluorescence rate for each emission line of each element. @@ -1506,7 +1522,7 @@ def _channelsToEnergy(self, x): def _channelsToXpol(self, x): return self.zero + self.gain * (x - x.mean()) - @linear_parameter + @independent_linear_parameter_group def linpol_coefficients(self): if isinstance(self.continuumModel, LinearPolynomialModel): return self.continuum_coefficients @@ -1518,7 +1534,23 @@ def linpol_coefficients(self, values): if isinstance(self.continuumModel, LinearPolynomialModel): self.continuum_coefficients = values - @parameter + @linpol_coefficients.counter + def linpol_coefficients(self): + continuum = self.continuum_name + if continuum is None: + return 0 + elif continuum == "Constant": + return 1 + elif continuum == "Linear": + return 2 + elif continuum == "Parabolic": + return 3 + elif continuum == "Linear Polynomial": + return self.config["fit"]["linpolorder"] + 1 + else: + return 0 + + @nonlinear_parameter_group def exppol_coefficients(self): if isinstance(self.continuumModel, ExponentialPolynomialModel): return self.continuum_coefficients @@ -1530,6 +1562,13 @@ def exppol_coefficients(self, values): if isinstance(self.continuumModel, ExponentialPolynomialModel): self.continuum_coefficients = values + @exppol_coefficients.counter + def exppol_coefficients(self): + if self.continuum_name == "Exp. Polynomial": + return self.config["fit"]["exppolorder"] + 1 + else: + return 0 + def _snip(self, signal, anchorslist): """Apply SNIP filtering to a signal""" _logger.debug("CALCULATING SNIP") @@ -1603,7 +1642,7 @@ def _smooth(self, y): ysmooth[-1] = 0.5 * (ysmooth[-1] + ysmooth[-2]) return ysmooth - @parameter + @nonlinear_parameter_group def zero(self): return self.config["detector"]["zero"] @@ -1620,7 +1659,7 @@ def zero(self): delta = self.config["detector"]["deltazero"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @nonlinear_parameter_group def gain(self): return self.config["detector"]["gain"] @@ -1637,7 +1676,7 @@ def gain(self): delta = self.config["detector"]["deltagain"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @nonlinear_parameter_group def noise(self): return self.config["detector"]["noise"] @@ -1645,7 +1684,16 @@ def noise(self): def noise(self, value): self.config["detector"]["noise"] = value - @parameter + @noise.constraints + def noise(self): + if self.config["detector"]["fixednoise"]: + return Gefit.CFIXED, 0, 0 + else: + value = self.noise + delta = self.config["detector"]["deltanoise"] + return Gefit.CQUOTED, value - delta, value + delta + + @nonlinear_parameter_group def fano(self): return self.config["detector"]["fano"] @@ -1662,7 +1710,7 @@ def fano(self): delta = self.config["detector"]["deltafano"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @dependent_linear_parameter_group def sum(self): if self.config["fit"]["sumflag"]: return self.config["detector"]["sum"] @@ -1682,7 +1730,7 @@ def sum(self): delta = self.config["detector"]["deltasum"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @nonlinear_parameter_group def eta_factor(self): return self.config["peakshape"]["eta_factor"] @@ -1706,7 +1754,7 @@ def eta_factor(self): delta = self.config["peakshape"]["deltaeta_factor"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @dependent_linear_parameter_group def step_heightratio(self): if self._hypermetStep: return self.config["peakshape"]["step_heightratio"] @@ -1733,7 +1781,7 @@ def step_heightratio(self): delta = self.config["peakshape"]["deltastep_heightratio"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @nonlinear_parameter_group def lt_sloperatio(self): return self.config["peakshape"]["lt_sloperatio"] @@ -1757,7 +1805,7 @@ def lt_sloperatio(self): delta = self.config["peakshape"]["deltalt_sloperatio"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @dependent_linear_parameter_group def lt_arearatio(self): if self._hypermetLongTail: return self.config["peakshape"]["lt_arearatio"] @@ -1784,7 +1832,7 @@ def lt_arearatio(self): delta = self.config["peakshape"]["deltalt_arearatio"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @dependent_linear_parameter_group def st_sloperatio(self): return self.config["peakshape"]["st_sloperatio"] @@ -1811,7 +1859,7 @@ def st_sloperatio(self): delta = self.config["peakshape"]["deltast_sloperatio"] return Gefit.CQUOTED, value - delta, value + delta - @parameter + @dependent_linear_parameter_group def st_arearatio(self): if self._hypermetShortTail: return self.config["peakshape"]["st_arearatio"] @@ -1842,10 +1890,17 @@ def _convert_parameter_names(self, names): linegroup_names = None for name in names: if "linegroup_areas" in name: - if not linegroup_names: - linegroup_names = self.emissionGroupNames + if linegroup_names is None: + linegroup_names = self.linegroup_names idx = int(name.replace("linegroup_areas", "")) - yield linegroup_names[idx] + name = linegroup_names[idx] + if "inelastic" in name: + i = int(name.replace("inelastic", "")) + name = "Scatter Compton%03d" % i + elif "elastic" in name: + i = int(name.replace("elastic", "")) + name = "Scatter Peak%03d" % i + yield name elif "linpol_coefficients" in name: if self.continuum < self.CONTINUUM_LIST.index("Linear Polynomial"): idx = int(name.replace("linpol_coefficients", "")) @@ -1874,16 +1929,9 @@ def _convert_parameter_names(self, names): else: yield name.capitalize() - @property - def parameter_names(self): - return list( - self._convert_parameter_names(super(McaTheory, self).parameter_names) - ) - - @property - def linear_parameter_names(self): - return list( - self._convert_parameter_names(super(McaTheory, self).linear_parameter_names) + def get_parameter_names(self, **paramtype): + return tuple( + self._convert_parameter_names(super().get_parameter_names(**paramtype)) ) def evaluate_fitmodel(self, xdata=None): @@ -1896,7 +1944,7 @@ def mcatheory( continuum=True, summing=True, selected_groups=None, - normalized_peakgroups=False, + normalized_fit_parameters=False, ): """Evaluate to MCA model (does not include the numerical background) @@ -1915,10 +1963,10 @@ def mcatheory( Hypermet: - F(x) = A * Gnorm(x, u, s) - + st_arearatio*A * Tnorm(x, u, s, st_sloperatio) - + lt_arearatio*A * Tnorm(x, u, s, lt_sloperatio) - + step_heightratio*A * u/(sqrt(2*pi)*s) * Snorm(x, u, s) + A*F(x) = A * Gnorm(x, u, s) + + st_arearatio*A * Tnorm(x, u, s, st_sloperatio) + + lt_arearatio*A * Tnorm(x, u, s, lt_sloperatio) + + step_heightratio*A * u/(sqrt(2*pi)*s) * Snorm(x, u, s) A: the area of the gaussian part, not the entire peak @@ -1937,7 +1985,7 @@ def mcatheory( :param bool continuum: :param bool summing: :param list selected_groups: - :param bool normalized_peakgroups: + :param bool normalized_fit_parameters: :returns array: """ # Emission lines, scatter peaks and escape peaks @@ -1945,7 +1993,7 @@ def mcatheory( xdata=xdata, hypermet=hypermet, selected_groups=selected_groups, - normalized_peakgroups=normalized_peakgroups, + normalized_fit_parameters=normalized_fit_parameters, ) # Analytical background @@ -1954,11 +2002,11 @@ def mcatheory( # Pile-up if summing and self.sum: - y += self.ypileup(y, xdata=xdata) + y += self.ypileup(ymodel=y, xdata=xdata) return y - def ypileup(self, ymodel, xdata=None): + def ypileup(self, ymodel=None, xdata=None, normalized_fit_parameters=False): """The ymodel contains the peaks and the continuum. Pileup is """ @@ -1968,19 +2016,27 @@ def ypileup(self, ymodel, xdata=None): return numpy.zeros(self.ndata) else: return numpy.zeros(len(xdata)) + if ymodel is None: + ymodel = self.mcatheory(xdata=xdata, summing=False) if xdata is None: - xdata = self.xdata + xmin = self.xmin + else: + xmin = min(xdata) + if normalized_fit_parameters: + pileupfactor = 1 return pileupfactor * SpecfitFuns.pileup( - ymodel, min(xdata), self.zero, self.gain, 0 + ymodel, int(xmin), self.zero, self.gain, 0 ) def _y_full_to_fit(self, y, xdata=None): """The fitting is done after subtracting the numerical background""" if self.hasNumBkg: y = y - self.ynumbkg(xdata=xdata) - if self.linear and self.hasPileUp: - ymodel = self.mcatheory(xdata=xdata, summing=False) - y = y - self.ypileup(ymodel, xdata=xdata) + if self.parameter_types == ParameterType.independent_linear and self.hasPileUp: + # Note: this is not strictly correct but we have + # no other choice until the pileup is implemented + # properly (emission line combinations instead of convolution) + y = y - self.ypileup(xdata=xdata) return y @property @@ -1994,105 +2050,52 @@ def _y_fit_to_full(self, y, xdata=None): else: return y - def linear_derivatives_fitmodel(self, xdata=None): - """Derivates to all linear parameters - - :param array xdata: length nxdata - :returns array: nparams x nxdata - """ - derivatives = [] - # Derivatives to peak group areas - for pgroupi in range(self._nLineGroups): - derivatives.append( - self.mcatheory( - xdata=xdata, - selected_groups=[pgroupi], - normalized_peakgroups=True, - continuum=False, - summing=False, - ) - ) - # Derivatives to polynomial coefficients - if isinstance(self.continuumModel, LinearPolynomialModel): - arr = self.continuumModel.linear_derivatives_fitmodel(xdata=xdata) - derivatives += arr.tolist() - return numpy.array(derivatives) - - @contextmanager - def _keep_parameters(self): - # TODO: wait for parameters refactoring - keep = self.parameters - try: - yield - finally: - self.parameters = keep - def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): - """Derivate to a specific parameter + """Derivate to a specific nonlinear_parameter_group :param int param_idx: - :param array xdata: length nxdata - :returns array: nxdata + :param array xdata: shape (ndata,) + :returns array: shape (ndata,) """ - with self._keep_parameters(): - name, pgroupi = self._parameter_name_from_index(param_idx) - print( - name, - "index=", - param_idx, - "index in group=", - pgroupi, - "value=", - self.parameters[param_idx], - ) - if name == "st_arearatio": - self.parameters[param_idx] = 1 - return self.mcatheory(xdata=xdata, hypermet=2, continuum=False) - elif name == "lt_arearatio": - self.parameters[param_idx] = 1 - return self.mcatheory(xdata=xdata, hypermet=4, continuum=False) - elif name == "step_heightratio": - self.parameters[param_idx] = 1 - return self.mcatheory(xdata=xdata, hypermet=8, continuum=False) - elif name == "linegroup_areas": - self.parameters[param_idx] = 1 - return self.mcatheory( - selected_groups=[pgroupi], - xdata=xdata, - continuum=False, - ) - elif name == "linpol": - # TODO: subtract param - return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) - elif name == "exppol": - # TODO: subtract param - return self.continuumModel.derivative_fitmodel(param_idx, xdata=xdata) - else: - return self._numerical_derivative(param_idx, xdata=xdata) - - def _numerical_derivative(self, param_idx, xdata=None): - with self._keep_parameters(): - parameters = self.parameters - p0 = parameters[param_idx] - - # Choose delta to be a small fraction of the - # parameter value but not too small, otherwise - # the derivative is zero. - delta = p0 * 1e-5 - if delta < 0: - delta = min(delta, -1e-12) - else: - delta = max(delta, 1e-12) + return self.numerical_derivative_fitmodel(param_idx, xdata=xdata, **paramtype) - parameters[param_idx] = p0 + delta - self.parameters = parameters - f1 = self.evaluate_fitmodel(xdata=xdata) - - parameters[param_idx] = p0 - delta - self.parameters = parameters - f2 = self.evaluate_fitmodel(xdata=xdata) - - return (f1 - f2) / (2.0 * delta) + group = self._group_from_parameter_index(param_idx, **paramtype) + name = group.property_name + if name == "st_arearatio": + return self.mcatheory( + xdata=xdata, hypermet=2, continuum=False, normalized_fit_parameters=True + ) + elif name == "lt_arearatio": + return self.mcatheory( + xdata=xdata, hypermet=4, continuum=False, normalized_fit_parameters=True + ) + elif name == "step_heightratio": + return self.mcatheory( + xdata=xdata, hypermet=8, continuum=False, normalized_fit_parameters=True + ) + elif name == "linegroup_areas": + i = group.parameter_index_in_group(param_idx) + return self.mcatheory( + selected_groups=[i], + normalized_fit_parameters=True, + xdata=xdata, + continuum=False, + ) + elif name == "sum": + return self.ypileup(xdata=xdata, normalized_fit_parameters=True) + elif name == "linpol" and False: + model = self.continuumModel + keep = model.get_parameter_values(**paramtype) + try: + model.set_parameter_values(self.continuum_coefficients, **paramtype) + i = group.parameter_index_in_group(param_idx) + return model.derivative_fitmodel(i, xdata=xdata, **paramtype) + finally: + model.set_parameter_values(keep, **paramtype) + else: + return self.numerical_derivative_fitmodel( + param_idx, xdata=xdata, **paramtype + ) @property def maxiter(self): @@ -2106,69 +2109,91 @@ def deltachi(self): def weightflag(self): return self.config["fit"]["fitweight"] - @property - def linear(self): - return self.config["fit"].get("linearfitflag", 0) + @linked_property + def linearfitflag(self): + linearfitflag = int(self.parameter_types == ParameterType.independent_linear) + self.config["fit"]["linearfitflag"] = linearfitflag + return linearfitflag - @linear.setter - def linear(self, value): - self.config["fit"]["linearfitflag"] = value + @linearfitflag.setter + def linearfitflag(self, value): + if value: + self.parameter_types = ParameterType.independent_linear + else: + self.parameter_types = AllParameterTypes + + @contextmanager + def linear_context(self, linear=None): + keep = self.linearfitflag + if linear is not None: + self.linearfitflag = linear + try: + yield + finally: + self.linearfitflag = keep + + @property + def parameter_types(self): + return super(type(self), type(self)).parameter_types.fget(self) + + @parameter_types.setter + def parameter_types(self, value): + super(type(self), type(self)).parameter_types.fset(self, value) + self.config["fit"]["linearfitflag"] = int( + bool(self.parameter_types & ParameterType.independent_linear) + ) @contextmanager - def _nonlinear_fit_context(self): - with super(McaTheory, self)._nonlinear_fit_context(): + def _custom_iterative_optimization_context(self): + with super()._custom_iterative_optimization_context(): if abs(self.zero) < 1.0e-10: self.zero = 0.0 yield - def startFit(self, digest=0, linear=None): + def startFit(self, digest=False, linear=None): """Fit with legacy output""" - if linear is not None: - keep = self.linear - self.linear = linear - result = super(McaTheory, self).fit(full_output=True) - - if linear is not None: - self.linear = linear - - self._last_fit_result = result - if digest: - return self._legacyresult(result), self.digestresult() - else: - return self._legacyresult(result) + with self.linear_context(linear=linear): + result = self.fit(full_output=True) + self._last_fit_result = result + if digest: + return self._legacyresult(result), self.digestresult() + else: + return self._legacyresult(result) @staticmethod def _legacyresult(result): - if result["linear"]: - return ( - result["parameters"].tolist(), - numpy.nan, - result["uncertainties"].tolist(), - 1, - numpy.nan, - ) - else: - return ( - result["parameters"].tolist(), - result["chi2_red"], - result["uncertainties"].tolist(), - result["niter"], - result["lastdeltachi"], - ) + return ( + result["parameters"], + result["chi2_red"], + result["uncertainties"], + result["niter"], + result["lastdeltachi"], + ) def digestresult(self): with self.use_fit_result_context(self._last_fit_result): - return dict() + result = { + "xdata": self.xdata, + "energy": self.xenergy, + "ydata": self.ydata, + "continuum": self.ybackground(), + "yfit": self.yfullmodel, + "ypileup": self.ypileup(), + "parameters": self.get_parameter_names(), + "fittedpar": self._last_fit_result["parameters"], + "chisq": self._last_fit_result["chi2_red"], + "sigmapar": self._last_fit_result["uncertainties"], + "config": self.config.copy(), + } + return result def imagingDigestResult(self): with self.use_fit_result_context(self._last_fit_result): return dict() -class MultiMcaTheory(ConcatModel): +class MultiMcaTheory(LeastSquaresCombinedFitModel): def __init__(self, ndetectors=1): - models = [McaTheory() for i in range(ndetectors)] - shared_attributes = [] # nothing shared yet - super(MultiMcaTheory, self).__init__( - models, shared_attributes=shared_attributes - ) + models = {f"detector{i}": McaTheory() for i in range(ndetectors)} + super().__init__(models) + # self._enable_property_link("concentrations") diff --git a/PyMca5/tests/FitPolModelTest.py b/PyMca5/tests/FitPolModelTest.py index 868b3483d..38511c6b0 100644 --- a/PyMca5/tests/FitPolModelTest.py +++ b/PyMca5/tests/FitPolModelTest.py @@ -2,6 +2,7 @@ import numpy from PyMca5.PyMcaMath.fitting.model import PolynomialModels from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType +from PyMca5.PyMcaMath.fitting.model.ParameterModel import AllParameterTypes class testFitPolModel(unittest.TestCase): @@ -32,9 +33,12 @@ def testLinearPol(self): numpy.testing.assert_array_equal(fitmodel.yfitdata, model.yfitmodel) numpy.testing.assert_array_equal(model.yfitmodel, model.yfullmodel) - for parameter_type in [ParameterType.independent_linear, None]: - with self.subTest(degree=degree, parameter_type=parameter_type): - fitmodel.parameter_type = parameter_type + for parameter_types in [ + ParameterType.independent_linear, + AllParameterTypes, + ]: + with self.subTest(degree=degree, parameter_types=parameter_types): + fitmodel.parameter_types = parameter_types fitmodel.coefficients = numpy.zeros_like(expected) self.assertEqual(fitmodel.degree, degree) result = fitmodel.fit()["parameters"] @@ -63,11 +67,14 @@ def testExpPol(self): model.yfitmodel, numpy.log(model.yfullmodel) ) - for parameter_type in [ParameterType.independent_linear, None]: - with self.subTest(degree=degree, parameter_type=parameter_type): - fitmodel.parameter_type = parameter_type + for parameter_types in [ + ParameterType.independent_linear, + AllParameterTypes, + ]: + with self.subTest(degree=degree, parameter_types=parameter_types): + fitmodel.parameter_types = parameter_types fitmodel.coefficients = numpy.zeros_like(expected) - if parameter_type is None: + if parameter_types == AllParameterTypes: fitmodel.coefficients[0] = 0.1 self.assertEqual(fitmodel.degree, degree) result = fitmodel.fit()["parameters"] diff --git a/PyMca5/tests/FitSimpleModelTest.py b/PyMca5/tests/FitSimpleModelTest.py index 703533236..82d267564 100644 --- a/PyMca5/tests/FitSimpleModelTest.py +++ b/PyMca5/tests/FitSimpleModelTest.py @@ -4,6 +4,7 @@ from PyMca5.tests.SimpleModel import SimpleModel from PyMca5.tests.SimpleModel import SimpleCombinedModel from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType +from PyMca5.PyMcaMath.fitting.model.ParameterModel import AllParameterTypes class testFitModel(unittest.TestCase): @@ -16,19 +17,21 @@ def testLinearFit(self): def testNonLinearFit(self): for _ in self._fit_model_subtests(): - self._test_fit(None) + self._test_fit(AllParameterTypes) - def _test_fit(self, parameter_type): - self.fitmodel.parameter_type = parameter_type - refined_params = self.fitmodel.get_parameter_values(parameter_type=None).copy() + def _test_fit(self, parameter_types): + self.fitmodel.parameter_types = parameter_types + refined_params = self.fitmodel.get_parameter_values( + parameter_types=AllParameterTypes + ).copy() lin_refined_params = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ).copy() - self._modify_random(parameter_type=parameter_type) + self._modify_random(parameter_types=parameter_types) - before = self.fitmodel.get_parameter_values(parameter_type=None) + before = self.fitmodel.get_parameter_values(parameter_types=AllParameterTypes) lin_before = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) # with self._profile("test"): @@ -36,19 +39,19 @@ def _test_fit(self, parameter_type): # Verify the expected fit parameters rtol = 1e-3 - if result["parameter_type"]: + if result["parameter_types"] == AllParameterTypes: numpy.testing.assert_allclose( - result["parameters"], lin_refined_params, rtol=rtol + result["parameters"], refined_params, rtol=rtol ) else: numpy.testing.assert_allclose( - result["parameters"], refined_params, rtol=rtol + result["parameters"], lin_refined_params, rtol=rtol ) # Check that the model has not been affected - after = self.fitmodel.get_parameter_values(parameter_type=None) + after = self.fitmodel.get_parameter_values(parameter_types=AllParameterTypes) lin_after = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) numpy.testing.assert_array_equal(before, after) numpy.testing.assert_array_equal(lin_before, lin_after) @@ -62,18 +65,22 @@ def _assert_model_not_refined(self, refined_params, lin_refined_params): self.assertTrue( not numpy.allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) ) - parameters = self.fitmodel.get_parameter_values(parameter_type=None) + parameters = self.fitmodel.get_parameter_values( + parameter_types=AllParameterTypes + ) self.assertTrue(not numpy.allclose(parameters, refined_params)) parameters = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) self.assertTrue(not numpy.allclose(parameters, lin_refined_params)) def _assert_model_refined(self, refined_params, lin_refined_params, rtol=1e-7): - parameters = self.fitmodel.get_parameter_values(parameter_type=None) + parameters = self.fitmodel.get_parameter_values( + parameter_types=AllParameterTypes + ) numpy.testing.assert_allclose(parameters, refined_params, rtol=rtol) parameters = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) numpy.testing.assert_allclose(parameters, lin_refined_params, rtol=rtol) numpy.testing.assert_allclose(self.fitmodel.ydata, self.fitmodel.yfullmodel) @@ -100,7 +107,7 @@ def _create_model(self, nmodels): self.fitmodel = SimpleModel() else: self.fitmodel = SimpleCombinedModel(ndetectors=nmodels) - self.assertTrue(not self.fitmodel.parameter_type) + self.assertTrue(self.fitmodel.parameter_types, AllParameterTypes) self._init_random() @@ -148,46 +155,50 @@ def _init_random_model(self, model): model.concentrations = self.random_state.uniform(low=0.5, high=1, size=npeaks) model.efficiency = self.random_state.uniform(low=5000, high=6000, size=npeaks) - def _modify_random(self, parameter_type): - self._modify_random_model(parameter_type) + def _modify_random(self, parameter_types): + self._modify_random_model(parameter_types) self._validate_models() - def _modify_random_model(self, parameter_type): - self.assertIn(parameter_type, (None, ParameterType.independent_linear)) + def _modify_random_model(self, parameter_types): + self.assertIn( + parameter_types, (AllParameterTypes, ParameterType.independent_linear) + ) pnonlinorg = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.non_linear + parameter_types=ParameterType.non_linear ) plinorg = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) - pallorg = self.fitmodel.get_parameter_values(parameter_type=None) + pallorg = self.fitmodel.get_parameter_values(parameter_types=AllParameterTypes) - if parameter_type is None: + if parameter_types is AllParameterTypes: pmod = pnonlinorg.copy() pmod *= self.random_state.uniform(0.95, 1, len(pmod)) self.fitmodel.set_parameter_values( - pmod, parameter_type=ParameterType.non_linear + pmod, parameter_types=ParameterType.non_linear ) parameters = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.non_linear + parameter_types=ParameterType.non_linear ) numpy.testing.assert_array_equal(parameters, pmod) pmod = plinorg.copy() pmod *= self.random_state.uniform(0.5, 0.8, len(pmod)) self.fitmodel.set_parameter_values( - pmod, parameter_type=ParameterType.independent_linear + pmod, parameter_types=ParameterType.independent_linear ) parameters = self.fitmodel.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) numpy.testing.assert_array_equal(parameters, pmod) - pall = self.fitmodel.get_parameter_values(parameter_type=None) - for group in self.fitmodel.get_parameter_groups(parameter_type=None): + pall = self.fitmodel.get_parameter_values(parameter_types=AllParameterTypes) + for group in self.fitmodel.get_parameter_groups( + parameter_types=AllParameterTypes + ): current = pall[group.index] expected = pallorg[group.index] - if parameter_type is None or group.is_independent_linear: + if parameter_types is AllParameterTypes or group.is_independent_linear: # Values are expected to be modified if group.count == 1: self.assertNotEqual(current, expected, msg=group.name) @@ -209,9 +220,11 @@ def _validate_models(self): def _validate_model(self, model, model_idx=None): is_combined_model = self.is_combined_model and model_idx is None - original_parameters = model.get_parameter_values(parameter_type=None).copy() + original_parameters = model.get_parameter_values( + parameter_types=AllParameterTypes + ).copy() original_linear_parameters = model.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ).copy() nonlin_expected = {"gain", "wgain", "wzero", "zero"} @@ -228,19 +241,19 @@ def _validate_model(self, model, model_idx=None): } lin_expected = {"concentrations"} all_expected = lin_expected | nonlin_expected - names = model.get_parameter_group_names(parameter_type=None) + names = model.get_parameter_group_names(parameter_types=AllParameterTypes) self.assertEqual(set(names), all_expected) names = model.get_parameter_group_names( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) self.assertEqual(set(names), lin_expected) lin_expected = {f"concentrations{i}" for i in range(self.npeaks)} all_expected = lin_expected | nonlin_expected - names = model.get_parameter_names(parameter_type=None) + names = model.get_parameter_names(parameter_types=AllParameterTypes) self.assertEqual(set(names), all_expected) names = model.get_parameter_names( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) self.assertEqual(set(names), lin_expected) @@ -248,14 +261,14 @@ def _validate_model(self, model, model_idx=None): nexpected = len(model.xdata) self.assertEqual(n, nexpected) - nexpected = len(model.get_parameter_values(parameter_type=None)) - n = model.get_n_parameters(parameter_type=None) + nexpected = len(model.get_parameter_values(parameter_types=AllParameterTypes)) + n = model.get_n_parameters(parameter_types=AllParameterTypes) self.assertEqual(n, nexpected) nexpected = len( - model.get_parameter_values(parameter_type=ParameterType.independent_linear) + model.get_parameter_values(parameter_types=ParameterType.independent_linear) ) - n = model.get_n_parameters(parameter_type=ParameterType.independent_linear) + n = model.get_n_parameters(parameter_types=ParameterType.independent_linear) self.assertEqual(n, nexpected) arr1 = model.evaluate_fullmodel() @@ -277,18 +290,18 @@ def _validate_model(self, model, model_idx=None): nexpected = self.npeaks + self.nshapeparams * nmodels else: nexpected = self.npeaks + self.nshapeparams - n = model.get_n_parameters(parameter_type=None) + n = model.get_n_parameters(parameter_types=AllParameterTypes) self.assertEqual(n, nexpected) - n = model.get_n_parameters(parameter_type=ParameterType.independent_linear) + n = model.get_n_parameters(parameter_types=ParameterType.independent_linear) self.assertEqual(n, self.npeaks) rtol = 1e-3 - for parameter_type in (ParameterType.independent_linear, None): - with model.parameter_type_context(parameter_type): - noncached = list(model.compare_derivatives()) - cached = list(model.compare_derivatives()) - err_fmt = "[parameter_type={}] {{}} and {{}} derivative of '{{}}' (model: {}) are not equal".format( - parameter_type, model_idx + for parameter_types in (ParameterType.independent_linear, AllParameterTypes): + with model.parameter_types_context(parameter_types): + noncached = model.compare_derivatives() + cached = model.compare_derivatives() + err_fmt = "[parameter_types={}] {{}} and {{}} derivative of '{{}}' (model: {}) are not equal".format( + parameter_types, model_idx ) for deriv, deriv_cached in zip(noncached, cached): param_name, calc, numerical = deriv @@ -309,10 +322,10 @@ def _validate_model(self, model, model_idx=None): calc, calc_cached, err_msg=err_msg, rtol=rtol ) - parameters = model.get_parameter_values(parameter_type=None) + parameters = model.get_parameter_values(parameter_types=AllParameterTypes) numpy.testing.assert_array_equal(original_parameters, parameters) parameters = model.get_parameter_values( - parameter_type=ParameterType.independent_linear + parameter_types=ParameterType.independent_linear ) numpy.testing.assert_array_equal(original_linear_parameters, parameters) diff --git a/PyMca5/tests/ParameterModelTest.py b/PyMca5/tests/ParameterModelTest.py index da522f4c7..49fd5a023 100644 --- a/PyMca5/tests/ParameterModelTest.py +++ b/PyMca5/tests/ParameterModelTest.py @@ -8,6 +8,7 @@ independent_linear_parameter_group, ) from PyMca5.PyMcaMath.fitting.model.ParameterModel import ParameterType +from PyMca5.PyMcaMath.fitting.model.ParameterModel import AllParameterTypes class Model1(ParameterModel): @@ -123,30 +124,34 @@ def setUp(self): self.concat_model = ConcatModel() def test_instantiation(self): - self.assertEqual(self.concat_model.parameter_type, None) self.assertEqual(self.concat_model.nmodels, 4) + self.assertEqual(self.concat_model.parameter_types, AllParameterTypes) + for model in self.concat_model.models: + self.assertEqual(model.parameter_types, AllParameterTypes) def test_parameter_type(self): - for parameter_type in ParameterType: + for parameter_types in ParameterType: for group in self.concat_model.get_parameter_groups( - parameter_type=parameter_type + parameter_types=parameter_types ): - self.assertEqual(group.type, parameter_type) + self.assertEqual(group.type, parameter_types) types = set(group.type for group in self.concat_model.get_parameter_groups()) expected = {ParameterType.independent_linear, ParameterType.non_linear} self.assertEqual(types, expected) def test_parameter_type_context(self): - with self.concat_model.parameter_type_context(ParameterType.independent_linear): + with self.concat_model.parameter_types_context( + ParameterType.independent_linear + ): self.assertEqual( - self.concat_model.parameter_type, ParameterType.independent_linear + self.concat_model.parameter_types, ParameterType.independent_linear ) for model in self.concat_model.models: - self.assertTrue(model.parameter_type, ParameterType.independent_linear) + self.assertTrue(model.parameter_types, ParameterType.independent_linear) - self.assertEqual(self.concat_model.parameter_type, None) + self.assertEqual(self.concat_model.parameter_types, AllParameterTypes) for model in self.concat_model.models: - self.assertEqual(model.parameter_type, None) + self.assertEqual(model.parameter_types, AllParameterTypes) def test_parameter_group_names(self): for cacheoptions in self._parameterize_nonlinear_test(): @@ -566,26 +571,29 @@ def _assertValuesEqual(self, values, expected): def _parameterize_linear_test(self): yield from self._parameterize_tests( [ - [ParameterType.independent_linear, None], - [NotImplemented, ParameterType.independent_linear], + [None, ParameterType.independent_linear], + [ParameterType.independent_linear, AllParameterTypes], ] ) def _parameterize_nonlinear_test(self): yield from self._parameterize_tests( - [[None, ParameterType.independent_linear], [NotImplemented, None]] + [ + [None, AllParameterTypes], + [AllParameterTypes, ParameterType.independent_linear], + ] ) - def _parameterize_tests(self, linear_options): - for local_type, global_type in linear_options: - for cached in [True, False]: + def _parameterize_tests(self, types): + for local_type, global_type in types: + for cached in [False, True]: with self.subTest( local_type=local_type, global_type=global_type, cached=cached, ): - self.concat_model.parameter_type = global_type - cacheoptions = {"parameter_type": local_type} + self.concat_model.parameter_types = global_type + cacheoptions = {"parameter_types": local_type} self._cached = cached if cached: with self.concat_model._propertyCachingContext(**cacheoptions): diff --git a/PyMca5/tests/SimpleModel.py b/PyMca5/tests/SimpleModel.py index 274ea5ee3..05a697d3a 100644 --- a/PyMca5/tests/SimpleModel.py +++ b/PyMca5/tests/SimpleModel.py @@ -5,6 +5,7 @@ from PyMca5.PyMcaMath.fitting.model import LeastSquaresFitModel from PyMca5.PyMcaMath.fitting.model import LeastSquaresCombinedFitModel from PyMca5.PyMcaMath.fitting.model.LinkedModel import linked_property +from PyMca5.PyMcaMath.fitting.model.ParameterModel import AllParameterTypes class SimpleModel(LeastSquaresFitModel): @@ -16,7 +17,7 @@ def __init__(self): self.config = { "detector": {"zero": 0.0, "gain": 1.0, "wzero": 0.0, "wgain": 1.0}, "matrix": {"positions": [], "concentrations": [], "efficiency": []}, - "fit": {"parameter_type": None}, + "fit": {"parameter_types": AllParameterTypes}, "xmin": 0.0, "xmax": 1.0, } @@ -97,12 +98,12 @@ def concentrations(self, value): self.config["matrix"]["concentrations"] = value @linked_property - def parameter_type(self): - return self.config["fit"]["parameter_type"] + def parameter_types(self): + return self.config["fit"]["parameter_types"] - @parameter_type.setter - def parameter_type(self, value): - self.config["fit"]["parameter_type"] = value + @parameter_types.setter + def parameter_types(self, value): + self.config["fit"]["parameter_types"] = value @property def idx_channels(self): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index d8aab532f..7c5713f45 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -321,7 +321,6 @@ def _readStainlessSteelData(self): return x, y, configuration def testTrainingDataFit(self): - from PyMca5.PyMcaIO import specfilewrapper as specfile from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import ConcentrationsTool @@ -544,20 +543,20 @@ def testStainlessSteelDataFit(self): "Strategy: Element %s discrepancy too large %.1f %%" % \ (element.split()[0], delta)) - def testCompareLegacyMcaTheory(self): + def testTrainingDataFitCompareLegacy(self): x, y, configuration = self._readTrainingData() - self._testCompareLegacyMcaTheory(x, y, configuration) - return + self._compareLegacyMcaTheory(x, y, configuration) + def testStainlessSteelDataFitCompareLegacy(self): x, y, configuration = self._readStainlessSteelData() configuration["concentrations"]['usematrix'] = 0 configuration["concentrations"]["usemultilayersecondary"] = 0 - self._testCompareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration) configuration["concentrations"]['usematrix'] = 1 configuration["concentrations"]["usemultilayersecondary"] = 2 - self._testCompareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration) configuration["concentrations"]['usematrix'] = 0 configuration["concentrations"]["usemultilayersecondary"] = 2 @@ -578,9 +577,9 @@ def testCompareLegacyMcaTheory(self): "Mn", "Fe", "Ni", "-", "-", "-","-","-"] - self._testCompareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration) - def _testCompareLegacyMcaTheory(self, x, y, configuration): + def _compareLegacyMcaTheory(self, x, y, configuration): from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import NewClassMcaTheory @@ -588,45 +587,19 @@ def _testCompareLegacyMcaTheory(self, x, y, configuration): t0 = time.time() mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() + mcaFitLegacy.enableOptimizedLinearFit() _, fitResult1, result1 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFitLegacy, tmpflag=True) + x, y, copy.deepcopy(configuration), mcaFitLegacy) + + legacy_linear = mcaFitLegacy.config['fit'].get("linearfitflag", 0) and mcaFitLegacy._batchFlag and (mcaFitLegacy.linearMatrix is not None) t1 = time.time() mcaFit = NewClassMcaTheory.McaTheory() + configuration2 = copy.deepcopy(configuration) + configuration2["fit"]["linearfitflag"] = int(legacy_linear) _, fitResult2, result2 = self._configAndFit( - x, y, copy.deepcopy(configuration), mcaFit, tmpflag=True) - - import matplotlib.pyplot as plt - from pprint import pprint - - if False: - mcaFit.linear = False - print(mcaFit.linegroup_areas) - result = mcaFit.fit(full_output=True) - pprint(result["niter"]) - mcaFit.use_fit_result(result) - print(mcaFit.linegroup_areas) - - if False: - plt.plot(mcaFit.ydata, label="data") - plt.plot(mcaFit.ynumbkg(), label="ynumbkg") - plt.plot(mcaFit.yfullmodel, label="model") - plt.yscale("log") - plt.legend() - plt.show() - - if True: - for i, name in enumerate(mcaFit.parameter_names): - if "linegroup" in name: - continue - yd = mcaFit.derivative_fitmodel(i) - yd_num = mcaFit._numerical_derivative(i) - plt.plot(yd, label="yd") - plt.plot(yd_num, label="yd_num") - plt.legend() - plt.title(name) - plt.show() + x, y, configuration2, mcaFit) t2 = time.time() @@ -654,7 +627,7 @@ def _testCompareLegacyMcaTheory(self, x, y, configuration): lst = config1["attenuators"]["Matrix"] lst.extend([0, lst[-1]+lst[-2]]) config1["fit"]["continuum_name"] = config2["fit"]["continuum_name"] - + config1["fit"]["linearfitflag"] = config2["fit"]["linearfitflag"] self._assertDeepEqual(config1, config2) # Compare fluo rate dictionaries @@ -689,26 +662,60 @@ def _testCompareLegacyMcaTheory(self, x, y, configuration): self.assertEqual(line1[2], line2[2]) # Compares parameter names - #self.assertEqual(mcaFitLegacy.PARAMETERS, mcaFit.parameter_names) + names = mcaFit.get_parameter_names() + legacy_names = mcaFitLegacy.PARAMETERS + if mcaFit.continuum > 2: + self.assertEqual(set(legacy_names), set(names)) + else: + self.assertEqual(set(legacy_names), set(names)|{"Constant", "1st Order"}) # Compare fit results - #self.assertEqual(fitResult1[0], fitResult2[0]) + parameters1 = dict(zip(legacy_names, fitResult1[0])) + parameters2 = dict(zip(legacy_names, fitResult2[0])) + uncertainties1 = dict(zip(legacy_names, fitResult1[2])) + uncertainties2 = dict(zip(legacy_names, fitResult2[2])) + print(parameters1["Cu K"], parameters2["Cu K"]) + for name in names: + self.assertEqual(parameters1[name], parameters2[name], name) + self.assertEqual(uncertainties1[name], uncertainties2[name], name) + + # Compare digested results #self.assertEqual(result1, result2) - def _configAndFit(self, x, y, configuration, mcaFit, tmpflag=False): + def _configAndFit(self, x, y, configuration, mcaFit): configuration = mcaFit.configure(configuration) mcaFit.setData(x, y, xmin=configuration["fit"]["xmin"], xmax=configuration["fit"]["xmax"]) - mcaFit.estimate() - if tmpflag: - return configuration, None, None + if hasattr(mcaFit, "parameter_types") and False: + mcaFit.continuum_name = "Parabolic" + print(mcaFit.get_parameter_names()) + for param_name, calc, numerical in mcaFit.compare_derivatives(exclude_fixed=True): + self._vis_compare(calc, numerical, param_name) + numpy.testing.assert_allclose( + calc, numerical, err_msg=param_name, rtol=1e-3 + ) fitResult1, result1 = mcaFit.startFit(digest=1) + + if True: + import matplotlib.pyplot as plt + plt.plot(result1["ydata"]) + plt.plot(result1["yfit"]) + plt.title(str(mcaFit)) + plt.show() return configuration, fitResult1, result1 + def _vis_compare(self, a, b, title): + import matplotlib.pyplot as plt + + plt.plot(a) + plt.plot(b) + plt.title(title) + plt.show() + def _assertDeepEqual(self, obj1, obj2, _nodes=tuple()): """Better verbosity than assertEqual for deep structures """ @@ -738,12 +745,15 @@ def getSuite(auto=True): testSuite.addTest(testXrf("testTrainingDataFilePresence")) testSuite.addTest(testXrf("testTrainingDataFit")) testSuite.addTest(testXrf("testStainlessSteelDataFit")) - testSuite.addTest(testXrf("testCompareLegacyMcaTheory")) + testSuite.addTest(testXrf("testTrainingDataFitCompareLegacy")) + testSuite.addTest(testXrf("testStainlessSteelDataFitCompareLegacy")) return testSuite + def test(auto=False): return unittest.TextTestRunner(verbosity=2).run(getSuite(auto=auto)) + if __name__ == '__main__': if len(sys.argv) > 1: auto = False From 384632af40b4d98b66e2800765a70158be9a8178 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 16 Aug 2021 16:56:43 +0200 Subject: [PATCH 70/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index fafcf309a..45da790b7 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -2057,8 +2057,6 @@ def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): :param array xdata: shape (ndata,) :returns array: shape (ndata,) """ - return self.numerical_derivative_fitmodel(param_idx, xdata=xdata, **paramtype) - group = self._group_from_parameter_index(param_idx, **paramtype) name = group.property_name if name == "st_arearatio": From e458033e269adb3c8985963baa8a8e0a0d6bf58d Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Mon, 16 Aug 2021 18:03:46 +0200 Subject: [PATCH 71/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 97 ++++++++++++++------ PyMca5/tests/XrfTest.py | 13 +-- 2 files changed, 77 insertions(+), 33 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 45da790b7..116149586 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -1072,12 +1072,6 @@ def _refreshAreasCache(self, force=False): self._estimateLineGroupAreas() self._lastAreasCacheParams = params - import matplotlib.pyplot as plt - - plt.plot(self.yfitdata) - plt.plot(self.yfitmodel) - plt.show() - @property def _areasCacheParams(self): """Any change in these parameter will invalidate the cache""" @@ -1095,38 +1089,87 @@ def _estimateLineGroupAreas(self): emin = xenergy.min() emax = xenergy.max() factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM / self.gain - - import matplotlib.pyplot as plt - - plt.figure() - plt.plot(xenergy, ydata) zeroarea = max(ydata) * 1e-7 lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups linegroup_areas = self._lineGroupAreas for i, (group, escgroup) in enumerate(zip(lineGroups, escapeLineGroups)): - if not escgroup: - escgroup = [[]] * len(group) - selected_energy = 0 - selected_rate = 0 - for (peakenergy, rate, _), esclines in zip(group, escgroup): - if peakenergy >= emin and peakenergy <= emax: - if rate > selected_rate: - selected_energy = peakenergy - for peakenergy, escrate, _ in esclines: - if peakenergy >= emin and peakenergy <= emax: - if rate * escrate > selected_rate: - selected_energy = peakenergy + selected_energy, _ = self.__select_highest_peak(group, escgroup, emin, emax) if selected_energy: chan = numpy.abs(xenergy - selected_energy).argmin() - height = ydata[chan] - plt.plot([selected_energy, selected_energy], [0, height]) + height = ydata[chan] # omit dividing by the rate fwhm = self._peakFWHM(selected_energy) # energy domain linegroup_areas[i] = height * fwhm * factor # Gaussian else: - linegroup_areas[i] = zeroarea - plt.title(f"NEW ESTIMATE {self.zero} {self.gain}") + linegroup_areas[i] = 0 # no peak within [emin, emax] + + def __select_highest_peak(self, group, escgroup, emin, emax): + if not escgroup: + escgroup = [[]] * len(group) + selected_energy = 0 + selected_rate = 0 + for (peakenergy, rate, _), esclines in zip(group, escgroup): + if peakenergy >= emin and peakenergy <= emax: + if rate > selected_rate: + selected_energy = peakenergy + selected_rate = rate + for peakenergy, escrate, _ in esclines: + if peakenergy >= emin and peakenergy <= emax: + if rate * escrate > selected_rate: + selected_energy = peakenergy + selected_rate = rate * escrate + return selected_energy, selected_rate + + def klm_markers(self, xenergy=None): + if xenergy is None: + xenergy = self.xenergy + emin = xenergy.min() + emax = xenergy.max() + factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM / self.gain + + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self._lineGroupAreas + linegroup_names = self._lineGroupNames + + for group, escgroup, label, area in zip( + lineGroups, escapeLineGroups, linegroup_names, linegroup_areas + ): + if not escgroup: + escgroup = [[]] * len(group) + peakenergy, rate = self.__select_highest_peak(group, escgroup, emin, emax) + if not peakenergy: + continue # no peak within [emin, emax] + fwhm = self._peakFWHM(peakenergy) # energy domain + height = (rate * area) / (fwhm * factor) + yield peakenergy, height, label + for (peakenergy, rate, _), esclines in zip(group, escgroup): + fwhm = self._peakFWHM(peakenergy) # energy domain + height = (rate * area) / (fwhm * factor) + yield peakenergy, height, None + for peakenergy, escrate, _ in esclines: + fwhm = self._peakFWHM(peakenergy) # energy domain + height = (rate * escrate * area) / (fwhm * factor) + yield peakenergy, height, None + + def plot(self, title=None, markers=False): + import matplotlib.pyplot as plt + + plt.plot(self.xenergy, self.yfitdata, label="data") + plt.plot(self.xenergy, self.yfitmodel, label="fit") + if markers: + from matplotlib.pyplot import cm + + colors = iter(cm.rainbow(numpy.linspace(0, 1, self._nLineGroups))) + for energy, height, label in self.klm_markers(): + if label: + color = next(colors) + plt.text(energy, height, label, color=color) + plt.plot([energy, energy], [0, height], label=label, color=color) + plt.legend() + if title: + plt.title(title) plt.show() def estimate(self): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 7c5713f45..d0e464cc3 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -698,14 +698,15 @@ def _configAndFit(self, x, y, configuration, mcaFit): calc, numerical, err_msg=param_name, rtol=1e-3 ) + if hasattr(mcaFit, "parameter_types") and False: + mcaFit.plot(title="estimated", markers=True) + fitResult1, result1 = mcaFit.startFit(digest=1) - if True: - import matplotlib.pyplot as plt - plt.plot(result1["ydata"]) - plt.plot(result1["yfit"]) - plt.title(str(mcaFit)) - plt.show() + if hasattr(mcaFit, "parameter_types") and False: + with mcaFit.use_fit_result_context(mcaFit._last_fit_result): + mcaFit.plot(title="fitted", markers=True) + return configuration, fitResult1, result1 def _vis_compare(self, a, b, title): From 089fcd2718b7b1157ff0aa02f9d49e0c1af78528 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 17 Aug 2021 15:10:37 +0200 Subject: [PATCH 72/74] fixup --- .../PyMcaMath/fitting/model/CachingModel.py | 6 +- .../fitting/model/LeastSquaresFitModel.py | 75 ++++++++--- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 121 +++++++++++------- PyMca5/tests/XrfTest.py | 107 ++++++++++------ 4 files changed, 207 insertions(+), 102 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/CachingModel.py b/PyMca5/PyMcaMath/fitting/model/CachingModel.py index c33a989fe..070d15b29 100644 --- a/PyMca5/PyMcaMath/fitting/model/CachingModel.py +++ b/PyMca5/PyMcaMath/fitting/model/CachingModel.py @@ -159,7 +159,11 @@ def _set_property_values_cache(self, values_cache, **cacheoptions): if caches is None: return False key = self._cache_manager._property_cache_key(**cacheoptions) - caches[key] = values_cache + cache = caches.get(key, None) + if cache is None: + caches[key] = values_cache + else: + cache[:] = values_cache return True def _create_start_property_values_cache(self, **cacheoptions): diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index bfa09dba9..bf5d795e0 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -97,7 +97,9 @@ def reduced_chi_squared(self): @property def degrees_of_freedom(self): - return self.ndata - self.get_n_parameters() + constraints = self.get_parameter_constraints() + nfree = sum(constraints[:, 0] != Gefit.CFIXED, dtype=int) + return self.ndata - nfree def evaluate_fullmodel(self, xdata=None): """Evaluate the full model. @@ -237,26 +239,35 @@ def iterative_optimization(self, full_output=False): :returns dict: """ with self._iterative_optimization_context(): - constraints = self.get_parameter_constraints().T + constraints = self.get_parameter_constraints() xdata = self.xdata ydata = self.yfitdata ystd = self.yfitstd for i in range(max(self.niter_non_leastsquares, 1)): parameters = self.get_parameter_values() - result = Gefit.LeastSquaresFit( - self._gefit_evaluate_fitmodel, - parameters, - model_deriv=self._gefit_derivative_fitmodel, - xdata=xdata, - ydata=ydata, - sigmadata=ystd, - constrains=constraints, - maxiter=self.maxiter, - weightflag=self.weightflag, - deltachi=self.deltachi, - fulloutput=full_output, - linear=self.only_linear_parameters, - ) + try: + result = Gefit.LeastSquaresFit( + self._gefit_evaluate_fitmodel, + parameters, + model_deriv=self._gefit_derivative_fitmodel, + xdata=xdata, + ydata=ydata, + sigmadata=ystd, + constrains=constraints.T, + maxiter=self.maxiter, + weightflag=self.weightflag, + deltachi=self.deltachi, + fulloutput=full_output, + linear=self.only_linear_parameters, + ) + except numpy.linalg.LinAlgError as e: + if "singular matrix" in str(e).lower(): + reason = self.__singular_matrix_reason( + xdata, parameters, constraints + ) + if reason: + raise RuntimeError(reason) from e + raise if self.niter_non_leastsquares: self.set_parameter_values(result[0]) self.non_leastsquares_increment() @@ -271,6 +282,38 @@ def iterative_optimization(self, full_output=False): ret["lastdeltachi"] = result[4] return ret + def __singular_matrix_reason(self, xdata, parameters, constraints): + print("\n" * 3) + fitted = constraints[:, 0] != Gefit.CFIXED + fitted_indices = fitted.nonzero()[0].tolist() + derivatives = numpy.vstack( + self._gefit_derivative_fitmodel(parameters, i, xdata) + for i in fitted_indices + ) + names = numpy.array(self.get_parameter_names()) + names = names[fitted_indices].tolist() + + allzeros = (numpy.abs(derivatives) < 1e-10).all(axis=1) + if allzeros.any(): + names = [ + name for i, name in enumerate(self.get_parameter_names()) if allzeros[i] + ] + return "Derivatives of {} are zero".format(names) + + for i1, deriv1 in enumerate(derivatives): + for i2, deriv2 in enumerate(derivatives): + if i1 == i2: + continue + if numpy.allclose(deriv1, deriv2): + import matplotlib.pyplot as plt + + plt.plot(deriv1) + plt.plot(deriv2) + plt.show() + return "Derivatives of '{}' and '{}' are equal".format( + names[i1], names[i2] + ) + @property def maxiter(self): return 100 diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index 116149586..be3cdf503 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -716,6 +716,8 @@ def ycontinuum(self, xdata=None): return numpy.zeros(self.ndata) else: return numpy.zeros(len(xdata)) + if xdata is not None: + xdata = self._channelsToXpol(xdata) return model.evaluate_fullmodel(xdata=xdata) @property @@ -852,20 +854,6 @@ def xpol(self): def _channelsToXpol(self, x): raise NotImplementedError - @property - def continuum_coefficients(self): - model = self.continuumModel - if model is None: - return list() - else: - return model.get_parameter_values() - - @continuum_coefficients.setter - def continuum_coefficients(self, values): - model = self.continuumModel - if model is not None: - model.set_parameter_values(values) - class McaTheory(McaTheoryBackground, McaTheoryLegacyApi, LeastSquaresFitModel): """Model for MCA data""" @@ -1088,8 +1076,12 @@ def _estimateLineGroupAreas(self): xenergy = self.xenergy emin = xenergy.min() emax = xenergy.max() - factor = numpy.sqrt(2 * numpy.pi) / self.GAUSS_SIGMA_TO_FWHM / self.gain - zeroarea = max(ydata) * 1e-7 + # Height of a peak with rate Ri and group area A (Gaussian approx.) + # Hi(cts) = A(cts) * Ri / (sqrt(2*pi)*sigma(cts)) + # sigma(cts) = FWHM(keV) / (gain*GAUSS_SIGMA_TO_FWHM) + # A(cts) = Hi(cts) / Ri * sqrt(2*pi) * FWHM(keV) / (gain*GAUSS_SIGMA_TO_FWHM) + # A(cts) = Hi(cts) / Ri * FWHM(keV) * factor + factor = numpy.sqrt(2 * numpy.pi) / (self.GAUSS_SIGMA_TO_FWHM * self.gain) lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups @@ -1098,11 +1090,11 @@ def _estimateLineGroupAreas(self): selected_energy, _ = self.__select_highest_peak(group, escgroup, emin, emax) if selected_energy: chan = numpy.abs(xenergy - selected_energy).argmin() - height = ydata[chan] # omit dividing by the rate + height = ydata[chan] # omit rate to approx. compensate for peak overlap fwhm = self._peakFWHM(selected_energy) # energy domain linegroup_areas[i] = height * fwhm * factor # Gaussian - else: - linegroup_areas[i] = 0 # no peak within [emin, emax] + else: # no peak within [emin, emax] + linegroup_areas[i] = 0 # fixed at zero (see constraints) def __select_highest_peak(self, group, escgroup, emin, emax): if not escgroup: @@ -1158,6 +1150,7 @@ def plot(self, title=None, markers=False): plt.plot(self.xenergy, self.yfitdata, label="data") plt.plot(self.xenergy, self.yfitmodel, label="fit") + plt.plot(self.xenergy, self.ypileup(), label="pileup") if markers: from matplotlib.pyplot import cm @@ -1167,6 +1160,10 @@ def plot(self, title=None, markers=False): color = next(colors) plt.text(energy, height, label, color=color) plt.plot([energy, energy], [0, height], label=label, color=color) + + plt.yscale("log") + plt.ylim(1, None) + plt.legend() if title: plt.title(title) @@ -1568,14 +1565,14 @@ def _channelsToXpol(self, x): @independent_linear_parameter_group def linpol_coefficients(self): if isinstance(self.continuumModel, LinearPolynomialModel): - return self.continuum_coefficients + return self.continuumModel.coefficients else: return list() @linpol_coefficients.setter def linpol_coefficients(self, values): if isinstance(self.continuumModel, LinearPolynomialModel): - self.continuum_coefficients = values + self.continuumModel.coefficients = values @linpol_coefficients.counter def linpol_coefficients(self): @@ -1595,15 +1592,17 @@ def linpol_coefficients(self): @nonlinear_parameter_group def exppol_coefficients(self): - if isinstance(self.continuumModel, ExponentialPolynomialModel): - return self.continuum_coefficients + model = self.continuumModel + if isinstance(model, ExponentialPolynomialModel): + return model.coefficients else: return list() @exppol_coefficients.setter def exppol_coefficients(self, values): - if isinstance(self.continuumModel, ExponentialPolynomialModel): - self.continuum_coefficients = values + model = self.continuumModel + if isinstance(model, ExponentialPolynomialModel): + model.coefficients = values @exppol_coefficients.counter def exppol_coefficients(self): @@ -1936,14 +1935,7 @@ def _convert_parameter_names(self, names): if linegroup_names is None: linegroup_names = self.linegroup_names idx = int(name.replace("linegroup_areas", "")) - name = linegroup_names[idx] - if "inelastic" in name: - i = int(name.replace("inelastic", "")) - name = "Scatter Compton%03d" % i - elif "elastic" in name: - i = int(name.replace("elastic", "")) - name = "Scatter Peak%03d" % i - yield name + yield self._convert_line_name(linegroup_names[idx]) elif "linpol_coefficients" in name: if self.continuum < self.CONTINUUM_LIST.index("Linear Polynomial"): idx = int(name.replace("linpol_coefficients", "")) @@ -1972,6 +1964,15 @@ def _convert_parameter_names(self, names): else: yield name.capitalize() + def _convert_line_name(self, name): + if "inelastic" in name: + i = int(name.replace("inelastic", "")) + return "Scatter Compton%03d" % i + elif "elastic" in name: + i = int(name.replace("elastic", "")) + return "Scatter Peak%03d" % i + return name + def get_parameter_names(self, **paramtype): return tuple( self._convert_parameter_names(super().get_parameter_names(**paramtype)) @@ -2050,9 +2051,7 @@ def mcatheory( return y def ypileup(self, ymodel=None, xdata=None, normalized_fit_parameters=False): - """The ymodel contains the peaks and the continuum. - Pileup is - """ + """The ymodel contains the peaks and the continuum.""" pileupfactor = self.sum if not pileupfactor: if xdata is None: @@ -2124,15 +2123,16 @@ def derivative_fitmodel(self, param_idx, xdata=None, **paramtype): ) elif name == "sum": return self.ypileup(xdata=xdata, normalized_fit_parameters=True) - elif name == "linpol" and False: + elif name == "linpol_coefficients": model = self.continuumModel - keep = model.get_parameter_values(**paramtype) - try: - model.set_parameter_values(self.continuum_coefficients, **paramtype) - i = group.parameter_index_in_group(param_idx) - return model.derivative_fitmodel(i, xdata=xdata, **paramtype) - finally: - model.set_parameter_values(keep, **paramtype) + i = group.parameter_index_in_group(param_idx) + xpol = xdata + if xpol is not None: + xpol = self._channelsToXpol(xpol) + deriv = model.derivative_fitmodel(i, xdata=xpol, **paramtype) + if self.sum: + deriv = deriv + self.ypileup(ymodel=deriv, xdata=xdata) + return deriv else: return self.numerical_derivative_fitmodel( param_idx, xdata=xdata, **paramtype @@ -2191,6 +2191,29 @@ def _custom_iterative_optimization_context(self): self.zero = 0.0 yield + @contextmanager + def _propertyCachingContext(self, **kw): + with super()._propertyCachingContext(**kw) as values_cache: + with self._link_continuum_model_cache(): + yield values_cache + + @contextmanager + def _link_continuum_model_cache(self): + model = self.continuumModel + if model is None: + yield + return + keep = model.coefficients + try: + # Replace the model's coefficient storage by the cache view + if isinstance(model, LinearPolynomialModel): + model._coefficients = self.linpol_coefficients + else: + model._coefficients = self.exppol_coefficients + yield + finally: + model._coefficients = keep + def startFit(self, digest=False, linear=None): """Fit with legacy output""" with self.linear_context(linear=linear): @@ -2219,13 +2242,21 @@ def digestresult(self): "ydata": self.ydata, "continuum": self.ybackground(), "yfit": self.yfullmodel, - "ypileup": self.ypileup(), + "pileup": self.ypileup(), "parameters": self.get_parameter_names(), "fittedpar": self._last_fit_result["parameters"], "chisq": self._last_fit_result["chi2_red"], + "lastdeltachi": self._last_fit_result["lastdeltachi"], + "niter": self._last_fit_result["niter"], "sigmapar": self._last_fit_result["uncertainties"], "config": self.config.copy(), } + # TODO: + for name in self._lineGroupNames: + name = self._convert_line_name(name) + result[name] = None + result["y" + name] = None + result["groups"] = None return result def imagingDigestResult(self): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index d0e464cc3..b6ff0d867 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -550,34 +550,52 @@ def testTrainingDataFitCompareLegacy(self): def testStainlessSteelDataFitCompareLegacy(self): x, y, configuration = self._readStainlessSteelData() - configuration["concentrations"]['usematrix'] = 0 - configuration["concentrations"]["usemultilayersecondary"] = 0 - self._compareLegacyMcaTheory(x, y, configuration) - - configuration["concentrations"]['usematrix'] = 1 - configuration["concentrations"]["usemultilayersecondary"] = 2 - self._compareLegacyMcaTheory(x, y, configuration) - - configuration["concentrations"]['usematrix'] = 0 - configuration["concentrations"]["usemultilayersecondary"] = 2 - configuration["attenuators"]["Matrix"] = [1, 'Fe', 1.0, 1.0, 45.0, 45.0] - configuration["fit"]["strategyflag"] = 1 - configuration["fit"]["strategy"] = "SingleLayerStrategy" - configuration["SingleLayerStrategy"] = {} - configuration["SingleLayerStrategy"]["layer"] = "Auto" - configuration["SingleLayerStrategy"]["iterations"] = 3 - configuration["SingleLayerStrategy"]["completer"] = "-" - configuration["SingleLayerStrategy"]["flags"] = [1, 1, 1, 1, 0, - 0, 0, 0, 0, 0] - configuration["SingleLayerStrategy"]["peaks"] = [ "Cr K", - "Mn K", "Fe Ka", - "Ni K", "-", "-", - "-","-","-","-"] - configuration["SingleLayerStrategy"]["materials"] = ["Cr", - "Mn", "Fe", - "Ni", "-", "-", - "-","-","-"] - self._compareLegacyMcaTheory(x, y, configuration) + with self.subTest("simple"): + configuration["concentrations"]['usematrix'] = 0 + configuration["concentrations"]["usemultilayersecondary"] = 0 + self._compareLegacyMcaTheory(x, y, configuration) + + with self.subTest("continuum"): + configuration["fit"]["continuum"] = 3 + self._compareLegacyMcaTheory(x, y, configuration) + configuration["fit"]["continuum"] = 0 + + with self.subTest("tails"): + configuration["fit"]["fitfunction"] = 1|2|4|8 + self._compareLegacyMcaTheory(x, y, configuration) + configuration["fit"]["fitfunction"] = 1 + + with self.subTest("voigt"): + configuration["fit"]["fitfunction"] = 0 + self._compareLegacyMcaTheory(x, y, configuration) + configuration["fit"]["fitfunction"] = 1 + + with self.subTest("usematrix"): + configuration["concentrations"]['usematrix'] = 1 + configuration["concentrations"]["usemultilayersecondary"] = 2 + self._compareLegacyMcaTheory(x, y, configuration) + + with self.subTest("strategy"): + configuration["concentrations"]['usematrix'] = 0 + configuration["concentrations"]["usemultilayersecondary"] = 2 + configuration["attenuators"]["Matrix"] = [1, 'Fe', 1.0, 1.0, 45.0, 45.0] + configuration["fit"]["strategyflag"] = 1 + configuration["fit"]["strategy"] = "SingleLayerStrategy" + configuration["SingleLayerStrategy"] = {} + configuration["SingleLayerStrategy"]["layer"] = "Auto" + configuration["SingleLayerStrategy"]["iterations"] = 3 + configuration["SingleLayerStrategy"]["completer"] = "-" + configuration["SingleLayerStrategy"]["flags"] = [1, 1, 1, 1, 0, + 0, 0, 0, 0, 0] + configuration["SingleLayerStrategy"]["peaks"] = [ "Cr K", + "Mn K", "Fe Ka", + "Ni K", "-", "-", + "-","-","-","-"] + configuration["SingleLayerStrategy"]["materials"] = ["Cr", + "Mn", "Fe", + "Ni", "-", "-", + "-","-","-"] + self._compareLegacyMcaTheory(x, y, configuration) def _compareLegacyMcaTheory(self, x, y, configuration): from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory @@ -669,18 +687,25 @@ def _compareLegacyMcaTheory(self, x, y, configuration): else: self.assertEqual(set(legacy_names), set(names)|{"Constant", "1st Order"}) - # Compare fit results + # Compare fit parameters parameters1 = dict(zip(legacy_names, fitResult1[0])) - parameters2 = dict(zip(legacy_names, fitResult2[0])) + parameters2 = dict(zip(names, fitResult2[0])) uncertainties1 = dict(zip(legacy_names, fitResult1[2])) - uncertainties2 = dict(zip(legacy_names, fitResult2[2])) - print(parameters1["Cu K"], parameters2["Cu K"]) - for name in names: - self.assertEqual(parameters1[name], parameters2[name], name) - self.assertEqual(uncertainties1[name], uncertainties2[name], name) + uncertainties2 = dict(zip(names, fitResult2[2])) + if False: + for name in names: + self.assertEqual(parameters1[name], parameters2[name], name) + self.assertEqual(uncertainties1[name], uncertainties2[name], name) - # Compare digested results - #self.assertEqual(result1, result2) + # Compare fit results + self.assertEqual(set(result1), set(result2)) + self._vis_compare(result1["continuum"], result2["continuum"], "continuum") + numpy.testing.assert_allclose(result1["xdata"], result2["xdata"]) + numpy.testing.assert_allclose(result1["ydata"], result2["ydata"]) + numpy.testing.assert_allclose(result1["energy"], result2["energy"], rtol=1e-3) + numpy.testing.assert_allclose(result1["continuum"], result2["continuum"]) + numpy.testing.assert_allclose(result1["yfit"], result2["yfit"], rtol=0.05) + numpy.testing.assert_allclose(result1["pileup"], result2["pileup"], atol=1, rtol=0.05) def _configAndFit(self, x, y, configuration, mcaFit): configuration = mcaFit.configure(configuration) @@ -689,10 +714,12 @@ def _configAndFit(self, x, y, configuration, mcaFit): xmax=configuration["fit"]["xmax"]) mcaFit.estimate() - if hasattr(mcaFit, "parameter_types") and False: - mcaFit.continuum_name = "Parabolic" + if hasattr(mcaFit, "parameter_types"): print(mcaFit.get_parameter_names()) - for param_name, calc, numerical in mcaFit.compare_derivatives(exclude_fixed=True): + + if hasattr(mcaFit, "parameter_types") and configuration["fit"]["continuum"] and False: + xdata = mcaFit.xdata[10:-10] + for param_name, calc, numerical in mcaFit.compare_derivatives(xdata=xdata, exclude_fixed=True): self._vis_compare(calc, numerical, param_name) numpy.testing.assert_allclose( calc, numerical, err_msg=param_name, rtol=1e-3 From 0f4b7b2339a9c33ec7408f3c65ad48e8b8fb5d3c Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 6 Oct 2021 15:22:48 +0200 Subject: [PATCH 73/74] fixup --- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 80 ++++++++++++-- PyMca5/tests/XrfTest.py | 104 ++++++++++--------- 2 files changed, 127 insertions(+), 57 deletions(-) diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index be3cdf503..eaf982d09 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -51,7 +51,7 @@ def defaultConfigFilename(): class McaTheoryConfigApi: - """API on top of an MCA configuration""" + """API on top of a ConfigDict for MCA configuration""" def __init__(self, initdict=None, filelist=None, **kw): if initdict is None: @@ -904,7 +904,7 @@ def _initializeConfig(self): super()._initializeConfig() self.linearfitflag = self.config["fit"][ "linearfitflag" - ] # synchronize parameter_types + ] # done to synchronize parameter_types self._preCalculateParameterIndependent() self._preCalculateParameterDependent() @@ -2234,7 +2234,7 @@ def _legacyresult(result): result["lastdeltachi"], ) - def digestresult(self): + def digestresult(self, outfile=None, info=None): with self.use_fit_result_context(self._last_fit_result): result = { "xdata": self.xdata, @@ -2251,17 +2251,77 @@ def digestresult(self): "sigmapar": self._last_fit_result["uncertainties"], "config": self.config.copy(), } - # TODO: - for name in self._lineGroupNames: - name = self._convert_line_name(name) - result[name] = None - result["y" + name] = None - result["groups"] = None + self._digestLineResults(result) + if outfile: + self.saveDigestedResult(result, outfile, info=info) + return result def imagingDigestResult(self): with self.use_fit_result_context(self._last_fit_result): - return dict() + result = {"chisq": self._last_fit_result["chi2_red"]} + self._digestLineResults(result, full=False) + return result + + def _digestLineResults(self, result, full=True): + groups = result["groups"] = list() + lineGroups = self._lineGroups + escapeLineGroups = self._escapeLineGroups + linegroup_areas = self._lineGroupAreas + linegroup_names = self._lineGroupNames + + for group, escgroup, groupname, area in zip( + lineGroups, escapeLineGroups, linegroup_names, linegroup_areas + ): + groupname = self._convert_line_name(groupname) + groups.append(groupname) + if full: + result["y" + groupname] = NotImplemented + result[groupname] = { + "fitarea": area, + "sigmaarea": NotImplemented, + } + if full: + result[groupname]["mcaarea"] = NotImplemented + result[groupname]["statistics"] = NotImplemented + peaks = result[groupname]["peaks"] = list() + + if full: + escapepeaks = result[groupname]["escapepeaks"] = list() + if not escgroup: + escgroup = [[]] * len(group) + for (energy, rate, linename), esclines in zip(group, escgroup): + fwhm = self._peakFWHM(energy) + peaks.append(linename) + result[groupname][linename] = { + "ratio": rate, + "energy": energy, + "fwhm": fwhm, + } + for escenergy, escrate, escname in esclines: + escname = "{} {}".format(linename, escname.replace(" ", "_")) + fwhm = self._peakFWHM(escenergy) + escapepeaks.append(escname) + escname += "esc" + result[groupname][escname] = { + "ratio": escrate, + "energy": escenergy, + "fwhm": fwhm, + } + else: + peaks.extend(linename for (_, _, linename) in group) + + @staticmethod + def saveDigestedResult(result: dict, filename: str, info=None): + try: + os.remove(filename) + except Exception: + pass + if info is None: + dcfg = ConfigDict.ConfigDict({"result": result}) + else: + dcfg = ConfigDict.ConfigDict({"result": result, "info": info}) + dcfg.write(filename) class MultiMcaTheory(LeastSquaresCombinedFitModel): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index b6ff0d867..1555f91b3 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -545,7 +545,7 @@ def testStainlessSteelDataFit(self): def testTrainingDataFitCompareLegacy(self): x, y, configuration = self._readTrainingData() - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Training data") def testStainlessSteelDataFitCompareLegacy(self): x, y, configuration = self._readStainlessSteelData() @@ -553,27 +553,27 @@ def testStainlessSteelDataFitCompareLegacy(self): with self.subTest("simple"): configuration["concentrations"]['usematrix'] = 0 configuration["concentrations"]["usemultilayersecondary"] = 0 - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (simple)") with self.subTest("continuum"): configuration["fit"]["continuum"] = 3 - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (with continuum)") configuration["fit"]["continuum"] = 0 with self.subTest("tails"): configuration["fit"]["fitfunction"] = 1|2|4|8 - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (with peak tails)") configuration["fit"]["fitfunction"] = 1 with self.subTest("voigt"): configuration["fit"]["fitfunction"] = 0 - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (voigt)") configuration["fit"]["fitfunction"] = 1 with self.subTest("usematrix"): configuration["concentrations"]['usematrix'] = 1 configuration["concentrations"]["usemultilayersecondary"] = 2 - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (quantitative)") with self.subTest("strategy"): configuration["concentrations"]['usematrix'] = 0 @@ -595,9 +595,9 @@ def testStainlessSteelDataFitCompareLegacy(self): "Mn", "Fe", "Ni", "-", "-", "-","-","-"] - self._compareLegacyMcaTheory(x, y, configuration) + self._compareLegacyMcaTheory(x, y, configuration, "Stainless steal (quantitative with strategy)") - def _compareLegacyMcaTheory(self, x, y, configuration): + def _compareLegacyMcaTheory(self, x, y, configuration, title): from PyMca5.PyMcaPhysics.xrf import LegacyMcaTheory from PyMca5.PyMcaPhysics.xrf import NewClassMcaTheory @@ -606,7 +606,7 @@ def _compareLegacyMcaTheory(self, x, y, configuration): mcaFitLegacy = LegacyMcaTheory.LegacyMcaTheory() mcaFitLegacy.enableOptimizedLinearFit() - _, fitResult1, result1 = self._configAndFit( + _, fitResult1, digestedResult1, imagingDigestResult1 = self._configAndFit( x, y, copy.deepcopy(configuration), mcaFitLegacy) legacy_linear = mcaFitLegacy.config['fit'].get("linearfitflag", 0) and mcaFitLegacy._batchFlag and (mcaFitLegacy.linearMatrix is not None) @@ -616,13 +616,15 @@ def _compareLegacyMcaTheory(self, x, y, configuration): mcaFit = NewClassMcaTheory.McaTheory() configuration2 = copy.deepcopy(configuration) configuration2["fit"]["linearfitflag"] = int(legacy_linear) - _, fitResult2, result2 = self._configAndFit( + _, fitResult2, digestedResult2, imagingDigestResult2 = self._configAndFit( x, y, configuration2, mcaFit) t2 = time.time() - print("\nLEGACY TIME", t1-t0) - print("NEW TIME", t2-t1) + print() + print(title) + print(" LEGACY TIME", t1-t0) + print(" NEW TIME", t2-t1) # Compare data numpy.testing.assert_array_equal(mcaFitLegacy.xdata.flat, mcaFit.xdata) @@ -687,25 +689,52 @@ def _compareLegacyMcaTheory(self, x, y, configuration): else: self.assertEqual(set(legacy_names), set(names)|{"Constant", "1st Order"}) - # Compare fit parameters + # Compare fit results parameters1 = dict(zip(legacy_names, fitResult1[0])) parameters2 = dict(zip(names, fitResult2[0])) uncertainties1 = dict(zip(legacy_names, fitResult1[2])) uncertainties2 = dict(zip(names, fitResult2[2])) if False: for name in names: - self.assertEqual(parameters1[name], parameters2[name], name) - self.assertEqual(uncertainties1[name], uncertainties2[name], name) - - # Compare fit results - self.assertEqual(set(result1), set(result2)) - self._vis_compare(result1["continuum"], result2["continuum"], "continuum") - numpy.testing.assert_allclose(result1["xdata"], result2["xdata"]) - numpy.testing.assert_allclose(result1["ydata"], result2["ydata"]) - numpy.testing.assert_allclose(result1["energy"], result2["energy"], rtol=1e-3) - numpy.testing.assert_allclose(result1["continuum"], result2["continuum"]) - numpy.testing.assert_allclose(result1["yfit"], result2["yfit"], rtol=0.05) - numpy.testing.assert_allclose(result1["pileup"], result2["pileup"], atol=1, rtol=0.05) + numpy.testing.assert_allclose(parameters1[name], parameters2[name], atol=1e-5, rtol=0.5, err_msg=name) + numpy.testing.assert_allclose(uncertainties1[name], uncertainties2[name], atol=1e-5, rtol=0.5, err_msg=name) + + # Compare digested results + self.assertEqual(set(digestedResult1), set(digestedResult2)) + + self.assertEqual(digestedResult1["groups"], digestedResult2["groups"]) + for groupname in digestedResult1["groups"]: + group1 = digestedResult1[groupname] + group2 = digestedResult2[groupname] + self.assertEqual(group1.keys(), group2.keys()) + self.assertEqual(group1["peaks"], group2["peaks"]) + for linename in group1["peaks"]: + line1 = group1[linename] + line2 = group2[linename] + self.assertEqual(line1.keys(), line1.keys()) + self.assertEqual(group1["escapepeaks"], group2["escapepeaks"]) + for linename in group1["escapepeaks"]: + line1 = group1[linename+"esc"] + line2 = group2[linename+"esc"] + self.assertEqual(line1.keys(), line1.keys()) + + #self._vis_compare(digestedResult1["yfit"], digestedResult2["yfit"], "yfit") + numpy.testing.assert_allclose(digestedResult1["xdata"], digestedResult2["xdata"]) + numpy.testing.assert_allclose(digestedResult1["ydata"], digestedResult2["ydata"]) + numpy.testing.assert_allclose(digestedResult1["energy"], digestedResult2["energy"], rtol=1e-3) + numpy.testing.assert_allclose(digestedResult1["continuum"], digestedResult2["continuum"], atol=1, rtol=0.05) + numpy.testing.assert_allclose(digestedResult1["yfit"], digestedResult2["yfit"], atol=1, rtol=0.05) + numpy.testing.assert_allclose(digestedResult1["pileup"], digestedResult2["pileup"], atol=1, rtol=0.05) + + # Compare image digested results + self.assertEqual(set(imagingDigestResult1), set(imagingDigestResult2)) + + self.assertEqual(imagingDigestResult1["groups"], imagingDigestResult2["groups"]) + for groupname in imagingDigestResult1["groups"]: + group1 = imagingDigestResult1[groupname] + group2 = imagingDigestResult2[groupname] + self.assertEqual(group1.keys(), group2.keys()) + self.assertEqual(group1["peaks"], group2["peaks"]) def _configAndFit(self, x, y, configuration, mcaFit): configuration = mcaFit.configure(configuration) @@ -713,28 +742,9 @@ def _configAndFit(self, x, y, configuration, mcaFit): xmin=configuration["fit"]["xmin"], xmax=configuration["fit"]["xmax"]) mcaFit.estimate() - - if hasattr(mcaFit, "parameter_types"): - print(mcaFit.get_parameter_names()) - - if hasattr(mcaFit, "parameter_types") and configuration["fit"]["continuum"] and False: - xdata = mcaFit.xdata[10:-10] - for param_name, calc, numerical in mcaFit.compare_derivatives(xdata=xdata, exclude_fixed=True): - self._vis_compare(calc, numerical, param_name) - numpy.testing.assert_allclose( - calc, numerical, err_msg=param_name, rtol=1e-3 - ) - - if hasattr(mcaFit, "parameter_types") and False: - mcaFit.plot(title="estimated", markers=True) - - fitResult1, result1 = mcaFit.startFit(digest=1) - - if hasattr(mcaFit, "parameter_types") and False: - with mcaFit.use_fit_result_context(mcaFit._last_fit_result): - mcaFit.plot(title="fitted", markers=True) - - return configuration, fitResult1, result1 + fitResult, digestedResult = mcaFit.startFit(digest=1) + imagingDigestResult = mcaFit.imagingDigestResult() + return configuration, fitResult, digestedResult, imagingDigestResult def _vis_compare(self, a, b, title): import matplotlib.pyplot as plt From 9ca7db9acde3cd7f724b121eadd548ad8d098bf7 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 6 Oct 2021 17:56:14 +0200 Subject: [PATCH 74/74] fixup --- .../fitting/model/LeastSquaresFitModel.py | 9 +- PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py | 228 +++++++++++++----- PyMca5/tests/XrfTest.py | 4 +- 3 files changed, 176 insertions(+), 65 deletions(-) diff --git a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py index bf5d795e0..a5dee7a05 100644 --- a/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py +++ b/PyMca5/PyMcaMath/fitting/model/LeastSquaresFitModel.py @@ -96,10 +96,13 @@ def reduced_chi_squared(self): return self.chi_squared / self.degrees_of_freedom @property - def degrees_of_freedom(self): + def nfree_parameters(self): constraints = self.get_parameter_constraints() - nfree = sum(constraints[:, 0] != Gefit.CFIXED, dtype=int) - return self.ndata - nfree + return numpy.sum(constraints[:, 0] < 3, dtype=int) + + @property + def degrees_of_freedom(self): + return self.ndata - self.nfree_parameters def evaluate_fullmodel(self, xdata=None): """Evaluate the full model. diff --git a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py index eaf982d09..b903ade14 100644 --- a/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py +++ b/PyMca5/PyMcaPhysics/xrf/NewClassMcaTheory.py @@ -1122,8 +1122,8 @@ def klm_markers(self, xenergy=None): lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups - linegroup_areas = self._lineGroupAreas - linegroup_names = self._lineGroupNames + linegroup_areas = self.linegroup_areas + linegroup_names = self.linegroup_names for group, escgroup, label, area in zip( lineGroups, escapeLineGroups, linegroup_names, linegroup_areas @@ -2235,81 +2235,189 @@ def _legacyresult(result): ) def digestresult(self, outfile=None, info=None): + return self._digestresult(outfile=outfile, info=info, full=True) + + def imagingDigestResult(self): + return self._digestresult(full=False) + + def _digestresult(self, outfile=None, info=None, full=True): with self.use_fit_result_context(self._last_fit_result): - result = { - "xdata": self.xdata, - "energy": self.xenergy, - "ydata": self.ydata, - "continuum": self.ybackground(), - "yfit": self.yfullmodel, - "pileup": self.ypileup(), - "parameters": self.get_parameter_names(), - "fittedpar": self._last_fit_result["parameters"], - "chisq": self._last_fit_result["chi2_red"], - "lastdeltachi": self._last_fit_result["lastdeltachi"], - "niter": self._last_fit_result["niter"], - "sigmapar": self._last_fit_result["uncertainties"], - "config": self.config.copy(), - } - self._digestLineResults(result) + + parameter_names = self.get_parameter_names() + parameter_std = self._last_fit_result["uncertainties"] + + chisq = self._last_fit_result["chi2_red"] + + if full: + xenergy = self.xenergy + yfit = self.yfullmodel + ydata = self.ydata + weighted_residuals = (ydata - yfit) / self.ystd + prechisq = weighted_residuals ** 2 + + result = { + "xdata": self.xdata, + "energy": xenergy, + "ydata": ydata, + "continuum": self.ybackground(), + "yfit": yfit, + "pileup": self.ypileup(), + "parameters": parameter_names, + "fittedpar": self._last_fit_result["parameters"], + "sigmapar": parameter_std, + "chisq": chisq, + "lastdeltachi": self._last_fit_result["lastdeltachi"], + "niter": self._last_fit_result["niter"], + "config": self.config.copy(), + } + + statistics_info = { + "xenergy": xenergy, + "ydata": ydata, + "yfit": yfit, + "prechisq": prechisq, + } + else: + statistics_info = None + result = {"chisq": chisq} + linegroup_result = self._digestLineGroupResults( + parameter_names, parameter_std, statistics_info=statistics_info + ) + result.update(linegroup_result) if outfile: self.saveDigestedResult(result, outfile, info=info) return result - def imagingDigestResult(self): - with self.use_fit_result_context(self._last_fit_result): - result = {"chisq": self._last_fit_result["chi2_red"]} - self._digestLineResults(result, full=False) - return result - - def _digestLineResults(self, result, full=True): + def _digestLineGroupResults( + self, parameter_names, parameter_std, statistics_info=None + ): + full = statistics_info is not None + result = dict() groups = result["groups"] = list() lineGroups = self._lineGroups escapeLineGroups = self._escapeLineGroups - linegroup_areas = self._lineGroupAreas - linegroup_names = self._lineGroupNames + linegroup_areas = self.linegroup_areas + linegroup_names = self.linegroup_names - for group, escgroup, groupname, area in zip( - lineGroups, escapeLineGroups, linegroup_names, linegroup_areas + area_multiplier = abs(1 + self.st_arearatio) + if statistics_info: + statistics_info["nfree_parameters"] = self.nfree_parameters + ndata = self.ndata + + for igroup, (group, escgroup, groupname, area) in enumerate( + zip(lineGroups, escapeLineGroups, linegroup_names, linegroup_areas) ): groupname = self._convert_line_name(groupname) groups.append(groupname) - if full: - result["y" + groupname] = NotImplemented + + # Total group area with error (in the channel domain) + pidx = parameter_names.index(groupname) + group_fitarea = area * area_multiplier + group_fitarea_std = parameter_std[pidx] * area_multiplier result[groupname] = { - "fitarea": area, - "sigmaarea": NotImplemented, + "fitarea": group_fitarea, + "sigmaarea": group_fitarea_std, } - if full: - result[groupname]["mcaarea"] = NotImplemented - result[groupname]["statistics"] = NotImplemented + if not full: + result[groupname]["peaks"] = [linename for (_, _, linename) in group] + continue + + # Total group spectrum + yfit_group = self.mcatheory( + selected_groups=[igroup], + continuum=False, + ) + result["y" + groupname] = yfit_group + + # Information per line peaks = result[groupname]["peaks"] = list() + escapepeaks = result[groupname]["escapepeaks"] = list() + ydatarest_group = 0 + group_channel_mask = numpy.zeros(ndata, dtype=bool) + if not escgroup: + escgroup = [[]] * len(group) + for (energy, rate, linename), esclines in zip(group, escgroup): + line_result, line_channel_mask, ydatarest_line = self._digestLineResult( + energy, + rate, + yfit_group, + group_fitarea, + group_fitarea_std, + statistics_info, + ) + line_result["ratio"] = rate + peaks.append(linename) + result[groupname][linename] = line_result + + group_channel_mask &= line_channel_mask + ydatarest_group += ydatarest_line + + for escenergy, escrate, escname in esclines: + ( + line_result, + line_channel_mask, + ydatarest_line, + ) = self._digestLineResult( + escenergy, + escrate * rate, + yfit_group, + group_fitarea, + group_fitarea_std, + statistics_info, + ) + line_result["ratio"] = escrate + escname = "{} {}".format(linename, escname.replace(" ", "_")) + escapepeaks.append(escname) + escname += "esc" + result[groupname][escname] = line_result + + group_channel_mask &= line_channel_mask + ydatarest_group += ydatarest_line + + # Line group statistics + yfitrest_group = statistics_info["yfit"] - yfit_group + ydata_group = statistics_info["ydata"] - yfitrest_group + group_dataarea = ydata_group[group_channel_mask].sum() + result[groupname]["mcaarea"] = group_dataarea + result[groupname]["statistics"] = ydatarest_group + max( + group_dataarea, group_fitarea + ) - if full: - escapepeaks = result[groupname]["escapepeaks"] = list() - if not escgroup: - escgroup = [[]] * len(group) - for (energy, rate, linename), esclines in zip(group, escgroup): - fwhm = self._peakFWHM(energy) - peaks.append(linename) - result[groupname][linename] = { - "ratio": rate, - "energy": energy, - "fwhm": fwhm, - } - for escenergy, escrate, escname in esclines: - escname = "{} {}".format(linename, escname.replace(" ", "_")) - fwhm = self._peakFWHM(escenergy) - escapepeaks.append(escname) - escname += "esc" - result[groupname][escname] = { - "ratio": escrate, - "energy": escenergy, - "fwhm": fwhm, - } - else: - peaks.extend(linename for (_, _, linename) in group) + return result + + def _digestLineResult( + self, + energy, + rate, + yfit_group, + group_fitarea, + group_fitarea_std, + statistics_info, + ): + fwhm = self._peakFWHM(energy) + denergy = 3 * fwhm / self.GAUSS_SIGMA_TO_FWHM + line_channel_mask = (statistics_info["xenergy"] >= (energy - denergy)) & ( + statistics_info["xenergy"] <= (energy + denergy) + ) + + ndof = line_channel_mask.sum() - statistics_info["nfree_parameters"] + chisq = statistics_info["prechisq"][line_channel_mask].sum() / ndof + + total_ydata_lineroi = statistics_info["ydata"][line_channel_mask].sum() + total_ygroup_lineroi = yfit_group[line_channel_mask].sum() + ydatarest_line = rate * abs(total_ydata_lineroi - total_ygroup_lineroi) + + line_result = { + "energy": energy, + "fwhm": fwhm, + "fitarea": group_fitarea * rate, + "sigmaarea": group_fitarea_std * rate, + "statistics": total_ydata_lineroi, + "chisq": chisq, + } + + return line_result, line_channel_mask, ydatarest_line @staticmethod def saveDigestedResult(result: dict, filename: str, info=None): diff --git a/PyMca5/tests/XrfTest.py b/PyMca5/tests/XrfTest.py index 1555f91b3..5636aa4c3 100644 --- a/PyMca5/tests/XrfTest.py +++ b/PyMca5/tests/XrfTest.py @@ -711,12 +711,12 @@ def _compareLegacyMcaTheory(self, x, y, configuration, title): for linename in group1["peaks"]: line1 = group1[linename] line2 = group2[linename] - self.assertEqual(line1.keys(), line1.keys()) + self.assertEqual(line1.keys(), line2.keys()) self.assertEqual(group1["escapepeaks"], group2["escapepeaks"]) for linename in group1["escapepeaks"]: line1 = group1[linename+"esc"] line2 = group2[linename+"esc"] - self.assertEqual(line1.keys(), line1.keys()) + self.assertEqual(line1.keys(), line2.keys()) #self._vis_compare(digestedResult1["yfit"], digestedResult2["yfit"], "yfit") numpy.testing.assert_allclose(digestedResult1["xdata"], digestedResult2["xdata"])