Smooth simulation (#62)

* add possibility to plot simulation with more points than measurements

* add ScatterPlot to Isensee example

* add replicates test

* skip one old test

* Apply suggestions from code review

Co-authored-by: ErikaDudkin <49193148+erikadudki@users.noreply.github.com>

* returns in docstrings

Co-authored-by: ErikaDudkin <49193148+erikadudki@users.noreply.github.com>

Author: plakrisenko

doc/example/example_Isensee/Isensee_visualizationSpecification.tsv

@@ -30,6 +30,6 @@ plot4	JI09_151102_Drg421_418_Age	LinePlot	MeanAndSD	JI09_151102_Drg421_418_Age__
 plot5	JI09_160126_Drg449_444_CycNuc	BarPlot	MeanAndSD	JI09_160126_Drg449_444_CycNuc__ctrl	condition	0	Condition	pRII_Western	0	pRII_Western	Ctrl	lin	lin
 plot5	JI09_160126_Drg449_444_CycNuc	BarPlot	MeanAndSD	JI09_160126_Drg449_444_CycNuc__Fsk10_and_IBMX100	condition	0	Condition	pRII_Western	0	pRII_Western	IBMX(100)/Fsk(10)	lin	lin
 plot5	JI09_160126_Drg449_444_CycNuc	BarPlot	MeanAndSD	JI09_160126_Drg449_444_CycNuc__Sp8_Br_cAMPS_AM10	condition	0	Condition	pRII_Western	0	pRII_Western	Sp8-Br-cAMPS-AM(10)	lin	lin
-plot6	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__ctrl	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Ctrl	lin	lin
-plot6	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Fsk	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Fsk(10)	lin	lin
-plot6	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Sp8_Br_cAMPS_AM	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Sp8-Br-cAMPS-AM(10)	lin	lin
+plot6	JI09_160201_Drg453-452_CycNuc	ScatterPlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__ctrl	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Ctrl	lin	lin
+plot6	JI09_160201_Drg453-452_CycNuc	ScatterPlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Fsk	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Fsk(10)	lin	lin
+plot6	JI09_160201_Drg453-452_CycNuc	ScatterPlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Sp8_Br_cAMPS_AM	time	0	Time [min]	Calpha_Microscopy	0	Calpha_Microscopy	Sp8-Br-cAMPS-AM(10)	lin	lin

doc/example/example_Isensee/Isensee_visualizationSpecification_replicates.tsv

@@ -0,0 +1,15 @@
+plotId	plotName	plotTypeSimulation	plotTypeData	datasetId	xValues	xOffset	yValues	yOffset	yLabel	legendEntry	xScale	yScale
+plot1	JI09_150302_Drg345_343_CycNuc	LinePlot	MeanAndSD	JI09_150302_Drg345_343_CycNuc__4_ABnOH_and_ctrl	fourABnOH_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	4-ABnOH/Ctrl	lin	lin
+plot1	JI09_150302_Drg345_343_CycNuc	LinePlot	MeanAndSD	JI09_150302_Drg345_343_CycNuc__4_ABnOH_and_Fsk	fourABnOH_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	4-ABnOH/Fsk(10)	lin	lin
+plot1	JI09_150302_Drg345_343_CycNuc	LinePlot	MeanAndSD	JI09_150302_Drg345_343_CycNuc__Rp_cAMPS_pAB_and_ctrl	Rp_cAMPS_pAB_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	Rp-cAMPS-pAB/Ctrl	lin	lin
+plot1	JI09_150302_Drg345_343_CycNuc	LinePlot	MeanAndSD	JI09_150302_Drg345_343_CycNuc__Rp_cAMPS_pAB_and_ctrl	Rp_cAMPS_pAB_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	Rp-cAMPS-pAB/Fsk(10)	lin	lin
+plot2	JI09_150302_Drg345_343_CycNuc	LinePlot	replicate	JI09_150302_Drg345_343_CycNuc__4_ABnOH_and_ctrl	fourABnOH_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	4-ABnOH/Ctrl	lin	lin
+plot2	JI09_150302_Drg345_343_CycNuc	LinePlot	replicate	JI09_150302_Drg345_343_CycNuc__4_ABnOH_and_Fsk	fourABnOH_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	4-ABnOH/Fsk(10)	lin	lin
+plot2	JI09_150302_Drg345_343_CycNuc	LinePlot	replicate	JI09_150302_Drg345_343_CycNuc__Rp_cAMPS_pAB_and_ctrl	Rp_cAMPS_pAB_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	Rp-cAMPS-pAB/Ctrl	lin	lin
+plot2	JI09_150302_Drg345_343_CycNuc	LinePlot	replicate	JI09_150302_Drg345_343_CycNuc__Rp_cAMPS_pAB_and_ctrl	Rp_cAMPS_pAB_incubation_time	0	pRII_Microscopy	0	pRII_Microscopy	Rp-cAMPS-pAB/Fsk(10)	lin	lin
+plot3	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__ctrl	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Ctrl	lin	lin
+plot3	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Fsk	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Fsk(10)	lin	lin
+plot3	JI09_160201_Drg453-452_CycNuc	LinePlot	MeanAndSD	JI09_160201_Drg453-452_CycNuc__Sp8_Br_cAMPS_AM	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Sp8-Br-cAMPS-AM(10)	lin	lin
+plot4	JI09_160201_Drg453-452_CycNuc	LinePlot	replicate	JI09_160201_Drg453-452_CycNuc__ctrl	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Ctrl	lin	lin
+plot4	JI09_160201_Drg453-452_CycNuc	LinePlot	replicate	JI09_160201_Drg453-452_CycNuc__Fsk	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Fsk(10)	lin	lin
+plot4	JI09_160201_Drg453-452_CycNuc	LinePlot	replicate	JI09_160201_Drg453-452_CycNuc__Sp8_Br_cAMPS_AM	time	0	Calpha_Microscopy	0	Calpha_Microscopy	Sp8-Br-cAMPS-AM(10)	lin	lin

petab/visualize/plotter.py

@@ -52,9 +52,10 @@ def generate_lineplot(self, ax, dataplot: DataPlot, plotTypeData):
         plotTypeData
         """
 
-        simu_colors = None
-        data_to_plot = self.data_provider.get_data_to_plot(
-            dataplot, plotTypeData == PROVIDED)
+        simu_color = None
+        measurements_to_plot, simulations_to_plot = \
+            self.data_provider.get_data_to_plot(dataplot,
+                                                plotTypeData == PROVIDED)
         noise_col = None
         # set type of noise
         if plotTypeData == MEAN_AND_SD:
@@ -66,66 +67,65 @@ def generate_lineplot(self, ax, dataplot: DataPlot, plotTypeData):
 
         label_base = dataplot.legendEntry
 
-        if data_to_plot.measurements_to_plot is not None:
+        if measurements_to_plot is not None:
             # plotting all measurement data
 
             if plotTypeData == REPLICATE:
                 replicates = np.stack(
-                    data_to_plot.measurements_to_plot.repl.values)
+                    measurements_to_plot.data_to_plot.repl.values)
 
                 # plot first replicate
                 p = ax.plot(
-                    data_to_plot.conditions.values,
+                    measurements_to_plot.conditions,
                     replicates[:, 0],
                     linestyle='-.',
                     marker='x', markersize=10, label=label_base
                 )
 
                 # plot other replicates with the same color
                 ax.plot(
-                    data_to_plot.conditions.values,
+                    measurements_to_plot.conditions,
                     replicates[:, 1:],
                     linestyle='-.',
                     marker='x', markersize=10, color=p[0].get_color()
                 )
 
             # construct errorbar-plots: noise specified above
             else:
-                # sort index for the case that indices of conditions and
-                # measurements differ if indep_var='time', conditions is a
-                # numpy array, for indep_var=observable its a Series
-                if isinstance(data_to_plot.conditions, np.ndarray):
-                    data_to_plot.conditions.sort()
-                elif isinstance(data_to_plot.conditions,
-                                pd.core.series.Series):
-                    data_to_plot.conditions.sort_index(inplace=True)
-                else:
-                    raise ValueError('Strange: conditions object is '
-                                     'neither numpy nor series...')
-                data_to_plot.measurements_to_plot.sort_index(inplace=True)
                 # sorts according to ascending order of conditions
                 scond, smean, snoise = \
                     zip(*sorted(zip(
-                        data_to_plot.conditions,
-                        data_to_plot.measurements_to_plot['mean'],
-                        data_to_plot.measurements_to_plot[noise_col])))
+                        measurements_to_plot.conditions,
+                        measurements_to_plot.data_to_plot['mean'],
+                        measurements_to_plot.data_to_plot[noise_col])))
                 p = ax.errorbar(
                     scond, smean, snoise,
                     linestyle='-.', marker='.', label=label_base
                 )
 
             # simulations should have the same colors if both measurements
             # and simulations are plotted
-            simu_colors = p[0].get_color()
+            simu_color = p[0].get_color()
 
         # construct simulation plot
-        if data_to_plot.simulations_to_plot is not None:
-
-            xs, ys = zip(*sorted(zip(data_to_plot.conditions,
-                                     data_to_plot.simulations_to_plot)))
+        if simulations_to_plot is not None:
+            # markers will be displayed only for points that have measurement
+            # counterpart
+            if measurements_to_plot is not None:
+                meas_conditions = measurements_to_plot.conditions.to_numpy() \
+                    if isinstance(measurements_to_plot.conditions, pd.Series) \
+                    else measurements_to_plot.conditions
+                every = [condition in meas_conditions
+                         for condition in simulations_to_plot.conditions]
+            else:
+                every = None
+            # sorts according to ascending order of conditions
+            xs, ys = zip(*sorted(zip(simulations_to_plot.conditions,
+                                     simulations_to_plot.data_to_plot['mean'])
+                                 ))
             ax.plot(
-                xs, ys, linestyle='-', marker='o',
-                label=label_base + " simulation", color=simu_colors
+                xs, ys, linestyle='-', marker='o', markevery=every,
+                label=label_base + " simulation", color=simu_color
             )
 
     def generate_barplot(self, ax, dataplot: DataPlot, plotTypeData: str):
@@ -149,12 +149,13 @@ def generate_barplot(self, ax, dataplot: DataPlot, plotTypeData: str):
             noise_col = 'noise_model'
 
         simu_colors = None
-        data_to_plot = self.data_provider.get_data_to_plot(
-            dataplot, plotTypeData == PROVIDED)
+        measurements_to_plot, simulations_to_plot = \
+            self.data_provider.get_data_to_plot(dataplot,
+                                                plotTypeData == PROVIDED)
 
         x_name = dataplot.legendEntry
 
-        if data_to_plot.simulations_to_plot:
+        if simulations_to_plot:
             bar_kwargs = {
                 'align': 'edge',
                 'width': -1/3,
@@ -167,16 +168,17 @@ def generate_barplot(self, ax, dataplot: DataPlot, plotTypeData: str):
 
         color = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
 
-        if data_to_plot.measurements_to_plot is not None:
-            p = ax.bar(x_name, data_to_plot.measurements_to_plot['mean'],
-                       yerr=data_to_plot.measurements_to_plot[noise_col],
+        if measurements_to_plot is not None:
+            p = ax.bar(x_name, measurements_to_plot.data_to_plot['mean'],
+                       yerr=measurements_to_plot.data_to_plot[noise_col],
                        color=color, **bar_kwargs, label='measurement')
             simu_colors = p[0].get_facecolor()
 
-        if data_to_plot.simulations_to_plot is not None:
+        if simulations_to_plot is not None:
             bar_kwargs['width'] = -bar_kwargs['width']
-            ax.bar(x_name, data_to_plot.simulations_to_plot, color='white',
-                   edgecolor=simu_colors, **bar_kwargs, label='simulation')
+            ax.bar(x_name, simulations_to_plot.data_to_plot['mean'],
+                   color='white', edgecolor=simu_colors, **bar_kwargs,
+                   label='simulation')
 
     def generate_scatterplot(self, ax, dataplot: DataPlot, plotTypeData: str):
         """
@@ -189,14 +191,15 @@ def generate_scatterplot(self, ax, dataplot: DataPlot, plotTypeData: str):
         plotTypeData
         """
 
-        data_to_plot = self.data_provider.get_data_to_plot(
-            dataplot, plotTypeData == PROVIDED)
+        measurements_to_plot, simulations_to_plot = \
+            self.data_provider.get_data_to_plot(dataplot,
+                                                plotTypeData == PROVIDED)
 
-        if data_to_plot.simulations_to_plot is None:
+        if simulations_to_plot is None:
             raise NotImplementedError('Scatter plots do not work without'
                                       ' simulation data')
-        ax.scatter(data_to_plot.measurements_to_plot['mean'],
-                   data_to_plot.simulations_to_plot,
+        ax.scatter(measurements_to_plot.data_to_plot['mean'],
+                   simulations_to_plot.data_to_plot['mean'],
                    label=getattr(dataplot, LEGEND_ENTRY))
         ax = self._square_plot_equal_ranges(ax)