apply log filtering to each instrument

Author: j-atkins

src/virtualship/instruments/__init__.py

@@ -1,5 +1,13 @@
 """Measurement instrument that can be used with Parcels."""
 
-from . import adcp, argo_float, ctd, drifter, ship_underwater_st, xbt
+from . import adcp, argo_float, ctd, ctd_bgc, drifter, ship_underwater_st, xbt
 
-__all__ = ["adcp", "argo_float", "ctd", "drifter", "ship_underwater_st", "xbt"]
+__all__ = [
+    "adcp",
+    "argo_float",
+    "ctd",
+    "ctd_bgc",
+    "drifter",
+    "ship_underwater_st",
+    "xbt",
+]

src/virtualship/instruments/adcp.py

@@ -1,12 +1,11 @@
 """ADCP instrument."""
 
-import logging
 from pathlib import Path
 
 import numpy as np
 from parcels import FieldSet, ParticleSet, ScipyParticle, Variable
 
-from ..log_filter import DuplicateFilter
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 # we specifically use ScipyParticle because we have many small calls to execute
@@ -32,7 +31,6 @@ def simulate_adcp(
     min_depth: float,
     num_bins: int,
     sample_points: list[Spacetime],
-    log_filter: bool = True,
 ) -> None:
     """
     Use Parcels to simulate an ADCP in a fieldset.
@@ -43,7 +41,6 @@ def simulate_adcp(
     :param min_depth: Minimum depth the ADCP can measure.
     :param num_bins: How many samples to take in the complete range between max_depth and min_depth.
     :param sample_points: The places and times to sample at.
-    :param log_filter: Whether to filter duplicate log messages (defaults to True). This is a bit of a hack, but it works and could be removed if changed in Parcels.
     """
     sample_points.sort(key=lambda p: p.time)
 
@@ -64,31 +61,29 @@ def simulate_adcp(
     # outputdt set to infinite as we just want to write at the end of every call to 'execute'
     out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf)
 
-    #  whether to filter parcels duplicate log messages
-    if log_filter:
-        external_logger = logging.getLogger("parcels.tools.loggers")
-        for handler in external_logger.handlers:
-            handler.addFilter(DuplicateFilter())
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
 
-    for point in sample_points:
-        particleset.lon_nextloop[:] = point.location.lon
-        particleset.lat_nextloop[:] = point.location.lat
-        particleset.time_nextloop[:] = fieldset.time_origin.reltime(
-            np.datetime64(point.time)
-        )
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        for point in sample_points:
+            particleset.lon_nextloop[:] = point.location.lon
+            particleset.lat_nextloop[:] = point.location.lat
+            particleset.time_nextloop[:] = fieldset.time_origin.reltime(
+                np.datetime64(point.time)
+            )
 
-        # perform one step using the particleset
-        # dt and runtime are set so exactly one step is made.
-        particleset.execute(
-            [_sample_velocity],
-            dt=1,
-            runtime=1,
-            verbose_progress=False,
-            output_file=out_file,
-        )
+            # perform one step using the particleset
+            # dt and runtime are set so exactly one step is made.
+            particleset.execute(
+                [_sample_velocity],
+                dt=1,
+                runtime=1,
+                verbose_progress=False,
+                output_file=out_file,
+            )
 
-    # turn off log filter after .execute(), to prevent being applied universally to all loggers
-    # separate if statement from above to prevent error if log_filter is False
-    if log_filter:
+    finally:
         for handler in external_logger.handlers:
             handler.removeFilter(handler.filters[0])

src/virtualship/instruments/argo_float.py

@@ -15,6 +15,7 @@
     Variable,
 )
 
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 
@@ -171,16 +172,26 @@ def simulate_argo_floats(
     else:
         actual_endtime = np.timedelta64(endtime)
 
-    # execute simulation
-    argo_float_particleset.execute(
-        [
-            _argo_float_vertical_movement,
-            AdvectionRK4,
-            _keep_at_surface,
-            _check_error,
-        ],
-        endtime=actual_endtime,
-        dt=DT,
-        output_file=out_file,
-        verbose_progress=True,
-    )
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        # execute simulation
+        argo_float_particleset.execute(
+            [
+                _argo_float_vertical_movement,
+                AdvectionRK4,
+                _keep_at_surface,
+                _check_error,
+            ],
+            endtime=actual_endtime,
+            dt=DT,
+            output_file=out_file,
+            verbose_progress=False,
+        )
+
+    finally:
+        for handler in external_logger.handlers:
+            handler.removeFilter(handler.filters[0])

src/virtualship/instruments/ctd.py

@@ -7,6 +7,7 @@
 import numpy as np
 from parcels import FieldSet, JITParticle, ParticleSet, Variable
 
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 
@@ -121,14 +122,23 @@ def simulate_ctd(
     # define output file for the simulation
     out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt)
 
-    # execute simulation
-    ctd_particleset.execute(
-        [_sample_salinity, _sample_temperature, _ctd_cast],
-        endtime=fieldset_endtime,
-        dt=DT,
-        verbose_progress=False,
-        output_file=out_file,
-    )
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        ctd_particleset.execute(
+            [_sample_salinity, _sample_temperature, _ctd_cast],
+            endtime=fieldset_endtime,
+            dt=DT,
+            verbose_progress=False,
+            output_file=out_file,
+        )
+
+    finally:
+        for handler in external_logger.handlers:
+            handler.removeFilter(handler.filters[0])
 
     # there should be no particles left, as they delete themselves when they resurface
     if len(ctd_particleset.particledata) != 0:

src/virtualship/instruments/ctd_bgc.py

@@ -7,6 +7,7 @@
 import numpy as np
 from parcels import FieldSet, JITParticle, ParticleSet, Variable
 
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 
@@ -127,14 +128,23 @@ def simulate_ctd_bgc(
     # define output file for the simulation
     out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt)
 
-    # execute simulation
-    ctd_bgc_particleset.execute(
-        [_sample_o2, _sample_chlorophyll, _ctd_bgc_cast],
-        endtime=fieldset_endtime,
-        dt=DT,
-        verbose_progress=False,
-        output_file=out_file,
-    )
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        ctd_bgc_particleset.execute(
+            [_sample_o2, _sample_chlorophyll, _ctd_bgc_cast],
+            endtime=fieldset_endtime,
+            dt=DT,
+            verbose_progress=False,
+            output_file=out_file,
+        )
+
+    finally:
+        for handler in external_logger.handlers:
+            handler.removeFilter(handler.filters[0])
 
     # there should be no particles left, as they delete themselves when they resurface
     if len(ctd_bgc_particleset.particledata) != 0:

src/virtualship/instruments/drifter.py

@@ -7,6 +7,7 @@
 import numpy as np
 from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable
 
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 
@@ -95,14 +96,24 @@ def simulate_drifters(
     else:
         actual_endtime = np.timedelta64(endtime)
 
-    # execute simulation
-    drifter_particleset.execute(
-        [AdvectionRK4, _sample_temperature, _check_lifetime],
-        endtime=actual_endtime,
-        dt=dt,
-        output_file=out_file,
-        verbose_progress=True,
-    )
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        # execute simulation
+        drifter_particleset.execute(
+            [AdvectionRK4, _sample_temperature, _check_lifetime],
+            endtime=actual_endtime,
+            dt=dt,
+            output_file=out_file,
+            verbose_progress=False,
+        )
+
+    finally:
+        for handler in external_logger.handlers:
+            handler.removeFilter(handler.filters[0])
 
     # if there are more particles left than the number of drifters with an indefinite endtime, warn the user
     if len(drifter_particleset.particledata) > len(

src/virtualship/instruments/ship_underwater_st.py

@@ -1,12 +1,11 @@
 """Ship salinity and temperature."""
 
-import logging
 from pathlib import Path
 
 import numpy as np
 from parcels import FieldSet, ParticleSet, ScipyParticle, Variable
 
-from ..log_filter import DuplicateFilter
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 # we specifically use ScipyParticle because we have many small calls to execute
@@ -34,7 +33,6 @@ def simulate_ship_underwater_st(
     out_path: str | Path,
     depth: float,
     sample_points: list[Spacetime],
-    log_filter: bool = True,
 ) -> None:
     """
     Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset.
@@ -43,7 +41,6 @@ def simulate_ship_underwater_st(
     :param out_path: The path to write the results to.
     :param depth: The depth at which to measure. 0 is water surface, negative is into the water.
     :param sample_points: The places and times to sample at.
-    :param log_filter: Whether to filter duplicate log messages (defaults to True). This is a bit of a hack, but it works and could be removed if changed in Parcels.
     """
     sample_points.sort(key=lambda p: p.time)
 
@@ -60,33 +57,31 @@ def simulate_ship_underwater_st(
     # outputdt set to infinie as we want to just want to write at the end of every call to 'execute'
     out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf)
 
-    #  whether to filter parcels duplicate log messages
-    if log_filter:
-        external_logger = logging.getLogger("parcels.tools.loggers")
-        for handler in external_logger.handlers:
-            handler.addFilter(DuplicateFilter())
-
-    # iterate over each point, manually set lat lon time, then
-    # execute the particle set for one step, performing one set of measurement
-    for point in sample_points:
-        particleset.lon_nextloop[:] = point.location.lon
-        particleset.lat_nextloop[:] = point.location.lat
-        particleset.time_nextloop[:] = fieldset.time_origin.reltime(
-            np.datetime64(point.time)
-        )
-
-        # perform one step using the particleset
-        # dt and runtime are set so exactly one step is made.
-        particleset.execute(
-            [_sample_salinity, _sample_temperature],
-            dt=1,
-            runtime=1,
-            verbose_progress=False,
-            output_file=out_file,
-        )
-
-    # turn off log filter after .execute(), to prevent being applied universally to all loggers
-    # separate if statement from above to prevent error if log_filter is False
-    if log_filter:
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        # iterate over each point, manually set lat lon time, then
+        # execute the particle set for one step, performing one set of measurement
+        for point in sample_points:
+            particleset.lon_nextloop[:] = point.location.lon
+            particleset.lat_nextloop[:] = point.location.lat
+            particleset.time_nextloop[:] = fieldset.time_origin.reltime(
+                np.datetime64(point.time)
+            )
+
+            # perform one step using the particleset
+            # dt and runtime are set so exactly one step is made.
+            particleset.execute(
+                [_sample_salinity, _sample_temperature],
+                dt=1,
+                runtime=1,
+                verbose_progress=False,
+                output_file=out_file,
+            )
+
+    finally:
         for handler in external_logger.handlers:
             handler.removeFilter(handler.filters[0])

src/virtualship/instruments/xbt.py

@@ -7,6 +7,7 @@
 import numpy as np
 from parcels import FieldSet, JITParticle, ParticleSet, Variable
 
+from ..log_filter import Filter, external_logger
 from ..spacetime import Spacetime
 
 
@@ -125,14 +126,24 @@ def simulate_xbt(
     # define output file for the simulation
     out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt)
 
-    # execute simulation
-    xbt_particleset.execute(
-        [_sample_temperature, _xbt_cast],
-        endtime=fieldset_endtime,
-        dt=DT,
-        verbose_progress=False,
-        output_file=out_file,
-    )
+    # filter out Parcels logging messages
+    for handler in external_logger.handlers:
+        handler.addFilter(Filter())
+
+    # try/finally to ensure filter is always removed even if .execute fails (to avoid filter being appled universally)
+    try:
+        # execute simulation
+        xbt_particleset.execute(
+            [_sample_temperature, _xbt_cast],
+            endtime=fieldset_endtime,
+            dt=DT,
+            verbose_progress=False,
+            output_file=out_file,
+        )
+
+    finally:
+        for handler in external_logger.handlers:
+            handler.removeFilter(handler.filters[0])
 
     # there should be no particles left, as they delete themselves when they finish profiling
     if len(xbt_particleset.particledata) != 0: