managing duplicates by better validation. Could use rules or preference order of match pattern families...

Author: mubaldino

src/main/python/opensextant/extractors/xcoord.py

@@ -119,9 +119,10 @@ def normalize(self):
 class DMSOrdinate:
     SYMBOLS = {"°", "º", "'", "\"", ":", "lat", "lon", "geo", "coord", "deg"}
 
-    def __init__(self, axis: str, text: str, slots=None):
+    def __init__(self, axis: str, text: str, fam: str, slots=None):
         self.axis = axis
         self.text = text
+        self.pattern_family = fam
         self.slots = slots
         self.degrees = None
         self.min = None
@@ -134,7 +135,7 @@ def __init__(self, axis: str, text: str, slots=None):
         self.normalize()
 
     def is_valid(self):
-        if not self.degrees:
+        if self.degrees is None:
             return False
         # Must have degrees, in range for the axis
         if self.axis == "lat":
@@ -144,9 +145,9 @@ def is_valid(self):
             if not -180 < self.degrees < 180:
                 return False
         # Min and Secs must be in range if specified
-        if self.min and not 0 < self.min < 60:
+        if self.min is not None and not 0 <= self.min < 60:
             return False
-        if self.seconds and not 0 < self.seconds < 60:
+        if self.seconds is not None and not 0 <= self.seconds < 60:
             return False
 
         return True
@@ -216,6 +217,14 @@ def _digest_slots(self, axis):
         """
         Fields or slots are named xxxLatxx or xxxLonxx
         """
+        if self.pattern_family == "DMS":
+            min_sec_sep = self.slots.get(f"ms{axis}Sep")
+            deg_min_sep = self.slots.get(f"dm{axis}Sep")
+            if min_sec_sep and deg_min_sep and min_sec_sep == "." and min_sec_sep != deg_min_sep:
+                # valid coordinate, but separators like "DD MM.ss" suggest more DM pattern
+                #                      whereas          "DD.MM.SS" with consistent separators is DMS.
+                return
+
         # DEGREES
         deg = self.get_int(f"deg{axis}", "deg")
         deg2 = self.get_int(f"dmsDeg{axis}", "deg")
@@ -310,6 +319,7 @@ def __init__(self, *args, **kwargs):
         self.lat_ordinate = None
         self.lon_ordinate = None
         self.filter = None
+        self.pattern_family = self.pattern_id.split("-", 1)[0]
 
     def __str__(self):
         return f"{self.text}"
@@ -361,10 +371,11 @@ def filter_out(self, mgrs: GeocoordMatch) -> tuple:
         #    - is not a recent date;
         #    - is not a rate ('NNN per LB');
         #    - is not time with 'sec'
+        # Lexical filters:
         if not mgrs.is_valid:
+            # parsed earlier as invalid.
             return True, "invalid"
 
-        # Lexical filters:
         if not (mgrs.text.isupper() and len(mgrs.text.replace(" ", "")) > 6):
             return True, "lexical"
         parts = set(mgrs.text.split())
@@ -405,18 +416,20 @@ def filter_out(self, dms: GeocoordMatch) -> tuple:
         Easy filter -- if puncutation matches, this is an easy pattern to ignore.
         :return: True if filtered out, false positive.
         """
-        if not dms.is_valid:
-            return True, "invalid"
-        if dms.text[0].isalpha():
+        if dms.is_valid:
+            if dms.text[0].isalpha():
+                return False, None
+            for fmt in self.date_formats:
+                try:
+                    dt = arrow.get(dms.text, fmt)
+                    # Recency matters not.  Tests are literal date formats
+                    return True, "date"
+                except Exception as err:
+                    pass
+            # Not filtered. Is valid.
             return False, None
-        for fmt in self.date_formats:
-            try:
-                dt = arrow.get(dms.text, fmt)
-                # Recency matters not.  Tests are literal date formats
-                return True, "date"
-            except Exception as err:
-                pass
-        return False, None
+        # Filter out. invalid.
+        return True, "invalid"
 
 
 mgrs_filter = MGRSFilter()
@@ -524,8 +537,8 @@ def normalize(self):
         #     < hemiLonPre >\s? < degLon > < dmLonSep >\s? < minLon > < fractMinLon >? < msLonSep >?
 
         # TODO: conditions that invalidate this pattern?
-        self.lat_ordinate = DMSOrdinate("lat", self.text, slots=self.attributes())
-        self.lon_ordinate = DMSOrdinate("lon", self.text, slots=self.attributes())
+        self.lat_ordinate = DMSOrdinate("lat", self.text, self.pattern_family, slots=self.attributes())
+        self.lon_ordinate = DMSOrdinate("lon", self.text, self.pattern_family, slots=self.attributes())
         self._make_coordinate()
         self.validate()
 
@@ -537,8 +550,8 @@ def __init__(self, *args, **kwargs):
 
     def normalize(self):
         GeocoordMatch.normalize(self)
-        self.lat_ordinate = DMSOrdinate("lat", self.text, slots=self.attributes())
-        self.lon_ordinate = DMSOrdinate("lon", self.text, slots=self.attributes())
+        self.lat_ordinate = DMSOrdinate("lat", self.text, self.pattern_family, slots=self.attributes())
+        self.lon_ordinate = DMSOrdinate("lon", self.text, self.pattern_family, slots=self.attributes())
         self._make_coordinate()
         self.validate()
 
@@ -571,7 +584,7 @@ def validate(self):
 
     def normalize(self):
         GeocoordMatch.normalize(self)
-        self.lat_ordinate = DMSOrdinate("lat", self.text, slots=self.attributes())
-        self.lon_ordinate = DMSOrdinate("lon", self.text, slots=self.attributes())
+        self.lat_ordinate = DMSOrdinate("lat", self.text, self.pattern_family, slots=self.attributes())
+        self.lon_ordinate = DMSOrdinate("lon", self.text, self.pattern_family, slots=self.attributes())
         self._make_coordinate()
         self.validate()

src/main/python/setup.py

@@ -14,7 +14,7 @@
 
 setup(
     name='opensextant',
-    version='1.6.0',
+    version='1.6.2',
 
     description='OpenSextant APIs and Utilities',
     long_description=long_description,

src/main/python/tests/test_xcoord.py

@@ -1,13 +1,21 @@
-
 from opensextant.extractors.xcoord import XCoord
 
 tester = XCoord(debug=True)
 # focused tests:
 mgrs = "10 JAN 94"
 matches = tester.extract(mgrs)
 for m in matches:
-    print (m, m.filtered_out)
+    print(m, m.filtered_out)
+
+dms = ["'18 51.1S 34 38.8W'",
+       "08 00.4S 30 35.2W",   # DM
+       "08.00.4S 30.35.2W"    # DMS pattern
+       ]
+for text in dms:
+    matches = tester.extract(text)
+    for m in matches:
+        print(f"Text=[{m.text}]", m.pattern_id, "Filtered=", m.filtered_out, "Duplicate", m.is_duplicate)
 
 results = XCoord(debug=True).default_tests()
 for res in results:
-    print(res)
+    print(res)