Use regex substitution to interpolate data from regexes into name/model/version values

Author: thinkwelltwd

README.md

@@ -182,6 +182,8 @@ cp $upstream/regexes/*.yml $pdd/device_detector/regexes/upstream/
 cp $upstream/Tests/fixtures/* $pdd/device_detector/tests/fixtures/upstream/
 cp $upstream/Tests/Parser/Client/fixtures/* $pdd/device_detector/tests/parser/fixtures/upstream/client/
 cp $upstream/Tests/Parser/Device/fixtures/* $pdd/device_detector/tests/parser/fixtures/upstream/device/
+
+./configure_regex_interpolators.sh
 ```
 
 After copying the fixtures, review the diffs and restore changes that were made, especially for app types, so tests pass.

configure_regex_interpolators.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Replace $1 / $2 regex interpolation placeholders
+# with \g<1>, \g<2> for native regex substitution
+for i in 1 2 3 4 5;
+  do
+    grep -rl --include=*.yml "\$$i" device_detector/regexes | xargs sed -i "s#\$$i#\\\g<$i>#g" 2> /dev/null
+done
+
+MALFORMED=$(grep -rl --include=*.yml "eZee'Tab\\\g<1>" device_detector/regexes)
+
+if [ $MALFORMED ]; then
+   echo "Invalid yaml value found in '${MALFORMED}'"
+   echo "Manually convert 'eZee'Tab\g<1>' to 'eZee'Tab\\\g<1>"
+fi

device_detector/parser/device/base.py

@@ -1,5 +1,4 @@
-from ..parser import Parser
-from device_detector.parser.extractors import ModelExtractor
+from ..parser import Parser, perform_substitutions
 from device_detector.enums import DeviceType
 
 MOBILE_DEVICE_TYPES = {
@@ -2140,24 +2139,16 @@ def extract_model(self) -> None:
         """
         user_agent = self.user_agent
         for model in self.ua_data.pop('models', []):
-            if not (matched := model['regex'].search(user_agent)):
-                continue
+            if model_matched := model['regex'].search(user_agent):
+                self.ua_data |= {k: v.strip() for k, v in model.items() if k != 'regex'}
+                self.ua_data['model'] = perform_model_substitutions(
+                    model['model'], model_matched, ' '
+                )
+                return
 
-            self.matched_regex = matched
-            self.ua_data |= {
-                k: v.strip().replace('_', ' ') for k, v in model.items() if k != 'regex'
-            }
-
-            # Must return after first match! Later patterns could match
-            # again and clobber the earlier, correct, values.
-            # i.e. Sony Ericsson should override Sony
-            break
-
-        if 'model' in self.ua_data and self.matched_regex:
-            if groups := self.matched_regex.groups():
-                self.ua_data['model'] = ModelExtractor(self.ua_data, groups).extract()
-
-        return None
+        if name := self.ua_data.get('model', ''):
+            if "\\g<" in name:
+                self.ua_data['model'] = perform_model_substitutions(name, self.matched_regex, ' ')
 
     def dtype(self) -> DeviceType | str:
         return self.DEVICE_TYPE
@@ -2173,7 +2164,7 @@ def set_details(self) -> None:
         """
         Set device data from UA or Client Hints.
         """
-        if self.matched_regex:
+        if self.ua_data:
             self.extract_model()
 
             self.ua_data |= {
@@ -2184,6 +2175,25 @@ def set_details(self) -> None:
             return super().set_details()
 
 
+def perform_model_substitutions(substring: str, regex_match, underscore_substitute: str) -> str:
+    """
+    Perform several normalizations after default regex substitution
+    """
+    value = perform_substitutions(substring, regex_match, underscore_substitute)
+    if not value or value == 'Build':
+        return ''
+
+    # normalize D510_TD / ETON-T730D_TD
+    # Tbook 16 Power(M5F8) Build
+    for suffix in (' TD', ' Build'):
+        value = value.removesuffix(suffix)
+
+    if value.endswith('))'):
+        return value[:-1]
+
+    return value
+
+
 __all__ = (
     'BaseDeviceParser',
     'DEVICE_BRANDS',

device_detector/parser/extractors.py

@@ -21,95 +21,6 @@
 )
 
 
-class DataExtractor:
-    """
-    Regex will define a string value or 1-based index
-    position of the desired metadata
-
-    - regex: '(?:Apple-)?(?:iPhone|iPad|iPod)(?:.*Mac OS X.*Version/(\\d+\\.\\d+)|; Opera)?'
-      name: 'iOS'
-      version: '$1'
-    """
-
-    __slots__ = (
-        'metadata',
-        'groups',
-        'user_agent',
-        'details',
-        '_app_id_pretty_names',
-    )
-
-    # metadata value to extract / return
-    # subclasses must override
-    key = ''
-
-    def __init__(self, metadata: dict, groups: tuple):
-        """
-        :param metadata: dict of regex and associated metadata
-            {'regex': <regex1>, 'name': 'iOS', 'version': '$1'}
-            {'regex': <regex2>, 'name': 'Windows', 'version': '10'}
-        :param groups: Tuple of groups from regex
-            ('Debian', None)
-            ('iOS', '8_2')
-        """
-        self.metadata = metadata
-        self.groups = groups
-
-    def get_value_from_regex(self, value: str) -> str:
-        """
-        Model / Name values may be in format of
-        $<int> or <NamePrefix> $<int>
-
-        'Xino Z$1 X$2
-
-        Replace %<int> section replaced with {} for format string
-
-        'Xino Z$1 X$2 -> 'Xino Z{} X{}'
-
-        Return interpolated string with value from regex group
-        """
-        chars = []
-        indices = []
-        index_int_next = False
-
-        for char in value:
-            if char != '$':
-                if index_int_next:
-                    indices.append(int(char) - 1)
-                    index_int_next = False
-                else:
-                    chars.append(char)
-            else:
-                chars.append('{}')
-                index_int_next = True
-
-        # collect regex group values, substituting empty string for None
-        group_values = []
-        for pos in indices:
-            try:
-                if not self.groups[pos]:
-                    group_values.append('')
-                else:
-                    group_values.append(self.groups[pos])
-            except IndexError:
-                return ''
-
-        fmt_string = ''.join(chars)
-        return fmt_string.format(*group_values).strip()
-
-    def extract(self) -> str:
-        value = str(self.metadata.get(self.key, ''))
-        if value and '$' in value:
-            return self.get_value_from_regex(value)
-        return value
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__} Extractor'
-
-    def __repr__(self) -> str:
-        return f'{self.__class__.__name__}({self.metadata}, {self.groups})'
-
-
 class ApplicationIDExtractor(RegexLoader):
     """
     Extract App Store IDs such as:
@@ -195,42 +106,6 @@ def __repr__(self) -> str:
         return f'{self.__class__.__name__}({self.user_agent!r})'
 
 
-class NameExtractor(DataExtractor):
-    key = 'name'
-
-
-class ModelExtractor(DataExtractor):
-    key = 'model'
-
-    def extract(self) -> str:
-        value = super().extract()
-        if not value:
-            return value
-
-        if value == 'Build':
-            return ''
-
-        # normalize D510_TD / ETON-T730D_TD
-        if value.endswith('_TD'):
-            value = value[:-3]
-        return value.replace('_', ' ').strip()
-
-
-class VersionExtractor(DataExtractor):
-    key = 'version'
-
-    def extract(self) -> str:
-        value = super().extract()
-        if not value:
-            return value
-
-        return value.replace('_', '.').strip('.')
-
-
-__all__ = (
+__all__ = [
     'ApplicationIDExtractor',
-    'DataExtractor',
-    'ModelExtractor',
-    'NameExtractor',
-    'VersionExtractor',
-)
+]

device_detector/parser/parser.py

@@ -3,12 +3,9 @@
 except ImportError:
     from typing_extensions import Self
 
+from regex._regex_core import error as RegexError
 from ..lazy_regex import RegexLazyIgnore
 from .client_hints import ClientHints
-from .extractors import (
-    NameExtractor,
-    VersionExtractor,
-)
 from ..yaml_loader import RegexLoader, app_pretty_names_types_data
 
 # Match regexes that ONLY values like:
@@ -137,30 +134,40 @@ def extract_version(self) -> None:
         Extract the version if UA Yaml files specify version regexes.
         See oss.yml for example file structure.
         """
-
+        user_agent = self.user_agent
         for version in self.ua_data.pop('versions', []):
-            if version['regex'].search(self.user_agent):
-                self.ua_data['version'] = version['version']
+            if version_regex_match := version['regex'].search(user_agent):
+                self.ua_data['version'] = perform_substitutions(
+                    version['version'], version_regex_match, '.'
+                )
                 return
 
+        self._set_data_from_field('version', '.')
+
     def set_details(self) -> None:
         """
         Override this method on subclasses.
 
         Update fields with interpolated values from regex data
         """
-        groups = self.matched_regex and self.matched_regex.groups() or None
-        if groups:
-            if 'name' in self.ua_data:
-                self.ua_data['name'] = NameExtractor(self.ua_data, groups).extract()
-
-            if 'version' in self.ua_data:
-                self.ua_data['version'] = VersionExtractor(self.ua_data, groups).extract()
+        if self.matched_regex:
+            self._set_data_from_field('name', '.')
 
         # no version should be considered valid if the name can't be parsed
         if not self.ua_data.get('name') and self.ua_data.get('version'):
             self.ua_data['version'] = ''
 
+    def _set_data_from_field(self, field: str, separator: str):
+        """
+        Check specified field value to see if it has a regex separator,
+        and if so, update the value to include the regex capture details.
+        """
+        if substring := self.ua_data.get(field, ''):
+            if "\\g<" in substring:
+                self.ua_data[field] = perform_substitutions(
+                    substring, self.matched_regex, separator
+                )
+
     def name(self) -> str:
         return self.ua_data.get('name', '')
 
@@ -198,8 +205,24 @@ def __repr__(self) -> str:
         return f'{klass}({self.user_agent!r}, {self.ua_data!r})'
 
 
+def perform_substitutions(substring: str, regex_match, separator: str) -> str:
+    """
+    Substitute the captured value from the regex for the regex placeholder.
+    """
+    regex_pattern = regex_match.re
+    capture = regex_match.captures()[0]
+    try:
+        value = regex_pattern.sub(substring, capture)
+        if value.endswith(('\\g<1>', '\\g<2>')):
+            value = value[: value.rfind('\\g<')]
+        return value.replace('_', separator).strip(' .')
+    except RegexError:
+        return substring
+
+
 __all__ = (
     'Parser',
+    'perform_substitutions',
     'IPHONE_ONLY_UA',
     'ENDSWITH_DARWIN',
 )

device_detector/regexes/local/client/antivirus.yml

@@ -14,7 +14,7 @@
 
 - regex: ^cis_([\d\.]+)_
   name: Comodo Internet Security
-  version: '$1'
+  version: '\g<1>'
 
 - regex: AV/,,ffl$
   name: Avast
@@ -26,11 +26,11 @@
 
 - regex: AV/(\d+[\.\d]+)
   name: Avast
-  version: '$1'
+  version: '\g<1>'
 
 - regex: CCleaner, ?(\d+[\.\d]+)
   name: CCleaner
-  version: '$1'
+  version: '\g<1>'
 
 - regex: panda ?security ?nano
   name: Panda Security Nano

device_detector/regexes/local/client/browsers.yml

@@ -8,12 +8,12 @@
 # Avastium Browser - Avast browser based on Chrome
 - regex: 'Avastium Chrome(?:\/(\d+[\.\d]+))'
   name: 'Avast SafeZone'
-  version: '$1'
+  version: '\g<1>'
   engine:
     default: 'WebKit'
 
 - regex: 'avastium[ \/]\(?(\d+[\.\d]+)'
   name: 'Avast SafeZone'
-  version: '$1'
+  version: '\g<1>'
   engine:
     default: 'WebKit'