audit and docs update of logging (#54)

Author: jvanasco

CHANGELOG.txt

@@ -59,6 +59,8 @@
             `InvalidDocument` no longer has a .message attribute
 
             Exceptions now invoke `super().__init__(args)`
+            
+            `tldextract` is now required for installation and used by default.
 
     New Functionality
 

README.rst

@@ -75,32 +75,48 @@ Logging
 
 This file utilizes extensive logging to help developers pinpoint problems.
 
-* ``log.debug``
+* ``log.debug`` (10)
   This log level is mostly used to handle library maintenance and
   troubleshooting, aka "Library Debugging".  Library Debugging is verbose, but
   is nested under ``if __debug__:`` statements, so it is compiled away when
   PYTHONOPTIMIZE is set.
   Several sections of logic useful to developers will also emit logging
   statements at the ``debug`` level, regardless of PYTHONOPTIMIZE.
 
-* ``log.info``
-  Currently unused
+* ``log.info`` (20)
+    This log level is only used during package initialization to notify if
+    the ``tldextract`` package is being utilized or not.
 
-* ``log.warning``
+* ``log.warning`` (30)
   Currently unused
 
-* ``log.error``
-  This log level is mostly used to alert developers of errors that were
+* ``log.error`` (40)
+  This log level will record each URL that a parse is attempted for.
+
+  This log level is mostly used to alert users of errors that were
   encountered during url fetching and document parsing, and often emits a log
   statement just before an Exception is raised. The log statements will contain
   at least the exception type, and may contain the active URL and additional
   debugging information, if any of that information is available.
+  
+  URLs that trigger error logging should be collected and run on a secondary
+  system that utilizes `log.debug` without PYTHONOPTIMIZE.
 
-* ``log.critical``
+
+* ``log.critical`` (50)
   Currently unused
 
 
-It is STRONGLY recommended to keep Python's logging at ``debug``.
+It is STRONGLY recommended to keep Python's logging at ``debug`` and not run
+PYTHONOPTIMIZE if you are new to this package.
+
+For experienced users, running under PYTHONOPTIMIZE to not emit debug logging is
+designed to make the system run as fast as possible.  The intent of
+``log.error`` is to present you with a feed of URLs as they are processed, and
+show any errors that arise.  Any issues that arise should then be run on a
+second system that enables debug logging to pinpoint the error.  This allows one
+to split a deployment into production and R&D/troubleshooting, to maximize
+the throughput of the production system.
 
 
 Optional Integrations

setup.py

@@ -30,6 +30,7 @@
     "BeautifulSoup4>4.13.0,<4.14.0",
     "requests>=2.19.1",
     "requests-toolbelt>=0.8.0",
+    "tldextract<6.0.0",  # `.registered_domain` deprecation
     "typing_extensions",
 ]
 
@@ -38,10 +39,11 @@
 
 tests_require = [
     "httpbin",
+    "flake8",
+    "flake8-import-order>=0.19.2",
     "pytest",
     "pytest-httpbin",
     "responses",
-    "tldextract<6.0.0",  # `.registered_domain` deprecation
     "types-beautifulsoup4",
     "types-requests",
     "werkzeug<2.1.0",  # httpbin compat issue

src/metadata_parser/__init__.py

@@ -52,6 +52,7 @@
 
 if TYPE_CHECKING:
     from bs4 import Tag as _bs4_Tag
+
     from .typing import TYPE_ENCODER
     from .typing import TYPE_REQUESTS_TIMEOUT
     from .typing import TYPE_URL_FETCH
@@ -86,6 +87,10 @@
 
     USE_TLDEXTRACT = True
 
+    log.info("`tldextract` support enabled.")
+else:
+    log.info("`tldextract` support disabled.")
+
 # ------------------------------------------------------------------------------
 
 
@@ -189,7 +194,7 @@ def is_parsed_valid_url(
         parsed = parsed.decode()
     assert isinstance(parsed, ParseResult)
     if __debug__:
-        log.debug("is_parsed_valid_url = %s", parsed)
+        log.debug("is_parsed_valid_url(parsed=%s", parsed)
     if not all((parsed.scheme, parsed.netloc)):
         if __debug__:
             log.debug(" FALSE - missing `scheme` or `netloc`")
@@ -528,6 +533,9 @@ def log(
         logger: Callable[..., None] = log.error,
     ) -> None:
         """
+        Invoked to log troubleshooting information when an error is encountered.
+        By default this goes to `log.error`.
+
         :param prefix: Prefix for logging, defaults to "ResponseHistory"
         :type prefix: str
         :param logger: default `log.error`
@@ -1476,7 +1484,9 @@ def make_soup(self, html, **kwargs_bs):
             doc = BeautifulSoup(html, "lxml", **kwargs_bs)
         except Exception as exc:  # noqa: F841
             if __debug__:
-                log.debug("`BeautifulSoup` could not parse with `lxml`")
+                log.error(
+                    "`BeautifulSoup` could not parse with `lxml`; attempting `html.parser`"
+                )
             doc = BeautifulSoup(html, "html.parser", **kwargs_bs)
         return doc
 
@@ -1566,7 +1576,7 @@ def parse(
                 pass
             except Exception as exc:
                 if __debug__:
-                    log.debug("Ran into a serious error parsing `og`: %s", exc)
+                    log.error("Ran into a serious error parsing `og`: %s", exc)
                 pass
 
         twitters = doc_searchpath.find_all("meta", attrs={"name": RE_prefix_twitter})

src/metadata_parser/exceptions.py

@@ -4,6 +4,7 @@
 
 if TYPE_CHECKING:
     import requests
+
     from . import MetadataParser
     from .typing import TYPES_RESPONSE
 

src/metadata_parser/requests_extensions.py

@@ -1,7 +1,7 @@
 import _socket  # noqa: I201
 
 # stdlib
-import cgi  # noqa: I100
+import cgi  # noqa: I202
 import logging
 import socket
 from typing import Optional
@@ -18,9 +18,9 @@
 from .utils import DummyResponse
 from .utils import safe_sample
 
-
 if TYPE_CHECKING:
     from requests.structures import CaseInsensitiveDict
+
     from .typing import TYPES_PEERNAME
     from .typing import TYPES_RESPONSE
 

src/metadata_parser/typing.py

@@ -11,8 +11,10 @@
 from typing_extensions import Protocol  # py38
 
 if TYPE_CHECKING:
-    import requests
     from urllib.parse import ParseResult
+
+    import requests
+
     from . import DummyResponse
     from . import ResponseHistory
 

src/metadata_parser/utils.py

@@ -24,7 +24,6 @@
 from . import config
 from .regex import RE_rfc3986_valid_characters
 
-
 if TYPE_CHECKING:
     from urllib.parse import ParseResult
 
@@ -156,7 +155,7 @@ def fix_unicode_url(
             candidate[_idx] = parsed[_idx]
             candidate[_idx] = url_quote(url_unquote(candidate[_idx]))
         except Exception as exc:
-            log.debug("fix_unicode_url failure: %s | %s | %s", url, encoding, exc)
+            log.error("fix_unicode_url failure: %s | %s | %s", url, encoding, exc)
             return url
     _url = urlunparse(candidate)
     return _url

tests/test_document_parsing.py

@@ -439,6 +439,9 @@ def test_default_encoder(self):
 
 class _TestDocumentParsingCore:
 
+    assertIn: Callable
+    assertEqual: Callable
+
     def _MakeOne(self, filename):
         """lazy cache of files as needed"""
         global CACHED_FILESYSTEM_DOCUMENTS