_rules: Time and log ruleset reduction

Closes #4

Author: jayvdb

https_everywhere/_rules.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+from timeit import default_timer as timer
 
 from cached_property import cached_property
 from logging_helper import setup_logging
@@ -470,18 +471,22 @@ def _reduce_ruleset(ruleset):
     return True
 
 
-def _reduce_rules(rulesets, check=False):
+def _reduce_rules(rulesets, check=False, simplify=False):
     if isinstance(rulesets, dict):
         rulesets = rulesets["rulesets"]
 
-    if check and not expand_pattern:
+    if (check or simplify) and not expand_pattern:
         logger.warning("Rule analysis and simplification only supported on Python 3")
-        check = False
+        check = simplify = False
 
     mapping = {}
     domains = set()
     prefix_targets = set()
     suffix_targets = set()
+    simplifications_performed = 0
+
+    logger.info("Importing HTTPSEverywhere rules")
+    start = timer()
 
     for ruleset in rulesets:
         orig_ruleset = ruleset.copy()
@@ -636,9 +641,11 @@ def _reduce_rules(rulesets, check=False):
         # Discard common data
         if rules == ONLY_FORCE_HTTPS_RULE_IN:
             rules = ONLY_FORCE_HTTPS_RULE_COMPILED
+            ruleset = rules, exclusions
 
-        else:
+        elif simplify:
             reduced_rules = []
+            original_rule_count = len(rules)
             for item in rules:
                 from_ = item["from"]
                 if from_ in _FIXME_REJECT_PATTERNS:
@@ -659,20 +666,26 @@ def _reduce_rules(rulesets, check=False):
             if rules[-1] == FORCE_HTTPS_RULE:
                 rules[-1] = FORCE_HTTPS_RULE_COMPILED
 
-        reduced_ruleset = _Ruleset(rules, exclusions, targets)
+            reduced_ruleset = _Ruleset(rules, exclusions, targets)
 
-        if check:
             _reduce_ruleset(reduced_ruleset)
+            final_rule_count = len(reduced_ruleset._rules)
+            simplifications_performed += final_rule_count
 
-        ruleset = (reduced_ruleset._rules, reduced_ruleset._exclusions)
+            ruleset = (reduced_ruleset._rules, reduced_ruleset._exclusions)
+
+        else:
+            rules = [(item["from"], item["to"]) for item in rules]
+            ruleset = rules, exclusions
 
         if ruleset == ONLY_FORCE_HTTPS_RULE_COMPILED_NO_EXCEPTIONS:
             ruleset = ONLY_FORCE_HTTPS_RULE_COMPILED_NO_EXCEPTIONS
 
         for target in targets:
             # https://github.com/EFForg/https-everywhere/issues/18897
             if (
-                name == "Vox Media.com (resources)"
+                check
+                and name == "Vox Media.com (resources)"
                 and target
                 in [
                     "voxmedia.com",
@@ -712,6 +725,17 @@ def _reduce_rules(rulesets, check=False):
         # TODO: re-enable or remove when new published ruleset is fixed
         # assert sorted(overlapping_prefixes) == _FIXME_MULTIPLE_RULEST_PREFIXES, sorted(overlapping_prefixes)
 
+    end = timer()
+    elapsed = end - start
+    simplifications_message = "; {} non-trivial simplifications".format(
+        simplifications_performed
+    )
+    logger.info(
+        "Finished importing HTTPSEverywhere rules after {:.2f}s{}".format(
+            elapsed, simplifications_message if simplifications_performed else ""
+        )
+    )
+
     return mapping
 
 
@@ -750,7 +774,7 @@ def _get_rulesets():
     global _DATA
     if not _DATA:
         data = fetch_update()
-        _DATA = _reduce_rules(data)
+        _DATA = _reduce_rules(data, simplify=True)
     return _DATA
 
 

tests/test_rules.py

@@ -19,11 +19,16 @@
 https_everything_project = os.path.join(project_root, "..", "https-everywhere")
 
 if not os.path.exists(https_everything_project):
-    raise unittest.SkipTest("Clone https-everywhere beside https-everywhere-py to run rule tests")
+    raise unittest.SkipTest(
+        "Clone https-everywhere beside https-everywhere-py to run rule tests"
+    )
 
-https_everywhere_checker_root = os.path.join(https_everything_project, "test", "rules", "src")
+https_everywhere_checker_root = os.path.join(
+    https_everything_project, "test", "rules", "src"
+)
 https_everywhere_checker_root_init = os.path.join(
-    https_everywhere_checker_root, "https_everywhere_checker", "__init__.py")
+    https_everywhere_checker_root, "https_everywhere_checker", "__init__.py"
+)
 
 if not os.path.exists(https_everywhere_checker_root_init):
     with open(https_everywhere_checker_root_init, "w") as f:
@@ -37,7 +42,9 @@
 except ImportError as e:
     raise unittest.SkipTest("https_everywhere_checker not importable: {}".format(e))
 
-rules_path = os.path.join(https_everything_project, *("src/chrome/content/rules".split("/")))
+rules_path = os.path.join(
+    https_everything_project, *("src/chrome/content/rules".split("/"))
+)
 ruleset_file = os.path.join(rules_path, "default.rulesets")
 
 if not os.path.exists(ruleset_file):
@@ -77,7 +84,9 @@ def _load_upstream_reduced_rulesets():
 
     with open(ruleset_file) as f:
         _upstream_data = json.load(f)
-        _upstream_reduced = _reduce_rules(_upstream_data, check=_run_check)
+        _upstream_reduced = _reduce_rules(
+            _upstream_data, check=_run_check, simplify=True
+        )
         return _upstream_reduced