1. docstring.

2. remove drop_url in the default config.
3. rename the bad designed class name.
4. simple PatternMaker APIs.
5. more conditions on validate pattern unit parseing.

Author: cfhamlet

src/os_urlpattern/VERSION

@@ -1 +1 @@
-0.1.8
+0.1.9

src/os_urlpattern/__init__.py

@@ -1,3 +1,7 @@
+"""os-urlpattern.
+
+Unsupervised URLs clustering, generate and match URL pattern.
+"""
 import sys
 __all__ = ['__version__', 'version_info']
 

src/os_urlpattern/cmdline.py

@@ -1,11 +1,19 @@
+"""Command line tools.
+
+pattern-make:
+    Load URLs, cluster and generate URL pattern.
+
+pattern-matcher:
+    Load pattern, match the URL and get matched results.
+
+"""
 from __future__ import print_function
 
 import argparse
-import logging
+import logging.config
 import sys
 import time
 from collections import Counter
-from logging.config import dictConfig
 
 from .compat import binary_stdin, binary_stdout
 from .config import get_default_config
@@ -15,11 +23,17 @@
 from .formatter import FORMATTERS
 from .pattern_maker import PatternMaker
 from .pattern_matcher import PatternMatcher
-from .utils import LogSpeedAdapter, pretty_counter, MemoryUsageFormatter
+from .utils import LogSpeedAdapter, MemoryUsageFormatter, pretty_counter
+
+_DEFAULT_LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': True,
+    'incremental': True,
+}
 
 
 def _config_logging(log_level):
-    dictConfig(_DEFAULT_LOGGING)
+    logging.config.dictConfig(_DEFAULT_LOGGING)
     if log_level == 'NOTSET':
         handler = logging.NullHandler()
     else:
@@ -89,6 +103,7 @@ def add_argument(self, parser):
                             type=lambda s: s.upper())
 
     def _load(self, pattern_maker, args):
+        load_url = args.formatter == 'CLUSTER'
         stats = Counter()
         speed_logger = LogSpeedAdapter(self._logger, 5000)
         for url in args.file[0]:
@@ -100,7 +115,8 @@ def _load(self, pattern_maker, args):
             speed_logger.debug('[LOADING]')
             try:
                 url = url.decode(DEFAULT_ENCODING)
-                _, is_new = pattern_maker.load(url)
+                _, is_new = pattern_maker.load(
+                    url, meta=url if load_url else None)
                 if is_new:
                     stats['UNIQ'] += 1
                 stats['VALID'] += 1
@@ -119,20 +135,18 @@ def _load(self, pattern_maker, args):
         self._logger.debug('[LOADED] %s', pretty_counter(stats))
 
     def _process(self, pattern_maker, args):
+        combine = args.formatter == 'ETE'
         formatter = FORMATTERS[args.formatter]()
         s = time.time()
-        combine = args.formatter == 'ETE'
         for maker in pattern_maker.makers:
-            for url_meta, root in maker.make(combine):
+            for root in maker.make(combine):
                 e = time.time()
                 self._logger.debug('[CLUSTER] %d %.2fs', root.count, e - s)
-                for record in formatter.format(url_meta, root):
+                for record in formatter.format(maker.url_meta, root):
                     print(record)
                 s = time.time()
 
     def _confirm_config(self, args):
-        if args.formatter != 'CLUSTER':
-            self._config.set('make', 'drop_url', 'true')
         self._config.freeze()
 
     def run(self, args):
@@ -165,7 +179,7 @@ def add_argument(self, parser):
     def _load(self, pattern_matcher, args):
         stats = Counter()
         io_input = args.pattern_file[0]
-        self._logger.debug('[LOAD] Start %s', io_input.name)
+        self._logger.debug('[LOAD] Pattrn file: %s', io_input.name)
         speed_logger = LogSpeedAdapter(self._logger, 1000)
         for line in io_input:
             speed_logger.debug('[LOADING]')
@@ -224,13 +238,6 @@ def run(self, args):
         self._match(pattern_matcher, args)
 
 
-_DEFAULT_LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'incremental': True,
-}
-
-
 def _execute(command, argv=None):
     argv = argv or sys.argv
     parser = argparse.ArgumentParser()

src/os_urlpattern/compat.py

@@ -1,3 +1,6 @@
+"""Compatible import.
+"""
+
 import operator
 import string
 import sys

src/os_urlpattern/config/__init__.py

@@ -1,3 +1,5 @@
+"""Configure.
+"""
 from ..compat import RawConfigParser
 try:
     from collections import OrderedDict as _default_dict

src/os_urlpattern/config/default_config.cfg

@@ -1,3 +1,2 @@
 [make]
-min_cluster_num = 3
-drop_url = false
+min_cluster_num = 3

src/os_urlpattern/definition.py

@@ -1,3 +1,6 @@
+"""Definition of global constant varialbles.
+"""
+
 import hashlib
 
 from .compat import (ascii_lowercase_unicode, ascii_uppercase_unicode,
@@ -42,7 +45,6 @@ class BasePatternRule(object):
     MULTI_DIGIT_AND_ASCII_UPPER = u'[0-9A-Z]+'
     MULTI_DIGIT_AND_ASCII = u'[0-9A-Za-z]+'
     DOT = u'\\.'
-    ALL_MATCH = u'.*?'
     EMPTY = u''
     SINGLE_QUESTION = u'[\\?]'
 
@@ -132,5 +134,4 @@ class BasePattern(object):
         BasePatternRule.MULTI_DIGIT_AND_ASCII_UPPER)
     MULTI_DIGIT_AND_ASCII = Pattern(BasePatternRule.MULTI_DIGIT_AND_ASCII)
     DOT = Pattern(BasePatternRule.DOT)
-    ALL_MATCH = Pattern(BasePatternRule.ALL_MATCH)
     EMPTY = Pattern(BasePatternRule.EMPTY)

src/os_urlpattern/exceptions.py

@@ -1,3 +1,7 @@
+"""Custom Exceptions.
+"""
+
+
 class IrregularURLException(Exception):
     pass
 

src/os_urlpattern/formatter.py

@@ -1,3 +1,5 @@
+"""Clustered record formatter.
+"""
 import json
 
 from .definition import BasePatternRule, Symbols
@@ -23,13 +25,23 @@ def format(self, url_meta, clusterd, **kwargs):
 
         """
         yield
-        return
 
 
 class PatternFormatter(Formatter):
     """Pattern only formatter."""
 
     def format(self, url_meta, root, **kwargs):
+        """Generate url pattern string.
+
+        Args:
+            url_meta (URLMeta): The URLMeta object.
+            root (TreeNode): Root of a clusterd piece tree.
+            **kwargs: Arbitray keyword arguments.
+
+        Yields:
+            str: URL pattern string.
+
+        """
         for nodes in dump_tree(root):
             yield pack(url_meta, [p.pattern for p in nodes[1:]])
             break
@@ -39,6 +51,18 @@ class ClusterFormatter(PatternFormatter):
     """Pattern and meta formatter."""
 
     def format(self, url_meta, root, **kwargs):
+        """Generate url pattern string and dumps bound meta data.
+
+        Args:
+            url_meta (URLMeta): The URLMeta object.
+            root (TreeNode): Root of a clusterd piece tree.
+            **kwargs: Arbitray keyword arguments.
+
+        Yields:
+            str: Yield URL pattern string first, then yield meta
+                data of this cluster.
+
+        """
         for r in super(ClusterFormatter, self).format(url_meta, root, **kwargs):
             yield r
 
@@ -51,6 +75,18 @@ class JsonFormatter(Formatter):
     """Json record of pattern info formatter."""
 
     def format(self, url_meta, root, **kwargs):
+        """Generate json format of URL pattern and relative info.
+
+        Args:
+            url_meta (URLMeta): The URLMeta object.
+            root (TreeNode): Root of a clusterd piece tree.
+            **kwargs: Arbitray keyword arguments.
+
+        Yields:
+            str: Json string, key-value:
+                ptn: URL pattern string.
+                cnt: Number of uniq path in the cluster.
+        """
         for nodes in dump_tree(root):
             p = pack(url_meta, [p.pattern for p in nodes[1:]])
             yield json.dumps({u'ptn': p, u'cnt': root.count})
@@ -61,7 +97,16 @@ class ETEFormatter(Formatter):
     """Ete tree formatter."""
 
     def format(self, url_meta, root, **kwargs):
+        """Generate ete tree string.
+
+        Args:
+            url_meta (URLMeta): The URLMeta object.
+            root (TreeNode): Root of a pattern tree.
+            **kwargs: Arbitray keyword arguments.
 
+        Yields:
+            str: An ete tree string.
+        """
         def f(pattern_node):
             sep = Symbols.EMPTY
             query_key = Symbols.EMPTY

src/os_urlpattern/parse_utils.py

@@ -1,3 +1,5 @@
+"""Utilitis for parsing URL and pattern.
+"""
 import hashlib
 from collections import namedtuple
 
@@ -225,7 +227,7 @@ def _filterd(x):
 
 
 def parse_query_string(query_string):
-    """Parse query string into keys and values
+    """Parse query string into keys and values.
 
     Args:
         query_string (str): The string to be parsed.
@@ -242,7 +244,7 @@ def parse_query_string(query_string):
         return BLANK_TUPLE, BLANK_TUPLE
     elif query_string.endswith(Symbols.AMPERSAND):
         raise IrregularURLException("Invalid '&' pos")
-    kv_type = True  # qkey True, qvalue False
+    kv_type = True  # query_key True, query_value False
     last_c = None
     kv_buf = {True: StringIO(), False: StringIO()}
     kv_list = {True: [], False: []}
@@ -526,6 +528,19 @@ def _reset(self):
         self._piece_list = []
 
     def parse(self, piece):
+        """Parse a string into small sub-pieces with rules.
+
+        The consecutive charactors in the same charactor space
+        will be joined into one sub-piece, the corresponding
+        rule(charactor space) can also be got.
+        
+        Args:
+            piece (str): A string to be parsed.
+        
+        Returns:
+            tuple: 2-tuple, (pieces, rules).
+        """
+
         self._reset()
         self._preprocess(piece)
         return self._create_parsed_piece()
@@ -614,7 +629,7 @@ def analyze_url_pattern_string(url_pattern_string):
         url_pattern_string (str): The URL pattern string to be parsed.
 
     Returns:
-        tuple: A 2-tuple, (url_meta, pattern_string_pieces).
+        tuple: A 2-tuple, (url_meta, pattern_strings).
     """
     result = parse_url_pattern_string(url_pattern_string)
     return unpack(result, False)
@@ -736,8 +751,9 @@ def parse_pattern_unit_string(pattern_unit_string):
                 n = 1
             rule = p_str[idx:idx + n]
             if rule not in RULE_SET:
-                raise InvalidPatternException(
-                    "Invalid pattern unit: %s" % pattern_unit_string)
+                raise InvalidPatternException("Invalid rule '%s'" % rule)
             rules.add(rule)
             idx += n
+    if (num > 0 and len(rules) > num) or num == 0:
+        raise InvalidPatternException('Insufficient number')
     return rules, num