Refactoring APIs, add TreeNode

Author: cfhamlet

README.rst

@@ -269,11 +269,12 @@ Cluster and generate URL Pattern:
    for url in urls:
        pattern_maker.load(url)
 
-   # dump pattern data
+   # dump patterns
    formatter = PatternFormatter()
-   for cluster in pattern_maker.make():
-       for pattern in formatter.format(cluster):
-           print(pattern)
+   for maker in pattern_maker.makers:
+       for url_meta, clusterd in maker.make():
+           for pattern in formatter.format(url_meta, clusterd)
+               print(pattern)
 
 
 Match URLs:

src/os_urlpattern/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

src/os_urlpattern/cmdline.py

@@ -100,7 +100,8 @@ def _load(self, pattern_maker, args):
             speed_logger.debug('[LOADING]')
             try:
                 url = url.decode(DEFAULT_ENCODING)
-                if pattern_maker.load(url):
+                _, is_new = pattern_maker.load(url)
+                if is_new:
                     stats['UNIQ'] += 1
                 stats['VALID'] += 1
             except (InvalidPatternException,
@@ -120,13 +121,14 @@ def _load(self, pattern_maker, args):
     def _process(self, pattern_maker, args):
         formatter = FORMATTERS[args.formatter]()
         s = time.time()
-        for pattern_tree in pattern_maker.make(combine=args.formatter == 'ETE'):
-            e = time.time()
-            self._logger.debug('[CLUSTER] %d %.2fs',
-                               pattern_tree.root.count, e - s)
-            for record in formatter.format(pattern_tree):
-                print(record)
-            s = time.time()
+        combine = args.formatter == 'ETE'
+        for maker in pattern_maker.makers:
+            for url_meta, root in maker.make(combine):
+                e = time.time()
+                self._logger.debug('[CLUSTER] %d %.2fs', root.count, e - s)
+                for record in formatter.format(url_meta, root):
+                    print(record)
+                s = time.time()
 
     def _confirm_config(self, args):
         if args.formatter != 'CLUSTER':
@@ -172,7 +174,7 @@ def _load(self, pattern_matcher, args):
                 continue
             try:
                 pattern = pattern.decode(DEFAULT_ENCODING)
-                pattern_matcher.load(pattern, pattern)
+                pattern_matcher.load(pattern, meta=pattern)
                 stats['VALID'] += 1
             except Exception as e:
                 self._logger.warn("%s, %s", str(e), line)
@@ -191,7 +193,7 @@ def _match_result(self, pattern_matcher, raw_url, args):
             if not args.all_matched:
                 sorted(result, reverse=True)
                 result = result[:1]
-            result = "\t".join([r.data for r in result]
+            result = "\t".join([r.meta for r in result]
                                ).encode(DEFAULT_ENCODING)
         except (InvalidPatternException,
                 IrregularURLException,

src/os_urlpattern/formatter.py

@@ -3,70 +3,83 @@
 from .compat import StringIO
 from .definition import BasePatternRule, Symbols
 from .parse_utils import pack
-from .utils import get_ete_tree
+from .utils import dump_tree
 
 
 class Formatter(object):
 
-    def format(self, tree, **kwargs):
+    def format(self, url_meta, tree, **kwargs):
         pass
 
 
 class PatternFormatter(Formatter):
-    def format(self, clusterd_tree, **kwargs):
-        url_meta = clusterd_tree.url_meta
-        for node_path in clusterd_tree.dump_paths():
-            yield pack(url_meta, [p.pattern for p in node_path[1:]])
+    def format(self, url_meta, root, **kwargs):
+        for nodes in dump_tree(root):
+            yield pack(url_meta, [p.pattern for p in nodes[1:]])
             break
 
 
 class ClusterFormatter(PatternFormatter):
-    def format(self, clusterd_tree, **kwargs):
-        for r in super(ClusterFormatter, self).format(clusterd_tree, **kwargs):
+    def format(self, url_meta, root, **kwargs):
+        for r in super(ClusterFormatter, self).format(url_meta, root, **kwargs):
             yield r
 
-        for node_path in clusterd_tree.dump_paths():
-            for url in node_path[-1].extra_data:
+        for nodes in dump_tree(root):
+            for url in nodes[-1].meta:
                 yield u'\t'.join((u'', url))
 
 
 class JsonFormatter(Formatter):
-    def format(self, clusterd_tree, **kwargs):
-        url_meta = clusterd_tree.url_meta
-        for node_path in clusterd_tree.dump_paths():
-            p = pack(url_meta, [p.pattern for p in node_path[1:]])
-            yield json.dumps({'ptn': p, 'cnt': clusterd_tree.count})
+    def format(self, url_meta, root, **kwargs):
+        for nodes in dump_tree(root):
+            p = pack(url_meta, [p.pattern for p in nodes[1:]])
+            yield json.dumps({u'ptn': p, u'cnt': root.count})
             break
 
 
 class ETEFormatter(Formatter):
-    def format(self, pattern_tree, **kwargs):
-        url_meta = pattern_tree.url_meta
+    def format(self, url_meta, root, **kwargs):
 
         def f(pattern_node):
             sep = Symbols.EMPTY
             query_key = Symbols.EMPTY
-            if url_meta.path_depth < pattern_node.current_level <= (url_meta.path_depth + url_meta.query_depth):
+            path_depth = url_meta.path_depth
+            query_depth = url_meta.query_depth
+            current_level = pattern_node.level
+            if path_depth < current_level \
+                    and current_level <= (path_depth + query_depth):
                 sep = Symbols.AMPERSAND
-                if pattern_node.current_level == url_meta.path_depth + 1:
+                if current_level == path_depth + 1:
                     sep = BasePatternRule.SINGLE_QUESTION
-                query_key = url_meta.query_keys[pattern_node.current_level -
-                                                url_meta.path_depth - 1]
-            elif pattern_node.current_level == url_meta.path_depth + url_meta.query_depth + 1:
+                query_key = url_meta.query_keys[current_level - path_depth - 1]
+            elif current_level == path_depth + query_depth + 1:
                 sep = Symbols.NUMBER
             return u' {sep}{query_key}{pattern_string}({count}) '.format(
                 count=pattern_node.count,
-                pattern_string=pattern_node,
+                pattern_string=pattern_node.value,
                 query_key=query_key,
                 sep=sep)
 
-        if pattern_tree.root.count <= 0:
+        if root.count <= 0:
             return
 
-        ete_tree = get_ete_tree(pattern_tree.root, format=f)
+        ete_tree = get_ete_tree(root, format=f)
         yield ete_tree.get_ascii(show_internal=True)
 
 
+def get_ete_tree(root_node, format=str):
+    from ete3 import Tree
+
+    def add_children(node, ete_node):
+        for child in node.children:
+            ete_child = ete_node.add_child(name=format(child))
+            add_children(child, ete_child)
+
+    ete_root_node = Tree(name=format(root_node))
+    add_children(root_node, ete_root_node)
+    return ete_root_node
+
+
 FORMATTERS = {
     'PATTERN': PatternFormatter,
     'CLUSTER': ClusterFormatter,

src/os_urlpattern/pattern_cluster.py

@@ -1,12 +1,13 @@
 from collections import Counter, OrderedDict, namedtuple
 
-from .compat import iteritems, itervalues
-from .parse_utils import URLMeta, digest, number_rule, wildcard_rule
-from .parsed_piece_view import (BaseView, LastDotSplitFuzzyView,
-                                MixedView)
+from .compat import itervalues
+from .parse_utils import (EMPTY_PARSED_PIECE, URLMeta, number_rule,
+                          wildcard_rule)
+from .parsed_piece_view import BaseView, LastDotSplitFuzzyView, MixedView
 from .pattern import Pattern
-from .piece_pattern_tree import PiecePatternNode, PiecePatternTree
-from .utils import Bag
+from .piece_pattern_node import (PiecePatternNode, build_from_parsed_pieces,
+                                 build_from_piece_pattern_nodes)
+from .utils import Bag, dump_tree
 
 
 class TBag(Bag):
@@ -112,7 +113,8 @@ def set_pattern(self, pattern):
 class ViewPieceBagBucket(PieceBagBucket):
     def __init__(self, url_meta):
         super(ViewPieceBagBucket, self).__init__()
-        self._tree = PiecePatternTree(url_meta)
+        self._url_meta = url_meta
+        self._root = PiecePatternNode(EMPTY_PARSED_PIECE)
 
     def add(self, view_piece_bag, build_tree=True):
         piece_bag = view_piece_bag.piece_bag
@@ -123,24 +125,23 @@ def add(self, view_piece_bag, build_tree=True):
             return
         view = view_piece_bag.view
 
-        self._tree.add_from_parsed_pieces(
-            view.parsed_pieces,
-            count=piece_bag.count,
-            uniq=False)
+        build_from_parsed_pieces(
+            self._root, view.parsed_pieces, count=piece_bag.count, uniq=False)
 
     def cluster(self, config, **kwargs):
-        for clustered_tree in cluster(config, self._tree, **kwargs):
-            yield self._transfer(clustered_tree)
+        for clustered in cluster(config, self._url_meta, self._root, **kwargs):
+            yield self._transfer(clustered)
 
-    def _transfer(self, clusterted_tree):
+    def _transfer(self, root):
         pattern = None
-        bucket = ViewPieceBagBucket(self._tree.url_meta)
-        for path in clusterted_tree.dump_paths():
-            piece = u''.join([p.piece for p in path[1:]])
+        bucket = ViewPieceBagBucket(self._url_meta)
+        for nodes in dump_tree(root):
+            piece = u''.join([p.piece for p in nodes[1:]])
             view_piece_bag = self[piece]
             bucket.add(view_piece_bag, False)
             if pattern is None:
-                pattern = Pattern(u''.join([str(p.pattern) for p in path[1:]]))
+                pattern = Pattern(
+                    u''.join([str(p.pattern) for p in nodes[1:]]))
         return bucket, pattern
 
 
@@ -599,7 +600,7 @@ def _process(self):
     def add(self, node, add_children=False):
         c = self.get_cluster(PiecePatternCluster)
         if add_children:
-            for child in node.iter_children():
+            for child in node.children:
                 c.add(child)
         else:
             c.add(node)
@@ -634,17 +635,16 @@ def _create_next_level_processors(self):
             processor.add(node, add_children=True)
 
 
-def split_by_pattern(piece_pattern_tree):
-    url_meta = piece_pattern_tree.url_meta
-    trees = {}
-    for path in piece_pattern_tree.dump_paths():
-        pid = digest(url_meta, [p.pattern for p in path[1:]])
-        if pid not in trees:
-            trees[pid] = PiecePatternTree(url_meta)
-        tree = trees[pid]
-        tree.add_from_piece_pattern_node_path(path[1:])
+def split_by_pattern(root):
+    tree_roots = {}
+    for nodes in dump_tree(root):
+        pid = hash(u"/".join([str(p.pattern) for p in nodes]))
+        if pid not in tree_roots:
+            tree_roots[pid] = PiecePatternNode(EMPTY_PARSED_PIECE)
+        sub_root = tree_roots[pid]
+        build_from_piece_pattern_nodes(sub_root, nodes[1:])
 
-    return itervalues(trees)
+    return itervalues(tree_roots)
 
 
 def _can_be_splited(processor):
@@ -662,19 +662,18 @@ def _can_be_splited(processor):
     return False
 
 
-def process(config, piece_pattern_tree, **kwargs):
-    url_meta = piece_pattern_tree.url_meta
+def process(config, url_meta, root, **kwargs):
     meta_info = MetaInfo(url_meta, 0)
     processor = ClusterProcessor(config, meta_info, None, **kwargs)
-    processor.add(piece_pattern_tree.root)
+    processor.add(root)
     processor.process()
     return _can_be_splited(processor)
 
 
-def cluster(config, piece_pattern_tree, **kwargs):
-    if not process(config, piece_pattern_tree, **kwargs):
-        yield piece_pattern_tree
+def cluster(config, url_meta, root, **kwargs):
+    if not process(config, url_meta, root, **kwargs):
+        yield root
         return
-    for sub_piece_pattern_tree in split_by_pattern(piece_pattern_tree):
-        for tree in cluster(config, sub_piece_pattern_tree, **kwargs):
-            yield tree
+    for sub_root in split_by_pattern(root):
+        for clustered in cluster(config, url_meta, sub_root, **kwargs):
+            yield clustered

src/os_urlpattern/pattern_maker.py

@@ -1,9 +1,9 @@
 from .compat import itervalues
-from .parse_utils import PieceParser, digest, parse_url
+from .definition import BasePattern
+from .parse_utils import EMPTY_PARSED_PIECE, PieceParser, digest, parse_url
 from .pattern_cluster import cluster
-from .pattern_tree import PatternTree
-from .piece_pattern_tree import PiecePatternTree
-from .utils import load_obj
+from .piece_pattern_node import PiecePatternNode, build_from_parsed_pieces
+from .utils import TreeNode, build_tree, dump_tree
 
 
 class PatternMaker(object):
@@ -13,46 +13,46 @@ def __init__(self, config):
         self._makers = {}
         self._drop_url = self._config.getboolean('make', 'drop_url')
 
+    @property
+    def makers(self):
+        return itervalues(self._makers)
+
     def load(self, url):
         url_meta, pieces = parse_url(url)
         parsed_pieces = [self._parser.parse(piece) for piece in pieces]
         sid = digest(url_meta, [p.fuzzy_rule for p in parsed_pieces])
         if sid not in self._makers:
             self._makers[sid] = Maker(self._config, url_meta)
         return self._makers[sid].load(parsed_pieces,
-                                      data=url if not self._drop_url else None)
-
-    def make(self, combine=False):
-        for maker in itervalues(self._makers):
-            for tree in maker.make(combine):
-                yield tree
+                                      meta=url if not self._drop_url else None)
 
 
 class Maker(object):
     def __init__(self, config, url_meta):
         self._config = config
         self._url_meta = url_meta
-        self._piece_pattern_tree = PiecePatternTree(url_meta)
+        self._root = PiecePatternNode(EMPTY_PARSED_PIECE)
 
-    def load(self, parsed_pieces, count=1, uniq=True, data=None):
-        return self._piece_pattern_tree.add_from_parsed_pieces(
-            parsed_pieces, count=count, uniq=uniq, data=data)
+    def load(self, parsed_pieces, count=1, meta=None, uniq=True):
+        return build_from_parsed_pieces(self._root,
+                                        parsed_pieces,
+                                        count=count,
+                                        meta=meta,
+                                        uniq=uniq)
 
-    def _path_dump_and_load(self, src, dest, index=0):
-        for path in src.dump_paths():
-            if path:
-                dest.load_path(path[index:])
+    def _cluster(self):
+        for clustered in cluster(self._config, self._url_meta, self._root):
+            yield clustered
 
-    def cluster(self):
-        for clusterd_tree in cluster(self._config, self._piece_pattern_tree):
-            yield clusterd_tree
-
-    def make(self, combine):
+    def make(self, combine=False):
         if combine:
-            pattern_tree = PatternTree(self._url_meta)
-            for clustered_tree in self.cluster():
-                self._path_dump_and_load(clustered_tree, pattern_tree, 1)
-            yield pattern_tree
+            root = TreeNode(BasePattern.EMPTY)
+            for clustered in self._cluster():
+                for nodes in dump_tree(clustered):
+                    build_tree(root, [(n.pattern, n.pattern)
+                                      for n in nodes[1:]], nodes[-1].count)
+
+            yield self._url_meta, root
         else:
-            for clustered_tree in self.cluster():
-                yield clustered_tree
+            for clustered in self._cluster():
+                yield self._url_meta, clustered