refactoring APIs, hide url_meta

Author: cfhamlet

README.rst

@@ -249,8 +249,8 @@ Usage
 
     # dump pattern data
     formatter = PatternFormatter()
-    for url_meta, cluster in pattern_maker.make():
-        for pattern in formatter.format(url_meta, cluster):
+    for cluster in pattern_maker.make():
+        for pattern in formatter.format(cluster):
             print(pattern)
 
 

src/os_urlpattern/cmdline.py

@@ -120,11 +120,11 @@ def _load(self, pattern_maker, args):
     def _process(self, pattern_maker, args):
         formatter = FORMATTERS[args.formatter]()
         s = time.time()
-        for url_meta, pattern_tree in pattern_maker.make(combine=args.formatter == 'ETE'):
+        for pattern_tree in pattern_maker.make(combine=args.formatter == 'ETE'):
             e = time.time()
             self._logger.debug('[CLUSTER] %d %.2fs',
                                pattern_tree.root.count, e - s)
-            for record in formatter.format(url_meta, pattern_tree):
+            for record in formatter.format(pattern_tree):
                 print(record)
             s = time.time()
 

src/os_urlpattern/formatter.py

@@ -8,20 +8,21 @@
 
 class Formatter(object):
 
-    def format(self, url_meta, tree, **kwargs):
+    def format(self, tree, **kwargs):
         pass
 
 
 class PatternFormatter(Formatter):
-    def format(self, url_meta, clusterd_tree, **kwargs):
+    def format(self, clusterd_tree, **kwargs):
+        url_meta = clusterd_tree.url_meta
         for node_path in clusterd_tree.dump_paths():
             yield pack(url_meta, [p.pattern for p in node_path[1:]])
             break
 
 
 class ClusterFormatter(PatternFormatter):
-    def format(self, url_meta, clusterd_tree, **kwargs):
-        for r in super(ClusterFormatter, self).format(url_meta, clusterd_tree, **kwargs):
+    def format(self, clusterd_tree, **kwargs):
+        for r in super(ClusterFormatter, self).format(clusterd_tree, **kwargs):
             yield r
 
         for node_path in clusterd_tree.dump_paths():
@@ -30,15 +31,17 @@ def format(self, url_meta, clusterd_tree, **kwargs):
 
 
 class JsonFormatter(Formatter):
-    def format(self, url_meta, clusterd_tree, **kwargs):
+    def format(self, clusterd_tree, **kwargs):
+        url_meta = clusterd_tree.url_meta
         for node_path in clusterd_tree.dump_paths():
             p = pack(url_meta, [p.pattern for p in node_path[1:]])
             yield json.dumps({'ptn': p, 'cnt': clusterd_tree.count})
             break
 
 
 class ETEFormatter(Formatter):
-    def format(self, url_meta, pattern_tree, **kwargs):
+    def format(self, pattern_tree, **kwargs):
+        url_meta = pattern_tree.url_meta
 
         def f(pattern_node):
             sep = Symbols.EMPTY

src/os_urlpattern/pattern_cluster.py

@@ -110,9 +110,9 @@ def set_pattern(self, pattern):
 
 
 class ViewPieceBagBucket(PieceBagBucket):
-    def __init__(self):
+    def __init__(self, url_meta):
         super(ViewPieceBagBucket, self).__init__()
-        self._tree = PiecePatternTree()
+        self._tree = PiecePatternTree(url_meta)
 
     def add(self, view_piece_bag, build_tree=True):
         piece_bag = view_piece_bag.piece_bag
@@ -122,21 +122,20 @@ def add(self, view_piece_bag, build_tree=True):
         if not build_tree:
             return
         view = view_piece_bag.view
+
         self._tree.add_from_parsed_pieces(
             view.parsed_pieces,
             count=piece_bag.count,
             uniq=False)
 
     def cluster(self, config, **kwargs):
-        p_num = len(self.pick().view.parsed_pieces)
-        url_meta = URLMeta(p_num, [], False)
-        for single_tree in cluster(config, url_meta, self._tree, **kwargs):
-            yield self._transfer(single_tree)
+        for clustered_tree in cluster(config, self._tree, **kwargs):
+            yield self._transfer(clustered_tree)
 
-    def _transfer(self, single_tree):
+    def _transfer(self, clusterted_tree):
         pattern = None
-        bucket = ViewPieceBagBucket()
-        for path in single_tree.dump_paths():
+        bucket = ViewPieceBagBucket(self._tree.url_meta)
+        for path in clusterted_tree.dump_paths():
             piece = u''.join([p.piece for p in path[1:]])
             view_piece_bag = self[piece]
             bucket.add(view_piece_bag, False)
@@ -252,7 +251,7 @@ def _add_to_forward_cluster(self, piece_bag):
 
         if vl == 3 and self._processor.meta_info.is_last_path():
             ldsf_view = LastDotSplitFuzzyView(parsed_piece)
-            if view.view == ldsf_view.view:
+            if view == ldsf_view:
                 view = ldsf_view
                 p_cls = LastDotSplitFuzzyPatternCluster
         elif vl > 3:
@@ -265,7 +264,7 @@ def _add_to_forward_cluster(self, piece_bag):
             elif vl - mvl >= self._min_cluster_num:
                 if mvl == 3 and self._processor.meta_info.is_last_path():
                     ldsf_view = LastDotSplitFuzzyView(parsed_piece)
-                    if mixed_view.view == ldsf_view.view:
+                    if mixed_view == ldsf_view:
                         view = ldsf_view
                         p_cls = LastDotSplitFuzzyPatternCluster
                     else:
@@ -376,9 +375,9 @@ def _set_pattern(self, bucket, pattern):
 
     def add(self, view_piece_bag):
         view = view_piece_bag.view
-        view = view.view
         if view not in self._buckets:
-            self._buckets[view] = ViewPieceBagBucket()
+            url_meta = URLMeta(len(view.parsed_pieces), [], False)
+            self._buckets[view] = ViewPieceBagBucket(url_meta)
         self._buckets[view].add(view_piece_bag)
 
 
@@ -394,7 +393,7 @@ def _add_to_forward_cluster(self, view_piece_bag):
 
         p_cls = MixedPatternCluster
 
-        if view.view == mixed_view.view:
+        if view == mixed_view:
             if self._processor.meta_info.is_last_path():
                 ldsf_view = LastDotSplitFuzzyView(parsed_piece)
                 if len(ldsf_view.parsed_pieces) == 1:
@@ -416,7 +415,7 @@ def _add_to_forward_cluster(self, view_piece_bag):
                 return
             elif mvl == 3 and self._processor.meta_info.is_last_path():
                 ldsf_view = LastDotSplitFuzzyView(parsed_piece)
-                if mixed_view.view == ldsf_view.view:
+                if mixed_view == ldsf_view:
                     view = ldsf_view
                     p_cls = LastDotSplitFuzzyPatternCluster
 
@@ -635,12 +634,13 @@ def _create_next_level_processors(self):
             processor.add(node, add_children=True)
 
 
-def split_by_pattern(url_meta, piece_pattern_tree):
+def split_by_pattern(piece_pattern_tree):
+    url_meta = piece_pattern_tree.url_meta
     trees = {}
     for path in piece_pattern_tree.dump_paths():
         pid = digest(url_meta, [p.pattern for p in path[1:]])
         if pid not in trees:
-            trees[pid] = PiecePatternTree()
+            trees[pid] = PiecePatternTree(url_meta)
         tree = trees[pid]
         tree.add_from_piece_pattern_node_path(path[1:])
 
@@ -662,18 +662,19 @@ def _can_be_splited(processor):
     return False
 
 
-def process(config, url_meta, piece_pattern_tree, **kwargs):
+def process(config, piece_pattern_tree, **kwargs):
+    url_meta = piece_pattern_tree.url_meta
     meta_info = MetaInfo(url_meta, 0)
     processor = ClusterProcessor(config, meta_info, None, **kwargs)
     processor.add(piece_pattern_tree.root)
     processor.process()
     return _can_be_splited(processor)
 
 
-def cluster(config, url_meta, piece_pattern_tree, **kwargs):
-    if not process(config, url_meta, piece_pattern_tree, **kwargs):
+def cluster(config, piece_pattern_tree, **kwargs):
+    if not process(config, piece_pattern_tree, **kwargs):
         yield piece_pattern_tree
         return
-    for sub_piece_pattern_tree in split_by_pattern(url_meta, piece_pattern_tree):
-        for tree in cluster(config, url_meta, sub_piece_pattern_tree, **kwargs):
+    for sub_piece_pattern_tree in split_by_pattern(piece_pattern_tree):
+        for tree in cluster(config, sub_piece_pattern_tree, **kwargs):
             yield tree

src/os_urlpattern/pattern_maker.py

@@ -32,7 +32,7 @@ class Maker(object):
     def __init__(self, config, url_meta):
         self._config = config
         self._url_meta = url_meta
-        self._piece_pattern_tree = PiecePatternTree()
+        self._piece_pattern_tree = PiecePatternTree(url_meta)
 
     def load(self, parsed_pieces, count=1, uniq=True, data=None):
         return self._piece_pattern_tree.add_from_parsed_pieces(
@@ -44,16 +44,15 @@ def _path_dump_and_load(self, src, dest, index=0):
                 dest.load_path(path[index:])
 
     def cluster(self):
-        for clusterd_tree in cluster(self._config, self._url_meta,
-                                     self._piece_pattern_tree):
+        for clusterd_tree in cluster(self._config, self._piece_pattern_tree):
             yield clusterd_tree
 
     def make(self, combine):
         if combine:
             pattern_tree = PatternTree(self._url_meta)
             for clustered_tree in self.cluster():
                 self._path_dump_and_load(clustered_tree, pattern_tree, 1)
-            yield self._url_meta, pattern_tree
+            yield pattern_tree
         else:
             for clustered_tree in self.cluster():
-                yield self._url_meta, clustered_tree
+                yield clustered_tree

src/os_urlpattern/piece_pattern_tree.py

@@ -119,8 +119,13 @@ def extra_data(self):
 
 
 class PiecePatternTree(object):
-    def __init__(self):
+    def __init__(self, url_meta):
         self._root = PiecePatternNode(EMPTY_PARSED_PIECE)
+        self._url_meta = url_meta
+
+    @property
+    def url_meta(self):
+        return self._url_meta
 
     @property
     def root(self):