Make buffer_cfg work for meta tile build (#401)

Author: peitili

tilequeue/process.py

@@ -12,7 +12,7 @@
 from tilequeue.tile import calc_meters_per_pixel_dim
 from tilequeue.tile import coord_to_mercator_bounds
 from tilequeue.tile import normalize_geometry_type
-from tilequeue.transform import mercator_point_to_lnglat
+from tilequeue.transform import mercator_point_to_lnglat, calc_max_padded_bounds
 from tilequeue.transform import transform_feature_layers_shape
 from tilequeue import utils
 from zope.dottedname.resolve import resolve
@@ -518,7 +518,7 @@ def process_coord(coord, nominal_zoom, feature_layers, post_process_data,
     return all_formatted_tiles, extra_data
 
 
-def convert_source_data_to_feature_layers(rows, layer_data, bounds, zoom):
+def convert_source_data_to_feature_layers(rows, layer_data, unpadded_bounds, zoom):
     # TODO we might want to fold in the other processing into this
     # step at some point. This will prevent us from having to iterate
     # through all the features again.
@@ -581,12 +581,9 @@ def convert_source_data_to_feature_layers(rows, layer_data, bounds, zoom):
     for layer_datum in layer_data:
         layer_name = layer_datum['name']
         features = features_by_layer[layer_name]
-        # TODO padded bounds
-        padded_bounds = dict(
-            polygon=bounds,
-            line=bounds,
-            point=bounds,
-        )
+        query_bounds_pad_fn = layer_datum['query_bounds_pad_fn']
+
+        padded_bounds = query_bounds_pad_fn(unpadded_bounds, calc_meters_per_pixel_dim(zoom))
         feature_layer = dict(
             name=layer_name,
             features=features,
@@ -748,24 +745,25 @@ def __init__(self, coord, metatile_zoom, fetch_fn, layer_data,
         self.cfg_tile_sizes = cfg_tile_sizes
         self.log_fn = None
 
-    def fetch(self):
-        unpadded_bounds = coord_to_mercator_bounds(self.coord)
+        self.unpadded_bounds = coord_to_mercator_bounds(self.coord)
+        meters_per_pixel_dim = calc_meters_per_pixel_dim(self.coord.zoom)
+        self.max_padded_bounds = calc_max_padded_bounds(self.unpadded_bounds, meters_per_pixel_dim, self.buffer_cfg)
 
+    def fetch(self):
         cut_coords_by_zoom = calculate_cut_coords_by_zoom(
             self.coord, self.metatile_zoom, self.cfg_tile_sizes, self.max_zoom)
         feature_layers_by_zoom = {}
 
         for nominal_zoom, _ in cut_coords_by_zoom.items():
-            source_rows = self.fetch_fn(nominal_zoom, unpadded_bounds)
+            source_rows = self.fetch_fn(nominal_zoom, self.max_padded_bounds)
             feature_layers = convert_source_data_to_feature_layers(
-                source_rows, self.layer_data, unpadded_bounds, self.coord.zoom)
+                source_rows, self.layer_data, self.unpadded_bounds, self.coord.zoom)
             feature_layers_by_zoom[nominal_zoom] = feature_layers
 
         self.cut_coords_by_zoom = cut_coords_by_zoom
         self.feature_layers_by_zoom = feature_layers_by_zoom
 
     def process_tiles(self):
-        unpadded_bounds = coord_to_mercator_bounds(self.coord)
 
         all_formatted_tiles = []
         all_extra_data = {}
@@ -782,7 +780,7 @@ def log_fn(data):
             feature_layers = self.feature_layers_by_zoom[nominal_zoom]
             formatted_tiles, extra_data = process_coord(
                 self.coord, nominal_zoom, feature_layers,
-                self.post_process_data, self.formats, unpadded_bounds,
+                self.post_process_data, self.formats, self.unpadded_bounds,
                 cut_coords, self.buffer_cfg, self.output_calc_mapping,
                 log_fn=log_fn,
             )

tilequeue/query/rawr.py

@@ -724,9 +724,11 @@ def _add_feature(source, feature):
 
         return source_features.iteritems()
 
-    def __call__(self, zoom, unpadded_bounds):
+    def __call__(self, zoom, bounds):
+        """ The bounds is either an unpadded bounds if buffer_cfg is not set
+        or a padded bounds if buffer_cfg is set upstream """
         read_rows = []
-        bbox = box(*unpadded_bounds)
+        bbox = box(*bounds)
 
         # check that the call is fetching data which is actually within the
         # bounds of the tile pyramid. we don't have data outside of that, so
@@ -735,25 +737,27 @@ def __call__(self, zoom, unpadded_bounds):
         # loaded?
         assert zoom <= self.tile_pyramid.max_z
         assert zoom >= self.tile_pyramid.z
-        assert bbox.within(self.tile_pyramid.bbox())
+        # assert bbox.within(self.tile_pyramid.bbox())
 
-        for source, features in self._lookup(zoom, unpadded_bounds):
+        for source, features in self._lookup(zoom, bounds):
             for (fid, shape, props, layer_min_zooms) in features:
                 read_row = self._parse_row(
-                    zoom, unpadded_bounds, bbox, source, fid, shape, props,
+                    zoom, bounds, bbox, source, fid, shape, props,
                     layer_min_zooms)
                 if read_row:
                     read_rows.append(read_row)
 
         return read_rows
 
-    def _parse_row(self, zoom, unpadded_bounds, bbox, source, fid, shape,
+    def _parse_row(self, zoom, bounds, bbox, source, fid, shape,
                    props, layer_min_zooms):
+        """ The bounds is either an unpadded bounds if buffer_cfg is not set
+            or a padded bounds if buffer_cfg is set upstream"""
         # reject any feature which doesn't intersect the given bounds
         if bbox.disjoint(shape):
             return None
 
-        # place for assembing the read row as if from postgres
+        # place for assembling the read row as if from postgres
         read_row = {}
         generate_label_placement = False
 
@@ -826,11 +830,15 @@ def _parse_row(self, zoom, unpadded_bounds, bbox, source, fid, shape,
 
             # if this is a water layer feature, then clip to an expanded
             # bounding box to avoid tile-edge artefacts.
+            # Note: As of Oct 2021 we added support for buffer_cfg to
+            # tilequeue such that this extra buffer for water can actually be
+            # configured as a water layer buffer_cfg. But we leave as is for
+            # now.
             clip_box = bbox
             if layer_name == 'water':
                 pad_factor = 1.1
                 clip_box = calculate_padded_bounds(
-                    pad_factor, unpadded_bounds)
+                    pad_factor, bounds)
             # don't need to clip if geom is fully within the clipping box
             if box(*shape.bounds).within(clip_box):
                 clip_shape = shape

tilequeue/transform.py

@@ -97,6 +97,31 @@ def calc_buffered_bounds(
     return bounds
 
 
+def calc_max_padded_bounds(bounds, meters_per_pixel_dim, buffer_cfg):
+    """
+    :return: The bounds expanded by the maximum value in buffer_cfg, default = 0
+    """
+    max_buffer = 0
+
+    if buffer_cfg is None:
+        return bounds
+
+    for _, format_cfg in buffer_cfg.items():
+        layer_cfg = format_cfg.get('layer', {})
+        if layer_cfg is not None:
+            for _, value in layer_cfg.items():
+                assert isinstance(value, Number)
+                max_buffer = max(max_buffer, value)
+
+        geometry_cfg = format_cfg.get('geometry', {})
+        if geometry_cfg is not None:
+            for _, value in geometry_cfg.items():
+                assert isinstance(value, Number)
+                max_buffer = max(max_buffer, value)
+
+    return bounds_buffer(bounds, meters_per_pixel_dim * max_buffer)
+
+
 def _intersect_multipolygon(shape, tile_bounds, clip_bounds):
     """
     Return the parts of the MultiPolygon shape which overlap the tile_bounds,