Merge pull request #323 from tilezen/check-metatile-first

Add option to check for metatile before processing

Author: rmarianski

config.yaml.sample

@@ -386,3 +386,5 @@ batch:
   job-queue: <job-queue>
   # optional number of retry attempts, overrides job definition value
   retry-attempts: 5
+  # check store if metatile already exists, and if so skip processing
+  check-metatile-exists: true

tilequeue/command.py

@@ -1948,6 +1948,8 @@ def tilequeue_batch_process(cfg, args):
     queue_zoom = batch_yaml.get('queue-zoom')
     assert queue_zoom, 'Missing batch queue-zoom config'
 
+    check_metatile_exists = bool(batch_yaml.get('check-metatile-exists'))
+
     queue_coord = deserialize_coord(coord_str)
     if not queue_coord:
         print >> sys.stderr, 'Invalid coordinate: %s' % coord_str
@@ -1996,6 +1998,10 @@ def find_job_coords_for(coord, target_zoom):
 
     batch_logger.begin_run(queue_coord)
 
+    layer = 'all'
+    zip_format = lookup_format_by_extension('zip')
+    assert zip_format
+
     job_coords = find_job_coords_for(queue_coord, group_by_zoom)
     for job_coord in job_coords:
 
@@ -2017,6 +2023,12 @@ def find_job_coords_for(coord, target_zoom):
             nominal_zoom = coord.zoom + cfg.metatile_zoom
             unpadded_bounds = coord_to_mercator_bounds(coord)
 
+            if check_metatile_exists:
+                existing_data = store.read_tile(coord, zip_format, layer)
+                if existing_data is not None:
+                    batch_logger.metatile_already_exists(coord)
+                    continue
+
             try:
                 source_rows = fetch(nominal_zoom, unpadded_bounds)
                 feature_layers = convert_source_data_to_feature_layers(

tilequeue/log.py

@@ -318,3 +318,6 @@ def tile_process_failed(self, exception, coord):
 
     def metatile_storage_failed(self, exception, coord):
         self._log_exception('metatile storage failed', exception, coord)
+
+    def metatile_already_exists(self, coord):
+        self._log('metatile already exists', coord)