Merge pull request #324 from tilezen/submit-job-overrides

Add optional batch memory override

Author: rmarianski

config.yaml.sample

@@ -388,3 +388,7 @@ batch:
   retry-attempts: 5
   # check store if metatile already exists, and if so skip processing
   check-metatile-exists: true
+  # optional override memory reserved for container when submiting job
+  # NOTE: in megabytes
+  # https://docs.aws.amazon.com/batch/latest/APIReference/API_ContainerOverrides.html
+  memory:

tilequeue/command.py

@@ -1895,6 +1895,7 @@ def tilequeue_batch_enqueue(cfg, peripherals):
     assert job_queue, 'Missing batch job-queue config'
 
     retry_attempts = batch_yaml.get('retry-attempts')
+    memory = batch_yaml.get('memory')
 
     dim = 2 ** queue_zoom
     z = queue_zoom
@@ -1911,22 +1912,23 @@ def tilequeue_batch_enqueue(cfg, peripherals):
                 '--config', '/etc/tilequeue/config.yaml',
                 '--tile', coord_str,
             ]
+            container_overrides = dict(command=job_cmd)
             job_opts = dict(
                 jobDefinition=job_def,
                 jobQueue=job_queue,
                 jobName=job_name,
-                containerOverrides={
-                    'command': job_cmd,
-                },
+                containerOverrides=container_overrides,
             )
             if retry_attempts is not None:
                 job_opts['retryStrategy'] = dict(attempts=retry_attempts)
+            if memory:
+                container_overrides['memory'] = memory
             resp = client.submit_job(**job_opts)
             assert resp['ResponseMetadata']['HTTPStatusCode'] == 200, \
                 'Failed to submit job: %s' % 'JobName'
             i += 1
-            if i % 10000 == 0:
-                print i
+            if i % 1000 == 0:
+                print '%d jobs submitted' % i
 
     logger.info('Batch enqueue ... done')