Merge pull request #11 from ggatward/chunked-publish

Add batched publish/promotion option

Author: ggatward

CHANGELOG.md

@@ -4,6 +4,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Allow limiting of Publish and Promotion to specified number of CV's at once.
+
 
 ## [1.1.0] - 2017-10-23
 ### Added

README.md

@@ -185,9 +185,12 @@ optional arguments:
   -s SINCE, --since SINCE
                         Export content since YYYY-MM-DD HH:MM:SS
   -l, --last            Display time of last export
-  -n, --nogpg           Skip GPG checking
+  -L, --list            List all successfully completed exports
+  --nogpg               Skip GPG checking
   -r, --repodata        Include repodata for repos with no incremental content
   -p, --puppetforge     Include puppet-forge-server format Puppet Forge repo
+  --notar               Do not archive the extracted content
+  --forcexport          Force export from an import-only (Disconnected) Satellite
 
 ```
 
@@ -367,12 +370,17 @@ the WebUI or Hammer CLI.
 The defaults are configured in the main config.yml file in a YAML block like this:
 ```
 publish:
+  batch: 10
   content_views:
     - RHEL Server
     - RHEL Workstation
 ```
 This configuration will publish only the two listed content views.
 
+The batch: parameter can be used to limit the number of content views that will be published at
+once, to aid in performance tuning.
+
+
 
 ```
 usage: publish_content_view.py [-h] [-o ORG] [-a] [-d]
@@ -416,13 +424,14 @@ the WebUI or Hammer CLI.
 The defaults are configured in the main config.yml file in a YAML block like this:
 ```
 promotion:
+  batch: 10
   lifecycle1:
     name: Quality
     content_views:
       - RHEL Server
       - RHEL Workstation
 
-  lifecyclec2:
+  lifecycle2:
     name: Desktop QA
     content_views:
       - RHEL Workstation
@@ -432,6 +441,9 @@ use of the config.yml definition is strongly recommended to avoid views being
 promoted into the wrong lifecycle stream. This is more likely to be an
 issue promoting views from the Library, as this is shared by all environments.
 
+The batch: parameter can be used to limit the number of content views that will be promoted at
+once, to aid in performance tuning.
+
 ```
 usage: promote_content_view.py [-h] -e ENV [-o ORG] [-a] [-d]
 

config/config.yml.example

@@ -19,11 +19,13 @@ import:
   syncbatch: 50
 
 publish:
+  batch: 10
   content_views:
     - RHEL Server
     - RHEL Workstation
 
 promotion:
+  batch: 10
   lifecycle1:
     name: Quality
     content_views:

helpers.py

@@ -51,6 +51,14 @@
     SYNCBATCH = CONFIG['import']['syncbatch']
 else:
     SYNCBATCH = 255
+if 'batch' in CONFIG['publish']:
+    PUBLISHBATCH = CONFIG['publish']['batch']
+else:
+    PUBLISHBATCH = 255
+if 'batch' in CONFIG['promotion']:
+    PROMOTEBATCH = CONFIG['promotion']['batch']
+else:
+    PROMOTEBATCH = 255
 if 'hostname' in CONFIG['puppet-forge-server']:
     PFSERVER = CONFIG['puppet-forge-server']['hostname']
 

man/sat6_scripts.8

@@ -69,8 +69,8 @@ This determines whether this satellite can export or import content.
 .B "  proxy: proxy.example.com:8080"
 .RS
 Definition of HTTP proxy to use in support of the
-.I download_manifest 
-script. 
+.I download_manifest
+script.
 .br
 If no proxy is required this parameter can be omitted from the config file completely.
 .RE
@@ -102,13 +102,18 @@ Directory to export content to, used by
 .B import:
 .br
 .B "  dir: /var/sat-content"
+.br
+.B "  syncbatch: 10"
 .RS
 Directory to read import content from, used by
 .IR sat_import .
+The syncbatch: parameter defines the maximum number of repositories to sync at once, for performance tuning.
 .RE
 
 .B publish:
 .br
+.B "  batch: 10"
+.br
 .B "  content_views:"
 .br
 .B "    - RHEL Server"
@@ -117,14 +122,18 @@ Directory to read import content from, used by
 .RS
 List of content views to publish, one per line. Used by
 .IR publish_content_views .
+The batch: keyword specifies the maximum number of content views to publish in a batch, for performance tuning.
 .RE
 
 .B promotion:
 .br
+.B "  batch: 10"
+.br
 .B "  lifecycle1:"
 .RS
 Lifecycle environment definition. There can be multiple lifecycles defined, however
 the names must be unique. Increment the number for each defined environment.
+The batch: keyword specifies the maximum number of content views to promote in a batch, for performance tuning.
 .RE
 
 .B "    name: Quality"

promote_content_views.py

@@ -109,7 +109,7 @@ def get_cv(org_id, target_env, env_list, prior_list, promote_list):
 
 
 # Promote a content view version
-def promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list, dry_run):
+def promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list, dry_run, quiet):
     """Promote Content View"""
     target_env_id = env_list[target_env]
     source_env_id = prior_list[target_env_id]
@@ -133,41 +133,51 @@ def promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list,
         helpers.log_msg(msg, 'WARNING')
         sys.exit(1)
 
-    for cvid in ver_list.keys():
-
-        # Check if there is a publish/promote already running on this content view
-        locked = helpers.check_running_publish(cvid, ver_descr[cvid])
-
-        if not locked:
-            msg = "Promoting '" + str(ver_descr[cvid]) + "' Version " + str(ver_version[cvid]) +\
-                " from " + prior_env + " to " + str(target_env)
-            helpers.log_msg(msg, 'INFO')
-            print helpers.HEADER + msg + helpers.ENDC
-
-        if not dry_run and not locked:
-            try:
-                task_id = helpers.post_json(
-                    helpers.KATELLO_API + "content_view_versions/" + str(ver_list[cvid]) +\
-                    "/promote/", json.dumps(
-                        {
-                            "environment_id": target_env_id
-                        }
-                        ))["id"]
-            except Warning:
-                msg = "Failed to initiate promotion of " + str(ver_descr[cvid])
-                helpers.log_msg(msg, 'WARNING')
-            else:
-                task_list.append(task_id)
-                ref_list[task_id] = ver_descr[cvid]
+    # Break repos to promote into batches as configured in config.yml
+    cvchunks = [ ver_list.keys()[i:i+helpers.PROMOTEBATCH] for i in range(0, len(ver_list), helpers.PROMOTEBATCH) ]
+
+    # Loop through the smaller subsets of repo id's
+    for chunk in cvchunks:
+        for cvid in chunk:
+
+            # Check if there is a publish/promote already running on this content view
+            locked = helpers.check_running_publish(cvid, ver_descr[cvid])
+
+            if not locked:
+                msg = "Promoting '" + str(ver_descr[cvid]) + "' Version " + str(ver_version[cvid]) +\
+                    " from " + prior_env + " to " + str(target_env)
+                helpers.log_msg(msg, 'INFO')
+                print helpers.HEADER + msg + helpers.ENDC
+
+            if not dry_run and not locked:
+                try:
+                    task_id = helpers.post_json(
+                        helpers.KATELLO_API + "content_view_versions/" + str(ver_list[cvid]) +\
+                        "/promote/", json.dumps(
+                            {
+                                "environment_id": target_env_id
+                            }
+                            ))["id"]
+                except Warning:
+                    msg = "Failed to initiate promotion of " + str(ver_descr[cvid])
+                    helpers.log_msg(msg, 'WARNING')
+                else:
+                    task_list.append(task_id)
+                    ref_list[task_id] = ver_descr[cvid]
+
+        # Exit in the case of a dry-run
+        if dry_run:
+            msg = "Dry run - not actually performing promotion"
+            helpers.log_msg(msg, 'WARNING')
+        else:
+            # Monitor the status of the promotion tasks
+            helpers.watch_tasks(task_list, ref_list, task_name, quiet)
 
     # Exit in the case of a dry-run
     if dry_run:
-        msg = "Dry run - not actually performing promotion"
-        helpers.log_msg(msg, 'WARNING')
         sys.exit(2)
-
-
-    return task_list, ref_list, task_name
+    else:
+        return
 
 
 def main(args):
@@ -183,7 +193,6 @@ def main(args):
     global vardir
     dir = os.path.dirname(__file__)
     vardir = os.path.join(dir, 'var')
-#    confdir = os.path.join(dir, 'config')
 
     # Check for sane input
     parser = argparse.ArgumentParser(
@@ -242,6 +251,8 @@ def main(args):
     promote_list = []
     if not args.all:
         for x in helpers.CONFIG['promotion']:
+            if x == 'batch':
+                continue
             if helpers.CONFIG['promotion'][x]['name'] == target_env:
                 promote_list = helpers.CONFIG['promotion'][x]['content_views']
 
@@ -264,16 +275,12 @@ def main(args):
         promote_list)
 
     # Promote to the given environment. Returns a list of task IDs.
-    (task_list, ref_list, task_name) = promote(target_env, ver_list, ver_descr, ver_version,
-        env_list, prior_list, dry_run)
+    promote(target_env, ver_list, ver_descr, ver_version, env_list, prior_list, dry_run, args.quiet)
 
     # Add/Update the promotion history dictionary so we can check when we last promoted
     phistory[target_env] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
     pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
 
-    # Monitor the status of the promotion tasks
-    helpers.watch_tasks(task_list, ref_list, task_name, args.quiet)
-
     # Exit cleanly
     sys.exit(0)
 

publish_content_views.py

@@ -53,7 +53,7 @@ def get_cv(org_id, publish_list):
     return ver_list, ver_descr, ver_version
 
 
-def publish(ver_list, ver_descr, ver_version, dry_run, runuser):
+def publish(ver_list, ver_descr, ver_version, dry_run, runuser, quiet):
     """Publish Content View"""
 
     # Set the task name to be displayed in the task monitoring stage
@@ -69,43 +69,53 @@ def publish(ver_list, ver_descr, ver_version, dry_run, runuser):
         helpers.log_msg(msg, 'ERROR')
         sys.exit(1)
 
-    for cvid in ver_list.keys():
-
-        # Check if there is a publish/promote already running on this content view
-        locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid])
-
-        if not locked:
-            msg = "Publishing '" + str(ver_descr[cvid]) + "' Version " + str(ver_version[cvid]) + ".0"
-            helpers.log_msg(msg, 'INFO')
-            print helpers.HEADER + msg + helpers.ENDC
-
-        # Set up the description that will be added to the published version
-        description = "Published by " + runuser + "\n via API script"
-
-        if not dry_run and not locked:
-            try:
-                task_id = helpers.post_json(
-                    helpers.KATELLO_API + "content_views/" + str(ver_list[cvid]) +\
-                    "/publish", json.dumps(
-                        {
-                            "description": description
-                        }
-                        ))["id"]
-            except Warning:
-                msg = "Failed to initiate publication of " + str(ver_descr[cvid])
-                helpers.log_msg(msg, 'WARNING')
-            else:
-                task_list.append(task_id)
-                ref_list[task_id] = ver_descr[cvid]
+    # Break repos to publish into batches as configured in config.yml
+    cvchunks = [ ver_list.keys()[i:i+helpers.PUBLISHBATCH] for i in range(0, len(ver_list), helpers.PUBLISHBATCH) ]
+
+    # Loop through the smaller subsets of repo id's
+    for chunk in cvchunks:
+        for cvid in chunk:
+
+            # Check if there is a publish/promote already running on this content view
+            locked = helpers.check_running_publish(ver_list[cvid], ver_descr[cvid])
+
+            if not locked:
+                msg = "Publishing '" + str(ver_descr[cvid]) + "' Version " + str(ver_version[cvid]) + ".0"
+                helpers.log_msg(msg, 'INFO')
+                print helpers.HEADER + msg + helpers.ENDC
+
+            # Set up the description that will be added to the published version
+            description = "Published by " + runuser + "\n via API script"
+
+            if not dry_run and not locked:
+                try:
+                    task_id = helpers.post_json(
+                        helpers.KATELLO_API + "content_views/" + str(ver_list[cvid]) +\
+                        "/publish", json.dumps(
+                            {
+                                "description": description
+                            }
+                            ))["id"]
+                except Warning:
+                    msg = "Failed to initiate publication of " + str(ver_descr[cvid])
+                    helpers.log_msg(msg, 'WARNING')
+                else:
+                    task_list.append(task_id)
+                    ref_list[task_id] = ver_descr[cvid]
+
+        # Notify user in the case of a dry-run
+        if dry_run:
+            msg = "Dry run - not actually performing publish"
+            helpers.log_msg(msg, 'WARNING')
+        else:
+            # Wait for the tasks to finish
+            helpers.watch_tasks(task_list, ref_list, task_name, quiet)
 
     # Exit in the case of a dry-run
     if dry_run:
-        msg = "Dry run - not actually performing publish"
-        helpers.log_msg(msg, 'WARNING')
         sys.exit(2)
-
-
-    return task_list, ref_list, task_name
+    else:
+        return
 
 
 def main(args):
@@ -191,15 +201,12 @@ def main(args):
     (ver_list, ver_descr, ver_version) = get_cv(org_id, publish_list)
 
     # Publish the content views. Returns a list of task IDs.
-    (task_list, ref_list, task_name) = publish(ver_list, ver_descr, ver_version, dry_run, runuser)
+    publish(ver_list, ver_descr, ver_version, dry_run, runuser, args.quiet)
 
     # Add/Update the promotion history dictionary so we can check when we last promoted
     phistory['Library'] = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d')
     pickle.dump(phistory, open(vardir + '/promotions.pkl', 'wb'))
 
-    # Monitor the status of the publish tasks
-    helpers.watch_tasks(task_list, ref_list, task_name, args.quiet)
-
     # Exit cleanly
     sys.exit(0)