Skip to content

Commit 2a078e2

Browse files
authored
Add parallel AWX-to-Ascender migration role (5x faster pg_restore) (#193)
* add awx_migrate_ascender role for parallel AWX 24.6.1 to Ascender 25.3.5 migration * add awx_templates_with_enabled_notifications lookup plugin
1 parent 6c1e9e3 commit 2a078e2

23 files changed

Lines changed: 2280 additions & 0 deletions
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
---
2+
name: "AWX - Migrate to Ascender"
3+
description: "Migrate AWX production database into Ascender"
4+
project: "Ascender Install"
5+
organization: "Default"
6+
inventory: "localhost"
7+
playbook: "playbooks/awx_migrate_ascender.yml"
8+
credentials: []
9+
execution_environment: ""
10+
instance_groups: []
11+
verbosity: 0
12+
ask_variables_on_launch: false
13+
job_tags: "prep,download,secrets,pg_restore,migrate,cleanup,patch_cr,startup,configure,verify"
14+
ask_tags_on_launch: true
15+
ask_skip_tags_on_launch: true
16+
survey_enabled: true
17+
survey:
18+
name: "AWX Migrate Ascender Survey"
19+
description: "Configure AWX to Ascender migration"
20+
spec:
21+
- question_name: "Ascender Namespace"
22+
question_description: "Kubernetes namespace where Ascender is installed"
23+
variable: "ascender_namespace"
24+
type: "text"
25+
required: true
26+
default: "ascender"
27+
- question_name: "Ascender Instance"
28+
question_description: "AWX CR instance name in the namespace"
29+
variable: "ascender_instance"
30+
type: "text"
31+
required: true
32+
default: "ascender"
33+
- question_name: "K8s API URL"
34+
question_description: "Kubernetes API URL for the target cluster"
35+
variable: "k8s_url"
36+
type: "text"
37+
required: true
38+
default: ""
39+
- question_name: "K8s API Token"
40+
question_description: "Service account token for K8s API authentication"
41+
variable: "k8s_token"
42+
type: "password"
43+
required: true
44+
default: ""
45+
- question_name: "S3 Endpoint"
46+
question_description: "S3-compatible storage endpoint URL"
47+
variable: "s3_endpoint"
48+
type: "text"
49+
required: true
50+
default: ""
51+
- question_name: "S3 Access Key"
52+
question_description: "S3 access key"
53+
variable: "s3_accesskey"
54+
type: "password"
55+
required: true
56+
default: ""
57+
- question_name: "S3 Secret Key"
58+
question_description: "S3 secret key"
59+
variable: "s3_secretkey"
60+
type: "password"
61+
required: true
62+
default: ""
63+
- question_name: "S3 Bucket"
64+
question_description: "S3 bucket containing AWX backups"
65+
variable: "s3_bucket"
66+
type: "text"
67+
required: true
68+
default: ""
69+
- question_name: "Source AWX Instance"
70+
question_description: "AWX CR instance name on the source cluster (for secret name mapping)"
71+
variable: "source_awx_instance"
72+
type: "text"
73+
required: true
74+
default: "awx"
75+
- question_name: "pg_restore Progress Interval"
76+
question_description: "Seconds between pg_restore progress logs showing DB size and active queries"
77+
variable: "restore_poll_interval"
78+
type: "integer"
79+
required: false
80+
default: 30
81+
min: 10
82+
max: 300
83+
- question_name: "pg_restore Parallel Jobs"
84+
question_description: "Number of parallel pg_restore workers (max limited by vCPU)"
85+
variable: "pg_restore_jobs"
86+
type: "integer"
87+
required: false
88+
default: 16
89+
min: 1
90+
max: 64

playbooks/awx_migrate_ascender.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
- name: Migrate AWX to Ascender
3+
hosts: localhost
4+
gather_facts: true
5+
connection: local
6+
7+
roles:
8+
- role: awx_migrate_ascender
Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
#!/usr/bin/python
2+
# -*- coding: utf-8 -*-
3+
4+
DOCUMENTATION = '''
5+
lookup: awx_templates_with_enabled_notifications
6+
short_description: Find AWX job templates with notifications
7+
description:
8+
- This lookup returns all AWX job templates that have notification templates configured
9+
options:
10+
host:
11+
description: The AWX host URL
12+
required: True
13+
username:
14+
description: The AWX username
15+
required: True
16+
password:
17+
description: The AWX password
18+
required: True
19+
max_workers:
20+
description: Maximum number of concurrent workers
21+
required: False
22+
default: 30
23+
type: int
24+
page_size:
25+
description: Number of items to fetch per page
26+
required: False
27+
default: 400
28+
type: int
29+
verify_ssl:
30+
description:
31+
- Whether to verify SSL certificates or not
32+
- Useful for environments with self-signed certificates or corporate proxies
33+
- Setting to false also disables SSL warnings
34+
required: False
35+
default: True
36+
type: bool
37+
notes:
38+
- Returns a dictionary with template information, detailed notification data, and performance metrics
39+
'''
40+
41+
EXAMPLES = '''
42+
- name: Show job templates with notifications
43+
debug:
44+
msg: "{{ lookup('awx_notification_templates', host='https://awx.example.com', username='admin', password='password') }}"
45+
46+
- name: Get job templates with notifications using 10 workers
47+
set_fact:
48+
notification_data: "{{ lookup('awx_notification_templates', host=awx_host, username=awx_username, password=awx_password, max_workers=10)[0] }}"
49+
50+
- name: Use specific parts of the results
51+
debug:
52+
msg: "Found {{ notification_data.count }} templates with notifications"
53+
54+
- name: Loop through templates with notifications
55+
debug:
56+
msg: "Template {{ item.name }} (ID: {{ item.id }}) has {{ item.notification_types | join(', ') }} notifications"
57+
loop: "{{ notification_data.job_templates }}"
58+
59+
- name: Use with SSL verification disabled
60+
debug:
61+
msg: "{{ lookup('awx_notification_templates', host='https://awx.example.com', username='admin', password='password', verify_ssl=false) }}"
62+
'''
63+
64+
RETURN = '''
65+
_list:
66+
description:
67+
- List containing a single dictionary with job template notification information
68+
type: list
69+
elements: dict
70+
contains:
71+
job_templates:
72+
description: List of dictionaries with template id, name, and notification types
73+
type: list
74+
returned: always
75+
count:
76+
description: Count of templates with notifications
77+
type: int
78+
returned: always
79+
total_templates:
80+
description: Total number of job templates in AWX
81+
type: int
82+
returned: always
83+
execution_time_seconds:
84+
description: Time taken to execute the lookup in seconds
85+
type: float
86+
returned: always
87+
workers_used:
88+
description: Number of concurrent workers used
89+
type: int
90+
returned: always
91+
'''
92+
93+
from ansible.plugins.lookup import LookupBase
94+
from ansible.errors import AnsibleError, AnsibleParserError
95+
from ansible.utils.display import Display
96+
from ansible.module_utils.parsing.convert_bool import boolean
97+
98+
import requests
99+
import json
100+
import time
101+
from urllib3.exceptions import InsecureRequestWarning
102+
from concurrent.futures import ThreadPoolExecutor
103+
import threading
104+
105+
display = Display()
106+
107+
class LookupModule(LookupBase):
108+
109+
def run(self, terms, variables=None, **kwargs):
110+
start_time = time.time()
111+
112+
self.set_options(var_options=variables, direct=kwargs)
113+
114+
# Get parameters
115+
host = self.get_option('host')
116+
username = self.get_option('username')
117+
password = self.get_option('password')
118+
max_workers = self.get_option('max_workers', 30)
119+
page_size = self.get_option('page_size', 400)
120+
verify_ssl = self.get_option('verify_ssl', True)
121+
122+
# Convert verify_ssl to boolean if it's not already
123+
if not isinstance(verify_ssl, bool):
124+
verify_ssl = boolean(verify_ssl)
125+
126+
# Disable SSL warnings if verify_ssl is False
127+
if not verify_ssl:
128+
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
129+
130+
if not all([host, username, password]):
131+
raise AnsibleError("Missing required parameters: host, username, and password are required")
132+
133+
try:
134+
# Setup authentication
135+
auth = (username, password)
136+
headers = {'Content-Type': 'application/json'}
137+
138+
display.vvv(f"Fetching job templates from {host} with page size {page_size}")
139+
140+
# Get all job templates with pagination
141+
template_info = [] # Store both id and name
142+
next_url = f"{host}/api/v2/job_templates/?page_size={page_size}"
143+
144+
while next_url:
145+
templates_response = requests.get(next_url, auth=auth, headers=headers, verify=verify_ssl)
146+
147+
if templates_response.status_code != 200:
148+
raise AnsibleError(f"Failed to get job templates: {templates_response.status_code}")
149+
150+
templates_data = templates_response.json()
151+
# Store both id and name for each template
152+
template_info.extend([{'id': template['id'], 'name': template['name']} for template in templates_data.get('results', [])])
153+
154+
# Get next page URL
155+
next_url = templates_data.get('next')
156+
if next_url and not next_url.startswith('http'):
157+
next_url = f"{host}{next_url}"
158+
159+
display.vvv(f"Found {len(template_info)} total job templates, checking for notifications with {max_workers} workers")
160+
161+
# Templates with notifications
162+
templates_with_notifications = []
163+
lock = threading.Lock()
164+
165+
# Create a session for connection pooling
166+
session = requests.Session()
167+
168+
def check_template(template_data):
169+
template_id = template_data['id']
170+
template_name = template_data['name']
171+
template_info = {'id': template_id, 'name': template_name, 'notification_types': []}
172+
has_notifications = False
173+
174+
# Check each notification type
175+
for notification_type in ['started', 'success', 'error']:
176+
url = f"{host}/api/v2/job_templates/{template_id}/notification_templates_{notification_type}/"
177+
response = session.get(url, auth=auth, headers=headers, verify=verify_ssl)
178+
179+
if response.status_code == 200:
180+
data = response.json()
181+
count = data.get('count', 0)
182+
183+
if count > 0:
184+
has_notifications = True
185+
template_info['notification_types'].append(notification_type)
186+
187+
if has_notifications:
188+
with lock:
189+
templates_with_notifications.append(template_info)
190+
191+
# Use ThreadPoolExecutor for concurrent processing
192+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
193+
executor.map(check_template, template_info)
194+
195+
elapsed_time = time.time() - start_time
196+
197+
# Return with enhanced information
198+
return [{
199+
"job_templates": templates_with_notifications,
200+
"count": len(templates_with_notifications),
201+
"total_templates": len(template_info),
202+
"execution_time_seconds": round(elapsed_time, 2),
203+
"workers_used": max_workers
204+
}]
205+
206+
except Exception as e:
207+
raise AnsibleError(f"Error in awx_notification_templates lookup: {str(e)}")

0 commit comments

Comments
 (0)