Skip to content

Commit cfdafb4

Browse files
committed
Merge branch 'feature_update_image_build_fixes' into 'master'
Add support for updating images and fix some bugs with image building See merge request evernym/utilities/devlab!8
2 parents e9eb251 + b8119cc commit cfdafb4

7 files changed

Lines changed: 124 additions & 78 deletions

File tree

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ persistent_data/
1010
build/
1111
artifacts/
1212

13+
### VSCode ###
14+
.vscode
15+
1316
### Kate ###
1417
# Swap Files #
1518
.*.kate-swp

devlab

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,17 @@ if __name__ == '__main__':
116116
PARSER_UP.set_defaults(func=devlab_bench.actions.up.action)
117117

118118
# Add subparser for update action
119-
PARSER_UPDATE = SUBPARSERS.add_parser('update', help='Update devlab to the latest released version')
120-
PARSER_UPDATE.add_argument('--uninstall', '-U', action='store_true', help='Instead of updating using the installer, uninstall it')
121-
PARSER_UPDATE.add_argument('--set-version', '-V', default=None, help='Update/Downgrade to a specific version of devlab')
119+
PARSER_UPDATE = SUBPARSERS.add_parser('update', help='Update images used by component using combination of docker pull and/or docker build')
120+
PARSER_UPDATE.add_argument('components', nargs='*', default='*', type=get_components, help='Update the image(s) used by one or more components based on name of glob match. COMPONENTS: {}'.format(', '.join(CUR_COMPONENTS)))
121+
PARSER_UPDATE.add_argument('--skip-base-images', '-B', action='store_true', help='Skip updating built-in base devlab images')
122122
PARSER_UPDATE.set_defaults(func=devlab_bench.actions.update.action)
123123

124+
# Add subparser for upgrade action
125+
PARSER_UPGRADE = SUBPARSERS.add_parser('upgrade', help='Upgrade devlab to the latest released version')
126+
PARSER_UPGRADE.add_argument('--uninstall', '-U', action='store_true', help='Instead of updating using the installer, uninstall it')
127+
PARSER_UPGRADE.add_argument('--set-version', '-V', default=None, help='Update/Downgrade to a specific version of devlab')
128+
PARSER_UPGRADE.set_defaults(func=devlab_bench.actions.upgrade.action)
129+
124130
# Add subparser for restart
125131
PARSER_RESTART = SUBPARSERS.add_parser('restart', help='Restart components')
126132
PARSER_RESTART.add_argument('components', nargs='*', default='*', type=get_components, help='Stop and start a specific component(s) or glob match. COMPONENTS: {}'.format(', '.join(CUR_COMPONENTS)))
@@ -144,7 +150,7 @@ if __name__ == '__main__':
144150

145151
#The 'update' action is special and doesn't need all of the checks or a devlab_bench.PROJ_ROOT etc..
146152
#it also will exit after executing
147-
if ARGS.func in [devlab_bench.actions.update.action, devlab_bench.actions.global_status.action, action_default]:
153+
if ARGS.func in [devlab_bench.actions.upgrade.action, devlab_bench.actions.global_status.action, action_default]:
148154
ARGS.func(**vars(ARGS))
149155

150156
if ARGS.project_root:
@@ -153,7 +159,7 @@ if __name__ == '__main__':
153159
if not devlab_bench.PROJ_ROOT:
154160
#Running Adhoc without a project won't get us a DockerHelper object.
155161
#this will create one if none has been set.
156-
if ARGS.func == devlab_bench.actions.shell.action and ARGS.components == ['adhoc']: #pylint: disable=comparison-with-callable
162+
if ARGS.func == devlab_bench.actions.shell.action and ARGS.components == ['adhoc']: #pylint: disable=bad-option-value,comparison-with-callable
157163
devlab_bench.PROJ_ROOT = os.path.abspath('.')
158164
devlab_bench.helpers.docker.DOCKER = DockerHelper(
159165
labels=[
@@ -177,7 +183,7 @@ if __name__ == '__main__':
177183
devlab_bench.CONFIG = get_config()
178184

179185
#If we're doing an 'up' action, check for and run wizard
180-
if ARGS.func == devlab_bench.actions.up.action: #pylint: disable=comparison-with-callable
186+
if ARGS.func == devlab_bench.actions.up.action: #pylint: disable=bad-option-value,comparison-with-callable
181187
if devlab_bench.CONFIG['wizard_enabled']:
182188
if os.path.isfile('{}/wizard'.format(devlab_bench.PROJ_ROOT)):
183189
LOGGER.debug("Running wizard, in case it needs to be run")

devlab_bench/actions/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import devlab_bench.actions.shell
1010
import devlab_bench.actions.status
1111
import devlab_bench.actions.up
12+
import devlab_bench.actions.upgrade
1213
import devlab_bench.actions.update
1314

1415
__all__ = [

devlab_bench/actions/build.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
from devlab_bench.helpers.common import get_config, get_ordinal_sorting
1212
from devlab_bench.helpers.docker import docker_obj_status, DockerHelper, get_needed_images
1313

14-
def action(images='*', clean=False, no_cache=False, pull=False, **kwargs):
14+
def action(images='*', clean=False, no_cache=False, pull=False, skip_pull_images=None, **kwargs):
1515
"""
1616
This is responsible for building all the docker images etc...
1717
@@ -23,6 +23,8 @@ def action(images='*', clean=False, no_cache=False, pull=False, **kwargs):
2323
building. Default=False
2424
pull: boolean indicating whether a --pull should happen to update base
2525
images. Default=False
26+
skip_pull_images: list of image names to skip when 'pull' is specified
27+
Default=None
2628
Returns:
2729
None
2830
"""
@@ -46,6 +48,8 @@ def action(images='*', clean=False, no_cache=False, pull=False, **kwargs):
4648
],
4749
common_domain=config['domain']
4850
)
51+
if not skip_pull_images:
52+
skip_pull_images = []
4953
if not docker_helper:
5054
log.warning("No docker_helper was passed, using default helper... This is probably not intended?")
5155
docker_helper = DockerHelper(filter_label='com.lab.type=devlab')
@@ -95,13 +99,14 @@ def action(images='*', clean=False, no_cache=False, pull=False, **kwargs):
9599
else:
96100
image_n_tag = '{}:{}'.format(image, images_dict[image]['tag'])
97101
image_status = docker_obj_status(image_n_tag, 'image', devlab_bench.helpers.docker.DOCKER, logger=log)[0]
98-
image_context = PROJ_ROOT
99-
if image in base_images_to_build:
102+
image_context = os.path.dirname('{}/{}'.format(PROJ_ROOT, images_dict[image]['docker_file']))
103+
docker_helper_obj = docker_helper
104+
build_context = PROJ_ROOT
105+
if image in base_images_to_build: #Override default build context for built-in images
106+
build_context = DEVLAB_ROOT
100107
image_context = DEVLAB_ROOT
101108
docker_helper_obj = docker_helper_base
102-
else:
103-
docker_helper_obj = docker_helper
104-
images_dict[image]['docker_file_full_path'] = '{}/{}'.format(image_context, images_dict[image]['docker_file'])
109+
images_dict[image]['docker_file_full_path'] = '{}/{}'.format(build_context, images_dict[image]['docker_file'])
105110
if 'build_opts' not in images_dict[image]:
106111
images_dict[image]['build_opts'] = []
107112
if image in base_images_to_build:
@@ -130,11 +135,11 @@ def action(images='*', clean=False, no_cache=False, pull=False, **kwargs):
130135
log.debug(line)
131136
log.debug("Successfully removed image: %s", image)
132137
if pull:
133-
with open(images_dict[image]['docker_file_full_path']) as dfile:
138+
with open(images_dict[image]['docker_file_full_path']) as dfile:
134139
local_image = False
135140
for line in dfile.readlines():
136141
if line.startswith('FROM '):
137-
if line.split()[1].split(':')[0] in images_to_build:
142+
if line.split()[1].split(':')[0] in images_to_build + skip_pull_images:
138143
local_image = True
139144
log.debug("Skipping pull, as devlab manages this image's base image")
140145
break

devlab_bench/actions/up.py

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import devlab_bench
1111
import devlab_bench.actions.reset
12+
from devlab_bench.actions.update import update_component_images
1213
from devlab_bench.helpers.docker import get_needed_images, docker_obj_status, check_custom_registry
1314
from devlab_bench.helpers.common import get_config, get_env_from_file, get_ordinal_sorting, get_shell_components, get_primary_ip, quote, save_env_file, script_runner, unnest_list
1415

@@ -69,7 +70,7 @@ def action(components='*', skip_provision=False, bind_to_host=False, keep_up_on_
6970
log.debug("The following components will be started in this order: %s", ', '.join(components_to_run))
7071
if update_images:
7172
log.info("Looking for and updating images needed by components: %s", ','.join(components_to_run))
72-
update_component_images(components=components_to_run)
73+
update_component_images(components=components_to_run, skip_base_images=True)
7374
needed_images = get_needed_images()
7475
if needed_images['base_images']['missing'] or needed_images['base_images']['needs_update']:
7576
base_to_build = needed_images['base_images']['missing'] + needed_images['base_images']['needs_update']
@@ -328,34 +329,3 @@ def component_up(name, comp_config, skip_provision=False, keep_up_on_error=False
328329
break
329330
break
330331
return not errors
331-
332-
def update_component_images(components=None, logger=None):
333-
"""
334-
Look through given components and try to build or pull new versions of the
335-
image layers etc...
336-
337-
Args:
338-
components: list, of components to use for finding images to update
339-
logger: Logger object to use for log messages
340-
341-
Returns:
342-
None
343-
"""
344-
if logger:
345-
log = logger
346-
else:
347-
log = logging.getLogger('update_images')
348-
log.debug('Looking up images being referenced in components')
349-
needed_images = get_needed_images(components, logger=log)
350-
ext_images = needed_images['external_images']['exists'] + needed_images['external_images']['missing']
351-
int_images = needed_images['base_images']['exists'] + needed_images['base_images']['missing']
352-
int_images += needed_images['runtime_images']['exists'] + needed_images['runtime_images']['missing']
353-
log_output = True
354-
log.info("Building/Updating devlab and project's managed images: '%s'", ','.join(int_images))
355-
devlab_bench.actions.build.action(int_images, clean=True, pull=True)
356-
for ext_image in ext_images:
357-
log.info("Pulling down any updates to image: '%s'", ext_image)
358-
pi_res = devlab_bench.helpers.docker.DOCKER.pull_image(ext_image, log_output=log_output, logger=log)
359-
if pi_res[0] != 0:
360-
log.error("Failed pulling updates for image: %s", ext_image)
361-
sys.exit(1)

devlab_bench/actions/update.py

Lines changed: 45 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,61 @@
11
"""
2-
Things dealing with the 'update' action
2+
Deals with updating images used by components
33
"""
44
import logging
55
import sys
66

7-
import devlab_bench
8-
from devlab_bench.helpers.command import Command
7+
import devlab_bench.helpers.docker
8+
import devlab_bench.actions.build
9+
from devlab_bench.helpers.docker import get_needed_images
910

10-
def action(uninstall=False, set_version=None, **kwargs):
11+
def action(components='*', skip_base_images=False, **kwargs):
1112
"""
12-
Attempt to update devlab. This will cause call sys.exit
13+
This is for updating images used by components
1314
14-
exit code is 0 upon success and 1 if not
15+
Args:
16+
components: list of components or image names to update, this can also be
17+
the string '*'
18+
include_base_images: Bool whether to inlcude the devlab base images when updating
19+
Returns:
20+
None
1521
"""
16-
if update_devlab(uninstall=uninstall, set_version=set_version, **kwargs):
17-
sys.exit(0)
18-
else:
19-
sys.exit(1)
22+
ignored_args = kwargs
23+
log = logging.getLogger("UpdateImages")
24+
if components == '*':
25+
components = None
26+
update_component_images(components, skip_base_images=skip_base_images, logger=log)
2027

21-
def update_devlab(uninstall=False, set_version=None, **kwargs):
28+
def update_component_images(components=None, skip_base_images=True, logger=None):
2229
"""
23-
Use the installer to try and update devlab to the latest version in the repo
30+
Look through given components and try to build or pull new versions of the
31+
image layers etc...
2432
2533
Args:
26-
uninstall: bool, indicating to uninstall instead of update
27-
set_version: str, indicating a specific version of devlab to install
34+
components: list, of components to use for finding images to update
35+
include_base_images: Bool whether to include base images when updating
36+
logger: Logger object to use for log messages
2837
2938
Returns:
30-
Bool, True if successful False if not
39+
None
3140
"""
32-
ignored_args = kwargs
33-
log = logging.getLogger("UpdateDevlab")
34-
log.debug("Running installer.py to check for updates etc...")
35-
command = '{}/installer.py'.format(devlab_bench.DEVLAB_ROOT)
36-
args = []
37-
if uninstall and set_version:
38-
log.error("Cannot uninstall a specific version. Uninstall takes no argument")
39-
return False
40-
if uninstall:
41-
args.append('uninstall')
42-
if set_version:
43-
args += ['install', '--set-version', set_version]
44-
inst_out = Command(command, args, interactive=True).run()
45-
if inst_out[0] != 0:
46-
log.error("Installer did not exit successfully... Aborting!")
47-
return False
48-
return True
41+
if logger:
42+
log = logger
43+
else:
44+
log = logging.getLogger('update_images')
45+
log.debug('Looking up images being referenced in components')
46+
needed_images = get_needed_images(components, logger=log)
47+
ext_images = needed_images['external_images']['exists'] + needed_images['external_images']['missing']
48+
int_images = []
49+
base_images = needed_images['base_images']['exists'] + needed_images['base_images']['missing']
50+
if not skip_base_images:
51+
int_images += base_images
52+
int_images += needed_images['runtime_images']['exists'] + needed_images['runtime_images']['missing']
53+
log_output = True
54+
log.info("Building/Updating devlab and project's managed images: '%s'", ','.join(int_images))
55+
devlab_bench.actions.build.action(int_images, skip_pull_images=base_images, clean=True, pull=True)
56+
for ext_image in ext_images:
57+
log.info("Pulling down any updates to image: '%s'", ext_image)
58+
pi_res = devlab_bench.helpers.docker.DOCKER.pull_image(ext_image, log_output=log_output, logger=log)
59+
if pi_res[0] != 0:
60+
log.error("Failed pulling updates for image: %s", ext_image)
61+
sys.exit(1)

devlab_bench/actions/upgrade.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
"""
2+
Things dealing with the 'update' action
3+
"""
4+
import logging
5+
import sys
6+
7+
import devlab_bench
8+
from devlab_bench.helpers.command import Command
9+
10+
def action(uninstall=False, set_version=None, **kwargs):
11+
"""
12+
Attempt to update devlab. This will call sys.exit
13+
14+
exit code is 0 upon success and 1 if not
15+
"""
16+
if update_devlab(uninstall=uninstall, set_version=set_version, **kwargs):
17+
sys.exit(0)
18+
else:
19+
sys.exit(1)
20+
21+
def update_devlab(uninstall=False, set_version=None, **kwargs):
22+
"""
23+
Use the installer to try and update devlab to the latest version in the repo
24+
25+
Args:
26+
uninstall: bool, indicating to uninstall instead of update
27+
set_version: str, indicating a specific version of devlab to install
28+
29+
Returns:
30+
Bool, True if successful False if not
31+
"""
32+
ignored_args = kwargs
33+
log = logging.getLogger("UpdateDevlab")
34+
log.debug("Running installer.py to check for updates etc...")
35+
command = '{}/installer.py'.format(devlab_bench.DEVLAB_ROOT)
36+
args = []
37+
if uninstall and set_version:
38+
log.error("Cannot uninstall a specific version. Uninstall takes no argument")
39+
return False
40+
if uninstall:
41+
args.append('uninstall')
42+
if set_version:
43+
args += ['install', '--set-version', set_version]
44+
inst_out = Command(command, args, interactive=True).run()
45+
if inst_out[0] != 0:
46+
log.error("Installer did not exit successfully... Aborting!")
47+
return False
48+
return True

0 commit comments

Comments
 (0)