From 5b7f5fcb20a92c91cd1e3458b35eb34f825fef81 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:05:59 -0600 Subject: [PATCH 1/7] Fix typo in comment --- ci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build.sh b/ci/build.sh index c4449f8..6789c16 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -35,7 +35,7 @@ else release_ver=$(echo "$legacy_last_release" | cut -d ' ' -f 3) fi fi -echo "Getting changes comit changes between: ${beg_range}${last_change}" +echo "Getting commit changes between: ${beg_range}${last_change}" changes_since_release=$(git log --pretty=format:'%s%n' ${beg_range}${last_change} | sed "/^Merge branch '.\+' into '.\+'/d ; /^Merge branch '.\+' of .\+/d; /^$/d") if [ $(echo "$changes_since_release" | wc -l) -gt 0 ] ; then From b487c9ec29196a1c98d890c5780a377c37b901e5 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:06:50 -0600 Subject: [PATCH 2/7] Make tmpfs volume mounts configurable --- devlab_bench/helpers/docker.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/devlab_bench/helpers/docker.py b/devlab_bench/helpers/docker.py index 8810a37..ccb825e 100644 --- a/devlab_bench/helpers/docker.py +++ b/devlab_bench/helpers/docker.py @@ -452,7 +452,7 @@ def rm_image(self, name): logger=self.log ).run() return cmd_ret - def run_container(self, image, name, network=None, ports=None, background=True, interactive=False, ignore_nonzero_rc=False, cmd=None, logger=None, mounts=None, systemd_support=False, run_opts=None, **kwargs): #pylint: disable=too-many-arguments + def run_container(self, image, name, network=None, ports=None, background=True, interactive=False, ignore_nonzero_rc=False, cmd=None, logger=None, mounts=None, systemd_support=False, systemd_tmpfs_args=None, run_opts=None, **kwargs): #pylint: disable=too-many-arguments """ Run a docker_container @@ -473,6 +473,8 @@ def run_container(self, image, name, network=None, ports=None, background=True, 'docker run'. (OPTIONAL) systemd_support: bool, whether to enable opts to let systemd work inside the container. (OPTIONAL) + systemd_tmpfs_args: comma separated string of arguments to pass to + the --tmpfs argument when systemd_support is True Returns: tuple where: First Element is the return code from docker @@ -498,10 +500,12 @@ def run_container(self, image, name, network=None, ports=None, background=True, if network: opts.append("--network={}".format(network)) if systemd_support: + if systemd_tmpfs_args: + systemd_tmpfs_args=':{}'.format(systemd_tmpfs_args) opts += [ - '--tmpfs=/run', - '--tmpfs=/run/lock', - '--tmpfs=/tmp', + '--tmpfs=/run{}'.format(systemd_tmpfs_args), + '--tmpfs=/run/lock{}'.format(systemd_tmpfs_args), + '--tmpfs=/tmp{}'.format(systemd_tmpfs_args), '--volume=/sys/fs/cgroup:/sys/fs/cgroup:ro', '-t' #This is needed so that 'docker logs' will show systemd output ] From ca25171d631741968d54ee7ce6f89024c390efb6 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:07:38 -0600 Subject: [PATCH 3/7] Fix typo in module doc string --- devlab_bench/actions/global_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlab_bench/actions/global_status.py b/devlab_bench/actions/global_status.py index 2e25ba5..e0b7df6 100644 --- a/devlab_bench/actions/global_status.py +++ b/devlab_bench/actions/global_status.py @@ -1,5 +1,5 @@ """ -Things dealing with the 'global_satus' action +Things dealing with the 'global_status' action """ import sys From b16e70d74462f6e9fe96e0937a7a3b857561201c Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:08:38 -0600 Subject: [PATCH 4/7] Make the build action actually use the shared pythong package vars --- devlab_bench/actions/build.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/devlab_bench/actions/build.py b/devlab_bench/actions/build.py index 73de138..10a9af6 100644 --- a/devlab_bench/actions/build.py +++ b/devlab_bench/actions/build.py @@ -6,8 +6,8 @@ import os import sys +import devlab_bench import devlab_bench.helpers.docker -from devlab_bench import DEVLAB_ROOT, IMAGES, PROJ_ROOT from devlab_bench.helpers.common import get_config, get_ordinal_sorting from devlab_bench.helpers.docker import docker_obj_status, DockerHelper, get_needed_images @@ -39,7 +39,7 @@ def action(images='*', clean=False, no_cache=False, pull=False, skip_pull_images log.debug("Will build with no_cache set to: %s", no_cache) log.debug("Will build with pull set to: %s", pull) config = get_config() - base_images_dict = dict(IMAGES) + base_images_dict = dict(devlab_bench.IMAGES) images_dict = dict(base_images_dict) docker_helper = devlab_bench.helpers.docker.DOCKER docker_helper_base = DockerHelper( @@ -99,12 +99,12 @@ def action(images='*', clean=False, no_cache=False, pull=False, skip_pull_images else: image_n_tag = '{}:{}'.format(image, images_dict[image]['tag']) image_status = docker_obj_status(image_n_tag, 'image', devlab_bench.helpers.docker.DOCKER, logger=log)[0] - image_context = os.path.dirname('{}/{}'.format(PROJ_ROOT, images_dict[image]['docker_file'])) + image_context = os.path.dirname('{}/{}'.format(devlab_bench.PROJ_ROOT, images_dict[image]['docker_file'])) docker_helper_obj = docker_helper - build_context = PROJ_ROOT + build_context = devlab_bench.PROJ_ROOT if image in base_images_to_build: #Override default build context for built-in images - build_context = DEVLAB_ROOT - image_context = DEVLAB_ROOT + build_context = devlab_bench.DEVLAB_ROOT + image_context = devlab_bench.DEVLAB_ROOT docker_helper_obj = docker_helper_base images_dict[image]['docker_file_full_path'] = '{}/{}'.format(build_context, images_dict[image]['docker_file']) if 'build_opts' not in images_dict[image]: From 2f671d74d5123fd1a9e9235f193d9099f90a73dd Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:09:38 -0600 Subject: [PATCH 5/7] Add global-restart action for updating multiple envs --- devlab | 12 ++++- devlab_bench/actions/__init__.py | 2 + devlab_bench/actions/global_restart.py | 62 ++++++++++++++++++++++++++ devlab_bench/actions/restart.py | 7 ++- 4 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 devlab_bench/actions/global_restart.py diff --git a/devlab b/devlab index 9826012..f2357a0 100755 --- a/devlab +++ b/devlab @@ -98,6 +98,11 @@ if __name__ == '__main__': PARSER_RESET.add_argument('--full', '-f', action='store_true', help='Remove all component specific files, wizard files, as well as devlab files AND potentially files you\'re working on. BE CAREFUL IF YOU HAVE MANUAL CHANGES IN PATHS DEFINED IN YOUR \'paths.reset_full\'!!') PARSER_RESET.set_defaults(func=devlab_bench.actions.reset.action) + #Add Subparser for global_restart action + PARSER_GLOBAL_RESTART = SUBPARSERS.add_parser('global-restart', help='Restart components across all environments managed by devlab') + PARSER_GLOBAL_RESTART.add_argument('--update-images', '-u', action='store_true', help='Look for images that components are using, and try to either build new versions, or pull new ones') + PARSER_GLOBAL_RESTART.set_defaults(func=devlab_bench.actions.global_restart.action) + #Add Subparser for global_status action PARSER_GLOBAL_STATUS = SUBPARSERS.add_parser('global-status', help='Get a global status of all environments where devlab has created containers') PARSER_GLOBAL_STATUS.set_defaults(func=devlab_bench.actions.global_status.action) @@ -150,7 +155,12 @@ if __name__ == '__main__': #The 'update' action is special and doesn't need all of the checks or a devlab_bench.PROJ_ROOT etc.. #it also will exit after executing - if ARGS.func in [devlab_bench.actions.upgrade.action, devlab_bench.actions.global_status.action, action_default]: + if ARGS.func in [ + devlab_bench.actions.upgrade.action, + devlab_bench.actions.global_restart.action, + devlab_bench.actions.global_status.action, + action_default + ]: ARGS.func(**vars(ARGS)) if ARGS.project_root: diff --git a/devlab_bench/actions/__init__.py b/devlab_bench/actions/__init__.py index da67a22..e65ea32 100644 --- a/devlab_bench/actions/__init__.py +++ b/devlab_bench/actions/__init__.py @@ -3,6 +3,7 @@ """ import devlab_bench.actions.build import devlab_bench.actions.down +import devlab_bench.actions.global_restart import devlab_bench.actions.global_status import devlab_bench.actions.restart import devlab_bench.actions.reset @@ -15,6 +16,7 @@ __all__ = [ 'build', 'down', + 'global_restart', 'global_status', 'restart', 'reset', diff --git a/devlab_bench/actions/global_restart.py b/devlab_bench/actions/global_restart.py new file mode 100644 index 0000000..a41af5d --- /dev/null +++ b/devlab_bench/actions/global_restart.py @@ -0,0 +1,62 @@ +""" +Things dealing with the 'global_restart' action +""" +import sys +import logging + +import devlab_bench +from devlab_bench.helpers.common import get_config +from devlab_bench.helpers.docker import DockerHelper +def action(**kwargs): + """ + Restart all containers spun up with devlab across all environments + """ + log = logging.getLogger('Global-Restart') + restart_args = kwargs + global_devlab_docker = DockerHelper( + filter_label='com.lab.type=devlab' + ) + containers = global_devlab_docker.get_containers()[1] + project_map = {} + log.info("containers='%s'", containers) + for cont in containers: + cont_name = cont['name'] + comp_name = cont_name + comp_logger = logging.getLogger('{}-{}'.format(log.name, cont_name)) + details = global_devlab_docker.inspect_container(cont_name)[0] + labels = details['Config']['Labels'] + container_project = None + for label in labels: + if label == 'com.lab.project': + container_project = labels[label] + break + if not container_project: + log.error('Container: "%s" does not have a project path defined in the label "com.lab.project"', cont_name) + sys.exit(1) + if cont_name.endswith('-devlab'): + comp_name = cont_name[0:len(cont_name)-7] + try: + project_map[container_project]['components'].append(comp_name) + except KeyError: + project_map[container_project] = { + 'components': [comp_name] + } + for project in project_map: + restart_args['components'] = [] + comp_logger.info('Performing restarts for project at path: %s', project) + comp_logger.debug('Project components to restart: "%s"', ','.join(project_map[project]['components'])) + #Switch to other Devlab env: + devlab_bench.PROJ_ROOT = project + devlab_bench.CONFIG = get_config(force_reload=True) + devlab_bench.helpers.docker.DOCKER = DockerHelper( + labels=[ + 'com.lab.type=devlab', + 'com.lab.project={}'.format(project), + devlab_bench.CONFIG['project_filter'] + ] + ) + #Customize the components to pass to the restart action + restart_args['components'] = project_map[project]['components'] + #Execute the restart action + devlab_bench.actions.restart.action(logger=logging.getLogger('{}-{}'.format(comp_logger.name, 'Restart')), **restart_args) + sys.exit(0) diff --git a/devlab_bench/actions/restart.py b/devlab_bench/actions/restart.py index a8a5ff9..502d9ea 100644 --- a/devlab_bench/actions/restart.py +++ b/devlab_bench/actions/restart.py @@ -6,7 +6,7 @@ import devlab_bench.actions.down import devlab_bench.actions.up from devlab_bench.helpers.common import get_components, unnest_list -def action(components='*', update_images=False, **kwargs): +def action(components='*', logger=None, update_images=False, **kwargs): """ Restart components by bringing them down and then back up again @@ -15,7 +15,10 @@ def action(components='*', update_images=False, **kwargs): """ #restart ignored_args = kwargs - log = logging.getLogger('Restart') + if logger: + log = logger + else: + log = logging.getLogger('Restart') components_to_restart = components rm = False if isinstance(components, str): From c953721462af6c10fb02c711321a2bbea9f15e2e Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:10:00 -0600 Subject: [PATCH 6/7] Update README.md to reflect recent updates --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b8d1ae1..118437a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If you would like contribute to this project, please do the following: 1. **Wizard**: A script that is run before the `up` action. After the wizard has been executed a proper [DevlabConfig.json or DevlabConfig.yaml](#devlab-configuration) should exist. It is normal for the wizard to result in more files than just a `DevlabConfig.json` or `DevlabConfig.yaml`, and those files can be added to the config so that devlab can reset the wizard (forcing it to run again) if so desired. # Usage -All actions have a `--help` which should display relevent help. The following options below are "global" options and *should* preceed any action: +All actions have a `--help` which should display relevent help. The following options below are "common" options and *should* preceed any action: ``` -h, --help show this help message and exit @@ -49,7 +49,7 @@ All actions have a `--help` which should display relevent help. The following op ``` The format of the command should be: -`devlab ` +`devlab ` The "wizard" logic will be invoked the first time you run a `devlab up` command. This can be invoked separately with `wizard` if you so desire. @@ -128,6 +128,7 @@ The structure looks like this: { "image": "", "systemd_support": false, + "systemd_tmpfs_args": "", "enabled": false, "cmd": "", "ports": [], @@ -153,6 +154,7 @@ All Keys that are in **bold** are required | --- | --- | --- | | **image** | String | A docker notation of docker image to use for the component. This can also reference a devlab base image, as well as a project's [runtime image](#runtime-image-structure) | | systemd_support | Boolean | If set to `true` then this will start the component with proper `/run`, `/run/lock`, `/tmp`, and `/sys/fs/cgroup` mounts so systemd can run | +| systemd_tmpfs_args | String | If `systemd_support` is set to `true`, and this argument is set, then the value is appended to the tmpfs mounts as arguments for systemd support. This way you can specify things like: `rw`, `exec`, etc... | | **enabled** | Boolean | Whether or not the component should be brought [up](#up-action) and images [built](#build-action) | | **_name_** | String | This is only supported for `foreground_components` but required. It indicates the name of the component | | type | String | This only only supported for `foreground_components`, but can be either `host` or `container`. If set to host then `cmd` is executed on the local system instead of a container | @@ -457,7 +459,7 @@ foreground_component: ``` # Devlab Argument documentation -All actions have a `--help` which should display relevent help. The following options below are "global" options and *should* preceed any action: +All actions have a `--help` which should display relevent help. The following options below are "common" options and *should* preceed any action: ``` -h, --help show this help message and exit @@ -466,7 +468,7 @@ All actions have a `--help` which should display relevent help. The following op ``` The format of the command should be: -`devlab ` +`devlab ` For example. To bring the environment up with debug level messages: `devlab -l debug up` @@ -481,6 +483,7 @@ For example. To bring the environment up with debug level messages: including persistent data. This is useful if you want to have a component start from scratch without re- running the wizard + global-restart Restart components across all environments managed by devlab global-status Get a global status of all environments where devlab has created containers status Get a status of the environment From 86391372b51ba2ab077af6ef6c013dcb89b95762 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 22 Jul 2022 15:23:07 -0600 Subject: [PATCH 7/7] Fix installer when pointing to a local directory --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index 278eada..bfe7ecb 100755 --- a/installer.py +++ b/installer.py @@ -395,9 +395,9 @@ def list_packages(path, logger): else: logger = logging.getLogger('list_packages') packages = {} - path = path.lower() found_files = [] if 'http' in path: + path = path.lower() log.debug("Repo path is an HTTP url") if 'github.com/' in path and path.endswith('releases'): log.debug('Repo path is a github releases url. Looking for devlab packages in releases')