Skip to content

Commit

Permalink
Merge branch 'add_support_for_yaml' into 'master'
Browse files Browse the repository at this point in the history 
Fixes for adhoc action, detection for rebuilding images, and support for yaml

See merge request te/devlab!12
  • Loading branch information
Dave Martinez committed Aug 20, 2020
2 parents d61c0a0 + c889e34 commit 5b2f0b3
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 38 deletions.
4 changes: 2 additions & 2 deletions ci/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ if [ ! -z "$CI_COMMIT_TAG" ] ; then
beg_range="${prev_release_tag}.."
else
#check for fallback logic of special commit names etc...
legacy_changes=$(git log --pretty=format:'%s%n' | sed "/^Merge branch '.\+' into '.\+'/d ; /^\$/d")
legacy_changes=$(git log --pretty=format:'%s%n' | sed "/^Merge branch '.\+' into '.\+'/d ; /^Merge branch '.\+' of .\+/d ; /^\$/d")
legacy_last_change=$(echo "$legacy_changes" | head -n 1)
legacy_last_release=$(echo "$legacy_changes" | grep -m 1 '^New release: ')
if [ "$legacy_last_change" == "$legacy_last_release" ] ; then
Expand All @@ -33,7 +33,7 @@ else
fi
fi
echo "Getting changes comit changes between: ${beg_range}${last_change}"
changes_since_release=$(git log --pretty=format:'%s%n' ${beg_range}${last_change} | sed "/^Merge branch '.\+' into '.\+'/d ; /^$/d")
changes_since_release=$(git log --pretty=format:'%s%n' ${beg_range}${last_change} | sed "/^Merge branch '.\+' into '.\+'/d ; /^Merge branch '.\+' of .\+/d; /^$/d")

if [ $(echo "$changes_since_release" | wc -l) -gt 0 ] ; then
echo -e "Building a package with the following NEW changes included:\n----"
Expand Down
181 changes: 146 additions & 35 deletions devlab
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,15 @@ except NameError:
quote = shlex.quote #pylint: disable=invalid-name
from pathlib import Path

try:
import yaml
YAML_SUPPORT = True
except ImportError:
YAML_SUPPORT = False

##- Variables -##
ARGS = None
CONFIG_FILE_NAMES = ('DevlabConfig.json', 'Devlabconfig.json')
CONFIG_FILE_NAMES = ('DevlabConfig.json', 'DevlabConfig.yaml', 'DevlabConfig.yml', 'Devlabconfig.json', 'Devlabconfig.yaml', 'Devlabconfig.yml')
CONFIG_DEF = {
'domain': 'devlab.lab',
'wizard_enabled': True,
Expand All @@ -56,6 +62,7 @@ CONFIG_DEF = {
'reprovisionable_components': [],
'runtime_images': {}
}
CONFIG = {}
DOCKER = None
PARSER = None
IMAGES = {
Expand Down Expand Up @@ -411,6 +418,7 @@ class DockerHelper(object):
cmd_ret = Command(
self.docker_bin_paths,
[
'container',
'inspect',
container
],
Expand All @@ -420,6 +428,29 @@ class DockerHelper(object):
if cmd_ret[0] == 0:
ret = json.loads(cmd_ret[1])
return ret
def inspect_image(self, image):
"""
Grabs the inspection data (docker inspect) for an image
Args:
image: String of the image you want to inspect
Return dict
"""
ret = {}
cmd_ret = Command(
self.docker_bin_paths,
[
'image',
'inspect',
image
],
split=False,
logger=self.log
).run()
if cmd_ret[0] == 0:
ret = json.loads(cmd_ret[1])
return ret
def prune_images(self, prune_all=False):
"""
Prune images from docker
Expand Down Expand Up @@ -1435,11 +1466,7 @@ def action_shell(components='*', adhoc_image=None, adhoc_name=None, command=None
else:
adhoc_image_parsed['host'] = ''
if not adhoc_name:
if adhoc_image_parsed['host']:
#Strip off the host part of the image for generating a more sane name
adhoc_name = adhoc_image_parsed['bare_image']
else:
adhoc_name = adhoc_image
adhoc_name = adhoc_image_parsed['bare_image']
adhoc_name = '{}-adhoc'.format(adhoc_name.replace('/', '_').strip('_'))
command = 'helper_container|{host}{bare_image}^{tag}^{adhoc_name}: {command}'.format(adhoc_name=adhoc_name, command=command, **adhoc_image_parsed)
log.debug("Built adhoc command: '%s'", command)
Expand Down Expand Up @@ -1774,9 +1801,11 @@ def action_up(components='*', skip_provision=False, bind_to_host=False, keep_up_
log.info("Looking for and updating images needed by components: %s", ','.join(components_to_run))
update_component_images(components=components_to_run)
needed_images = get_needed_images()
if needed_images['base_images']['missing']:
base_to_build = needed_images['base_images']['missing']
if needed_images['base_images']['missing'] or needed_images['base_images']['needs_update']:
base_to_build = needed_images['base_images']['missing'] + needed_images['base_images']['needs_update']
log.debug("Images: '%s' not found in list of current images", base_to_build)
if needed_images['base_images']['needs_update']:
log.info("Found newer dockerfile(s), will update the following base images: %s", ','.join(needed_images['base_images']['needs_update']))
log.info("Need to build some base images before trying to start containers")
action_build(images=base_to_build)
if config['network']['name']:
Expand All @@ -1787,10 +1816,12 @@ def action_up(components='*', skip_provision=False, bind_to_host=False, keep_up_
if not network_status['exists']:
log.info("Custom user network: '%s' not found. Creating", config['network']['name'])
DOCKER.create_network(**config['network'])
if needed_images['runtime_images']['missing']:
runtime_to_build = needed_images['runtime_images']['missing']
if needed_images['runtime_images']['missing'] or needed_images['runtime_images']['needs_update']:
runtime_to_build = needed_images['runtime_images']['missing'] + needed_images['runtime_images']['needs_update']
log.debug("Runtime Images: '%s' not found in list of current images", runtime_to_build)
log.info("Need to build some runtime images before trying to start containers")
if needed_images['runtime_images']['needs_update']:
log.info("Found newer dockerfile(s), will update the following runtime images: %s", ','.join(needed_images['runtime_images']['needs_update']))
action_build(images=runtime_to_build)
if not os.path.isdir('{}/{}'.format(PROJ_ROOT, config['paths']['component_persistence'])):
os.mkdir('{}/{}'.format(PROJ_ROOT, config['paths']['component_persistence']))
Expand Down Expand Up @@ -1952,6 +1983,55 @@ def check_custom_registry(components=None, config=None, logger=None):
return True
return False

def check_build_image_need_update(image, dockerfile, logger=None):
"""
Determing if an image needs to be updated based on a dockerfile's
'last_modified' label. If there is no last_modified label in the dockerfile
then return will be False.
Args:
image: str, of the name of the build image
dockerfile: str, path to the dockerfile to compare against
logger: Logger, for to use for logging
Returns:
Bool:
True, if the image needs to be updated
False, if not
"""
if logger:
log = logger
else:
log = logging.getLogger('check_build_image_need_update')
last_modified = 0
log.debug("Determining if image: '%s' needs to be rebuilt based on 'last_modified' label", image)
with open(dockerfile) as dfile:
dfile_contents = dfile.read()
for df_line in dfile_contents.splitlines():
if df_line.startswith('LABEL'):
log.debug("Found LABEL line: '%s'", df_line)
if 'last_modified' in df_line:
log.debug("Found desired 'last_modified' LABEL")
last_modified = df_line.split(' ')[1].split('=')[1].strip('"')
log.debug("Last modified value in dockerfile: '%s' is '%s'", dockerfile, last_modified)
if last_modified: #Check if the image has the latest 'last_modified' label
log.debug("Looking for current last_modified label on existing image: %s", image)
image_details = DOCKER.inspect_image('{}:latest'.format(image))[0]['Config']
cur_last_modified = None
for ilabel, ivalue in image_details['Labels'].items():
log.debug("Found existing label: '%s' on image", ilabel)
if ilabel == 'last_modified':
log.debug("Found desired existing 'last_modified' LABEL with a value of '%s'", ivalue)
cur_last_modified = ivalue
if last_modified == cur_last_modified:
return False
else:
break
log.debug("Last modified value in dockerfile: '%s' is not the same as current image: '%s'. Update needed", last_modified, cur_last_modified)
return True
else:
log.debug("No last_modified label defined in dockerfile, no update needed")
return False

def component_up(name, comp_config, skip_provision=False, keep_up_on_error=False, current_containers=None, background=True, network=None, logger=None):
"""
Bring a component up
Expand Down Expand Up @@ -2119,22 +2199,33 @@ def docker_obj_status(name, obj_type, docker_helper, logger=None):
result.append(res)
return result

def get_config(fallback_default=False):
def get_config(force_reload=False, fallback_default=False):
"""
Try to load the main config file
"""
global IMAGES
config = deepcopy(CONFIG_DEF)
global CONFIG
if CONFIG and not force_reload:
return CONFIG
loaded_config = {}
CONFIG = deepcopy(CONFIG_DEF)
for cfile_name in CONFIG_FILE_NAMES:
if os.path.isfile('{}/{}'.format(PROJ_ROOT, cfile_name)):
with open('{}/{}'.format(PROJ_ROOT, cfile_name), 'r') as config_file:
cfile_path = '{}/{}'.format(PROJ_ROOT, cfile_name)
cfile_name_split = os.path.splitext(cfile_name)
if os.path.isfile(cfile_path):
if cfile_name_split[1] in ('.yaml', 'yml'):
if not YAML_SUPPORT:
print("Found devlab config: {} in yaml format, but the 'yaml' python module is NOT installed. Please install the yaml python module and try again".format(cfile_path))
sys.exit(1)
with open(cfile_path, 'r') as config_file:
try:
loaded_config = json.load(config_file)
except json.decoder.JSONDecodeError:
if YAML_SUPPORT:
loaded_config = yaml.load(config_file, Loader=yaml.SafeLoader)
else:
loaded_config = json.load(config_file)
except Exception: #pylint: disable=broad-except
exc_type, exc_value = sys.exc_info()[:2]
exc_str = "Failed loading JSON config file: '{cfile_name}' {exc_type}: {exc_val}".format(
cfile_name=cfile_name,
exc_str = "Failed loading config file: '{cfile_path}' {exc_type}: {exc_val}".format(
cfile_path=cfile_path,
exc_type=exc_type.__name__,
exc_val=exc_value
)
Expand All @@ -2145,19 +2236,22 @@ def get_config(fallback_default=False):
if os.path.isfile('{}/defaults/{}'.format(PROJ_ROOT, cfile_name)):
with open('{}/defaults/{}'.format(PROJ_ROOT, cfile_name), 'r') as config_file:
try:
loaded_config = json.load(config_file)
except json.decoder.JSONDecodeError:
if YAML_SUPPORT:
loaded_config = yaml.load(config_file, Loader=yaml.SafeLoader)
else:
loaded_config = json.load(config_file)
except Exception: #pylint: disable=broad-except
exc_type, exc_value = sys.exc_info()[:2]
exc_str = "Failed loading JSON config file: '{cfile_name}' {exc_type}: {exc_val}".format(
cfile_name=cfile_name,
exc_str = "Failed loading config file: '{cfile_path}' {exc_type}: {exc_val}".format(
cfile_path=cfile_path,
exc_type=exc_type.__name__,
exc_val=exc_value
)
print(exc_str)
sys.exit(1)
break
config.update(loaded_config)
return config
CONFIG.update(loaded_config)
return CONFIG

def get_components(filter_list=None, virtual_components=None, enabled_only=True, match_virtual=False, logger=None):
"""
Expand Down Expand Up @@ -2302,16 +2396,19 @@ def get_needed_images(components=None, logger=None):
'missing': [],
'exists': [],
'exists_owned': [],
'needs_update': []
},
'base_images': {
'missing': [],
'exists': [],
'exists_owned': [],
'needs_update': []
},
'external_images': {
'missing': [],
'exists': [],
'exists_owned': []
'exists_owned': [],
'needs_update': []
}
}
"""
Expand All @@ -2330,16 +2427,19 @@ def get_needed_images(components=None, logger=None):
'missing': [],
'exists': [],
'exists_owned': [],
'needs_update': []
},
'base_images': {
'missing': [],
'exists': [],
'exists_owned': [],
'needs_update': []
},
'external_images': {
'missing': [],
'exists': [],
'exists_owned': []
'exists_owned': [],
'needs_update': []
}
}
if logger:
Expand All @@ -2361,6 +2461,8 @@ def get_needed_images(components=None, logger=None):
continue
else:
result['base_images']['exists'].append(image['name'])
if check_build_image_need_update(image=image['name'], dockerfile='{}/{}'.format(DEVLAB_ROOT, IMAGES[image['name']]['docker_file'])):
result['base_images']['needs_update'].append(image['name'])
if image['owned']:
result['base_images']['exists_owned'].append(image['name'])
try:
Expand Down Expand Up @@ -2465,6 +2567,16 @@ def get_needed_images(components=None, logger=None):
result['runtime_images']['missing'].append(rt_image['name'].split(':')[0])
continue
else:
if check_build_image_need_update(
image=rt_image['name'],
dockerfile='{}/{}'.format(
PROJ_ROOT,
runtime_images_dict[
rt_image['name'].split(':')[0]
]['docker_file']
)
):
result['runtime_images']['needs_update'].append(rt_image['name'])
result['runtime_images']['exists'].append(rt_image['name'].split(':')[0])
if rt_image['owned']:
result['runtime_images']['exists_owned'].append(rt_image['name'].split(':')[0])
Expand Down Expand Up @@ -2567,7 +2679,7 @@ def get_proj_root(start_dir=None):
Args:
start_dir: String of the path where to start traversing backwards
looking for the DevlabConfig.json, config.json and equivalents in defaults/
looking for the DevlabConfig.json, DevlabConfig.yaml and equivalents in defaults/
Returns:
String of the path found, or None if not found
Expand Down Expand Up @@ -2671,7 +2783,6 @@ def logging_init(level):
"""
Initialize and create initial LOGGER
level is a String of one of:
'trace'
'debug'
'info'
'warning'
Expand Down Expand Up @@ -3051,7 +3162,7 @@ if __name__ == '__main__':
PARSER = argparse.ArgumentParser(description='Main interface for devlab')
PARSER.add_argument('--log-level', '-l', choices=list(LOGGING_LEVELS.keys()), default='info', help='Set the log-level output')
PARSER.add_argument('--version', '-v', action='store_true', help='Display the version of devlab and exit')
PARSER.add_argument('--project-root', '-P', default=None, help='Force project root to a specific path instead of searching for DevlabConfig.json')
PARSER.add_argument('--project-root', '-P', default=None, help='Force project root to a specific path instead of searching for DevlabConfig.json/DevlabConfig.yaml etc...')
SUBPARSERS = PARSER.add_subparsers(help='Actions')

#Add Subparser for dummy default action
Expand Down Expand Up @@ -3085,7 +3196,7 @@ if __name__ == '__main__':
PARSER_RESET = SUBPARSERS.add_parser('reset', help='Reset a specific component, getting rid of all data including persistent data. This is useful if you want to have a component start from scratch without re-running the wizard')
PARSER_RESET.add_argument('targets', nargs='*', default='default', type=get_reset_components, help='Reset the specific target(s) or glob matches. * means all components, but this does NOT inlcude other targets like \'devlab\'. TARGETS: {}'.format(', '.join(CUR_COMPONENTS + ['devlab'])))
PARSER_RESET.add_argument('--reset-wizard', '-r', action='store_true', help='Also remove wizard related files so that the wizard will run again for the specified component')
PARSER_RESET.add_argument('--full', '-f', action='store_true', help='Remove all component specific files, wizard files, as well as devlab files AND potentially files you\'re working on. BE CAREFUL IF YOU HAVE MANUAL CHANGES PATHS DEFINED IN in \'paths.reset_full\'!!')
PARSER_RESET.add_argument('--full', '-f', action='store_true', help='Remove all component specific files, wizard files, as well as devlab files AND potentially files you\'re working on. BE CAREFUL IF YOU HAVE MANUAL CHANGES IN PATHS DEFINED IN YOUR \'paths.reset_full\'!!')
PARSER_RESET.set_defaults(func=action_reset)

#Add Subparser for global_status action
Expand Down Expand Up @@ -3160,7 +3271,7 @@ if __name__ == '__main__':
ARGS.func(**vars(ARGS))
sys.exit(0)
else:
LOGGER.error("Aborting... could not determine project root. Please create a DevlabConfig.json")
LOGGER.error("Aborting... could not determine project root. Please create a DevlabConfig.json or DevlabConfig.yaml etc...")
sys.exit(1)

#Load config
Expand All @@ -3175,15 +3286,15 @@ if __name__ == '__main__':
if WIZ_OUT[0] != 0:
LOGGER.error("Wizard did not exit successfully... Aborting!")
sys.exit(1)
CONFIG = get_config()
CONFIG = get_config(force_reload=True)
else:
LOGGER.warning("WARNING!!!!WARNING!!! No wizard found!!!")

#See if we have enough details in our config
if not CONFIG['components'] and 'foreground_component' not in CONFIG:
LOGGER.warning("No DevlabConfig.json was found yet.")
LOGGER.warning("No devlab configuration was found yet.")
LOGGER.info("Trying to load from 'defaults/'")
CONFIG = get_config(fallback_default=True)
CONFIG = get_config(force_reload=True, fallback_default=True)
if not CONFIG['components']:
LOGGER.error("No configured components found!... aborting")
sys.exit(1)
Expand Down
1 change: 1 addition & 0 deletions docker/base.Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
FROM ubuntu:18.04
LABEL "com.lab.devlab"="base"
LABEL "last_modified"=1597871407

ARG APT="/usr/bin/apt-get --no-install-recommends -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef"
ARG PACKAGES="sudo less screen tmux vim ca-certificates git curl iproute2 less gnupg2"
Expand Down
Loading

0 comments on commit 5b2f0b3

Please sign in to comment.