From ed3276989e867445273baadc602d73720cb180b2 Mon Sep 17 00:00:00 2001 From: Colby Williams Date: Thu, 5 Jan 2023 14:39:56 -0600 Subject: [PATCH] Add support for AzureDevOps --- README.md | 2 +- bake/HISTORY.rst | 4 ++++ bake/azext_bake/_constants.py | 27 ++++++++++----------- bake/azext_bake/_params.py | 8 +++++-- bake/azext_bake/_repos.py | 18 +++++++------- bake/azext_bake/_utils.py | 2 +- bake/azext_bake/_validators.py | 43 +++++++++++++++++++++++++++------- bake/azext_bake/custom.py | 32 +++++++++++++++++-------- bake/setup.py | 2 +- builder/Dockerfile | 2 +- 10 files changed, 93 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 22c3da2..11d5a3d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Microsoft Azure CLI Extension for creating (or _"baking"_) custom virtual machin To install the Azure CLI Custom Image Helper extension, simply run the following command: ```sh -az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.3-py3-none-any.whl -y +az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.4-py3-none-any.whl -y ``` ### Update diff --git a/bake/HISTORY.rst b/bake/HISTORY.rst index 2c53fb3..850b81a 100644 --- a/bake/HISTORY.rst +++ b/bake/HISTORY.rst @@ -3,6 +3,10 @@ Release History =============== +0.3.4 +++++++ ++ Add support for AzureDevOps + 0.3.3 ++++++ + Fix choco bug in builder diff --git a/bake/azext_bake/_constants.py b/bake/azext_bake/_constants.py index 45dc232..4ceb773 100644 --- a/bake/azext_bake/_constants.py +++ b/bake/azext_bake/_constants.py @@ -222,7 +222,7 @@ def tag_key(key): }} ''' - +GITHUB_PROVIDER_NAME = 'GitHub' GITHUB_WORKFLOW_FILE = 'bake_images.yml' GITHUB_WORKFLOW_DIR = '.github/workflows' GITHUB_WORKFLOW_CONTENT = '''name: Bake Images @@ -264,6 +264,7 @@ def tag_key(key): run: az bake repo build --verbose --repo . ''' +DEVOPS_PROVIDER_NAME = 'AzureDevOps' DEVOPS_PIPELINE_FILE = 'azure-pipelines.yml' DEVOPS_PIPELINE_DIR = '.azure' DEVOPS_PIPELINE_CONTENT = '''name: Bake Images @@ -272,30 +273,26 @@ def tag_key(key): - main pool: - vmImage: 'ubuntu-latest' + vmImage: ubuntu-latest -# stages: -# - stage: Bake Images -# jobs: -# - job: bakeImages -# displayName: Bake Images steps: - - displayName: Login to Azure + - script: az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID + displayName: Login to Azure env: AZURE_CLIENT_ID: $(AZURE_CLIENT_ID) AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET) AZURE_TENANT_ID: $(AZURE_TENANT_ID) - bash: az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_CLIENT_SECRET --tenant $AZURE_TENANT_ID - - displayName: Install az bake # get the latest version of az bake from the github releases and install it - bash: | - az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.3-py3-none-any.whl -y - az bake upgrade + - script: az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.4-py3-none-any.whl -y + displayName: Install az bake + + - script: az bake upgrade + displayName: Update az bake - - displayName: Run az bake + - script: az bake repo build --verbose --repo . + displayName: Run az bake env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) - bash: az bake repo build --verbose --repo . ''' diff --git a/bake/azext_bake/_params.py b/bake/azext_bake/_params.py index 0d693fe..dc4982a 100644 --- a/bake/azext_bake/_params.py +++ b/bake/azext_bake/_params.py @@ -5,11 +5,12 @@ # pylint: disable=line-too-long, too-many-statements from argcomplete.completers import DirectoriesCompleter, FilesCompleter -from azure.cli.core.commands.parameters import (file_type, get_location_type, get_resource_group_completion_list, - tags_type) +from azure.cli.core.commands.parameters import (file_type, get_enum_type, get_location_type, + get_resource_group_completion_list, tags_type) from knack.arguments import CLIArgumentType from ._completers import get_version_completion_list +from ._constants import DEVOPS_PROVIDER_NAME, GITHUB_PROVIDER_NAME from ._validators import (bake_source_version_validator, gallery_resource_id_validator, repository_path_validator, sandbox_resource_group_name_validator, yaml_out_validator) @@ -128,6 +129,9 @@ def load_arguments(self, _): with self.argument_context('bake repo setup') as c: c.argument('repository_path', options_list=['--repo-path', '--repo'], type=file_type, default='./', validator=repository_path_validator, help='Path to the locally cloned repository.') + c.argument('repository_provider', get_enum_type([GITHUB_PROVIDER_NAME, DEVOPS_PROVIDER_NAME], default=None), + options_list=['--repo-provider', '--provider'], required=False, + help='Repository provider. If not specified, will attempt to detect provider.') c.argument('sandbox_resource_group_name', sandbox_resource_group_name_type) c.argument('gallery_resource_id', gallery_resource_id_type) c.ignore('sandbox') diff --git a/bake/azext_bake/_repos.py b/bake/azext_bake/_repos.py index 3f0483e..fece0d4 100644 --- a/bake/azext_bake/_repos.py +++ b/bake/azext_bake/_repos.py @@ -11,10 +11,12 @@ from azure.cli.core.azclierror import CLIError +from ._constants import DEVOPS_PROVIDER_NAME, GITHUB_PROVIDER_NAME + @dataclass class CI: - provider: Literal['github', 'devops'] = None + provider: Literal['GitHub', 'AzureDevOps'] = None url: str = None token: str = None ref: str = None @@ -28,7 +30,7 @@ def is_ci(): def __init__(self) -> None: if os.environ.get('CI', False) and os.environ.get('GITHUB_ACTION', False): - self.provider = 'github' + self.provider = GITHUB_PROVIDER_NAME self.token = os.environ.get('GITHUB_TOKEN', None) self.ref = os.environ.get('GITHUB_REF', None) self.revision = os.environ.get('GITHUB_SHA', None) @@ -41,7 +43,7 @@ def __init__(self) -> None: raise CLIError('Could not determine GitHub repository url from environment variables.') elif os.environ.get('TF_BUILD', False): - self.provider = 'devops' + self.provider = DEVOPS_PROVIDER_NAME self.token = os.environ.get('SYSTEM_ACCESSTOKEN', None) self.ref = os.environ.get('BUILD_SOURCEBRANCH', None) self.revision = os.environ.get('BUILD_SOURCEVERSION', None) @@ -58,7 +60,7 @@ def __init__(self) -> None: class Repo: # required properties url: str - provider: Literal['github', 'devops'] = field(init=False) + provider: Literal['GitHub', 'AzureDevOps'] = field(init=False) org: str = field(init=False) repo: str = field(init=False) # optional properties @@ -137,11 +139,11 @@ def _parse_github_url(self, url): def __post_init__(self): if 'github.com' in self.url.lower(): - self.provider = 'github' + self.provider = GITHUB_PROVIDER_NAME self._parse_github_url(self.url) self.clone_url = self.url.replace('https://', f'https://gituser:{self.token}@') if self.token else self.url elif 'dev.azure.com' in self.url.lower() or 'visualstudio.com' in self.url.lower(): - self.provider = 'devops' + self.provider = DEVOPS_PROVIDER_NAME self._parse_devops_url(self.url) self.clone_url = self.url.replace( 'https://', f'https://azurereposuser:{self.token}@') if self.token else self.url @@ -164,11 +166,11 @@ def __post_init__(self): for test in test_urls: repository = Repo(url=test, token='mytoken') - if repository.provider not in ['github', 'devops']: + if repository.provider not in [GITHUB_PROVIDER_NAME, DEVOPS_PROVIDER_NAME]: raise CLIError(f'{repository.provider} is not a valid provider') if repository.org != 'colbylwilliams': raise CLIError(f'{repository.org} is not a valid organization') - if repository.provider == 'devops' and repository.project != 'myproject': + if repository.provider == DEVOPS_PROVIDER_NAME and repository.project != 'myproject': raise CLIError(f'{repository.project} is not a valid project') if repository.repo != 'az-bake': raise CLIError(f'{repository.repo} is not a valid repository') diff --git a/bake/azext_bake/_utils.py b/bake/azext_bake/_utils.py index 3341d77..393597a 100644 --- a/bake/azext_bake/_utils.py +++ b/bake/azext_bake/_utils.py @@ -26,7 +26,7 @@ def get_logger(name): if IN_BUILDER and STORAGE_DIR.is_dir(): import logging - log_file = OUTPUT_DIR / 'builder.txt' + log_file = OUTPUT_DIR / 'builder.log' formatter = logging.Formatter('{asctime} [{name:^28}] {levelname:<8}: {message}', datefmt='%m/%d/%Y %I:%M:%S %p', style='{',) fh = logging.FileHandler(log_file) diff --git a/bake/azext_bake/_validators.py b/bake/azext_bake/_validators.py index b8f522e..d42350f 100644 --- a/bake/azext_bake/_validators.py +++ b/bake/azext_bake/_validators.py @@ -18,8 +18,8 @@ from azure.cli.core.extension import get_extension from azure.mgmt.core.tools import is_valid_resource_id, parse_resource_id -from ._constants import (AZ_BAKE_BUILD_IMAGE_NAME, AZ_BAKE_IMAGE_BUILDER, AZ_BAKE_IMAGE_BUILDER_VERSION, IN_BUILDER, - REPO_DIR, STORAGE_DIR, tag_key) +from ._constants import (AZ_BAKE_BUILD_IMAGE_NAME, AZ_BAKE_IMAGE_BUILDER, AZ_BAKE_IMAGE_BUILDER_VERSION, + DEVOPS_PROVIDER_NAME, GITHUB_PROVIDER_NAME, IN_BUILDER, REPO_DIR, STORAGE_DIR, tag_key) from ._data import BakeConfig, Gallery, Image from ._github import get_github_latest_release_version, github_release_version_exists from ._packer import check_packer_install @@ -63,7 +63,7 @@ def process_bake_repo_build_namespace(cmd, ns): ci = CI() if ci.token is None: - env_key = 'GITHUB_TOKEN' if ci.provider == 'github' else 'SYSTEM_ACCESSTOKEN' + env_key = 'GITHUB_TOKEN' if ci.provider == GITHUB_PROVIDER_NAME else 'SYSTEM_ACCESSTOKEN' logger.warning(f'WARNING: {env_key} environment variable not set. This is required for private repositories.') repo = Repo(url=ci.url, token=ci.token, ref=ci.ref, revision=ci.revision) @@ -135,7 +135,7 @@ def builder_validator(cmd, ns): def repository_images_validator(cmd, ns): if not ns.repository_path: - raise RequiredArgumentMissingError('--repository-path/-r is required') + raise RequiredArgumentMissingError('--repo-path/--repo is required') images_path = _validate_dir_path(ns.repository_path / 'images', name='images') @@ -171,13 +171,40 @@ def repository_images_validator(cmd, ns): def repository_path_validator(cmd, ns): '''Ensure the repository path is valid, transforms to a path object, and validates a .git directory exists''' if not ns.repository_path: - raise RequiredArgumentMissingError('--repository-path/-r is required') + raise RequiredArgumentMissingError('--repo-path/--repo is required') repo_path = _validate_dir_path(ns.repository_path, name='repository') ns.repository_path = repo_path - git_path = repo_path / '.git' - git_path = _validate_dir_path(git_path, name='.git') + git_path = _validate_dir_path(repo_path / '.git', name='.git') + if hasattr(ns, 'git_path'): + ns.git_path = git_path + + if hasattr(ns, 'repository_provider'): + if ns.repository_provider and ns.repository_provider not in [GITHUB_PROVIDER_NAME, DEVOPS_PROVIDER_NAME]: + raise InvalidArgumentValueError(f'--repo-provider/--provider must be one of {GITHUB_PROVIDER_NAME} ' + f'or {DEVOPS_PROVIDER_NAME}') + + # if the repository provider is not specified, try to determine it from the git config + git_config = _validate_file_path(git_path / 'config', 'git config') + config_lines = git_config.read_text().splitlines() + remote_url = None + for line in config_lines: + line_clean = line.strip() + if line_clean.startswith('url = '): + remote_url = line_clean.replace('url = ', '') + + if not remote_url: + raise ValidationError('Unable to determine repository provider from git config. ' + 'Please specify --repo-provider/--provider') + + if 'github.com' in remote_url: + ns.repository_provider = GITHUB_PROVIDER_NAME + elif 'dev.azure.com' in remote_url or 'visualstudio.com' in remote_url: + ns.repository_provider = DEVOPS_PROVIDER_NAME + else: + raise ValidationError('Unable to determine repository provider from git config. ' + 'Please specify --repo-provider/--provider') def image_names_validator(cmd, ns): @@ -240,7 +267,7 @@ def bake_yaml_validator(cmd, ns, path=None): # should have already run the repository_path_validator path = get_yaml_file_path(ns.repository_path, 'bake', required=True) else: - raise RequiredArgumentMissingError('usage error: --repository-path is required.') + raise RequiredArgumentMissingError('--repo-path/--repo is required.') bake_config = get_yaml_file_data(BakeConfig, path) diff --git a/bake/azext_bake/custom.py b/bake/azext_bake/custom.py index 0dfd272..f5ca6cf 100644 --- a/bake/azext_bake/custom.py +++ b/bake/azext_bake/custom.py @@ -11,16 +11,17 @@ import yaml +from azure.cli.core.azclierror import CLIError, InvalidArgumentValueError from azure.cli.core.extension.operations import show_extension, update_extension -from knack.util import CLIError from packaging.version import parse from ._arm import (create_image_definition, create_resource_group, deploy_arm_template_at_resource_group, ensure_gallery_permissions, get_arm_output, get_gallery, get_image_definition, get_resource_group_by_name, image_version_exists) from ._client_factory import cf_container, cf_container_groups -from ._constants import (BAKE_YAML_SCHEMA, GITHUB_WORKFLOW_CONTENT, GITHUB_WORKFLOW_DIR, GITHUB_WORKFLOW_FILE, - IMAGE_DEFAULT_BASE_WINDOWS, IMAGE_YAML_SCHEMA, IN_BUILDER) +from ._constants import (BAKE_YAML_SCHEMA, DEVOPS_PIPELINE_CONTENT, DEVOPS_PIPELINE_DIR, DEVOPS_PIPELINE_FILE, + DEVOPS_PROVIDER_NAME, GITHUB_PROVIDER_NAME, GITHUB_WORKFLOW_CONTENT, GITHUB_WORKFLOW_DIR, + GITHUB_WORKFLOW_FILE, IMAGE_DEFAULT_BASE_WINDOWS, IMAGE_YAML_SCHEMA, IN_BUILDER) from ._data import Gallery, Image, Sandbox, get_dict from ._github import get_github_latest_release_version, get_github_release, get_release_templates, get_template_url from ._packer import (copy_packer_files, inject_choco_provisioners, inject_powershell_provisioner, @@ -180,7 +181,7 @@ def bake_repo_build(cmd, repository_path, image_names: Sequence[str] = None, san logger.warning(f' - Azure Portal: {portal}') logger.warning('') - if repo and repo.provider == 'github': + if repo and repo.provider == GITHUB_PROVIDER_NAME: github_step_summary = os.environ.get('GITHUB_STEP_SUMMARY', None) if github_step_summary: summary = [ @@ -201,17 +202,28 @@ def bake_repo_validate(cmd, repository_path, sandbox: Sandbox = None, gallery: G def bake_repo_setup(cmd, sandbox_resource_group_name: str, gallery_resource_id: str, repository_path='./', - sandbox: Sandbox = None, gallery: Gallery = None): + repository_provider: str = None, sandbox: Sandbox = None, gallery: Gallery = None): logger.info('Setting up repository') + + # logger.warning(repository_provider) _bake_yaml_export(sandbox=sandbox, gallery=gallery, outdir=repository_path) - workflows_dir = repository_path / GITHUB_WORKFLOW_DIR + if repository_provider == GITHUB_PROVIDER_NAME: + workflows_dir = repository_path / GITHUB_WORKFLOW_DIR + + if not workflows_dir.exists(): + workflows_dir.mkdir(parents=True, exist_ok=True) - if not workflows_dir.exists(): - workflows_dir.mkdir(parents=True, exist_ok=True) + with open(workflows_dir / GITHUB_WORKFLOW_FILE, 'w', encoding='utf-8') as f: + f.write(GITHUB_WORKFLOW_CONTENT) - with open(workflows_dir / GITHUB_WORKFLOW_FILE, 'w', encoding='utf-8') as f: - f.write(GITHUB_WORKFLOW_CONTENT) + elif repository_provider == DEVOPS_PROVIDER_NAME: + + with open(repository_path / DEVOPS_PIPELINE_FILE, 'w', encoding='utf-8') as f: + f.write(DEVOPS_PIPELINE_CONTENT) + else: + raise InvalidArgumentValueError(f'--repo-provider/--provider must be one of {GITHUB_PROVIDER_NAME} ' + f'or {DEVOPS_PROVIDER_NAME}') # ---------------- diff --git a/bake/setup.py b/bake/setup.py index fe7a1b5..b028370 100644 --- a/bake/setup.py +++ b/bake/setup.py @@ -17,7 +17,7 @@ logger.warn("Wheel is not available, disabling bdist_wheel hook") # Must match a HISTORY.rst entry. -VERSION = '0.3.3' +VERSION = '0.3.4' # The full list of classifiers is available at # https://pypi.python.org/pypi?%3Aaction=list_classifiers diff --git a/builder/Dockerfile b/builder/Dockerfile index a4620d5..ad03bd3 100644 --- a/builder/Dockerfile +++ b/builder/Dockerfile @@ -38,7 +38,7 @@ LABEL maintainer="Microsoft" \ RUN apk add --no-cache packer --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community # install az-bake -RUN az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.3-py3-none-any.whl -y +RUN az extension add --source https://github.com/colbylwilliams/az-bake/releases/latest/download/bake-0.3.4-py3-none-any.whl -y # Terminate container on stop STOPSIGNAL SIGTERM