From 44b9b7967608d51274aaf052b3eee40291cc8ce0 Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 27 Sep 2018 09:40:22 +0200 Subject: [PATCH 01/14] add token to login_to_registry --- conu/backend/origin/backend.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index dabb9495..cb2c838e 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -63,7 +63,7 @@ def http_request(self, path="/", method="GET", host=None, port=None, json=False, """ perform a HTTP request - :param path: str, path within the reqest, e.g. "/api/version" + :param path: str, path within the request, e.g. "/api/version" :param method: str, HTTP method :param host: str, if None, set to 127.0.0.1 :param port: str or int, if None, set to 8080 @@ -257,13 +257,15 @@ def clean_project(self, app_name): raise ConuException("Cleanup failed: %s" % ex) @staticmethod - def login_to_registry(username): + def login_to_registry(username, token=None): """ Login within docker daemon to docker registry running in this OpenShift cluster :return: """ + + token = token or get_oc_api_token() + with DockerBackend() as backend: - token = get_oc_api_token() backend.login(username, password=token, registry=OpenshiftBackend.get_internal_registry_ip(), reauth=True) From 1c3672c34503778eac8e6b7a51f9f4aa5e3e6fed Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 27 Sep 2018 10:04:14 +0200 Subject: [PATCH 02/14] create new registry module --- conu/backend/origin/backend.py | 52 ++---------------------- conu/backend/origin/registry.py | 62 +++++++++++++++++++++++++++++ tests/integration/test_openshift.py | 7 ++-- 3 files changed, 70 insertions(+), 51 deletions(-) create mode 100644 conu/backend/origin/registry.py diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index cb2c838e..51e6cddd 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -27,14 +27,12 @@ from requests.exceptions import ConnectionError +from conu.backend.origin.registry import push_to_registry from conu.backend.k8s.backend import K8sBackend -from conu.backend.docker.backend import DockerBackend from conu.exceptions import ConuException from conu.utils import oc_command_exists, run_cmd -from conu.backend.origin.constants import PORT from conu.utils.http_client import get_url from conu.utils.probes import Probe, ProbeTimeout -from conu.utils import get_oc_api_token logger = logging.getLogger(__name__) @@ -123,8 +121,7 @@ def new_app(self, image, project, source=None, template=None, name_in_template=N other_images, oc_new_app_args, project) else: - new_image = OpenshiftBackend.push_to_registry(image, image.name.split('/')[-1], - image.tag, project) + new_image = push_to_registry(image, image.name.split('/')[-1], image.tag, project) c = self._oc_command( ["new-app"] + oc_new_app_args + [new_image.name + "~" + source] + @@ -170,14 +167,13 @@ def _create_app_from_template(self, image, name, template, name_in_template, """ # push images to registry repository, tag = list(name_in_template.items())[0] - OpenshiftBackend.push_to_registry(image, repository, tag, project) + push_to_registry(image, repository, tag, project) other_images = other_images or [] for o in other_images: image, tag = list(o.items())[0] - OpenshiftBackend.push_to_registry(image, tag.split(':')[0], tag.split(':')[1], - project) + push_to_registry(image, tag.split(':')[0], tag.split(':')[1], project) oc_new_app_args += ["-p", "NAME=%s" % name, "-p", "NAMESPACE=%s" % project] @@ -255,43 +251,3 @@ def clean_project(self, app_name): logger.info(line) except subprocess.CalledProcessError as ex: raise ConuException("Cleanup failed: %s" % ex) - - @staticmethod - def login_to_registry(username, token=None): - """ - Login within docker daemon to docker registry running in this OpenShift cluster - :return: - """ - - token = token or get_oc_api_token() - - with DockerBackend() as backend: - backend.login(username, password=token, - registry=OpenshiftBackend.get_internal_registry_ip(), reauth=True) - - @staticmethod - def push_to_registry(image, repository, tag, project): - """ - :param image: DockerImage, image to push - :param repository: str, new name of image - :param tag: str, new tag of image - :param project: str, oc project - :return: DockerImage, new docker image - """ - return image.push("%s/%s/%s" % (OpenshiftBackend.get_internal_registry_ip(), - project, repository), tag=tag) - - @staticmethod - def get_internal_registry_ip(): - """ - Search for `docker-registry` IP - :return: str, ip address - """ - with OpenshiftBackend() as origin_backend: - services = origin_backend.list_services() - for service in services: - if service.name == 'docker-registry': - logger.debug("Internal docker-registry IP: %s", - "{ip}:{port}".format(ip=service.get_ip(), port=PORT)) - return "{ip}:{port}".format(ip=service.get_ip(), port=PORT) - return None diff --git a/conu/backend/origin/registry.py b/conu/backend/origin/registry.py new file mode 100644 index 00000000..6b98683c --- /dev/null +++ b/conu/backend/origin/registry.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +""" +This are the methods helpful while working with OpenShift internal docker registry +""" + +from conu.backend.origin.constants import PORT +from conu.backend.docker.backend import DockerBackend +from conu.utils import get_oc_api_token + + +def login_to_registry(username, token=None): + """ + Login within docker daemon to docker registry running in this OpenShift cluster + :return: + """ + + token = token or get_oc_api_token() + + with DockerBackend() as backend: + backend.login(username, password=token, + registry=get_internal_registry_ip(), reauth=True) + + +def push_to_registry(image, repository, tag, project): + """ + :param image: DockerImage, image to push + :param repository: str, new name of image + :param tag: str, new tag of image + :param project: str, oc project + :return: DockerImage, new docker image + """ + return image.push("%s/%s/%s" % (get_internal_registry_ip(), project, repository), tag=tag) + + +def get_internal_registry_ip(): + """ + Search for `docker-registry` IP + :return: str, ip address + """ + with OpenshiftBackend() as origin_backend: + services = origin_backend.list_services() + for service in services: + if service.name == 'docker-registry': + logger.debug("Internal docker-registry IP: %s", + "{ip}:{port}".format(ip=service.get_ip(), port=PORT)) + return "{ip}:{port}".format(ip=service.get_ip(), port=PORT) + return None diff --git a/tests/integration/test_openshift.py b/tests/integration/test_openshift.py index 0cb4cb1a..90bfcd81 100644 --- a/tests/integration/test_openshift.py +++ b/tests/integration/test_openshift.py @@ -23,6 +23,7 @@ from conu.backend.origin.backend import OpenshiftBackend from conu.backend.docker.backend import DockerBackend +from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token, oc_command_exists, is_oc_cluster_running @@ -37,7 +38,7 @@ def test_oc_s2i_remote(self): with DockerBackend(logging_level=logging.DEBUG) as backend: python_image = backend.ImageClass("centos/python-36-centos7") - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer', token=api_key) app_name = openshift_backend.new_app( python_image, @@ -59,7 +60,7 @@ def test_oc_s2i_local(self): with DockerBackend(logging_level=logging.DEBUG) as backend: python_image = backend.ImageClass("centos/python-36-centos7") - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer', token=api_key) app_name = openshift_backend.new_app( python_image, @@ -82,7 +83,7 @@ def test_oc_s2i_template(self): python_image = backend.ImageClass("centos/python-36-centos7", tag="latest") psql_image = backend.ImageClass("centos/postgresql-96-centos7", tag="9.6") - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer', token=api_key) app_name = openshift_backend.new_app( image=python_image, From 7409d9f1bb8ce56cda51a58a17463e8da761180f Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 27 Sep 2018 10:14:09 +0200 Subject: [PATCH 03/14] use random string generator from conu.utils --- conu/backend/k8s/backend.py | 8 ++------ conu/backend/origin/backend.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/conu/backend/k8s/backend.py b/conu/backend/k8s/backend.py index 92e9871a..b8b1c0fe 100644 --- a/conu/backend/k8s/backend.py +++ b/conu/backend/k8s/backend.py @@ -17,8 +17,6 @@ """ This is backend for kubernetes """ -import string -import random import logging import enum @@ -31,6 +29,7 @@ import conu.backend.k8s.client as k8s_client from conu.exceptions import ConuException from conu.utils.probes import Probe +from conu.utils import random_str from kubernetes import client from kubernetes.client.rest import ApiException @@ -110,10 +109,7 @@ def create_namespace(self): Create namespace with random name :return: name of new created namespace """ - random_string = ''.join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(4)) - - name = 'namespace-{random_string}'.format(random_string=random_string) + name = 'namespace-{random_string}'.format(random_string=random_str(5)) namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=name)) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 51e6cddd..2280e056 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -20,8 +20,6 @@ import logging import subprocess -import string -import random import os.path import requests @@ -30,7 +28,7 @@ from conu.backend.origin.registry import push_to_registry from conu.backend.k8s.backend import K8sBackend from conu.exceptions import ConuException -from conu.utils import oc_command_exists, run_cmd +from conu.utils import oc_command_exists, run_cmd, random_str from conu.utils.http_client import get_url from conu.utils.probes import Probe, ProbeTimeout @@ -107,9 +105,7 @@ def new_app(self, image, project, source=None, template=None, name_in_template=N raise ConuException('cannot combine template parameter with source parameter') # app name is generated randomly - random_string = ''.join( - random.choice(string.ascii_lowercase + string.digits) for _ in range(4)) - name = 'app-{random_string}'.format(random_string=random_string) + name = 'app-{random_string}'.format(random_string=random_str(5)) oc_new_app_args = oc_new_app_args or [] From b8aeb00e7b62bc9c7eae1fe2dd2e8d9bcc3b78aa Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 27 Sep 2018 10:31:47 +0200 Subject: [PATCH 04/14] add option to delete all objects in clean_project method --- conu/backend/origin/backend.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 2280e056..cbea7ffc 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -233,17 +233,30 @@ def wait_for_service(self, app_name, expected_output=None, timeout=100): Probe(timeout=timeout, fnc=self.request_service, app_name=app_name, expected_output=expected_output, expected_retval=True).run() - def clean_project(self, app_name): + def clean_project(self, app_name=None, delete_all=False): """ - Delete all objects in current project in OpenShift cluster + Delete objects in current project in OpenShift cluster. If both parameters are passed, + delete all objects in project. + :param app_name: str, name of app + :param delete_all: bool, if true delete all objects in current project :return: None """ - logger.info('Deleting app') + + if not app_name and not delete_all: + ConuException("You need to specify either app_name or set delete_all=True") + + if delete_all: + args = ["--all"] + logger.info('Deleting all objects in current project') + else: + args = "-l app=%s" % app_name + logger.info('Deleting all objects with label app=%s' % app_name) + try: - o = run_cmd(self._oc_command(["delete", "all", "-l app=%s" % app_name]), + o = run_cmd(self._oc_command(["delete", "all", args]), return_output=True) o_lines = o.split('\n') for line in o_lines: logger.info(line) except subprocess.CalledProcessError as ex: - raise ConuException("Cleanup failed: %s" % ex) + raise ConuException("Cleanup failed because of exception: %s" % ex) From c5ae36c875785e1f41ef1222e5d545411a5982dc Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 27 Sep 2018 11:16:16 +0200 Subject: [PATCH 05/14] create method for start-build --- conu/backend/origin/backend.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index cbea7ffc..58a0682d 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -132,17 +132,7 @@ def new_app(self, image, project, source=None, template=None, name_in_template=N raise ConuException("oc new-app failed: %s" % ex) if os.path.isdir(source): - c = self._oc_command(["-n"] + [project] + ["start-build"] + - [name] + ["--from-dir=%s" % source]) - - logger.info("Build application from local source in project %s", project) - - try: - Probe(timeout=-1, pause=5, count=2, - expected_exceptions=subprocess.CalledProcessError, - expected_retval=None, fnc=run_cmd, cmd=c).run() - except ProbeTimeout as e: - raise ConuException("Cannot start build of application: %s" % e) + self.start_build(name, ["-n", project, "start-build", "--from-dir=%s" % source]) return name @@ -186,9 +176,22 @@ def _create_app_from_template(self, image, name, template, name_in_template, except subprocess.CalledProcessError as ex: raise ConuException("oc new-app failed: %s" % ex) - c = self._oc_command(["start-build"] + [name]) + self.start_build(name) + + def start_build(self, build, args=None): + """ + Start new build, raise exception if build failed + :param build: str, name of the build + :param args: list of str, another args of 'oc start-build' commands + :return: None + """ + + args = args or [] + + c = self._oc_command(["start-build"] + [build] + args) - logger.info("Build application from local source in project %s", project) + logger.info("Executing build %s" % build) + logger.info("Build command: %s" % " ".join(c)) try: Probe(timeout=-1, pause=5, count=2, expected_exceptions=subprocess.CalledProcessError, From 9e2678defe126315bde7158579caf40ceeae9a42 Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Mon, 1 Oct 2018 08:33:32 +0200 Subject: [PATCH 06/14] add port option to request service --- conu/backend/origin/backend.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 58a0682d..f206df05 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -199,19 +199,22 @@ def start_build(self, build, args=None): except ProbeTimeout as e: raise ConuException("Cannot start build of application: %s" % e) - def request_service(self, app_name, expected_output=None): + def request_service(self, app_name, port, expected_output=None): """ Make request on service of app. If there is connection error function return False. :param app_name: str, name of app :param expected_output: str, If not None method will check output returned from request and try to find matching string. + :param port: str or int, port of the service :return: bool, True if connection was established False if there was connection error """ + + # get ip of service ip = [service.get_ip() for service in self.list_services() if service.name == app_name][0] try: - output = self.http_request(host=ip) + output = self.http_request(host=ip, port=port) if expected_output is not None: if expected_output not in output.text: raise ConuException( From a918ae7a5d896ff9d2471d8533a903ab5310161c Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Mon, 1 Oct 2018 08:57:51 +0200 Subject: [PATCH 07/14] move hardcoded NAME and NAMESPACE setup from origin backend --- conu/backend/origin/backend.py | 2 -- conu/backend/origin/constants.py | 2 +- conu/backend/origin/registry.py | 6 +++--- tests/integration/test_openshift.py | 8 ++++++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index f206df05..882faef4 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -161,8 +161,6 @@ def _create_app_from_template(self, image, name, template, name_in_template, image, tag = list(o.items())[0] push_to_registry(image, tag.split(':')[0], tag.split(':')[1], project) - oc_new_app_args += ["-p", "NAME=%s" % name, "-p", "NAMESPACE=%s" % project] - c = self._oc_command(["new-app"] + [template] + oc_new_app_args + ["-n"] + [project] + ["--name=%s" % name]) diff --git a/conu/backend/origin/constants.py b/conu/backend/origin/constants.py index 8c2c0963..34eabf23 100644 --- a/conu/backend/origin/constants.py +++ b/conu/backend/origin/constants.py @@ -18,4 +18,4 @@ This is constants used in origin backend """ -PORT = "5000" +INTERNAL_REGISTRY_PORT = "5000" diff --git a/conu/backend/origin/registry.py b/conu/backend/origin/registry.py index 6b98683c..a9f0cd93 100644 --- a/conu/backend/origin/registry.py +++ b/conu/backend/origin/registry.py @@ -18,7 +18,7 @@ This are the methods helpful while working with OpenShift internal docker registry """ -from conu.backend.origin.constants import PORT +from conu.backend.origin.constants import INTERNAL_REGISTRY_PORT from conu.backend.docker.backend import DockerBackend from conu.utils import get_oc_api_token @@ -57,6 +57,6 @@ def get_internal_registry_ip(): for service in services: if service.name == 'docker-registry': logger.debug("Internal docker-registry IP: %s", - "{ip}:{port}".format(ip=service.get_ip(), port=PORT)) - return "{ip}:{port}".format(ip=service.get_ip(), port=PORT) + "{ip}:{port}".format(ip=service.get_ip(), port=INTERNAL_REGISTRY_PORT)) + return "{ip}:{port}".format(ip=service.get_ip(), port=INTERNAL_REGISTRY_PORT) return None diff --git a/tests/integration/test_openshift.py b/tests/integration/test_openshift.py index 90bfcd81..651e2a6d 100644 --- a/tests/integration/test_openshift.py +++ b/tests/integration/test_openshift.py @@ -85,16 +85,20 @@ def test_oc_s2i_template(self): login_to_registry('developer', token=api_key) + project = 'myproject' + app_name = openshift_backend.new_app( image=python_image, template="https://raw.githubusercontent.com/sclorg/django-ex" "/master/openshift/templates/django-postgresql.json", - oc_new_app_args=["-p", "SOURCE_REPOSITORY_REF=master", "-p", + oc_new_app_args=["-p", "NAMESPACE=%s" % project, + "-p", "NAME=django-psql-example", + "-p", "SOURCE_REPOSITORY_REF=master", "-p", "PYTHON_VERSION=3.6", "-p", "POSTGRESQL_VERSION=9.6"], name_in_template={"python": "3.6"}, other_images=[{psql_image: "postgresql:9.6"}], - project='myproject') + project=project) try: openshift_backend.wait_for_service( From b683d0139cafdb8cc6017130ea993682feef14da Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Tue, 2 Oct 2018 16:01:44 +0200 Subject: [PATCH 08/14] add logs and status to origin backend --- conu/backend/origin/backend.py | 50 +++++++++++++++++++++++++++------ conu/backend/origin/registry.py | 6 +++- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 882faef4..9f5a281c 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -132,7 +132,7 @@ def new_app(self, image, project, source=None, template=None, name_in_template=N raise ConuException("oc new-app failed: %s" % ex) if os.path.isdir(source): - self.start_build(name, ["-n", project, "start-build", "--from-dir=%s" % source]) + self.start_build(name, ["-n", project, "--from-dir=%s" % source]) return name @@ -174,8 +174,6 @@ def _create_app_from_template(self, image, name, template, name_in_template, except subprocess.CalledProcessError as ex: raise ConuException("oc new-app failed: %s" % ex) - self.start_build(name) - def start_build(self, build, args=None): """ Start new build, raise exception if build failed @@ -200,7 +198,7 @@ def start_build(self, build, args=None): def request_service(self, app_name, port, expected_output=None): """ Make request on service of app. If there is connection error function return False. - :param app_name: str, name of app + :param app_name: str, name of the app :param expected_output: str, If not None method will check output returned from request and try to find matching string. :param port: str or int, port of the service @@ -223,19 +221,55 @@ def request_service(self, app_name, port, expected_output=None): except ConnectionError: return False - def wait_for_service(self, app_name, expected_output=None, timeout=100): + def wait_for_service(self, app_name, port, expected_output=None, timeout=100): """ Block until service is not ready to accept requests, raises an exc ProbeTimeout if timeout is reached - :param app_name: str, name of app + :param app_name: str, name of the app + :param port: str or int, port of the service :param expected_output: If not None method will check output returned from request and try to find matching string. :param timeout: int or float (seconds), time to wait for pod to run :return: None """ logger.info('Waiting for service to get ready') - Probe(timeout=timeout, fnc=self.request_service, - app_name=app_name, expected_output=expected_output, expected_retval=True).run() + try: + + Probe(timeout=timeout, fnc=self.request_service, app_name=app_name, + port=port, expected_output=expected_output, expected_retval=True).run() + except ProbeTimeout: + logger.warning("Timeout: Request to service unsuccessful.") + ConuException("Timeout: Request to service unsuccessful.") + + def get_status(self): + """ + Get status of OpenShift cluster, similar to `oc cluster status` + :return: str + """ + try: + c = self._oc_command(["cluster", "status"]) + o = run_cmd(c, return_output=True) + for line in o.split('\n'): + logger.info(line) + return o + except subprocess.CalledProcessError as ex: + raise ConuException("Cannot obtain OpenShift cluster status: %s" % ex) + + def get_logs(self, name): + """ + Get logs from all pods + :param name: str, name of app + :return: str, cluster status and logs from all pods + """ + logs = self.get_status() + + for pod in self.list_pods(): + if name in pod.name: # get just logs from pods related to app + pod_logs = pod.get_logs() + if pod_logs: + logs += pod_logs + + return logs def clean_project(self, app_name=None, delete_all=False): """ diff --git a/conu/backend/origin/registry.py b/conu/backend/origin/registry.py index a9f0cd93..a599c356 100644 --- a/conu/backend/origin/registry.py +++ b/conu/backend/origin/registry.py @@ -17,10 +17,14 @@ """ This are the methods helpful while working with OpenShift internal docker registry """ +import logging from conu.backend.origin.constants import INTERNAL_REGISTRY_PORT from conu.backend.docker.backend import DockerBackend from conu.utils import get_oc_api_token +import conu.backend.origin.backend + +logger = logging.getLogger(__name__) def login_to_registry(username, token=None): @@ -52,7 +56,7 @@ def get_internal_registry_ip(): Search for `docker-registry` IP :return: str, ip address """ - with OpenshiftBackend() as origin_backend: + with conu.backend.origin.backend.OpenshiftBackend() as origin_backend: services = origin_backend.list_services() for service in services: if service.name == 'docker-registry': From b614997b6e10f193810f3c3d3082c868fbbb99cb Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Wed, 3 Oct 2018 14:17:06 +0200 Subject: [PATCH 09/14] split oc new-app to independent methods --- conu/backend/origin/backend.py | 117 ++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 46 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 9f5a281c..2a962edd 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -28,7 +28,7 @@ from conu.backend.origin.registry import push_to_registry from conu.backend.k8s.backend import K8sBackend from conu.exceptions import ConuException -from conu.utils import oc_command_exists, run_cmd, random_str +from conu.utils import oc_command_exists, run_cmd, random_str, check_port from conu.utils.http_client import get_url from conu.utils.probes import Probe, ProbeTimeout @@ -72,7 +72,9 @@ def http_request(self, path="/", method="GET", host=None, port=None, json=False, port = port or 8080 url = get_url(host=host, port=port, path=path) - return self.http_session.request(method, url, json=json, data=data) + logger.info("Request url: %s" % url) + + return self.http_session.request(method, url, json=json, data=data, verify=False) def _oc_command(self, args): """ @@ -84,63 +86,77 @@ def _oc_command(self, args): oc_command_exists() return ["oc"] + args - def new_app(self, image, project, source=None, template=None, name_in_template=None, - other_images=None, oc_new_app_args=None): + def deploy_image(self, image, oc_new_app_args, project, name=None): + """ + Deploy image in OpenShift cluster using 'oc new-app' + :param image: DockerImage, image to be deployed + :param oc_new_app_args: additional parameters for the `oc new-app`, env variables etc. + :param project: project where app should be created + :param name:str, name of application, if None random name is generated + :return: str, name of the app """ - Deploy app in OpenShift cluster using 'oc new-app' + + # app name is generated randomly + name = name or 'app-{random_string}'.format(random_string=random_str(5)) + + oc_new_app_args = oc_new_app_args or [] + + new_image = push_to_registry(image, image.name.split('/')[-1], image.tag, project) + + c = self._oc_command( + ["new-app"] + oc_new_app_args + [new_image.name] + + ["-n"] + [project] + ["--name=%s" % name]) + + logger.info("Creating new app in project %s", project) + + try: + run_cmd(c) + except subprocess.CalledProcessError as ex: + raise ConuException("oc new-app failed: %s" % ex) + + return name + + def create_new_app_from_source(self, image, project, source=None, oc_new_app_args=None): + """ + Deploy app using source-to-image in OpenShift cluster using 'oc new-app' :param image: image to be used as builder image :param project: project where app should be created :param source: source used to extend the image, can be path or url - :param template: str, url or local path to a template to use - :param name_in_template: dict, {repository:tag} image name used in the template - :param other_images: list of dict, some templates need other image to be pushed into the - OpenShift registry, specify them in this parameter as list of dict [{:}], - where "" is DockerImage and "" is a tag under which the image should be - available in the OpenShift registry. :param oc_new_app_args: additional parameters for the `oc new-app` - :return: str, name of app + :return: str, name of the app """ - if template is not None and source is not None: - raise ConuException('cannot combine template parameter with source parameter') - # app name is generated randomly name = 'app-{random_string}'.format(random_string=random_str(5)) oc_new_app_args = oc_new_app_args or [] - if template is not None: - if name_in_template is None: - raise ConuException('You need to specify name_in_template') + new_image = push_to_registry(image, image.name.split('/')[-1], image.tag, project) - self._create_app_from_template(image, name, template, name_in_template, - other_images, oc_new_app_args, project) + c = self._oc_command( + ["new-app"] + [new_image.name + "~" + source] + oc_new_app_args + + ["-n"] + [project] + ["--name=%s" % name]) - else: - new_image = push_to_registry(image, image.name.split('/')[-1], image.tag, project) - - c = self._oc_command( - ["new-app"] + oc_new_app_args + [new_image.name + "~" + source] + - ["-n"] + [project] + ["--name=%s" % name]) - - logger.info("Creating new app in project %s", project) + logger.info("Creating new app in project %s", project) - try: - o = run_cmd(c, return_output=True) - logger.debug(o) - except subprocess.CalledProcessError as ex: - raise ConuException("oc new-app failed: %s" % ex) + try: + o = run_cmd(c, return_output=True) + logger.debug(o) + except subprocess.CalledProcessError as ex: + raise ConuException("oc new-app failed: %s" % ex) - if os.path.isdir(source): - self.start_build(name, ["-n", project, "--from-dir=%s" % source]) + # build from local source + if os.path.isdir(source): + self.start_build(name, ["-n", project, "--from-dir=%s" % source]) return name - def _create_app_from_template(self, image, name, template, name_in_template, - other_images, oc_new_app_args, project): + def create_app_from_template(self, image, name, template, name_in_template, + other_images, oc_new_app_args, project): """ Helper function to create app from template :param image: image to be used as builder image + :param name: name of app from template :param template: str, url or local path to a template to use :param name_in_template: dict, {repository:tag} image name used in the template :param other_images: list of dict, some templates need other image to be pushed into the @@ -174,6 +190,8 @@ def _create_app_from_template(self, image, name, template, name_in_template, except subprocess.CalledProcessError as ex: raise ConuException("oc new-app failed: %s" % ex) + return name + def start_build(self, build, args=None): """ Start new build, raise exception if build failed @@ -209,17 +227,24 @@ def request_service(self, app_name, port, expected_output=None): ip = [service.get_ip() for service in self.list_services() if service.name == app_name][0] - try: - output = self.http_request(host=ip, port=port) - if expected_output is not None: + # make http request to obtain output + if expected_output is not None: + try: + output = self.http_request(host=ip, port=port) + logger.info("Requesting service %s with IP address: %s" % (app_name, ip)) + logger.info("Answer: %s" % output.text) if expected_output not in output.text: raise ConuException( "Connection to service established, but didn't match expected output") else: logger.info("Connection to service established and return expected output!") + except ConnectionError as e: + logger.info("Connection to service failed %s!" % e) + return False + elif check_port(port, host=ip): # check if port is open return True - except ConnectionError: - return False + + return False def wait_for_service(self, app_name, port, expected_output=None, timeout=100): """ @@ -235,19 +260,19 @@ def wait_for_service(self, app_name, port, expected_output=None, timeout=100): logger.info('Waiting for service to get ready') try: - Probe(timeout=timeout, fnc=self.request_service, app_name=app_name, + Probe(timeout=timeout, pause=5, count=10, fnc=self.request_service, app_name=app_name, port=port, expected_output=expected_output, expected_retval=True).run() except ProbeTimeout: logger.warning("Timeout: Request to service unsuccessful.") - ConuException("Timeout: Request to service unsuccessful.") + raise ConuException("Timeout: Request to service unsuccessful.") def get_status(self): """ - Get status of OpenShift cluster, similar to `oc cluster status` + Get status of OpenShift cluster, similar to `oc status` :return: str """ try: - c = self._oc_command(["cluster", "status"]) + c = self._oc_command(["status"]) o = run_cmd(c, return_output=True) for line in o.split('\n'): logger.info(line) From e2c118e865b9957eadc9d94a24e641ec0384196f Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Wed, 3 Oct 2018 15:01:23 +0200 Subject: [PATCH 10/14] all_pods_are_ready in openshift backend --- conu/backend/origin/backend.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 2a962edd..9e005f24 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -260,12 +260,30 @@ def wait_for_service(self, app_name, port, expected_output=None, timeout=100): logger.info('Waiting for service to get ready') try: - Probe(timeout=timeout, pause=5, count=10, fnc=self.request_service, app_name=app_name, + Probe(timeout=timeout, pause=10, fnc=self.request_service, app_name=app_name, port=port, expected_output=expected_output, expected_retval=True).run() except ProbeTimeout: logger.warning("Timeout: Request to service unsuccessful.") raise ConuException("Timeout: Request to service unsuccessful.") + def all_pods_are_ready(self, app_name): + """ + Check if all pods are ready for specific app + :param app_name: str, name of the app + :return: bool + """ + app_pod_exists = False + for pod in self.list_pods(): + if app_name in pod.name and 'build' not in pod.name and 'deploy' not in pod.name: + app_pod_exists = True + if not pod.is_ready(): + return False + if app_pod_exists: + logger.info("All pods are ready!") + return True + + return False + def get_status(self): """ Get status of OpenShift cluster, similar to `oc status` From bd73d629bcb193c9739f4f70b59c644bc0e5ba51 Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 4 Oct 2018 15:04:51 +0200 Subject: [PATCH 11/14] update tests, docs and examples according to new openshift functionality --- README.md | 4 + conu/backend/origin/backend.py | 7 +- .../mysql-cfg/myconfig.cnf | 3 + .../extend-mariadb-image/mysql-data/init.sql | 4 + .../mysql-init/80-add-arbitrary-users.sh | 17 +++ .../mysql-init/90-init-db.sh | 12 ++ .../80-check-arbitrary-users.sh | 10 ++ .../examples/openshift/openshift_database.py | 56 +++++++++ .../examples/openshift/openshift_s2i_local.py | 12 +- .../openshift/openshift_s2i_remote.py | 16 ++- .../examples/openshift/openshift_template.py | 22 +++- tests/constants.py | 9 ++ tests/integration/test_openshift.py | 118 ++++++++++++++---- 13 files changed, 250 insertions(+), 40 deletions(-) create mode 100755 docs/source/examples/openshift/extend-mariadb-image/mysql-cfg/myconfig.cnf create mode 100755 docs/source/examples/openshift/extend-mariadb-image/mysql-data/init.sql create mode 100755 docs/source/examples/openshift/extend-mariadb-image/mysql-init/80-add-arbitrary-users.sh create mode 100755 docs/source/examples/openshift/extend-mariadb-image/mysql-init/90-init-db.sh create mode 100755 docs/source/examples/openshift/extend-mariadb-image/mysql-pre-init/80-check-arbitrary-users.sh create mode 100644 docs/source/examples/openshift/openshift_database.py diff --git a/README.md b/README.md index 42a5da00..5510895b 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,16 @@ usercont/conu:0.5.0 python3 /app/my_source.py ## OpenShift - create new app using `oc new-app` command + - deploy pure image into openshift - support building s2i images from remote repository - support building s2i images from local path - support creating new applications using OpenShift templates - push images to internal OpenShift registry - request service - waiting until service is ready +- obtain logs from all pods +- get status of application +- check readiness of pods - cleanup objects of specific application in current namespace # Docker example diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 9e005f24..49bbdc5c 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -72,9 +72,7 @@ def http_request(self, path="/", method="GET", host=None, port=None, json=False, port = port or 8080 url = get_url(host=host, port=port, path=path) - logger.info("Request url: %s" % url) - - return self.http_session.request(method, url, json=json, data=data, verify=False) + return self.http_session.request(method, url, json=json, data=data) def _oc_command(self, args): """ @@ -231,13 +229,12 @@ def request_service(self, app_name, port, expected_output=None): if expected_output is not None: try: output = self.http_request(host=ip, port=port) - logger.info("Requesting service %s with IP address: %s" % (app_name, ip)) - logger.info("Answer: %s" % output.text) if expected_output not in output.text: raise ConuException( "Connection to service established, but didn't match expected output") else: logger.info("Connection to service established and return expected output!") + return True except ConnectionError as e: logger.info("Connection to service failed %s!" % e) return False diff --git a/docs/source/examples/openshift/extend-mariadb-image/mysql-cfg/myconfig.cnf b/docs/source/examples/openshift/extend-mariadb-image/mysql-cfg/myconfig.cnf new file mode 100755 index 00000000..7764adf9 --- /dev/null +++ b/docs/source/examples/openshift/extend-mariadb-image/mysql-cfg/myconfig.cnf @@ -0,0 +1,3 @@ +[mysqld] +query-cache-limit=262144 + diff --git a/docs/source/examples/openshift/extend-mariadb-image/mysql-data/init.sql b/docs/source/examples/openshift/extend-mariadb-image/mysql-data/init.sql new file mode 100755 index 00000000..31599829 --- /dev/null +++ b/docs/source/examples/openshift/extend-mariadb-image/mysql-data/init.sql @@ -0,0 +1,4 @@ +CREATE TABLE products (id INTEGER, name VARCHAR(256), price FLOAT, variant INTEGER); +CREATE TABLE products_variant (id INTEGER, name VARCHAR(256)); +INSERT INTO products_variant (id, name) VALUES ('1', 'blue'), ('2', 'green'); + diff --git a/docs/source/examples/openshift/extend-mariadb-image/mysql-init/80-add-arbitrary-users.sh b/docs/source/examples/openshift/extend-mariadb-image/mysql-init/80-add-arbitrary-users.sh new file mode 100755 index 00000000..55ae2d29 --- /dev/null +++ b/docs/source/examples/openshift/extend-mariadb-image/mysql-init/80-add-arbitrary-users.sh @@ -0,0 +1,17 @@ +create_arbitrary_users() { + # Do not care what option is compulsory here, just create what is specified + log_info "Creating user specified by MYSQL_OPERATIONS_USER (${MYSQL_OPERATIONS_USER}) ..." +mysql $mysql_flags <. +# + +import logging + +from conu.backend.origin.backend import OpenshiftBackend +from conu.backend.docker.backend import DockerBackend +from conu.backend.origin.registry import login_to_registry +from conu.utils import get_oc_api_token + +api_key = get_oc_api_token() +with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + + openshift_backend.get_status() + + with DockerBackend(logging_level=logging.DEBUG) as backend: + mariadb_image = backend.ImageClass("centos/mariadb-102-centos7") + + login_to_registry('developer', token=api_key) + + app_name = openshift_backend.create_new_app_from_source( + mariadb_image, + oc_new_app_args=[ + "--env", "MYSQL_ROOT_PASSWORD=test", + "--env", "MYSQL_OPERATIONS_USER=test1", + "--env", "MYSQL_OPERATIONS_PASSWORD=test1", + "--env", "MYSQL_DATABASE=testdb", + "--env", "MYSQL_USER=user1", + "--env", "MYSQL_PASSWORD=user1"], + source="examples/openshift/extend-mariadb-image", + project='myproject') + + openshift_backend.get_status() + + try: + openshift_backend.wait_for_service( + app_name=app_name, + port=3306, + timeout=300) + assert openshift_backend.all_pods_are_ready(app_name) + finally: + openshift_backend.get_logs(app_name) + openshift_backend.clean_project(app_name) diff --git a/docs/source/examples/openshift/openshift_s2i_local.py b/docs/source/examples/openshift/openshift_s2i_local.py index 8ee3d7f6..747af2c6 100644 --- a/docs/source/examples/openshift/openshift_s2i_local.py +++ b/docs/source/examples/openshift/openshift_s2i_local.py @@ -18,6 +18,7 @@ from conu.backend.origin.backend import OpenshiftBackend from conu.backend.docker.backend import DockerBackend +from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token api_key = get_oc_api_token() @@ -27,17 +28,20 @@ python_image = backend.ImageClass("centos/python-36-centos7") # docker login inside OpenShift internal registry - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer') # create new app from local source in OpenShift cluster - app_name = openshift_backend.new_app(python_image, - source="examples/openshift/standalone-test-app", - project='myproject') + app_name = openshift_backend.create_new_app_from_source( + python_image, + source="examples/openshift/standalone-test-app", + project='myproject') try: # wait until service is ready to accept requests openshift_backend.wait_for_service( app_name=app_name, + port=8080, expected_output="Hello World from standalone WSGI application!") finally: + openshift_backend.get_logs(app_name) openshift_backend.clean_project(app_name) diff --git a/docs/source/examples/openshift/openshift_s2i_remote.py b/docs/source/examples/openshift/openshift_s2i_remote.py index b77ffc8d..b665d2ff 100644 --- a/docs/source/examples/openshift/openshift_s2i_remote.py +++ b/docs/source/examples/openshift/openshift_s2i_remote.py @@ -18,27 +18,35 @@ from conu.backend.origin.backend import OpenshiftBackend from conu.backend.docker.backend import DockerBackend +from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token api_key = get_oc_api_token() with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + + openshift_backend.get_status() + with DockerBackend(logging_level=logging.DEBUG) as backend: # builder image python_image = backend.ImageClass("centos/python-36-centos7") # docker login inside OpenShift internal registry - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer') # create new app from remote source in OpenShift cluster - app_name = openshift_backend.new_app(python_image, - source="https://github.com/openshift/django-ex.git", - project='myproject') + app_name = openshift_backend.create_new_app_from_source( + python_image, + source="https://github.com/openshift/django-ex.git", + project='myproject') try: # wait until service is ready to accept requests + openshift_backend.wait_for_service( app_name=app_name, + port=8080, expected_output='Welcome to your Django application on OpenShift', timeout=300) finally: + openshift_backend.get_logs(app_name) openshift_backend.clean_project(app_name) diff --git a/docs/source/examples/openshift/openshift_template.py b/docs/source/examples/openshift/openshift_template.py index d18f002a..d0ebfaf2 100644 --- a/docs/source/examples/openshift/openshift_template.py +++ b/docs/source/examples/openshift/openshift_template.py @@ -18,6 +18,7 @@ from conu.backend.origin.backend import OpenshiftBackend from conu.backend.docker.backend import DockerBackend +from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token @@ -30,24 +31,33 @@ psql_image = backend.ImageClass("centos/postgresql-96-centos7", tag="9.6") # docker login inside OpenShift internal registry - OpenshiftBackend.login_to_registry('developer') + login_to_registry('developer') # create new app from remote source in OpenShift cluster - app_name = openshift_backend.new_app( + project = 'myproject' + app_name_in_template = 'django-psql-example' + + openshift_backend.create_app_from_template( image=python_image, + name=app_name_in_template, template="https://raw.githubusercontent.com/sclorg/django-ex" "/master/openshift/templates/django-postgresql.json", - oc_new_app_args=["-p", "SOURCE_REPOSITORY_REF=master", "-p", "PYTHON_VERSION=3.6", + oc_new_app_args=["-p", "NAMESPACE=%s" % project, + "-p", "NAME=%s" % app_name_in_template, + "-p", "SOURCE_REPOSITORY_REF=master", "-p", + "PYTHON_VERSION=3.6", "-p", "POSTGRESQL_VERSION=9.6"], name_in_template={"python": "3.6"}, other_images=[{psql_image: "postgresql:9.6"}], - project='myproject') + project=project) try: # wait until service is ready to accept requests openshift_backend.wait_for_service( - app_name=app_name, + app_name=app_name_in_template, + port=8080, expected_output='Welcome to your Django application on OpenShift', timeout=300) finally: - openshift_backend.clean_project(app_name) + openshift_backend.get_logs(app_name_in_template) + openshift_backend.clean_project(app_name_in_template) diff --git a/tests/constants.py b/tests/constants.py index 2082bfb5..958d5f02 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -24,3 +24,12 @@ S2I_IMAGE = "punchbag" THE_HELPER_IMAGE = "rudolph" + + +CENTOS_PYTHON_3 = "centos/python-36-centos7" +CENTOS_MARIADB_10_2 = "centos/mariadb-102-centos7" +MY_PROJECT = "myproject" +OC_CLUSTER_USER = "developer" +CENTOS_POSTGRES_9_6 = "centos/postgresql-96-centos7" +CENTOS_POSTGRES_9_6_TAG = "9.6" +DJANGO_POSTGRES_TEMPLATE = "django-psql-example" diff --git a/tests/integration/test_openshift.py b/tests/integration/test_openshift.py index 651e2a6d..cf255960 100644 --- a/tests/integration/test_openshift.py +++ b/tests/integration/test_openshift.py @@ -25,86 +25,162 @@ from conu.backend.docker.backend import DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token, oc_command_exists, is_oc_cluster_running +from ..constants import CENTOS_MARIADB_10_2, CENTOS_PYTHON_3, MY_PROJECT, OC_CLUSTER_USER, \ + CENTOS_POSTGRES_9_6, CENTOS_POSTGRES_9_6_TAG, DJANGO_POSTGRES_TEMPLATE @pytest.mark.skipif(not oc_command_exists(), reason="OpenShift is not installed!") @pytest.mark.skipif(not is_oc_cluster_running(), reason="OpenShift cluster is not running!") class TestOpenshift(object): + def test_deploy_image(self): + api_key = get_oc_api_token() + with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + + with DockerBackend(logging_level=logging.DEBUG) as backend: + # builder image + mariadb_image = backend.ImageClass(CENTOS_MARIADB_10_2) + + # docker login inside OpenShift internal registry + login_to_registry(OC_CLUSTER_USER, token=api_key) + + # create new app from remote source in OpenShift cluster + app_name = openshift_backend.deploy_image(mariadb_image, + oc_new_app_args=[ + "--env", "MYSQL_ROOT_PASSWORD=test"], + project=MY_PROJECT) + + try: + # wait until service is ready to accept requests + openshift_backend.wait_for_service( + app_name=app_name, + port=3306, + timeout=300) + assert openshift_backend.all_pods_are_ready(app_name) + finally: + openshift_backend.get_logs(app_name) + openshift_backend.clean_project(app_name) + + def test_oc_s2i_local_mariadb(self): + api_key = get_oc_api_token() + with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + + openshift_backend.get_status() + + with DockerBackend(logging_level=logging.DEBUG) as backend: + mariadb_image = backend.ImageClass(CENTOS_MARIADB_10_2) + + login_to_registry(OC_CLUSTER_USER, token=api_key) + + app_name = openshift_backend.create_new_app_from_source( + mariadb_image, + oc_new_app_args=[ + "--env", "MYSQL_ROOT_PASSWORD=test", + "--env", "MYSQL_OPERATIONS_USER=test1", + "--env", "MYSQL_OPERATIONS_PASSWORD=test1", + "--env", "MYSQL_DATABASE=testdb", + "--env", "MYSQL_USER=user1", + "--env", "MYSQL_PASSWORD=user1"], + source="examples/openshift/extend-mariadb-image", + project=MY_PROJECT) + + openshift_backend.get_status() + + try: + openshift_backend.wait_for_service( + app_name=app_name, + port=3306, + timeout=300) + assert openshift_backend.all_pods_are_ready(app_name) + finally: + openshift_backend.get_logs(app_name) + openshift_backend.clean_project(app_name) + def test_oc_s2i_remote(self): api_key = get_oc_api_token() with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + openshift_backend.get_status() + with DockerBackend(logging_level=logging.DEBUG) as backend: - python_image = backend.ImageClass("centos/python-36-centos7") + python_image = backend.ImageClass(CENTOS_PYTHON_3) - login_to_registry('developer', token=api_key) + login_to_registry(OC_CLUSTER_USER, token=api_key) - app_name = openshift_backend.new_app( + app_name = openshift_backend.create_new_app_from_source( python_image, source="https://github.com/openshift/django-ex.git", - project='myproject') + project=MY_PROJECT) try: openshift_backend.wait_for_service( app_name=app_name, + port=8080, expected_output='Welcome to your Django application on OpenShift', timeout=300) finally: + openshift_backend.get_logs(app_name) openshift_backend.clean_project(app_name) def test_oc_s2i_local(self): api_key = get_oc_api_token() with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: + openshift_backend.get_status() + with DockerBackend(logging_level=logging.DEBUG) as backend: - python_image = backend.ImageClass("centos/python-36-centos7") + python_image = backend.ImageClass(CENTOS_PYTHON_3) - login_to_registry('developer', token=api_key) + login_to_registry(OC_CLUSTER_USER, token=api_key) - app_name = openshift_backend.new_app( + app_name = openshift_backend.create_new_app_from_source( python_image, source="examples/openshift/standalone-test-app", - project='myproject') + project=MY_PROJECT) try: openshift_backend.wait_for_service( app_name=app_name, + port=8080, expected_output="Hello World from standalone WSGI application!", timeout=300) finally: + openshift_backend.get_logs(app_name) openshift_backend.clean_project(app_name) def test_oc_s2i_template(self): api_key = get_oc_api_token() with OpenshiftBackend(api_key=api_key, logging_level=logging.DEBUG) as openshift_backend: - with DockerBackend(logging_level=logging.DEBUG) as backend: - python_image = backend.ImageClass("centos/python-36-centos7", tag="latest") - psql_image = backend.ImageClass("centos/postgresql-96-centos7", tag="9.6") + openshift_backend.get_status() - login_to_registry('developer', token=api_key) + with DockerBackend(logging_level=logging.DEBUG) as backend: + python_image = backend.ImageClass(CENTOS_PYTHON_3, tag="latest") + psql_image = backend.ImageClass(CENTOS_POSTGRES_9_6, tag=CENTOS_POSTGRES_9_6_TAG) - project = 'myproject' + login_to_registry(OC_CLUSTER_USER, token=api_key) - app_name = openshift_backend.new_app( + openshift_backend.create_app_from_template( image=python_image, + name=DJANGO_POSTGRES_TEMPLATE, template="https://raw.githubusercontent.com/sclorg/django-ex" "/master/openshift/templates/django-postgresql.json", - oc_new_app_args=["-p", "NAMESPACE=%s" % project, - "-p", "NAME=django-psql-example", + oc_new_app_args=["-p", "NAMESPACE=%s" % MY_PROJECT, + "-p", "NAME=%s" % DJANGO_POSTGRES_TEMPLATE, "-p", "SOURCE_REPOSITORY_REF=master", "-p", "PYTHON_VERSION=3.6", - "-p", "POSTGRESQL_VERSION=9.6"], + "-p", "POSTGRESQL_VERSION=%s" % CENTOS_POSTGRES_9_6_TAG], name_in_template={"python": "3.6"}, - other_images=[{psql_image: "postgresql:9.6"}], - project=project) + other_images=[{psql_image: "postgresql:%s" % CENTOS_POSTGRES_9_6_TAG}], + project=MY_PROJECT) try: openshift_backend.wait_for_service( - app_name=app_name, + app_name=DJANGO_POSTGRES_TEMPLATE, + port=8080, expected_output='Welcome to your Django application on OpenShift', timeout=300) finally: + openshift_backend.get_logs(DJANGO_POSTGRES_TEMPLATE) # pass name from template as argument - openshift_backend.clean_project('django-psql-example') + openshift_backend.clean_project(DJANGO_POSTGRES_TEMPLATE) From bad1675c01efa4a0b43980d9092645ce0eb71a4f Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Mon, 8 Oct 2018 08:53:06 +0200 Subject: [PATCH 12/14] specify string format arguments as logging function parameters --- conu/backend/origin/backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 49bbdc5c..676f9766 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -202,8 +202,8 @@ def start_build(self, build, args=None): c = self._oc_command(["start-build"] + [build] + args) - logger.info("Executing build %s" % build) - logger.info("Build command: %s" % " ".join(c)) + logger.info("Executing build %s", build) + logger.info("Build command: %s", " ".join(c)) try: Probe(timeout=-1, pause=5, count=2, expected_exceptions=subprocess.CalledProcessError, @@ -236,7 +236,7 @@ def request_service(self, app_name, port, expected_output=None): logger.info("Connection to service established and return expected output!") return True except ConnectionError as e: - logger.info("Connection to service failed %s!" % e) + logger.info("Connection to service failed %s!", e) return False elif check_port(port, host=ip): # check if port is open return True @@ -328,7 +328,7 @@ def clean_project(self, app_name=None, delete_all=False): logger.info('Deleting all objects in current project') else: args = "-l app=%s" % app_name - logger.info('Deleting all objects with label app=%s' % app_name) + logger.info('Deleting all objects with label app=%s', app_name) try: o = run_cmd(self._oc_command(["delete", "all", args]), From 8a86f22e0bedf490b60b12266049f173deb12d3d Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Wed, 17 Oct 2018 13:12:08 +0200 Subject: [PATCH 13/14] better UX for importin K8s and OpenShift backend --- conu/__init__.py | 6 ++++++ docs/source/examples/k8s_deployment.py | 2 +- docs/source/examples/k8s_pod.py | 4 ++-- docs/source/examples/openshift/openshift_database.py | 4 ++-- docs/source/examples/openshift/openshift_s2i_local.py | 4 ++-- docs/source/examples/openshift/openshift_s2i_remote.py | 4 ++-- docs/source/examples/openshift/openshift_template.py | 4 ++-- tests/integration/test_k8s.py | 4 ++-- tests/integration/test_openshift.py | 4 ++-- 9 files changed, 21 insertions(+), 15 deletions(-) diff --git a/conu/__init__.py b/conu/__init__.py index 3309e5e0..629c8302 100644 --- a/conu/__init__.py +++ b/conu/__init__.py @@ -26,6 +26,12 @@ DockerImage, S2IDockerImage, DockerImagePullPolicy, DockerImageViaArchiveFS ) +# k8s backend +from conu.backend.k8s.backend import K8sBackend, K8sCleanupPolicy + +# OpenShift +from conu.backend.origin.backend import OpenshiftBackend + # utils from conu.utils.filesystem import Directory from conu.utils.probes import Probe, ProbeTimeout, CountExceeded diff --git a/docs/source/examples/k8s_deployment.py b/docs/source/examples/k8s_deployment.py index 6f2a58a1..7f281335 100644 --- a/docs/source/examples/k8s_deployment.py +++ b/docs/source/examples/k8s_deployment.py @@ -19,7 +19,7 @@ """ import logging -from conu.backend.k8s.backend import K8sBackend +from conu import K8sBackend from conu.backend.k8s.deployment import Deployment from conu.utils import get_oc_api_token diff --git a/docs/source/examples/k8s_pod.py b/docs/source/examples/k8s_pod.py index d59df972..de406e11 100644 --- a/docs/source/examples/k8s_pod.py +++ b/docs/source/examples/k8s_pod.py @@ -16,8 +16,8 @@ import logging -from conu.backend.k8s.backend import K8sBackend -from conu.backend.docker.backend import DockerBackend +from conu import K8sBackend, \ + DockerBackend from conu.backend.k8s.pod import PodPhase from conu.utils import get_oc_api_token diff --git a/docs/source/examples/openshift/openshift_database.py b/docs/source/examples/openshift/openshift_database.py index 08b242ae..94e7578c 100644 --- a/docs/source/examples/openshift/openshift_database.py +++ b/docs/source/examples/openshift/openshift_database.py @@ -16,8 +16,8 @@ import logging -from conu.backend.origin.backend import OpenshiftBackend -from conu.backend.docker.backend import DockerBackend +from conu import OpenshiftBackend, \ + DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token diff --git a/docs/source/examples/openshift/openshift_s2i_local.py b/docs/source/examples/openshift/openshift_s2i_local.py index 747af2c6..eefcf3c7 100644 --- a/docs/source/examples/openshift/openshift_s2i_local.py +++ b/docs/source/examples/openshift/openshift_s2i_local.py @@ -16,8 +16,8 @@ import logging -from conu.backend.origin.backend import OpenshiftBackend -from conu.backend.docker.backend import DockerBackend +from conu import OpenshiftBackend, \ + DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token diff --git a/docs/source/examples/openshift/openshift_s2i_remote.py b/docs/source/examples/openshift/openshift_s2i_remote.py index b665d2ff..84fe0c41 100644 --- a/docs/source/examples/openshift/openshift_s2i_remote.py +++ b/docs/source/examples/openshift/openshift_s2i_remote.py @@ -16,8 +16,8 @@ import logging -from conu.backend.origin.backend import OpenshiftBackend -from conu.backend.docker.backend import DockerBackend +from conu import OpenshiftBackend, \ + DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token diff --git a/docs/source/examples/openshift/openshift_template.py b/docs/source/examples/openshift/openshift_template.py index d0ebfaf2..ec49d93c 100644 --- a/docs/source/examples/openshift/openshift_template.py +++ b/docs/source/examples/openshift/openshift_template.py @@ -16,8 +16,8 @@ import logging -from conu.backend.origin.backend import OpenshiftBackend -from conu.backend.docker.backend import DockerBackend +from conu import OpenshiftBackend, \ + DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 7e878b5c..b62763a4 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -21,8 +21,8 @@ import urllib3 import pytest -from conu import DockerBackend -from conu.backend.k8s.backend import K8sBackend, K8sCleanupPolicy +from conu import DockerBackend, \ + K8sBackend, K8sCleanupPolicy from conu.backend.k8s.pod import PodPhase from conu.backend.k8s.service import Service from conu.backend.k8s.deployment import Deployment diff --git a/tests/integration/test_openshift.py b/tests/integration/test_openshift.py index cf255960..2388042b 100644 --- a/tests/integration/test_openshift.py +++ b/tests/integration/test_openshift.py @@ -21,8 +21,8 @@ import logging import pytest -from conu.backend.origin.backend import OpenshiftBackend -from conu.backend.docker.backend import DockerBackend +from conu import OpenshiftBackend, \ + DockerBackend from conu.backend.origin.registry import login_to_registry from conu.utils import get_oc_api_token, oc_command_exists, is_oc_cluster_running from ..constants import CENTOS_MARIADB_10_2, CENTOS_PYTHON_3, MY_PROJECT, OC_CLUSTER_USER, \ From d8d35ca7dafad6cde536968956a120563998eb13 Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Wed, 17 Oct 2018 13:37:23 +0200 Subject: [PATCH 14/14] use logger.debug for debugging --- conu/backend/k8s/pod.py | 8 ++++---- conu/backend/origin/backend.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/conu/backend/k8s/pod.py b/conu/backend/k8s/pod.py index 221be202..124072c9 100644 --- a/conu/backend/k8s/pod.py +++ b/conu/backend/k8s/pod.py @@ -101,14 +101,14 @@ def get_logs(self): """ try: api_response = self.core_api.read_namespaced_pod_log(self.name, self.namespace) - logger.info("Logs from pod: %s in namespace: %s", self.name, self.namespace) + logger.debug("Logs from pod: %s in namespace: %s", self.name, self.namespace) for line in api_response.split('\n'): - logger.info(line) + logger.debug(line) return api_response except ApiException as e: # no reason to throw exception when logs cannot be obtain, just notify user - logger.info("Cannot get pod logs because of " - "exception during calling Kubernetes API %s\n", e) + logger.debug("Cannot get pod logs because of " + "exception during calling Kubernetes API %s\n", e) return None diff --git a/conu/backend/origin/backend.py b/conu/backend/origin/backend.py index 676f9766..3c2fe4bf 100644 --- a/conu/backend/origin/backend.py +++ b/conu/backend/origin/backend.py @@ -290,15 +290,16 @@ def get_status(self): c = self._oc_command(["status"]) o = run_cmd(c, return_output=True) for line in o.split('\n'): - logger.info(line) + logger.debug(line) return o except subprocess.CalledProcessError as ex: raise ConuException("Cannot obtain OpenShift cluster status: %s" % ex) def get_logs(self, name): """ - Get logs from all pods - :param name: str, name of app + Obtain cluster status and logs from all pods and print them using logger. + This method is useful for debugging. + :param name: str, name of app generated by oc new-app :return: str, cluster status and logs from all pods """ logs = self.get_status()