From ccd57c1a56764d29265f453260ca162dbc8c9abf Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Wed, 17 Jul 2019 23:43:22 +0200 Subject: [PATCH 1/7] add snapshots repo URL templates Signed-off-by: Mikael Arguedas --- docker_templates/packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker_templates/packages.py b/docker_templates/packages.py index f17d5a4..38e00f2 100644 --- a/docker_templates/packages.py +++ b/docker_templates/packages.py @@ -30,6 +30,8 @@ 'gazebo_packages': string.Template('http://packages.osrfoundation.org/gazebo/$os_name-$release/dists/$os_code_name/main/binary-$arch/Packages'), 'ros_packages': string.Template('http://packages.ros.org/ros/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'), 'ros2_packages': string.Template('http://packages.ros.org/ros2/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'), + 'ros_packages_snapshots': string.Template('http://snapshots.ros.org/$rosdistro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'), + 'ros2_packages_snapshots': string.Template('http://snapshots.ros.org/$ros2distro_name/final/ubuntu/dists/$os_code_name/main/binary-$arch/Packages'), } packageNameVersionTemplateLookup = { From 3ecd2c8d7e71ac5712d60027652047d5a6d3b7d7 Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Wed, 17 Jul 2019 23:46:00 +0200 Subject: [PATCH 2/7] add function to determine if base distro of ros distro is EOL Signed-off-by: Mikael Arguedas --- docker_templates/eol_distro.py | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docker_templates/eol_distro.py diff --git a/docker_templates/eol_distro.py b/docker_templates/eol_distro.py new file mode 100644 index 0000000..ee83a94 --- /dev/null +++ b/docker_templates/eol_distro.py @@ -0,0 +1,53 @@ +# Copyright 2019 Mikael Arguedas +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def isDistroEol(ros_distro_name, os_distro_name): + eol_ros_distros = [ + # ROS 1 + 'boxturtle', + 'cturtle', + 'diamondback', + 'electric', + 'fuerte', + 'groovy', + 'hydro', + 'indigo', + 'jade', + 'lunar', + # ROS 2 + 'ardent', + ] + eol_base_images = [ + # Ubuntu + 'lucid', + 'maverick', + 'natty', + 'oneiric', + 'precise', + 'quantal', + 'raring', + 'saucy', + 'trusty', + 'utopic', + 'vivid', + 'wily', + 'yakkety', + 'zesty', + 'artful', + 'cosmic', + # Debian + 'wheezy', + 'jessie', + ] + return os_distro_name in eol_base_images or ros_distro_name in eol_ros_distros From 14b0c32edb75a5050a1ac02214bf13686994f6ba Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Wed, 17 Jul 2019 23:48:28 +0200 Subject: [PATCH 3/7] determine if distro is EOL and update index URL template appropriately Signed-off-by: Mikael Arguedas --- docker_templates/eol_distro.py | 2 +- docker_templates/packages.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker_templates/eol_distro.py b/docker_templates/eol_distro.py index ee83a94..729adff 100644 --- a/docker_templates/eol_distro.py +++ b/docker_templates/eol_distro.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -def isDistroEol(ros_distro_name, os_distro_name): +def isDistroEOL(ros_distro_name, os_distro_name): eol_ros_distros = [ # ROS 1 'boxturtle', diff --git a/docker_templates/packages.py b/docker_templates/packages.py index 38e00f2..4ed155a 100644 --- a/docker_templates/packages.py +++ b/docker_templates/packages.py @@ -16,6 +16,8 @@ import re import urllib.request +from docker_templates.eol_distro import isDistroEOL + # TODO: think of a better version pattern like # r'\d(?!Version\:\s)(.+)(?=(~\w+\n))' but works without a trailing ~ version_pattern = r'(?<=Version: )\d+\.\d+\.\d+\-\d+' @@ -110,7 +112,17 @@ def getPackageVersions(data, package_index, packages, package_type): def expandPackages(data): for package_type in indexUrlTemplateLookup: if package_type in data: - package_index_url_template = indexUrlTemplateLookup[package_type] + # determine if distro is eol and apply the appropriate index URL template + ros_distro_name = '' + if package_type == 'ros_packages': + ros_distro_name = data['rosdistro_name'] + elif package_type == 'ros2_packages': + ros_distro_name = data['ros2distro_name'] + eol = isDistroEOL(ros_distro_name, data['os_code_name']) + if eol: + package_index_url_template = indexUrlTemplateLookup[package_type + '_snapshots'] + else: + package_index_url_template = indexUrlTemplateLookup[package_type] package_index_url = package_index_url_template.substitute(data) package_index = getPackageIndex(data, package_index_url) package_versions = getPackageVersions(data, package_index, data[package_type], package_type) From 2339d0dccdb2e682f1a935260df50283ca735972 Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Wed, 17 Jul 2019 23:50:05 +0200 Subject: [PATCH 4/7] add docker snippet to setup ros apt repo add the snapshot repo is the distro is EOL use the snippet in templates needing it Signed-off-by: Mikael Arguedas --- .../create_ros_core_image.Dockerfile.em | 13 +++++----- .../create_ros_core_image.Dockerfile.em | 14 ++++++----- ...create_ros_ros1_bridge_image.Dockerfile.em | 12 +++++---- .../snippet/setup_ros_sources.Dockerfile.em | 25 +++++++++++++++++++ 4 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em diff --git a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em index 54e8b83..c79eb48 100644 --- a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em @@ -24,7 +24,6 @@ template_dependencies = [ 'dirmngr', 'gnupg2', - 'lsb-release', ] }@ @(TEMPLATE( @@ -33,11 +32,13 @@ template_dependencies = [ upstream_packages=upstream_packages if 'upstream_packages' in locals() else [], ))@ @ -# setup keys -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 - -# setup sources.list -RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list +@(TEMPLATE( + 'snippet/setup_ros_sources.Dockerfile.em', + os_name=os_name, + os_code_name=os_code_name, + rosdistro_name=rosdistro_name, + ros_version=ros_version, +))@ @(TEMPLATE( 'snippet/install_ros_bootstrap_tools.Dockerfile.em', diff --git a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em index 89392bb..047bc67 100644 --- a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em @@ -22,7 +22,6 @@ template_dependencies = [ 'dirmngr', 'gnupg2', - 'lsb-release', ] # add 'python3-pip' to 'template_dependencies' if pip dependencies are declared if 'pip3_install' in locals(): @@ -35,11 +34,14 @@ if 'pip3_install' in locals(): upstream_packages=upstream_packages if 'upstream_packages' in locals() else [], ))@ @ -# setup ros2 keys -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 - -# setup sources.list -RUN echo "deb http://packages.ros.org/ros2/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros2-latest.list +@(TEMPLATE( + 'snippet/setup_ros_sources.Dockerfile.em', + os_name=os_name, + os_code_name=os_code_name, + ros2distro_name=ros2distro_name, + rosdistro_name='', + ros_version=ros_version, +))@ @(TEMPLATE( 'snippet/install_ros_bootstrap_tools.Dockerfile.em', diff --git a/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em index 1bd51f2..803f846 100644 --- a/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em @@ -35,11 +35,13 @@ RUN pip3 install -U \ @[ end if]@ @[end if]@ -# setup keys -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9FF1F717815A3895523BAEEB01FA116 - -# setup sources.list -RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list +@(TEMPLATE( + 'snippet/setup_ros_sources.Dockerfile.em', + os_name=os_name, + os_code_name=os_code_name, + rosdistro_name=rosdistro_name, + ros_version='1', +))@ ENV ROS1_DISTRO @rosdistro_name ENV ROS2_DISTRO @ros2distro_name diff --git a/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em b/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em new file mode 100644 index 0000000..3566ca6 --- /dev/null +++ b/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em @@ -0,0 +1,25 @@ +@{ +from docker_templates.eol_distro import isDistroEOL +if int(ros_version) == 2: + ros_distro_name = ros2distro_name +elif int(ros_version) == 1: + ros_distro_name = rosdistro_name + +if isDistroEOL(ros_distro_name, os_code_name): + repo_url = 'http://snapshots.ros.org/' + str(ros_distro_name) + '/final/' + 'ubuntu' + '/' + repo_key = 'AD19BAB3CBF125EA' + source_suffix = 'snapshots' +else: + repo_key = 'C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' + source_suffix = 'latest' + + if int(ros_version) == 1: + repo_url = 'http://packages.ros.org/ros/ubuntu' + elif int(ros_version) == 2: + repo_url = 'http://packages.ros.org/ros2/ubuntu' +}@ +# setup keys +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys @(repo_key) + +# setup sources.list +RUN echo "deb @(repo_url) @(os_code_name) main" > /etc/apt/sources.list.d/ros@(ros_version)-@(source_suffix).list From 52fa00aa0871eb4df938689afaee2e78cedb4015 Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Thu, 18 Jul 2019 02:52:59 +0200 Subject: [PATCH 5/7] add old release set snippet to ros/ros2 core image template Signed-off-by: Mikael Arguedas --- .../docker_images/create_ros_core_image.Dockerfile.em | 7 +++++++ .../create_ros_core_image.Dockerfile.em | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em index c79eb48..8a4b5c4 100644 --- a/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em @@ -14,6 +14,13 @@ maintainer_name=maintainer_name, ))@ @ +@(TEMPLATE( + 'snippet/old_release_set.Dockerfile.em', + template_packages=template_packages, + os_name=os_name, + os_code_name=os_code_name, +))@ +@ @(TEMPLATE( 'snippet/setup_tzdata.Dockerfile.em', os_name=os_name, diff --git a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em index 047bc67..98e2128 100644 --- a/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em @@ -13,11 +13,20 @@ base_image=base_image, maintainer_name=maintainer_name, ))@ +@ +@(TEMPLATE( + 'snippet/old_release_set.Dockerfile.em', + template_packages=template_packages, + os_name=os_name, + os_code_name=os_code_name, +))@ +@ @(TEMPLATE( 'snippet/setup_tzdata.Dockerfile.em', os_name=os_name, os_code_name=os_code_name, ))@ +@ @{ template_dependencies = [ 'dirmngr', From c99dd844a23cd66241cdf42f117ed3642c829aed Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Sat, 27 Jul 2019 21:34:43 +0200 Subject: [PATCH 6/7] update legacy template as well Signed-off-by: Mikael Arguedas --- .../create_ros_core_image.Dockerfile.em | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker_templates/templates/docker_images_legacy/create_ros_core_image.Dockerfile.em b/docker_templates/templates/docker_images_legacy/create_ros_core_image.Dockerfile.em index 89197fa..9f2461e 100644 --- a/docker_templates/templates/docker_images_legacy/create_ros_core_image.Dockerfile.em +++ b/docker_templates/templates/docker_images_legacy/create_ros_core_image.Dockerfile.em @@ -32,11 +32,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ @[ end if]@ @[end if]@ -# setup keys -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys AD19BAB3CBF125EA - -# setup sources.list -RUN echo "deb http://snapshots.ros.org/@rosdistro_name/final/@os_name @os_code_name main" > /etc/apt/sources.list.d/ros-snapshots.list +@ +@(TEMPLATE( + 'snippet/setup_ros_sources.Dockerfile.em', + os_name=os_name, + os_code_name=os_code_name, + rosdistro_name=rosdistro_name, + ros_version=ros_version, +))@ # setup environment RUN locale-gen en_US.UTF-8 From 49cc868ea68c0a165d82efa19d861931943baa19 Mon Sep 17 00:00:00 2001 From: Mikael Arguedas Date: Sat, 27 Jul 2019 21:44:47 +0200 Subject: [PATCH 7/7] remove hardcoded ubuntu and cleanup snapshot url construction logic Signed-off-by: Mikael Arguedas --- .../templates/snippet/setup_ros_sources.Dockerfile.em | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em b/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em index 3566ca6..34cbf89 100644 --- a/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em +++ b/docker_templates/templates/snippet/setup_ros_sources.Dockerfile.em @@ -1,4 +1,6 @@ @{ +import os + from docker_templates.eol_distro import isDistroEOL if int(ros_version) == 2: ros_distro_name = ros2distro_name @@ -6,7 +8,12 @@ elif int(ros_version) == 1: ros_distro_name = rosdistro_name if isDistroEOL(ros_distro_name, os_code_name): - repo_url = 'http://snapshots.ros.org/' + str(ros_distro_name) + '/final/' + 'ubuntu' + '/' + repo_url = os.path.join( + 'http://snapshots.ros.org', + str(ros_distro_name), + 'final', + str(os_name) + ) repo_key = 'AD19BAB3CBF125EA' source_suffix = 'snapshots' else: