From c7cb42cfd9f1f6c711a102120ceeb1cbc0d19bd7 Mon Sep 17 00:00:00 2001 From: Robert Schweikert Date: Fri, 19 Jan 2024 09:52:13 -0500 Subject: [PATCH 01/24] Disable authentication for license files We want to have a where the client, for example SUSEConnect, can display license information to the user prior to registering a specific module or product. With the intend to have this display prior to registration the license file to be displayed cannot be in a path that requires authentication. Update the nginx configuration for Public Cloud setup to skip authentication for the license path. --- package/files/nginx-pubcloud/nginx-https.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/package/files/nginx-pubcloud/nginx-https.conf b/package/files/nginx-pubcloud/nginx-https.conf index 0a8c44f39..07cae2e82 100644 --- a/package/files/nginx-pubcloud/nginx-https.conf +++ b/package/files/nginx-pubcloud/nginx-https.conf @@ -22,6 +22,12 @@ server { autoindex off; } + # no authentication for license files + location ~ ^.*/product.license/.*$ { + autoindex on; + log_not_found off; + } + location /repo { autoindex on; log_not_found off; From 62e586800bad24b2d8ed3d412f7e0aee5d5f34f7 Mon Sep 17 00:00:00 2001 From: Thomas Schmidt Date: Tue, 30 Jan 2024 13:24:58 +0100 Subject: [PATCH 02/24] Add development support for sqlite --- Gemfile | 1 + Gemfile.lock | 2 ++ bin/rmt-cli | 5 +++++ lib/rmt/cli/base.rb | 1 + lib/rmt/lockfile.rb | 2 ++ 5 files changed, 11 insertions(+) diff --git a/Gemfile b/Gemfile index 152167724..04d320c6e 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,7 @@ end gem 'puma', '~> 5.6.2' gem 'mysql2', '~> 0.5.3' +gem 'sqlite3' gem 'nokogiri', '< 1.13' # Locked because of Ruby >= 2.6 dependency gem 'thor' diff --git a/Gemfile.lock b/Gemfile.lock index fd319f55d..5f64bf548 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -282,6 +282,7 @@ GEM spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) + sqlite3 (1.4.4) strong_migrations (0.7.9) activerecord (>= 5) sync (0.5.0) @@ -358,6 +359,7 @@ DEPENDENCIES spring spring-commands-rspec spring-watcher-listen (~> 2.0.0) + sqlite3 strong_migrations terminal-table (~> 3.0) thor diff --git a/bin/rmt-cli b/bin/rmt-cli index e2499b47a..b71c9ceb6 100755 --- a/bin/rmt-cli +++ b/bin/rmt-cli @@ -45,6 +45,11 @@ end db_config = RMT::Config.db_config ActiveRecord::Base.establish_connection(db_config) +if ActiveRecord::Base.connection.adapter_name != "Mysql2" + warn "Running with experimental support for #{ActiveRecord::Base.connection.adapter_name}." + warn 'RMT is running without locking operations, make sure not to run it in multipe processes.' +end + begin RMT::CLI::Main.start(ARGV) rescue Interrupt diff --git a/lib/rmt/cli/base.rb b/lib/rmt/cli/base.rb index b6a2cba6c..f8929712a 100644 --- a/lib/rmt/cli/base.rb +++ b/lib/rmt/cli/base.rb @@ -1,6 +1,7 @@ require 'rmt/lockfile' require 'rmt/cli/decorators' require 'etc' +require 'mysql2' class RMT::CLI::Base < Thor diff --git a/lib/rmt/lockfile.rb b/lib/rmt/lockfile.rb index db275881a..4fb6bf868 100644 --- a/lib/rmt/lockfile.rb +++ b/lib/rmt/lockfile.rb @@ -3,6 +3,8 @@ class RMT::Lockfile class << self def lock(lock_name = nil) + yield and return if ActiveRecord::Base.connection.adapter_name != 'Mysql2' + lock_name = ['rmt-cli', lock_name].compact.join('-') is_lock_obtained = obtain_lock(lock_name) From cf1a359b2230855f3dee64881185be0760283643 Mon Sep 17 00:00:00 2001 From: Thomas Schmidt Date: Tue, 30 Jan 2024 13:32:15 +0100 Subject: [PATCH 03/24] Add note to README how to use sqlite --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c14ed17cc..697f53540 100644 --- a/README.md +++ b/README.md @@ -23,19 +23,27 @@ Please view our [guide](docs/installation.md) to assist you in the RMT installat sudo zypper in libxml2-devel libxslt-devel libmariadb-devel gcc ``` 2. Install the ruby version specified in the `.ruby-version` [file](.ruby-version). -3. Install and start either the MariaDB or MySQL server: +3. Install and setup the database: + + **Default: MariaDB or MySQL server** ``` sudo zypper in mariadb sudo systemctl enable mariadb sudo systemctl start mariadb ``` -4. Log into the MariaDB or MySQL server as root and create the RMT database user: + Log into the MariaDB or MySQL server as root and create the RMT database user: ``` mysql -u root -p < Date: Tue, 30 Jan 2024 17:40:18 +0100 Subject: [PATCH 04/24] Ignore sqlite db files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 830afd8a1..abcfff7ff 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ # Ignore the default SQLite database. /db/*.sqlite3 /db/*.sqlite3-journal +/rmt_development +/rmt_test +/rmt_production + # Ignore all logfiles and tempfiles. /log/* From 81748ccdf12eee648b4a34c4ea77e9551c027564 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 17:14:08 +0100 Subject: [PATCH 05/24] Initial CI overhaul attempt --- ci/Dockerfile | 33 ++++++++++++++++++++ ci/README.md | 3 ++ ci/rmt-build-rpm | 42 +++++++++++++++++++++++++ ci/rmt-configure | 67 ++++++++++++++++++++++++++++++++++++++++ ci/rmt-console | 8 +++++ ci/rmt-run-feature-tests | 18 +++++++++++ features/import_spec.rb | 4 ++- 7 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 ci/Dockerfile create mode 100644 ci/README.md create mode 100755 ci/rmt-build-rpm create mode 100755 ci/rmt-configure create mode 100755 ci/rmt-console create mode 100755 ci/rmt-run-feature-tests diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 000000000..f4776e1be --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,33 @@ +FROM opensuse/leap:15.5 + +RUN zypper --non-interactive in make chrpath fdupes gcc libcurl-devel libffi-devel libmysqlclient-devel \ + libxml2-devel libxslt-devel nodejs sqlite-devel ruby2.5-rubygem-ronn \ + rpmbuild systemd gzip tar bzip2 vim curl wget ruby2.5 ruby2.5-devel \ + ruby2.5-rubygem-bundler + +# Set default bundler to the required version. This way we can easily update ruby in case required +RUN update-alternatives --install /usr/bin/bundle bundle /usr/bin/bundle.ruby2.5 5 && \ + update-alternatives --install /usr/bin/bundler bundler /usr/bin/bundler.ruby2.5 5 + +VOLUME /usr/src/rmt-server + +ARG SCC_USERNAME +ARG SCC_PASSWORD + +# Prepare ruby environment +ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 + +COPY Gemfile /usr/src/ +COPY Gemfile.lock /usr/src + +RUN bundle install --gemfile=/usr/src/Gemfile + +# Add rmt ci/development scripts +RUN mkdir -p /usr/src/rmt-server /usr/local/bin + +COPY ci/rmt-build-rpm /usr/local/bin/ +COPY ci/rmt-configure /usr/local/bin/ +COPY ci/rmt-console /usr/local/bin/ +COPY ci/rmt-run-feature-tests /usr/local/bin/ + +WORKDIR /usr/src/rmt-server diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000..24bcd1199 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,3 @@ +## RMT CI setup + +tbd diff --git a/ci/rmt-build-rpm b/ci/rmt-build-rpm new file mode 100755 index 000000000..d00b750a4 --- /dev/null +++ b/ci/rmt-build-rpm @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +# Where is the rmt-server source code located +SOURCE=${SOURCE:-/usr/src/rmt-server} + +# Where is the rpm build environment to be found +BUILD_DIR=${BUILD_DIR:-/usr/src/packages} + +# Where copy the built rpm files after a successful build +ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} + +# Current version we are working with +VERSION=$(ruby -e "require '$SOURCE/lib/rmt.rb'; print RMT::VERSION") + +group() { echo "::group::$@"; } +groupend() { echo "::groupend::"; } +fail() { echo "::error::$@"; exit 1;} + +# Setup artifact dir +mkdir -p $ARTIFACT_DIR + +group "create rmt-server tarball" +pushd $SOURCE + make dist + cp $SOURCE/package/obs/rmt-server-$VERSION.tar.bz2 $ARTIFACT_DIR +popd +group + +group "prepare build environment (in $BUILD_DIR)" +pushd $BUILD_DIR + mkdir -p {BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS,OTHERS} + cp -r $SOURCE/package/obs/* SOURCES/ +popd +groupend + +group "build rmt-server-$VERSION.x86_64.rpm" +pushd $BUILD_DIR + rpmbuild -ba --define '_srcdefattr (-,root,root)' --nosignature --undefine _enable_debug_packages SOURCES/rmt-server.spec + cp -r RPMS/x86_64/rmt-server-$VERSION*.rpm $ARTIFACT_DIR/ +popd +groupend diff --git a/ci/rmt-configure b/ci/rmt-configure new file mode 100755 index 000000000..6140ccea4 --- /dev/null +++ b/ci/rmt-configure @@ -0,0 +1,67 @@ +#!/bin/bash +set -e + +# Where is the rmt-server source code located +SOURCE=${SOURCE:-/usr/src/rmt-server} + +# Where is the rpm build environment to be found +BUILD_DIR=${BUILD_DIR:-/usr/src/packages} + +# Where copy the built rpm files after a successful build +ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} + +# Current version we are working with +VERSION=$(ruby -e "require '$SOURCE/lib/rmt.rb'; print RMT::VERSION") + +# Default RPMS which will be installed +RPMS=${RPMS:-rmt-server-config-${VERSION}-*.x86_64.rpm rmt-server-${VERSION}-*.x86_64.rpm} + +# Installation path to RMT installed via rpm +INSTALL_DIR=${INSTALL_DIR:-/usr/share/rmt} + +# MYSQL environment variables are needed to configure RMT correctly +MYSQL_HOST=${MYSQL_HOST:-127.0.0.1} +MYSQL_USER=${MYSQL_USER:-rmt} +MYSQL_PASSWORD=${MYSQL_PASSWORD:-rmt} +MYSQL_DATABASE=${MYSQL_DATABASE:-rmt_features} +MYSQL_PORT=${MYSQL_PORT:-3306} + +group() { echo "::group::$@"; } +groupend() { echo "::groupend::"; } +fail() { echo "::error::$@"; exit 1; } + +if [ "x$SCC_PASSWORD" = "x" ] || [ "x$SCC_USERNAME" = "x" ]; then + fail "To correctly configure RMT environment variables \$SCC_USERNAME and \$SCC_PASSWORD are required. Check your environment!" +fi + +group "install rmt-server rpms" +pushd $BUILD_DIR/RPMS/x86_64 + zypper --non-interactive --no-refresh install --allow-unsigned-rpm $RPMS +popd +groupend + +group "create /etc/rmt.conf" +cat > /etc/rmt.conf << EOL +--- +database: + host: $MYSQL_HOST + username: $MYSQL_USER + password: $MYSQL_PASSWORD + database: $MYSQL_DATABASE +scc: + username: $SCC_USERNAME + password: $SCC_PASSWORD + sync_systems: false +mirroring: + mirror_src: false + verify_rpm_checksums: false + dedup_method: hardlink +EOL +groupend + +group "setup database" +pushd $INSTALL_DIR + RAILS_ENV=production DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bundle exec rails db:drop db:create db:migrate +popd +groupend + diff --git a/ci/rmt-console b/ci/rmt-console new file mode 100755 index 000000000..065a0588d --- /dev/null +++ b/ci/rmt-console @@ -0,0 +1,8 @@ +#!/bin/bash + +# Installation path to RMT installed via rpm +INSTALL_DIR=${INSTALL_DIR:-/usr/share/rmt} + +pushd $INSTALL_DIR + RAILS_ENV=production bundle exec rails console +popd diff --git a/ci/rmt-run-feature-tests b/ci/rmt-run-feature-tests new file mode 100755 index 000000000..66cb4d61d --- /dev/null +++ b/ci/rmt-run-feature-tests @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Where is the rmt-server source code located +SOURCE=${SOURCE:-/usr/src/rmt-server} + +group() { echo "::group::$@"; } +groupend() { echo "::groupend::"; } + +group "synchronize SCC product/repositories" + rmt-cli sync +groupend + +group "run feature tests" +pushd $SOURCE + NO_COVERAGE=true bundle exec rspec features/ +popd +groupend diff --git a/features/import_spec.rb b/features/import_spec.rb index 816e23747..33a0f6abc 100644 --- a/features/import_spec.rb +++ b/features/import_spec.rb @@ -2,8 +2,10 @@ describe 'rmt data importer' do describe 'import repo from smt' do + let(:fixtures) { File.expand_path('../spec/fixtures/files/dummy_export/', File.dirname(__FILE__)) } + before do - command '/usr/bin/rmt-data-import --no-systems --no-hwinfo -d /tmp/rmt-server/spec/fixtures/files/dummy_export' + command "/usr/bin/rmt-data-import --no-systems --no-hwinfo -d #{fixtures}" end after do `/usr/bin/rmt-cli repos disable 3114` From 3275fb6a19376fe8bc633e80244efbbcfc554f1f Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 17:20:37 +0100 Subject: [PATCH 06/24] Use ci scripts from current pwd directory rather ci/ --- ci/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index f4776e1be..2b0acc3ef 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -10,6 +10,7 @@ RUN update-alternatives --install /usr/bin/bundle bundle /usr/bin/bundle.ruby2.5 update-alternatives --install /usr/bin/bundler bundler /usr/bin/bundler.ruby2.5 5 VOLUME /usr/src/rmt-server +VOLUME /usr/ ARG SCC_USERNAME ARG SCC_PASSWORD @@ -25,9 +26,9 @@ RUN bundle install --gemfile=/usr/src/Gemfile # Add rmt ci/development scripts RUN mkdir -p /usr/src/rmt-server /usr/local/bin -COPY ci/rmt-build-rpm /usr/local/bin/ -COPY ci/rmt-configure /usr/local/bin/ -COPY ci/rmt-console /usr/local/bin/ -COPY ci/rmt-run-feature-tests /usr/local/bin/ +COPY rmt-build-rpm /usr/local/bin/ +COPY rmt-configure /usr/local/bin/ +COPY rmt-console /usr/local/bin/ +COPY rmt-run-feature-tests /usr/local/bin/ WORKDIR /usr/src/rmt-server From 5be58dcfac3421636f76b7dda573173a9374de90 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 17:28:48 +0100 Subject: [PATCH 07/24] allow obs to discover the base container --- ci/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/Dockerfile b/ci/Dockerfile index 2b0acc3ef..2094ba2de 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,3 +1,5 @@ +#!BuildTag: rmt-ci-container +#!UseOBSRepositories FROM opensuse/leap:15.5 RUN zypper --non-interactive in make chrpath fdupes gcc libcurl-devel libffi-devel libmysqlclient-devel \ From 18cca4117fca18366f942996408b508876646ab0 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 18:38:09 +0100 Subject: [PATCH 08/24] No network when building containers. Skip bundling then for now --- ci/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index 2094ba2de..e8e5b9fc3 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -20,11 +20,6 @@ ARG SCC_PASSWORD # Prepare ruby environment ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 -COPY Gemfile /usr/src/ -COPY Gemfile.lock /usr/src - -RUN bundle install --gemfile=/usr/src/Gemfile - # Add rmt ci/development scripts RUN mkdir -p /usr/src/rmt-server /usr/local/bin From c98a451c2956619749798114f1a39e617fb0897a Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 18:57:38 +0100 Subject: [PATCH 09/24] Add feature test pipeline --- .github/workflows/features.yml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/features.yml diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml new file mode 100644 index 000000000..f6dd622c0 --- /dev/null +++ b/.github/workflows/features.yml @@ -0,0 +1,48 @@ +--- + +name: RMT feature tests +on: + pull_request: + branches: [master] + +jobs: + feature-tests: + runs-on: ubuntu-latest + container: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest + services: + mysql: + image: mysql:5.7 + env: + MYSQL_DATABASE: rmt_features + MYSQL_USER: rmt + MYSQL_PASSWORD: rmt + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=3 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + + - name: build rpm + run: | + rmt-build-rpm + + - name: configure rmt + run: | + rmt-configure + + - name: run feature tests + run: | + rmt-run-feature-tests From b25601310e258512c2043d6ba985c5bb62d0b223 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 19:05:26 +0100 Subject: [PATCH 10/24] Disable caching for now --- .github/workflows/features.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index f6dd622c0..083ae4286 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -29,12 +29,6 @@ jobs: with: fetch-depth: 0 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true - - name: build rpm run: | rmt-build-rpm From 116993c24ec9c92707d86c498829bd5719182fd6 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 19:13:46 +0100 Subject: [PATCH 11/24] Run the inner container as root --- .github/workflows/features.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 083ae4286..2fca743db 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -8,7 +8,9 @@ on: jobs: feature-tests: runs-on: ubuntu-latest - container: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest + container: + image: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest + options: --user root services: mysql: image: mysql:5.7 From ebed7b5a24fd455e199e3322d0618ef799307afc Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 19:17:46 +0100 Subject: [PATCH 12/24] Default to bash and set correct working directory --- .github/workflows/features.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 2fca743db..066ae6eae 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -8,6 +8,10 @@ on: jobs: feature-tests: runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: /usr/src/rmt-server container: image: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest options: --user root From 87169e84c15bf887aa5a514cba2615b9a1a096e2 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Mon, 5 Feb 2024 19:24:54 +0100 Subject: [PATCH 13/24] Do not install ci scripts into bin/ --- .github/workflows/features.yml | 41 ++++++++++++++++++++++++++++------ Makefile | 2 +- ci/Dockerfile | 18 +-------------- ci/rmt-build-rpm | 3 ++- ci/rmt-configure | 11 +++++++++ 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 066ae6eae..02aca8ab6 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -1,16 +1,25 @@ --- +name: feature tests -name: RMT feature tests on: pull_request: branches: [master] +env: + SOURCE: /usr/src/rmt-server + SCC_USERNAME: ${{ secrets.SCC_USERNAME }} + SCC_PASSWORD: ${{ secrets.SCC_PASSWORD }} + SYSTEM_UUID: ${{ secrets.SYSTEM_UUID }} + + # Within containters Github Actions does create a bridged network exposing + # the service named after its label + MYSQL_HOST: mysql + jobs: feature-tests: runs-on: ubuntu-latest defaults: run: - shell: bash working-directory: /usr/src/rmt-server container: image: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest @@ -35,14 +44,32 @@ jobs: with: fetch-depth: 0 - - name: build rpm + - name: move source to /usr/src/rmt-server + run: | + [ -d $SOURCE ] && rm -r $SOURCE + cp -r $GITHUB_WORKSPACE $SOURCE + + - name: build RPM package + run: | + bash ci/rmt-build-rpm + + - name: gather RPM build artifacts + uses: actions/upload-artifact@v4 + with: + name: rmt-server-rpms + path: ${{ env.SOURCE }}/tmp/artifacts/*.rpm + + - name: configure RMT to run feature tests run: | - rmt-build-rpm + bash ci/rmt-configure - - name: configure rmt + # Yes, yes we could use setup-ruby for this BUT: + # it will require to make local ruby available within + # our custom container. This here is more easy and clean! + - name: install test depdencies run: | - rmt-configure + bundle install - name: run feature tests run: | - rmt-run-feature-tests + bash ci/rmt-run-feature-tests diff --git a/Makefile b/Makefile index 2e46640c1..e53563184 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ clean: rm -rf $(NAME)-$(VERSION)/ man: - bundle exec ronn --roff --pipe --manual RMT MANUAL.md > rmt-cli.8 && gzip -f rmt-cli.8 + ronn --roff --pipe --manual RMT MANUAL.md > rmt-cli.8 && gzip -f rmt-cli.8 mv rmt-cli.8.gz package/obs dist: clean man diff --git a/ci/Dockerfile b/ci/Dockerfile index e8e5b9fc3..14e3319ef 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -11,21 +11,5 @@ RUN zypper --non-interactive in make chrpath fdupes gcc libcurl-devel libffi-dev RUN update-alternatives --install /usr/bin/bundle bundle /usr/bin/bundle.ruby2.5 5 && \ update-alternatives --install /usr/bin/bundler bundler /usr/bin/bundler.ruby2.5 5 -VOLUME /usr/src/rmt-server -VOLUME /usr/ - -ARG SCC_USERNAME -ARG SCC_PASSWORD - -# Prepare ruby environment -ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 - -# Add rmt ci/development scripts -RUN mkdir -p /usr/src/rmt-server /usr/local/bin - -COPY rmt-build-rpm /usr/local/bin/ -COPY rmt-configure /usr/local/bin/ -COPY rmt-console /usr/local/bin/ -COPY rmt-run-feature-tests /usr/local/bin/ - +RUN mkdir /usr/src/rmt-server WORKDIR /usr/src/rmt-server diff --git a/ci/rmt-build-rpm b/ci/rmt-build-rpm index d00b750a4..b73aad466 100755 --- a/ci/rmt-build-rpm +++ b/ci/rmt-build-rpm @@ -25,7 +25,7 @@ pushd $SOURCE make dist cp $SOURCE/package/obs/rmt-server-$VERSION.tar.bz2 $ARTIFACT_DIR popd -group +groupend group "prepare build environment (in $BUILD_DIR)" pushd $BUILD_DIR @@ -38,5 +38,6 @@ group "build rmt-server-$VERSION.x86_64.rpm" pushd $BUILD_DIR rpmbuild -ba --define '_srcdefattr (-,root,root)' --nosignature --undefine _enable_debug_packages SOURCES/rmt-server.spec cp -r RPMS/x86_64/rmt-server-$VERSION*.rpm $ARTIFACT_DIR/ + cp -r RPMS/x86_64/rmt-server-config-$VERSION*.rpm $ARTIFACT_DIR/ popd groupend diff --git a/ci/rmt-configure b/ci/rmt-configure index 6140ccea4..072c44911 100755 --- a/ci/rmt-configure +++ b/ci/rmt-configure @@ -26,6 +26,10 @@ MYSQL_PASSWORD=${MYSQL_PASSWORD:-rmt} MYSQL_DATABASE=${MYSQL_DATABASE:-rmt_features} MYSQL_PORT=${MYSQL_PORT:-3306} +# The systems uuid to use to identify this proxy (namely to not +# create a new proxy each time this is running!) +SYSTEM_UUID=${SYSTEM_UUID:-$(uuidgen)} + group() { echo "::group::$@"; } groupend() { echo "::groupend::"; } fail() { echo "::error::$@"; exit 1; } @@ -34,6 +38,12 @@ if [ "x$SCC_PASSWORD" = "x" ] || [ "x$SCC_USERNAME" = "x" ]; then fail "To correctly configure RMT environment variables \$SCC_USERNAME and \$SCC_PASSWORD are required. Check your environment!" fi +group "setup system uuid" + echo "saving uuid to /var/lib/rmt/system_uuid" + mkdir -p /var/lib/rmt/ + echo "$SYSTEM_UUID" > /var/lib/rmt/system_uuid +groupend + group "install rmt-server rpms" pushd $BUILD_DIR/RPMS/x86_64 zypper --non-interactive --no-refresh install --allow-unsigned-rpm $RPMS @@ -41,6 +51,7 @@ popd groupend group "create /etc/rmt.conf" + echo "saving /etc/rmt.conf" cat > /etc/rmt.conf << EOL --- database: From 4fb683db872d6eea94e1be9a55cc8c7478744cf4 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Tue, 6 Feb 2024 12:59:38 +0100 Subject: [PATCH 14/24] Remove old CI files and rename integrations.yml to lint-unit.yml --- .../{integrations.yml => lint-unit.yml} | 2 +- Jenkinsfile | 23 ---- ci/README.md | 45 ++++++- .../check-version-matches.rb | 1 + ci/rmt-console | 8 -- integration/Dockerfile.test | 37 ------ integration/Jenkinsfile | 27 ---- integration/create-oscrc.sh | 15 --- integration/create-rmt-conf.sh | 11 -- integration/install.sh | 16 --- integration/prophet/.prophet_ci.yml | 5 - integration/prophet/ci_executor.rb | 115 ------------------ integration/prophet/prophet.rb | 66 ---------- integration/run.sh | 22 ---- 14 files changed, 46 insertions(+), 347 deletions(-) rename .github/workflows/{integrations.yml => lint-unit.yml} (98%) delete mode 100644 Jenkinsfile rename integration/ci_check.rb => ci/check-version-matches.rb (99%) delete mode 100755 ci/rmt-console delete mode 100644 integration/Dockerfile.test delete mode 100644 integration/Jenkinsfile delete mode 100644 integration/create-oscrc.sh delete mode 100644 integration/create-rmt-conf.sh delete mode 100644 integration/install.sh delete mode 100644 integration/prophet/.prophet_ci.yml delete mode 100644 integration/prophet/ci_executor.rb delete mode 100755 integration/prophet/prophet.rb delete mode 100755 integration/run.sh diff --git a/.github/workflows/integrations.yml b/.github/workflows/lint-unit.yml similarity index 98% rename from .github/workflows/integrations.yml rename to .github/workflows/lint-unit.yml index ae9535be0..634e429de 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/lint-unit.yml @@ -72,5 +72,5 @@ jobs: - name: Run version checks run: | echo "::group::Version verification checks" - ruby integration/ci_check.rb + ruby ci/check-version-matches.rb echo "::endgroup::" diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 633a2d17a..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,23 +0,0 @@ -node('scc-jenkins-node-chucker') { - stage('checkout') { - git url: 'https://github.com/suse/rmt.git', branch: 'master' - } - - stage('build and push the image') { - sh 'docker build -t registry.scc.suse.de/rmt:latest .' - sh 'docker push registry.scc.suse.de/rmt:latest' - } - - stage('staging deploy') { - sh 'ssh root@rmt.scc.suse.de -t "docker pull registry.scc.suse.de/rmt:latest"' - try { - sh 'ssh root@rmt.scc.suse.de -t "docker stop rmt_production && docker rm rmt_production || true"' - sh 'ssh root@rmt.scc.suse.de -t "docker stop rmt_cron && docker rm rmt_cron || true"' - } - finally { - sh 'ssh root@rmt.scc.suse.de -t "docker run --restart=always -d --name rmt_production --network=rmt_network -e SECRET_KEY_BASE=\\$SECRET_KEY_BASE -e RMT_ORGANIZATION_USERNAME=\\$RMT_ORGANIZATION_USERNAME -e RMT_ORGANIZATION_PASSWORD=\\$RMT_ORGANIZATION_PASSWORD -v /media/rmt-data/:/srv/www/rmt/public/ -v /var/run/mysql/mysql.sock:/var/run/mysql/mysql.sock registry.scc.suse.de/rmt"' - sh 'ssh root@rmt.scc.suse.de -t "docker exec rmt_production bundle exec rails db:migrate"' - sh 'ssh root@rmt.scc.suse.de -t "docker run --restart=always -d --name rmt_cron --network=rmt_network -e SECRET_KEY_BASE=\\$SECRET_KEY_BASE -e RMT_ORGANIZATION_USERNAME=\\$RMT_ORGANIZATION_USERNAME -e RMT_ORGANIZATION_PASSWORD=\\$RMT_ORGANIZATION_PASSWORD -v /media/rmt-data/:/srv/www/rmt/public/ -v /var/www/rmt/shared/config/crontab:/etc/config/crontab -v /var/run/mysql/mysql.sock:/var/run/mysql/mysql.sock registry.scc.suse.de/rmt cron -n /etc/config/crontab"' - } - } -} diff --git a/ci/README.md b/ci/README.md index 24bcd1199..3070d1434 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,3 +1,46 @@ ## RMT CI setup -tbd +Our CI setup runs the following steps: + +### Lint and unit tests + +workflow definition: [.github/workflows/lint-unit.yml](https://github.com/SUSE/rmt/blob/master/.github/workflows/lint-unit.yml) + +This workflow runs rubocop to check if the source is well formated and afterwards unit tests and engine unit tests. At last it checks +if version in RMT and the rpm spec file matches. + +**Running it locally** + +There is no special mechanism needed to run these steps locally. Check the workflow for hints how to run unit tests + +### CLI feature tests + +workflow definition: [.github/workflows/features.yml](https://github.com/SUSE/rmt/blob/master/.github/workflows/features.yml) + +This workflow runs our simple CLI feature tests und build the rpm beforehand to see the system working with an installed RMT rpm. + +**Running it locally** + +To run feature tests locally, you need: + +- A checkout of RMT +- A running mysql database +- Proxy credentials to synchronize product information with SCC + +``` +# Fetch the CI container +$ export IMAGE="registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest" + +# Build RMT rpms with the CI container the resulting rpms are in tmp/artifacts/ +$ docker run --rm -it -v $(pwd):/usr/src/rmt-server $IMAGE 'ci/rmt-build-rpm' + +# Run feature tests in the CI container +# Note: Running --network=host isn't stricly required if you setup mysql access otherwise +$ docker run --rm -it -v $(pwd):/usr/src/rmt-server --network=host $IMAGE bash -c 'ci/rmt-build-rpm && ci/rmt-configure && ci/rmt-run-feature-tests' +``` + +### The CI container + +Our CI container is built here: https://build.opensuse.org/package/show/systemsmanagement:SCC:containers/rmt-ci-container + +On push to Github master the rebuilt of the container is triggered diff --git a/integration/ci_check.rb b/ci/check-version-matches.rb similarity index 99% rename from integration/ci_check.rb rename to ci/check-version-matches.rb index 539744890..3a6b1fef0 100755 --- a/integration/ci_check.rb +++ b/ci/check-version-matches.rb @@ -1,4 +1,5 @@ #! /usr/bin/env ruby + def modified_files `git fetch origin master` `git diff --name-only origin/master`.strip.split "\n" diff --git a/ci/rmt-console b/ci/rmt-console deleted file mode 100755 index 065a0588d..000000000 --- a/ci/rmt-console +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Installation path to RMT installed via rpm -INSTALL_DIR=${INSTALL_DIR:-/usr/share/rmt} - -pushd $INSTALL_DIR - RAILS_ENV=production bundle exec rails console -popd diff --git a/integration/Dockerfile.test b/integration/Dockerfile.test deleted file mode 100644 index 056e72ec4..000000000 --- a/integration/Dockerfile.test +++ /dev/null @@ -1,37 +0,0 @@ -FROM registry.scc.suse.de/suse/sles15:15.3 - -RUN useradd --no-log-init --create-home scc - -ARG REGCODE -ARG OBS_USER -ARG OBS_PASSWORD -ARG MYSQL_HOST -ARG MYSQL_USER -ARG MYSQL_PASSWORD -ARG MYSQL_DATABASE -ARG SCC_USERNAME -ARG SCC_PASSWORD -ARG SYSTEM_UUID - -RUN mkdir /tmp/rmt-server -WORKDIR /tmp/rmt-server -RUN chown -R scc /tmp/rmt-server -ADD integration/install.sh /tmp/rmt-server/integration/install.sh -RUN sh integration/install.sh - -ENV DOCKERIZE_VERSION v0.6.0 -RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ - && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz - -ADD integration/create-oscrc.sh /tmp/rmt-server/integration/create-oscrc.sh -RUN sh integration/create-oscrc.sh -ADD integration/create-rmt-conf.sh /tmp/rmt-server/integration/create-rmt-conf.sh -RUN sh integration/create-rmt-conf.sh -RUN mkdir -p /var/lib/rmt/ -RUN printf "$SYSTEM_UUID" >> /var/lib/rmt/system_uuid - -COPY ./ /tmp/rmt-server -RUN NOKOGIRI_USE_SYSTEM_LIBRARIES=1 bundle install - -CMD dockerize -wait tcp://$MYSQL_HOST:3306 -timeout 60s true diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile deleted file mode 100644 index 283c5096c..000000000 --- a/integration/Jenkinsfile +++ /dev/null @@ -1,27 +0,0 @@ -pipeline { - agent { - label 'scc-happy-customer' - } - - stages { - stage('checkout') { - steps { - script { - currentBuild.displayName = "#${BUILD_NUMBER}: ${branch}" - } - - git url: 'git@github.com:SUSE/rmt.git', branch: "${branch}" - } - } - stage('copy configuration') { - steps { - sh 'cp /home/jenkins/options-local-connect-prophet.yml integration/prophet/options-local.yml' - } - } - stage('run prophet') { - steps { - sh 'ruby ./integration/prophet/prophet.rb' - } - } - } -} diff --git a/integration/create-oscrc.sh b/integration/create-oscrc.sh deleted file mode 100644 index 5eadf7d58..000000000 --- a/integration/create-oscrc.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -xe -printf "[general]\n\ -build-root = /oscbuild/%(repo)s-%(arch)s\n\ -packagecachedir = /oscbuild/packagecache\n\ -[https://api.suse.de]\n\ -user=$OBS_USER\n\ -pass=$OBS_PASSWORD\n\ -sslcertck = 0\n\ -trusted_prj=SLE_12 SUSE:SLE-12:GA\n\ -[https://api.opensuse.org]\n\ -user=$OBS_USER\n\ -pass=$OBS_PASSWORD\n\ -sslcertck = 0\n\ -trusted_prj=SLE_12 SUSE:SLE-12:GA\n\ -" >> ~/.oscrc diff --git a/integration/create-rmt-conf.sh b/integration/create-rmt-conf.sh deleted file mode 100644 index fde2f4549..000000000 --- a/integration/create-rmt-conf.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -xe -mkdir /etc/rmt -printf "database:\n\ - host: $MYSQL_HOST\n\ - username: $MYSQL_USER\n\ - password: $MYSQL_PASSWORD\n\ - database: $MYSQL_DATABASE\n\ -scc:\n\ - username: $SCC_USERNAME\n\ - password: $SCC_PASSWORD\n\ -" >> /etc/rmt.conf diff --git a/integration/install.sh b/integration/install.sh deleted file mode 100644 index 9bf912af6..000000000 --- a/integration/install.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -xe -if [ -z "$REGCODE" ]; then - echo "Required REGCODE env is not set" - exit 1 -fi -SUSEConnect -r $REGCODE -SUSEConnect -p sle-module-desktop-applications/15.3/x86_64 -SUSEConnect -p sle-module-development-tools/15.3/x86_64 # this and above is needed for 'rpm-build' package -zypper --non-interactive ar http://download.opensuse.org/repositories/openSUSE:/Tools/SLE_15/openSUSE:Tools.repo -zypper --non-interactive --gpg-auto-import-keys ref -zypper --non-interactive up -zypper --non-interactive in -t pattern devel_osc_build -zypper --non-interactive install --no-recommends wget curl timezone \ - osc gcc-c++ libffi-devel git-core zlib-devel libxml2-devel libxslt-devel libmariadb-devel \ - mariadb-client mariadb ruby2.5-rubygem-bundler make build sudo ruby-devel nginx obs-service-format_spec_file -SUSEConnect -d diff --git a/integration/prophet/.prophet_ci.yml b/integration/prophet/.prophet_ci.yml deleted file mode 100644 index 8dc87e9bb..000000000 --- a/integration/prophet/.prophet_ci.yml +++ /dev/null @@ -1,5 +0,0 @@ -projects: - rmt: - docker-compose-build-images: docker-compose -f docker-compose-test.yml build - docker-compose-run-tests: docker-compose -f docker-compose-test.yml run rmt_test /tmp/rmt-server/integration/run.sh - docker-compose-cleanup: docker-compose -f docker-compose-test.yml down -v --rmi all --remove-orphans diff --git a/integration/prophet/ci_executor.rb b/integration/prophet/ci_executor.rb deleted file mode 100644 index 707ff991d..000000000 --- a/integration/prophet/ci_executor.rb +++ /dev/null @@ -1,115 +0,0 @@ -module SCC - - # FailedStepExecution - class FailedStepExecution < RuntimeError; end - - # RunStep - class RunStep - attr_accessor :name, :command, :status, :project - - def initialize(name: nil, command: nil, project:) - @name = name - @command = command - @project = project - @status = true - end - - def run! - @status = system(command) - raise FailedStepExecution, "Failed fast in project #{project.name} on step #{name}." unless @status - end - - def text_status - status ? "passed -- #{name}" : "failed -- #{name}" - end - end - - # Project - class Project - attr_accessor :name, :steps - - def initialize(name: nil, steps: []) - @name = name - @steps = steps - end - - def run! - steps.each(&:run!) - end - - def status - steps.all?(&:status) - end - - def text_status - status ? "#{name} is all green" : "#{name} failed" - end - - def requires_testing? - if defined? @requires_testing - @requires_testing - else - @requires_testing = begin - any_different = system('git diff HEAD..origin/master --exit-code --quiet -- .') - if any_different - CiExecutor.logger.info "Skipping #{name} - as no changes detected" - false - else - CiExecutor.logger.info "Testing #{name} - as changes detected" - true - end - end - end - end - end - - # CiExecutor - class CiExecutor - class << self - def logger - Thread.current[:logger] - end - - def logger=(logger) - Thread.current[:logger] = logger - end - end - - attr_accessor :status, :projects, :logger, :fail_message - - def initialize(logger: nil) - self.class.logger = logger - @projects = [] - YAML.load_file(File.expand_path('.prophet_ci.yml', __dir__))['projects'].each_pair do |project_name, run_steps| - project = Project.new(name: project_name) - run_steps.each_pair { |name, cmd| project.steps << RunStep.new(name: name, command: cmd, project: project) } - @projects << project - end - end - - def run! - projects.each(&:run!) - rescue FailedStepExecution => e - @fail_message = e.to_s - CiExecutor.logger.info e if CiExecutor.logger - end - - def inspect_failed - failed_projects.each do |prj| - CiExecutor.logger.info prj.text_status - prj.steps.each do |step| - CiExecutor.logger.info '|----' + step.text_status - break unless step.status - end - end - end - - def success? - projects.all?(&:status) - end - - def failed_projects - projects.reject(&:status) - end - end -end diff --git a/integration/prophet/prophet.rb b/integration/prophet/prophet.rb deleted file mode 100755 index 45e497128..000000000 --- a/integration/prophet/prophet.rb +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env ruby - -require 'prophet' -require 'logger' -require 'yaml' -require_relative 'ci_executor' - -Prophet.setup do |config| - # Setup Github access. - CONFIG_FILE = File.expand_path('options-local.yml', __dir__) - - if File.exist?(CONFIG_FILE) - options = YAML.load_file(CONFIG_FILE) - # The GitHub (GH) username/password to use for commenting on a successful run. - config.username_pass = options['default']['gh_username'] - config.access_token_pass = options['default']['gh_token'] - - # The GH credentials for commenting on failing runs (can be the same as above). - # NOTE: If you specify two different accounts with different avatars, it's - # a lot easier to spot failing test runs at first glance. - config.username_fail = options['default']['gh_username_fail'] - config.access_token_fail = options['default']['gh_token_fail'] - end - - # Setup logging. - config.logger = log = @logger = Logger.new(STDOUT) - log.level = Logger::INFO - - # Now that GitHub has fixed their notifications system, we can dare to increase - # Prophet's verbosity and use a new comment for every run. - config.reuse_comments = false - - # Set failure / success messages and add Jenkins URL if available. - jenkins_url = `echo $BUILD_URL`.chomp - if jenkins_url.empty? - message = '' - else - message = "\n#{jenkins_url}console\nIf the given link has expired," - message += 'you can force a Prophet rerun by just deleting this comment.' - end - config.comment_failure = 'Prophet reports failure.' + message - config.comment_success = 'Well Done! Your tests are still passing.' + message - - # Specify which tests to run. (Defaults to `rake test`.) - # NOTE: Either ensure the last call in that block runs your tests - # or manually set @result to a boolean inside this block. - config.execution do - executor = SCC::CiExecutor.new(logger: config.logger) - executor.run! - - config.success = executor.success? - - if config.success - log.info 'All tests are passing.' - else - config.comment_failure += "\n#{executor.fail_message}" - log.info 'Some tests are failing.' - executor.inspect_failed - - throw RuntimeError, config.comment_failure - end - end -end - -# Finally, run Prophet! -Prophet.run diff --git a/integration/run.sh b/integration/run.sh deleted file mode 100755 index 9fc32fddb..000000000 --- a/integration/run.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -xe - -# update project -rm -r /tmp/rmt-server/* -cp -r /tmp/workdir/* /tmp/rmt-server -chown -R scc /tmp/rmt-server - -cd /tmp/rmt-server/ -NOKOGIRI_USE_SYSTEM_LIBRARIES=1 bundle install -make dist -mkdir ~/obs -cd ~/obs -osc co systemsmanagement:SCC:RMT rmt-server -cd /tmp/rmt-server/package -cp obs/* ~/obs/systemsmanagement:SCC:RMT/rmt-server -cd ~/obs/systemsmanagement:SCC:RMT/rmt-server && osc addremove && osc build SLE_15 x86_64 --no-verify --trust-all-projects --clean && cd .. && -find /oscbuild/SLE_15-x86_64/home/abuild/rpmbuild/RPMS/x86_64/ -name '*.rpm' -not -name '*pubcloud*' -exec zypper --non-interactive --no-gpg-checks in --no-recommends {} \+ -cd /usr/share/rmt -RAILS_ENV=production DISABLE_DATABASE_ENVIRONMENT_CHECK=1 /usr/share/rmt/bin/rails db:drop db:create db:migrate -/usr/bin/rmt-cli sync -cd /tmp/rmt-server/ -NO_COVERAGE=true rspec features/ From 99449b8b9d921be3498a4b6b30d5535f3a9a6f86 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Tue, 6 Feb 2024 13:33:18 +0100 Subject: [PATCH 15/24] Add obs workflow to build the ci-container automatically on master push --- .github/workflows/features.yml | 2 +- .obs/workflows.yml | 11 +++++++++++ ci/check-version-matches.rb | 10 +++++----- 3 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 .obs/workflows.yml diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 02aca8ab6..4c3f3b2a1 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -66,7 +66,7 @@ jobs: # Yes, yes we could use setup-ruby for this BUT: # it will require to make local ruby available within # our custom container. This here is more easy and clean! - - name: install test depdencies + - name: install test dependencies run: | bundle install diff --git a/.obs/workflows.yml b/.obs/workflows.yml new file mode 100644 index 000000000..0d03cdc16 --- /dev/null +++ b/.obs/workflows.yml @@ -0,0 +1,11 @@ +--- +update-ci-container: + filters: + event: push + steps: + - trigger_services: + project: systemsmanagement:SCC:containers + package: rmt-ci-container + - rebuild_package: + project: systemsmanagement:SCC:containers + package: rmt-ci-container diff --git a/ci/check-version-matches.rb b/ci/check-version-matches.rb index 3a6b1fef0..f255d4855 100755 --- a/ci/check-version-matches.rb +++ b/ci/check-version-matches.rb @@ -6,9 +6,9 @@ def modified_files end def spec_version - return @_spec_version if defined?(@_spec_version) + return @spec_version if defined?(@spec_version) - @_spec_version = File.open('package/obs/rmt-server.spec', 'r') do |f| + @spec_version = File.open('package/obs/rmt-server.spec', 'r') do |f| f.each_line do |line| break line.split(':').last.strip if /^Version/.match?(line) end @@ -16,10 +16,10 @@ def spec_version end def rmt_version - return @_rmt_version if defined?(@_rmt_version) + return @rmt_version if defined?(@rmt_version) - require_relative '../lib/rmt.rb' - @_rmt_version = RMT::VERSION + require_relative '../lib/rmt' + @rmt_version = RMT::VERSION end def failure(msg) From 2ba89d1f4fc2722c44ac9e428414514cccfe7163 Mon Sep 17 00:00:00 2001 From: Felix Schnizlein Date: Tue, 6 Feb 2024 17:07:00 +0100 Subject: [PATCH 16/24] Joses great suggestions! Co-authored-by: Jose D. Gomez R. --- .github/workflows/features.yml | 2 +- ci/check-version-matches.rb | 11 +++++------ ci/rmt-build-rpm | 2 +- ci/rmt-configure | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 4c3f3b2a1..70b2ecfb7 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -40,7 +40,7 @@ jobs: --health-timeout=5s --health-retries=3 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/ci/check-version-matches.rb b/ci/check-version-matches.rb index f255d4855..f81088c2d 100755 --- a/ci/check-version-matches.rb +++ b/ci/check-version-matches.rb @@ -8,15 +8,14 @@ def modified_files def spec_version return @spec_version if defined?(@spec_version) - @spec_version = File.open('package/obs/rmt-server.spec', 'r') do |f| - f.each_line do |line| - break line.split(':').last.strip if /^Version/.match?(line) - end - end + @spec_version ||= File.read('package/obs/rmt-server.spec') + .match(/^Version:\s+(.+)\s*$/) + .captures + .first end def rmt_version - return @rmt_version if defined?(@rmt_version) + @rmt_version ||= (require_relative 'lib/rmt' and RMT::VERSION) require_relative '../lib/rmt' @rmt_version = RMT::VERSION diff --git a/ci/rmt-build-rpm b/ci/rmt-build-rpm index b73aad466..128824f16 100755 --- a/ci/rmt-build-rpm +++ b/ci/rmt-build-rpm @@ -11,7 +11,7 @@ BUILD_DIR=${BUILD_DIR:-/usr/src/packages} ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} # Current version we are working with -VERSION=$(ruby -e "require '$SOURCE/lib/rmt.rb'; print RMT::VERSION") +VERSION=$(ruby -r "$SOURCE/lib/rmt" -e 'print RMT::VERSION') group() { echo "::group::$@"; } groupend() { echo "::groupend::"; } diff --git a/ci/rmt-configure b/ci/rmt-configure index 072c44911..35f619e4e 100755 --- a/ci/rmt-configure +++ b/ci/rmt-configure @@ -11,7 +11,7 @@ BUILD_DIR=${BUILD_DIR:-/usr/src/packages} ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} # Current version we are working with -VERSION=$(ruby -e "require '$SOURCE/lib/rmt.rb'; print RMT::VERSION") +VERSION=$(ruby -r "$SOURCE/lib/rmt" -e 'print RMT::VERSION') # Default RPMS which will be installed RPMS=${RPMS:-rmt-server-config-${VERSION}-*.x86_64.rpm rmt-server-${VERSION}-*.x86_64.rpm} From 212be2268402ab93861572044d87a1d61185436c Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Tue, 6 Feb 2024 17:31:46 +0100 Subject: [PATCH 17/24] Adding shell linting + fixing check-version-matches.rb --- .github/workflows/lint-unit.yml | 9 ++++++--- ci/check-version-matches.rb | 5 +---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index 634e429de..e6249533a 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -38,9 +38,7 @@ jobs: --health-retries=3 steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Ruby uses: ruby/setup-ruby@v1 @@ -53,6 +51,11 @@ jobs: sudo mkdir -p /var/lib/rmt ruby -e "require 'yaml'; puts({'database_test'=>{'host' => '127.0.0.1', 'port' => 3306, 'username'=>'rmt','password'=>'rmt','database'=>'rmt_test','adapter'=>'mysql2','encoding'=>'utf8','timeout'=>5000,'pool'=>5}}.to_yaml)" > config/rmt.local.yml + - name: Run Shell lint + uses: azohra/shell-linter@latest + with: + path: "ci/rmt-*" + - name: Run Lint run: | bundle exec rubocop -D diff --git a/ci/check-version-matches.rb b/ci/check-version-matches.rb index f81088c2d..10fd87015 100755 --- a/ci/check-version-matches.rb +++ b/ci/check-version-matches.rb @@ -15,10 +15,7 @@ def spec_version end def rmt_version - @rmt_version ||= (require_relative 'lib/rmt' and RMT::VERSION) - - require_relative '../lib/rmt' - @rmt_version = RMT::VERSION + @rmt_version ||= (require_relative '../lib/rmt' and RMT::VERSION) end def failure(msg) From e9f4e2f5f8bca1bdc129f24dfeddaae1566c5995 Mon Sep 17 00:00:00 2001 From: Jesus Bermudez Velazquez Date: Tue, 6 Feb 2024 16:39:27 +0000 Subject: [PATCH 18/24] set a timeout for regsharing requests Default timeout for that request is 60 seconds, which is longer than the regsharing timer (30 seconds) Set a timout shorter than regsharing timer --- .../registration_sharing/lib/registration_sharing/client.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/engines/registration_sharing/lib/registration_sharing/client.rb b/engines/registration_sharing/lib/registration_sharing/client.rb index e10dbcd35..d6a154646 100644 --- a/engines/registration_sharing/lib/registration_sharing/client.rb +++ b/engines/registration_sharing/lib/registration_sharing/client.rb @@ -1,4 +1,7 @@ class RegistrationSharing::Client + # default is 60 seconds, should be shorter than regsharing timer + REGSHARING_TIMEOUT = 25.freeze + def initialize(peer, system_login) @peer = peer @system_login = system_login @@ -42,6 +45,7 @@ def make_request(method, params) 'Authorization' => "Bearer #{RegistrationSharing.config_api_secret}", 'User-Agent' => "RMT::Regsharing/#{RMT::VERSION}" }, + timeout: REGSHARING_TIMEOUT, body: JSON.dump(params), capath: RegistrationSharing.config_ca_path ) From 208d85b15de0091af8e78c07ce11486841743e7a Mon Sep 17 00:00:00 2001 From: Jesus Bermudez Velazquez Date: Tue, 6 Feb 2024 16:44:25 +0000 Subject: [PATCH 19/24] Remove freeze --- engines/registration_sharing/lib/registration_sharing/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/registration_sharing/lib/registration_sharing/client.rb b/engines/registration_sharing/lib/registration_sharing/client.rb index d6a154646..5b8987865 100644 --- a/engines/registration_sharing/lib/registration_sharing/client.rb +++ b/engines/registration_sharing/lib/registration_sharing/client.rb @@ -1,6 +1,6 @@ class RegistrationSharing::Client # default is 60 seconds, should be shorter than regsharing timer - REGSHARING_TIMEOUT = 25.freeze + REGSHARING_TIMEOUT = 25 def initialize(peer, system_login) @peer = peer From d8f118ef39ccfb2d3f06d8ec53b9be91c7faadd1 Mon Sep 17 00:00:00 2001 From: Jesus Bermudez Velazquez Date: Tue, 6 Feb 2024 16:53:20 +0000 Subject: [PATCH 20/24] Update comment --- engines/registration_sharing/lib/registration_sharing/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/registration_sharing/lib/registration_sharing/client.rb b/engines/registration_sharing/lib/registration_sharing/client.rb index 5b8987865..3028facb9 100644 --- a/engines/registration_sharing/lib/registration_sharing/client.rb +++ b/engines/registration_sharing/lib/registration_sharing/client.rb @@ -1,5 +1,5 @@ class RegistrationSharing::Client - # default is 60 seconds, should be shorter than regsharing timer + # default is 60 seconds, should be shorter than regsharing timer (30 seconds) REGSHARING_TIMEOUT = 25 def initialize(peer, system_login) From 0f5e8a484d2b5f23644361aac0458c8a086d0dca Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Tue, 6 Feb 2024 17:49:07 +0100 Subject: [PATCH 21/24] shell lint fixes and trailing whitespace --- .github/workflows/features.yml | 19 +++++++++++++++---- .github/workflows/lint-unit.yml | 23 +++++++++++++++++++---- ci/rmt-build-rpm | 20 ++++++++++---------- ci/rmt-configure | 15 ++++++--------- ci/rmt-run-feature-tests | 4 ++-- 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 70b2ecfb7..e08009445 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -1,6 +1,20 @@ --- name: feature tests +# About security when running the tests and NOT exposing +# the secrets to externals. Currently Github Actions does +# NOT expose the secrets if the branch is coming from a forked +# repository. +# See: https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ +# See: https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions +# +# An alternate would be to set, pull_request_target but this takes the CI code +# from master removing the ability to change the code in a PR easily. +# +# Aditionally, since 2021 pull requests from new contributors will not +# trigger workflows automatically but will wait for approval from somebody +# with write access. +# See: https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks on: pull_request: branches: [master] @@ -26,7 +40,7 @@ jobs: options: --user root services: mysql: - image: mysql:5.7 + image: registry.suse.com/suse/mariadb:10.6 env: MYSQL_DATABASE: rmt_features MYSQL_USER: rmt @@ -63,9 +77,6 @@ jobs: run: | bash ci/rmt-configure - # Yes, yes we could use setup-ruby for this BUT: - # it will require to make local ruby available within - # our custom container. This here is more easy and clean! - name: install test dependencies run: | bundle install diff --git a/.github/workflows/lint-unit.yml b/.github/workflows/lint-unit.yml index e6249533a..9ac44b6eb 100644 --- a/.github/workflows/lint-unit.yml +++ b/.github/workflows/lint-unit.yml @@ -1,5 +1,19 @@ -name: RMT lint, unit + engines tests +name: lint + unit + engine tests +# About security when running the tests and NOT exposing +# the secrets to externals. Currently Github Actions does +# NOT expose the secrets if the branch is coming from a forked +# repository. +# See: https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ +# See: https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions +# +# An alternate would be to set, pull_request_target but this takes the CI code +# from master removing the ability to change the code in a PR easily. +# +# Aditionally, since 2021 pull requests from new contributors will not +# trigger workflows automatically but will wait for approval from somebody +# with write access. +# See: https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks on: pull_request: branches: [master] @@ -18,12 +32,12 @@ jobs: # those are: SLE15 (2.5.8, 2.5.9) ruby-version: ['2.5.8', '2.5.9'] # skipping Ruby3 tests for now, because RMT isn't Ruby3 compatible yet, and the failing - # tests confuse the team. + # tests confuse the team. # ruby-version: ['2.5.8', '2.5.9', '3.0.3'] services: mysql: - image: mysql:5.7 + image: registry.suse.com/suse/mariadb:10.6 env: MYSQL_DATABASE: rmt_test MYSQL_USER: rmt @@ -40,11 +54,12 @@ jobs: steps: - uses: actions/checkout@v4 + # Run bundle install and cache the installed gems - name: Set up Ruby uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true # runs 'bundle install' and caches installed gems automatically + bundler-cache: true - name: Setup database config file run: | diff --git a/ci/rmt-build-rpm b/ci/rmt-build-rpm index 128824f16..35d87db53 100755 --- a/ci/rmt-build-rpm +++ b/ci/rmt-build-rpm @@ -13,31 +13,31 @@ ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} # Current version we are working with VERSION=$(ruby -r "$SOURCE/lib/rmt" -e 'print RMT::VERSION') -group() { echo "::group::$@"; } +group() { echo "::group::$1"; } groupend() { echo "::groupend::"; } -fail() { echo "::error::$@"; exit 1;} +fail() { echo "::error::$1"; exit 1;} # Setup artifact dir -mkdir -p $ARTIFACT_DIR +mkdir -p "$ARTIFACT_DIR" group "create rmt-server tarball" -pushd $SOURCE +pushd "$SOURCE" make dist - cp $SOURCE/package/obs/rmt-server-$VERSION.tar.bz2 $ARTIFACT_DIR + cp "$SOURCE/package/obs/rmt-server-$VERSION.tar.bz2" "$ARTIFACT_DIR" popd groupend group "prepare build environment (in $BUILD_DIR)" -pushd $BUILD_DIR +pushd "$BUILD_DIR" mkdir -p {BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS,OTHERS} - cp -r $SOURCE/package/obs/* SOURCES/ + cp -r "$SOURCE/package/obs"/* SOURCES/ popd groupend group "build rmt-server-$VERSION.x86_64.rpm" -pushd $BUILD_DIR +pushd "$BUILD_DIR" rpmbuild -ba --define '_srcdefattr (-,root,root)' --nosignature --undefine _enable_debug_packages SOURCES/rmt-server.spec - cp -r RPMS/x86_64/rmt-server-$VERSION*.rpm $ARTIFACT_DIR/ - cp -r RPMS/x86_64/rmt-server-config-$VERSION*.rpm $ARTIFACT_DIR/ + cp -r "RPMS/x86_64/rmt-server-$VERSION"*.rpm "$ARTIFACT_DIR/" + cp -r "RPMS/x86_64/rmt-server-config-$VERSION"*.rpm "$ARTIFACT_DIR/" popd groupend diff --git a/ci/rmt-configure b/ci/rmt-configure index 35f619e4e..1208e2232 100755 --- a/ci/rmt-configure +++ b/ci/rmt-configure @@ -13,9 +13,6 @@ ARTIFACT_DIR=${ARTIFACT_DIR:-$SOURCE/tmp/artifacts} # Current version we are working with VERSION=$(ruby -r "$SOURCE/lib/rmt" -e 'print RMT::VERSION') -# Default RPMS which will be installed -RPMS=${RPMS:-rmt-server-config-${VERSION}-*.x86_64.rpm rmt-server-${VERSION}-*.x86_64.rpm} - # Installation path to RMT installed via rpm INSTALL_DIR=${INSTALL_DIR:-/usr/share/rmt} @@ -30,11 +27,11 @@ MYSQL_PORT=${MYSQL_PORT:-3306} # create a new proxy each time this is running!) SYSTEM_UUID=${SYSTEM_UUID:-$(uuidgen)} -group() { echo "::group::$@"; } +group() { echo "::group::$1"; } groupend() { echo "::groupend::"; } -fail() { echo "::error::$@"; exit 1; } +fail() { echo "::error::$1"; exit 1; } -if [ "x$SCC_PASSWORD" = "x" ] || [ "x$SCC_USERNAME" = "x" ]; then +if [ "$SCC_PASSWORD" = "" ] || [ "$SCC_USERNAME" = "" ]; then fail "To correctly configure RMT environment variables \$SCC_USERNAME and \$SCC_PASSWORD are required. Check your environment!" fi @@ -45,8 +42,8 @@ group "setup system uuid" groupend group "install rmt-server rpms" -pushd $BUILD_DIR/RPMS/x86_64 - zypper --non-interactive --no-refresh install --allow-unsigned-rpm $RPMS +pushd "$BUILD_DIR/RPMS/x86_64" + zypper --non-interactive --no-refresh install --allow-unsigned-rpm "rmt-server-config-${VERSION}"-*.x86_64.rpm "rmt-server-${VERSION}-"*.x86_64.rpm popd groupend @@ -71,7 +68,7 @@ EOL groupend group "setup database" -pushd $INSTALL_DIR +pushd "$INSTALL_DIR" RAILS_ENV=production DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bundle exec rails db:drop db:create db:migrate popd groupend diff --git a/ci/rmt-run-feature-tests b/ci/rmt-run-feature-tests index 66cb4d61d..6e5ee8dca 100755 --- a/ci/rmt-run-feature-tests +++ b/ci/rmt-run-feature-tests @@ -4,7 +4,7 @@ set -e # Where is the rmt-server source code located SOURCE=${SOURCE:-/usr/src/rmt-server} -group() { echo "::group::$@"; } +group() { echo "::group::$1"; } groupend() { echo "::groupend::"; } group "synchronize SCC product/repositories" @@ -12,7 +12,7 @@ group "synchronize SCC product/repositories" groupend group "run feature tests" -pushd $SOURCE +pushd "$SOURCE" NO_COVERAGE=true bundle exec rspec features/ popd groupend From cb7338ef9fa483a67b439cfbd3ffbdca8d3f15ec Mon Sep 17 00:00:00 2001 From: "depfu[bot]" <23717796+depfu[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 22:46:15 +0000 Subject: [PATCH 22/24] Update webmock to version 3.20.0 --- Gemfile.lock | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1f165a671..c663b9a34 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -30,10 +30,11 @@ GEM minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - addressable (2.8.5) + addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) ast (2.4.2) awesome_print (1.9.2) + bigdecimal (3.1.6) builder (3.2.4) byebug (11.1.3) case_transform (0.2) @@ -49,7 +50,8 @@ GEM term-ansicolor (~> 1.3) thor (>= 0.19.4, < 2.0) tins (~> 1.6) - crack (0.4.5) + crack (0.4.6) + bigdecimal rexml crass (1.0.6) deep_merge (1.2.1) @@ -129,7 +131,7 @@ GEM guard (~> 2.1) guard-compat (~> 1.1) rspec (>= 2.99.0, < 4.0) - hashdiff (1.0.1) + hashdiff (1.1.0) hpricot (0.8.6) i18n (1.14.1) concurrent-ruby (~> 1.0) @@ -306,7 +308,7 @@ GEM activesupport (>= 3) railties (>= 3) yard (~> 0.9.20) - webmock (3.19.1) + webmock (3.20.0) addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) From d861954be340a4d573535c37259dc1a45e7e9b32 Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Thu, 8 Feb 2024 15:36:37 +0100 Subject: [PATCH 23/24] 2.15 pre release We are almost there, just waiting for SLL7 script changes to release 2.15 finally! --- package/obs/rmt-server.changes | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/package/obs/rmt-server.changes b/package/obs/rmt-server.changes index 4d8b7ebf9..70aa81357 100644 --- a/package/obs/rmt-server.changes +++ b/package/obs/rmt-server.changes @@ -1,14 +1,15 @@ ------------------------------------------------------------------- -Thu Jan 25 17:40:00 UTC 2024 - Felix Schnizlein +Thu Feb 08 15:33:00 UTC 2024 - Felix Schnizlein - Version 2.15: * Moving system hardware information to systems database table to allow transmitting system information dynamically. (jsc#PED-3734) * Dropping Rails Secrets facilities and related config files (bsc#1215176) - * rmt-client-setup-res script: fix for CentOS8 clients (bsc#1214709) * Updated supportconfig script (bsc#1216389) * Support zstd compression for repository metadata (bsc#1218775) - * Do not add credential handling to normal repository URLs (#1219153) + * Do not add credential handling to normal repository URLs (bsc#1219153) + * Disable authentication for license files in pubcloud context + * Higher registration sharing timeout ------------------------------------------------------------------- Thu Jun 06 15:44:00 UTC 2023 - Luís Caparroz From 188bb8e21ea68adc1ad3b5299af5515775f61f8e Mon Sep 17 00:00:00 2001 From: Felix Schizlein Date: Thu, 8 Feb 2024 16:24:15 +0100 Subject: [PATCH 24/24] Add missing build require for sqlite; the real pre-release --- package/obs/rmt-server.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/package/obs/rmt-server.spec b/package/obs/rmt-server.spec index 28c84df55..577087ba5 100644 --- a/package/obs/rmt-server.spec +++ b/package/obs/rmt-server.spec @@ -49,6 +49,7 @@ BuildRequires: gcc BuildRequires: libcurl-devel BuildRequires: libffi-devel BuildRequires: libmysqlclient-devel +BuildRequires: sqlite-devel BuildRequires: libxml2-devel BuildRequires: libxslt-devel BuildRequires: pkgconfig(systemd)