Merge branch 'master' into mysql_ssl_connection

.github/workflows/features.yml

@@ -0,0 +1,86 @@
+---
+name: feature tests
+
+# About security when running the tests and NOT exposing
+# the secrets to externals. Currently Github Actions does
+# NOT expose the secrets if the branch is coming from a forked
+# repository.
+# See: https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/
+# See: https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions
+#
+# An alternate would be to set, pull_request_target but this takes the CI code
+# from master removing the ability to change the code in a PR easily.
+#
+# Aditionally, since 2021 pull requests from new contributors will not
+# trigger workflows automatically but will wait for approval from somebody
+# with write access.
+# See: https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks
+on:
+  pull_request:
+    branches: [master]
+
+env:
+  SOURCE: /usr/src/rmt-server
+  SCC_USERNAME: ${{ secrets.SCC_USERNAME }}
+  SCC_PASSWORD: ${{ secrets.SCC_PASSWORD }}
+  SYSTEM_UUID: ${{ secrets.SYSTEM_UUID }}
+
+  # Within containters Github Actions does create a bridged network exposing
+  # the service named after its label
+  MYSQL_HOST: mysql
+
+jobs:
+  feature-tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: /usr/src/rmt-server
+    container:
+      image: registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest
+      options: --user root
+    services:
+      mysql:
+        image: registry.suse.com/suse/mariadb:10.6
+        env:
+          MYSQL_DATABASE: rmt_features
+          MYSQL_USER: rmt
+          MYSQL_PASSWORD: rmt
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: move source to /usr/src/rmt-server
+        run: |
+          [ -d $SOURCE ] && rm -r $SOURCE
+          cp -r $GITHUB_WORKSPACE $SOURCE
+
+      - name: build RPM package
+        run: |
+          bash ci/rmt-build-rpm
+
+      - name: gather RPM build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: rmt-server-rpms
+          path: ${{ env.SOURCE }}/tmp/artifacts/*.rpm
+
+      - name: configure RMT to run feature tests
+        run: |
+          bash ci/rmt-configure
+
+      - name: install test dependencies
+        run: |
+          bundle install
+
+      - name: run feature tests
+        run: |
+          bash ci/rmt-run-feature-tests

.github/workflows/integrations.yml → .github/workflows/lint-unit.yml

@@ -1,5 +1,19 @@
-name: RMT lint, unit + engines tests
+name: lint + unit + engine tests
 
+# About security when running the tests and NOT exposing
+# the secrets to externals. Currently Github Actions does
+# NOT expose the secrets if the branch is coming from a forked
+# repository.
+# See: https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/
+# See: https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions
+#
+# An alternate would be to set, pull_request_target but this takes the CI code
+# from master removing the ability to change the code in a PR easily.
+#
+# Aditionally, since 2021 pull requests from new contributors will not
+# trigger workflows automatically but will wait for approval from somebody
+# with write access.
+# See: https://docs.github.com/en/actions/managing-workflow-runs/approving-workflow-runs-from-public-forks
 on:
   pull_request:
     branches: [master]
@@ -18,12 +32,12 @@ jobs:
         # those are: SLE15 (2.5.8, 2.5.9)
         ruby-version: ['2.5.8', '2.5.9']
         # skipping Ruby3 tests for now, because RMT isn't Ruby3 compatible yet, and the failing 
-        # tests confuse the team. 
+        # tests confuse the team.
         # ruby-version: ['2.5.8', '2.5.9', '3.0.3']
 
     services:
       mysql:
-        image: mysql:5.7
+        image: registry.suse.com/suse/mariadb:10.6
         env:
           MYSQL_DATABASE: rmt_test
           MYSQL_USER: rmt
@@ -38,21 +52,25 @@ jobs:
           --health-retries=3
 
     steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
+      - uses: actions/checkout@v4
 
+      # Run bundle install and cache the installed gems
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
-          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+          bundler-cache: true
 
       - name: Setup database config file
         run: |
           sudo mkdir -p /var/lib/rmt
           ruby -e "require 'yaml'; puts({'database_test'=>{'host' => '127.0.0.1', 'port' => 3306, 'username'=>'rmt','password'=>'rmt','database'=>'rmt_test','adapter'=>'mysql2','encoding'=>'utf8','timeout'=>5000,'pool'=>5}}.to_yaml)" > config/rmt.local.yml
 
+      - name: Run Shell lint
+        uses: azohra/shell-linter@latest
+        with:
+          path: "ci/rmt-*"
+
       - name: Run Lint
         run: |
           bundle exec rubocop -D
@@ -72,5 +90,5 @@ jobs:
       - name: Run version checks
         run: |
           echo "::group::Version verification checks"
-          ruby integration/ci_check.rb
+          ruby ci/check-version-matches.rb
           echo "::endgroup::"

.gitignore

@@ -10,6 +10,10 @@
 # Ignore the default SQLite database.
 /db/*.sqlite3
 /db/*.sqlite3-journal
+/rmt_development
+/rmt_test
+/rmt_production
+
 
 # Ignore all logfiles and tempfiles.
 /log/*

.obs/workflows.yml

@@ -0,0 +1,11 @@
+---
+update-ci-container:
+  filters:
+    event: push
+  steps:
+    - trigger_services:
+        project: systemsmanagement:SCC:containers
+        package: rmt-ci-container
+    - rebuild_package:
+        project: systemsmanagement:SCC:containers
+        package: rmt-ci-container

Gemfile

@@ -7,6 +7,7 @@ end
 
 gem 'puma', '~> 5.6.2'
 gem 'mysql2', '~> 0.5.3'
+gem 'sqlite3'
 
 gem 'nokogiri', '< 1.13' # Locked because of Ruby >= 2.6 dependency
 gem 'thor', '<= 1.2.2' # Locked because of Ruby >= 2.6 dependency

Gemfile.lock

@@ -30,10 +30,11 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.8.5)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     awesome_print (1.9.2)
+    bigdecimal (3.1.6)
     builder (3.2.4)
     byebug (11.1.3)
     case_transform (0.2)
@@ -49,7 +50,8 @@ GEM
       term-ansicolor (~> 1.3)
       thor (>= 0.19.4, < 2.0)
       tins (~> 1.6)
-    crack (0.4.5)
+    crack (0.4.6)
+      bigdecimal
       rexml
     crass (1.0.6)
     deep_merge (1.2.1)
@@ -129,7 +131,7 @@ GEM
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    hashdiff (1.0.1)
+    hashdiff (1.1.0)
     hpricot (0.8.6)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
@@ -283,6 +285,7 @@ GEM
     spring-watcher-listen (2.0.1)
       listen (>= 2.7, < 4.0)
       spring (>= 1.2, < 3.0)
+    sqlite3 (1.4.4)
     strong_migrations (0.7.9)
       activerecord (>= 5)
     sync (0.5.0)
@@ -305,7 +308,7 @@ GEM
       activesupport (>= 3)
       railties (>= 3)
       yard (~> 0.9.20)
-    webmock (3.19.1)
+    webmock (3.20.0)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -360,6 +363,7 @@ DEPENDENCIES
   spring
   spring-commands-rspec
   spring-watcher-listen (~> 2.0.0)
+  sqlite3
   strong_migrations
   terminal-table (~> 3.0)
   thor (<= 1.2.2)

Makefile

@@ -10,7 +10,7 @@ clean:
 	rm -rf $(NAME)-$(VERSION)/
 
 man:
-	bundle exec ronn --roff --pipe --manual RMT MANUAL.md > rmt-cli.8 && gzip -f rmt-cli.8
+	ronn --roff --pipe --manual RMT MANUAL.md > rmt-cli.8 && gzip -f rmt-cli.8
 	mv rmt-cli.8.gz package/obs
 
 dist: clean man

README.md

@@ -23,19 +23,27 @@ Please view our [guide](docs/installation.md) to assist you in the RMT installat
     sudo zypper in libxml2-devel libxslt-devel libmariadb-devel gcc
     ```
 2. Install the ruby version specified in the `.ruby-version` [file](.ruby-version).
-3. Install and start either the MariaDB or MySQL server:
+3. Install and setup the database:
+
+   **Default: MariaDB or MySQL server**
     ```
     sudo zypper in mariadb
     sudo systemctl enable mariadb
     sudo systemctl start mariadb
     ```
-4. Log into the MariaDB or MySQL server as root and create the RMT database user:
+    Log into the MariaDB or MySQL server as root and create the RMT database user:
     ```
     mysql -u root -p <<EOFF
     GRANT ALL PRIVILEGES ON \`rmt%\`.* TO rmt@localhost IDENTIFIED BY 'rmt';
     FLUSH PRIVILEGES;
     EOFF
     ```
+
+    **Experimental: SQLite**
+
+    For development purposes it can be easier to run with SQLite, to avoid extra dependencies.
+    To run RMT with SQLite, switch the database adapter in `config/rmt.yml` to `sqlite3`.
+
 5. Clone the RMT repository:
     ```
     git clone git@github.com:SUSE/rmt.git

bin/rmt-cli

@@ -45,6 +45,11 @@ end
 db_config = RMT::Config.db_config
 ActiveRecord::Base.establish_connection(db_config)
 
+if ActiveRecord::Base.connection.adapter_name != "Mysql2"
+  warn "Running with experimental support for #{ActiveRecord::Base.connection.adapter_name}."
+  warn 'RMT is running without locking operations, make sure not to run it in multipe processes.'
+end
+
 begin
   RMT::CLI::Main.start(ARGV)
 rescue Interrupt

ci/Dockerfile

@@ -0,0 +1,15 @@
+#!BuildTag: rmt-ci-container
+#!UseOBSRepositories
+FROM opensuse/leap:15.5
+
+RUN zypper --non-interactive in make chrpath fdupes gcc libcurl-devel libffi-devel libmysqlclient-devel \
+                                libxml2-devel libxslt-devel nodejs sqlite-devel ruby2.5-rubygem-ronn    \
+                                rpmbuild systemd gzip tar bzip2 vim curl wget ruby2.5 ruby2.5-devel     \
+                                ruby2.5-rubygem-bundler
+
+# Set default bundler to the required version. This way we can easily update ruby in case required
+RUN update-alternatives --install /usr/bin/bundle bundle /usr/bin/bundle.ruby2.5 5 && \
+    update-alternatives --install /usr/bin/bundler bundler /usr/bin/bundler.ruby2.5 5
+
+RUN mkdir /usr/src/rmt-server
+WORKDIR /usr/src/rmt-server

ci/README.md

@@ -0,0 +1,46 @@
+## RMT CI setup
+
+Our CI setup runs the following steps:
+
+### Lint and unit tests
+
+workflow definition: [.github/workflows/lint-unit.yml](https://github.com/SUSE/rmt/blob/master/.github/workflows/lint-unit.yml)
+
+This workflow runs rubocop to check if the source is well formated and afterwards unit tests and engine unit tests. At last it checks
+if version in RMT and the rpm spec file matches.
+
+**Running it locally**
+
+There is no special mechanism needed to run these steps locally. Check the workflow for hints how to run unit tests
+
+### CLI feature tests
+
+workflow definition: [.github/workflows/features.yml](https://github.com/SUSE/rmt/blob/master/.github/workflows/features.yml)
+
+This workflow runs our simple CLI feature tests und build the rpm beforehand to see the system working with an installed RMT rpm.
+
+**Running it locally**
+
+To run feature tests locally, you need:
+
+- A checkout of RMT
+- A running mysql database
+- Proxy credentials to synchronize product information with SCC
+
+```
+# Fetch the CI container
+$ export IMAGE="registry.opensuse.org/systemsmanagement/scc/containers/15.5/rmt-ci-container:latest"
+
+# Build RMT rpms with the CI container the resulting rpms are in tmp/artifacts/
+$ docker run --rm -it -v $(pwd):/usr/src/rmt-server $IMAGE 'ci/rmt-build-rpm'
+
+# Run feature tests in the CI container
+# Note: Running --network=host isn't stricly required if you setup mysql access otherwise
+$ docker run --rm -it -v $(pwd):/usr/src/rmt-server --network=host $IMAGE bash -c 'ci/rmt-build-rpm && ci/rmt-configure && ci/rmt-run-feature-tests'
+```
+
+### The CI container
+
+Our CI container is built here: https://build.opensuse.org/package/show/systemsmanagement:SCC:containers/rmt-ci-container
+
+On push to Github master the rebuilt of the container is triggered