Tebako 0.5.4

.cirrus.yml

@@ -25,16 +25,20 @@
 
 test_task_template: &TEST_TASK_TEMPLATE
   testset-1_script: |
-    tests/scripts/functional-tests.sh
+    RUBY_VER=3.1.4 tests/scripts/functional-tests.sh
 
   testset-2_script: |
-    ruby tests-2/tebako-test.rb
+    RUBY_VER=3.1.4 ruby tests-2/tebako-test.rb
+
 
   checkout-packed-mn_script: |
-    git clone --depth 1 https://github.com/metanorma/packed-mn.git packed-mn
+    git clone https://github.com/metanorma/packed-mn.git packed-mn
+    pushd packed-mn
+    git checkout maxirmx-tebako-0.5.4
+    popd
 
   package-metanorma_script: |
-    exe/tebako press --root=packed-mn --entry-point=bin/metanorma --output=metanorma-package
+    exe/tebako press --root=packed-mn --entry-point=bin/metanorma --output=metanorma-package -R 3.1.4
 
   smoke-test_script: |
     ./metanorma-package help
@@ -100,7 +104,7 @@ task:
     bundle install
     bundle exec rake generate_version_txt
     exe/tebako clean_ruby
-    exe/tebako setup
+    exe/tebako setup -R 3.1.4
 
   upload_caches:
     - tebako

.github/actions/test-package-metanorma/action.yml

@@ -33,6 +33,7 @@ runs:
       with:
         repository: metanorma/packed-mn
         path: ${{github.workspace}}/packed-mn
+        ref: 'maxirmx-tebako-0.5.4'
 
     - name: Download gem
       uses: actions/download-artifact@v3

.github/workflows/alpine.yml

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 01
+  CACHE_VER: 02
   VCPKG_FORCE_SYSTEM_BINARIES: 1
   TZ: "Etc/UTC"
   VERBOSE: no

.github/workflows/gem-test-and-release.yml

@@ -49,7 +49,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 01
+  CACHE_VER: 02
   DEBIAN_FRONTEND: "noninteractive"
   TZ: "Etc/UTC"
 # show cmake output
@@ -187,7 +187,7 @@ jobs:
     name: Release gem
     needs: [ package-metanorma-ubuntu, package-metanorma-alpine, package-metanorma-macos ]
     runs-on: ubuntu-latest
-    if: contains(github.ref, 'refs/tags/v')
+    if: contains(github.ref, 'refs/tags/v') && !contains(github.ref, 'pre')
     steps:
     - uses: actions/checkout@v3
       with:

.github/workflows/macos.yml

@@ -50,7 +50,7 @@ concurrency:
 env:
   CC: clang
   CXX: clang++
-  CACHE_VER: 01
+  CACHE_VER: 02
   VERBOSE: no
 
 jobs:

.github/workflows/ubuntu.yml

@@ -48,7 +48,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 01
+  CACHE_VER: 02
   DEBIAN_FRONTEND: "noninteractive"
   TZ: "Etc/UTC"
 # show cmake output (yes/no)

CMakeLists.txt

@@ -73,8 +73,6 @@ include(${CMAKE_SOURCE_DIR}/tools/cmake-scripts/macos-environment.cmake)
 include(${CMAKE_SOURCE_DIR}/tools/cmake-scripts/msys-environment.cmake)
 include(${CMAKE_SOURCE_DIR}/tools/cmake-scripts/def-external-project.cmake)
 
-include(${CMAKE_SOURCE_DIR}/cmake/def_tty_colours.cmake)
-
 option(SETUP_MODE "Tebako setup" OFF)
 if(NOT LOG_LEVEL)
   set(LOG_LEVEL "error")
@@ -85,7 +83,7 @@ set(START_WITH_PACKAGED_FILESYSTEM OFF CACHE BOOL "Start workflow from 'packaged
 if (NOT ${SETUP_MODE})
   message("Running tebako press script")
   if (${START_WITH_PACKAGED_FILESYSTEM})
-    message("${ColourBoldRed}*** Starting workflow from 'packaged filesystem' target ***${ColourReset}")
+    message("*** Starting workflow from 'packaged filesystem' target ***")
   endif(${START_WITH_PACKAGED_FILESYSTEM})
 
   if ("-${ROOT}" STREQUAL "-")
@@ -174,7 +172,12 @@ elseif("${OSTYPE_TXT}" MATCHES "^msys*")
   set(RUBY_MAKEFILE "--file=GNUmakefile")
 elseif("${OSTYPE_TXT}" MATCHES "^darwin.*")
   set(IS_DARWIN ON)
-  set(BUILD_OPENSSL_ROOT_DIR "${BUILD_BREW_PREFIX}/opt/openssl@1.1")
+  if(${RUBY_VER} VERSION_LESS "3.2.0")
+    set(OPENSSL_VER "1.1")
+  else(${RUBY_VER} VERSION_LESS "3.2.0")
+    set(OPENSSL_VER "3")
+  endif(${RUBY_VER} VERSION_LESS "3.2.0")
+  set(BUILD_OPENSSL_ROOT_DIR "${BUILD_BREW_PREFIX}/opt/openssl@${OPENSSL_VER}")
 endif()
 
 # Various locations for external projects
@@ -217,7 +220,7 @@ if(IS_MSYS)
 endif(IS_MSYS)
 
 def_ext_prj_g(INCBIN "348e36b")
-def_ext_prj_g(DWARFS_WR "v0.3.4")
+def_ext_prj_g(DWARFS_WR "v0.3.5")
 
 if (DEFINED ENV{BUNDLER_VER})
  set(BUNDLER_VER $ENV{BUNDLER_VER})
@@ -411,11 +414,6 @@ endif()
 set(C_FLAGS_DEST cppflags)
 
 if(IS_DARWIN)
-  if(${RUBY_VER} VERSION_LESS "3.2.2")
-    set(OPENSSL_VER "1.1")
-  else(${RUBY_VER} VERSION_LESS "3.2.2")
-    set(OPENSSL_VER "3")
-  endif(${RUBY_VER} VERSION_LESS "3.2.2")
   string(CONCAT RUBY_C_FLAGS ${RUBY_C_FLAGS} " -I${BUILD_BREW_PREFIX}/opt/openssl@${OPENSSL_VER}/include -I${BUILD_BREW_PREFIX}/opt/zlib/include -I${BUILD_BREW_PREFIX}/include")
   string(CONCAT RUBY_L_FLAGS ${RUBY_L_FLAGS} " -L${BUILD_BREW_PREFIX}/opt/openssl@${OPENSSL_VER}/lib -L${BUILD_BREW_PREFIX}/opt/zlib/lib  -L${BUILD_BREW_PREFIX}/lib")
   set(OPENSSL_RUBY_OPTION "--with-openssl-dir=${BUILD_BREW_PREFIX}/opt/openssl@${OPENSSL_VER}")
@@ -488,7 +486,7 @@ else (${SETUP_MODE})
   list(LENGTH GEMS     GLENGTH)
 
   add_custom_target(clean_filesystem
-      COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR}
+      COMMAND ruby ${EXE}/tebako-packager deploy ${RUBY_STASH_DIR} ${DATA_SRC_DIR} ${DATA_PRE_DIR} ${DATA_BIN_DIR} ${TBD} ${TGD}
       DEPENDS ${RUBY_PRJ}
   )
   if(GSLENGTH GREATER 0)

common.env

@@ -1,6 +1,6 @@
 BUILD_TYPE=Release
 DEPS=deps
 INCBIN_TAG=348e36b
-DWARFS_WR_TAG=v0.3.4
+DWARFS_WR_TAG=v0.3.5
 BUNDLER_VER=2.3.22
 RUBY_VER=3.1.4

exe/tebako-packager

@@ -78,11 +78,13 @@ begin
     #       ARGV[2] -- DATA_SRC_DIR
     #       ARGV[3] -- DATA_PRE_DIR
     #       ARGV[4] -- DATA_BIN_DIR
-    unless ARGV.length == 5
+    #       ARGV[5] -- TARGET_BIN_DIR (TBD)
+    #       ARGV[6] -- TARGET_GEM_DIR (TGD)
+    unless ARGV.length == 7
       raise Tebako::Error,
-            "tebako-packager deploy command expects 5 arguments, #{ARGV.length} has been provided."
+            "tebako-packager deploy command expects 7 arguments, #{ARGV.length} has been provided."
     end
-    Tebako::Packager.deploy(ARGV[1], ARGV[2], ARGV[3], ARGV[4])
+    Tebako::Packager.deploy(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
   else
     raise Tebako::Error, "tebako-packager cannot process #{ARGV[0]} command"
   end

lib/tebako/packager.rb

@@ -42,6 +42,7 @@ module Packager
       file.c
       io.c
       tool/mkconfig.rb
+      gem_prelude.rb
     ].freeze
 
     FILES_TO_RESTORE_MSYS = %w[
@@ -55,6 +56,12 @@ module Packager
       thread_pthread.c
     ].freeze
 
+    DEPLOY_ENV = {
+      "GEM_HOME" => nil,
+      "GEM_PATH" => nil,
+      "TEBAKO_PASS_THROUGH" => "1"
+    }.freeze
+
     class << self
       # Pass1
       # Executed before Ruby build, patching ensures that Ruby itself is linked statically
@@ -83,7 +90,7 @@ def pass2(ostype, ruby_source_dir, deps_lib_dir, ruby_ver)
       # Saves pristine Ruby environment that is used to deploy applications for packaging
       def stash(src_dir, stash_dir)
         puts "-- Running stash script"
-        #  .... this code snippet is executed 'outdside' of Ruby scripts
+        #  .... this code snippet is executed 'outside' of Ruby scripts
         # shall be reconsidered
         #    FileUtils.cd ruby_source_dir do
         #        puts "   ... creating pristine ruby environment at #{src_dir} [patience, it will take some time]"
@@ -98,18 +105,40 @@ def stash(src_dir, stash_dir)
       end
 
       # Deploy
-      # To be extended
-      # Now it just recreates Ruby prostine environment from stash
-      def deploy(stash_dir, src_dir, pre_dir, bin_dir)
+      def deploy(stash_dir, src_dir, pre_dir, bin_dir, tbd)
         puts "-- Running deploy script"
 
         puts "   ... creating packaging environment at #{src_dir}"
         recreate([src_dir, pre_dir, bin_dir])
         FileUtils.cp_r "#{stash_dir}/.", src_dir
+
+        install_gem tbd, "tebako-runtime"
       end
 
       private
 
+      def install_gem(tbd, name)
+        puts "   ... installing #{name} gem"
+        with_env(DEPLOY_ENV) do
+          out, st = Open3.capture2e("#{tbd}/gem", "install", name.to_s, "--no-doc")
+          raise Tebako::Error, "Failed to install #{name} (#{st}):\n #{out}" unless st.exitstatus.zero?
+        end
+      end
+
+      def do_patch(patch_map, root)
+        patch_map.each { |fname, mapping| patch_file("#{root}/#{fname}", mapping) }
+      end
+
+      def patch_file(fname, mapping)
+        raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname)
+
+        puts "   ... patching #{fname}"
+        restore_and_save(fname)
+        contents = File.read(fname)
+        mapping.each { |pattern, subst| contents.sub!(pattern, subst) }
+        File.open(fname, "w") { |file| file << contents }
+      end
+
       def recreate(dirname)
         FileUtils.rm_rf(dirname, noop: nil, verbose: nil, secure: true)
         FileUtils.mkdir(dirname)
@@ -132,18 +161,20 @@ def restore_and_save_files(files, ruby_source_dir)
         end
       end
 
-      def patch_file(fname, mapping)
-        raise Tebako::Error, "Could not patch #{fname} because it does not exist." unless File.exist?(fname)
-
-        puts "   ... patching #{fname}"
-        restore_and_save(fname)
-        contents = File.read(fname)
-        mapping.each { |pattern, subst| contents.sub!(pattern, subst) }
-        File.open(fname, "w") { |file| file << contents }
-      end
-
-      def do_patch(patch_map, root)
-        patch_map.each { |fname, mapping| patch_file("#{root}/#{fname}", mapping) }
+      # Sets up temporary environment variables and yields to the
+      # block. When the block exits, the environment variables are set
+      # back to their original values.
+      def with_env(hash)
+        old = {}
+        hash.each do |k, v|
+          old[k] = ENV.fetch(k, nil)
+          ENV[k] = v
+        end
+        begin
+          yield
+        ensure
+          hash.each { |k, _v| ENV[k] = old[k] }
+        end
       end
     end
   end

lib/tebako/packager/pass2.rb

@@ -34,18 +34,8 @@ module Packager
     # Ruby patching definitions (pass2)
     module Pass2
       class << self
-        def get_dir_c_patch(ostype)
-          dir_c_patch = patch_c_file(ostype =~ /msys/ ? "/* define system APIs */" : "#ifdef HAVE_GETATTRLIST")
-          dir_c_patch.merge!(DIR_C_BASE_PATCH)
-        end
-
         def get_patch_map(ostype, deps_lib_dir, ruby_ver)
-          patch_map = {
-            "template/Makefile.in" => template_makefile_in_patch(ostype, deps_lib_dir, ruby_ver),
-            "main.c" => MAIN_C_PATCH,
-            "tool/mkconfig.rb" => TOOL_MKCONFIG_RB_PATCH,
-            "dir.c" => get_dir_c_patch(ostype)
-          }
+          patch_map = get_patch_map_base(ostype, deps_lib_dir, ruby_ver)
 
           C_FILES_TO_PATCH.each { |patch| patch_map.store(patch[0], patch_c_file(patch[1])) }
           patch_map.store("thread_pthread.c", LINUX_MUSL_THREAD_PTHREAD_PATCH) if ostype =~ /linux-musl/
@@ -58,6 +48,21 @@ def get_patch_map(ostype, deps_lib_dir, ruby_ver)
 
         include Tebako::Packager::PatchLiterals
 
+        def get_dir_c_patch(ostype)
+          dir_c_patch = patch_c_file(ostype =~ /msys/ ? "/* define system APIs */" : "#ifdef HAVE_GETATTRLIST")
+          dir_c_patch.merge!(DIR_C_BASE_PATCH)
+        end
+
+        def get_patch_map_base(ostype, deps_lib_dir, ruby_ver)
+          {
+            "template/Makefile.in" => template_makefile_in_patch(ostype, deps_lib_dir, ruby_ver),
+            "main.c" => MAIN_C_PATCH,
+            "tool/mkconfig.rb" => TOOL_MKCONFIG_RB_PATCH,
+            "gem_prelude.rb" => GEM_PRELUDE_RB_PATCH,
+            "dir.c" => get_dir_c_patch(ostype)
+          }
+        end
+
         def process_brew_libs!(libs, brew_libs)
           brew_libs.each { |lib| libs << "#{PatchHelpers.get_prefix(lib[0]).chop}/lib/lib#{lib[1]}.a " }
         end
@@ -73,16 +78,12 @@ def darwin_libs(deps_lib_dir, ruby_ver)
           SUBST
         end
 
-        def yaml_reference(ruby_ver)
-          PatchHelpers.ruby32?(ruby_ver) ? "-l:libyaml.a" : ""
-        end
-
         def linux_gnu_libs(ruby_ver)
           <<~SUBST
             -l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -l:libfolly.a -l:libfsst.a -l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a \
             -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a -l:libiberty.a -l:libacl.a -l:libssl.a -l:libcrypto.a -l:liblz4.a -l:libz.a \
             -l:libzstd.a -l:libgdbm.a -l:libreadline.a -l:libtinfo.a -l:libffi.a -l:libncurses.a -l:libjemalloc.a -l:libunwind.a -l:libcrypt.a -l:libanl.a -l:liblzma.a \
-            #{yaml_reference(ruby_ver)} -l:libboost_system.a -l:libstdc++.a -l:librt.a -ldl -lpthread
+            #{PatchHelpers.yaml_reference(ruby_ver)} -l:libboost_system.a -l:libstdc++.a -l:librt.a -ldl -lpthread
           SUBST
         end
 
@@ -91,7 +92,7 @@ def linux_musl_libs(ruby_ver)
             -l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -l:libfolly.a -l:libfsst.a -l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a \
             -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a -l:libiberty.a -l:libacl.a -l:libssl.a -l:libcrypto.a -l:liblz4.a -l:libz.a \
             -l:libzstd.a -l:libgdbm.a -l:libreadline.a -l:libffi.a -l:libncurses.a -l:libjemalloc.a -l:libunwind.a -l:libcrypt.a -l:liblzma.a \
-            #{yaml_reference(ruby_ver)} -l:libboost_system.a -l:libstdc++.a -l:librt.a -ldl -lpthread
+            #{PatchHelpers.yaml_reference(ruby_ver)} -l:libboost_system.a -l:libstdc++.a -l:librt.a -ldl -lpthread
           SUBST
         end
 
@@ -100,7 +101,7 @@ def msys_libs(ruby_ver)
             -l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -l:libfolly.a -l:libfsst.a -l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a \
             -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a -l:libssl.a -l:libcrypto.a -l:liblz4.a -l:libz.a \
             -l:libzstd.a -l:libffi.a -l:libgdbm.a -l:libncurses.a -l:libjemalloc.a -l:libunwind.a -l:liblzma.a -l:libiberty.a \
-            #{yaml_reference(ruby_ver)} -l:libstdc++.a -l:libdl.a -lole32 -loleaut32 -luuid
+            #{PatchHelpers.yaml_reference(ruby_ver)} -l:libstdc++.a -l:libdl.a -lole32 -loleaut32 -luuid
           SUBST
         end
 
@@ -138,14 +139,7 @@ def patch_c_file(pattern)
         end
 
         def template_makefile_in_patch(ostype, deps_lib_dir, ruby_ver)
-          base_patch = {
-            "LIBS = @LIBS@ $(EXTLIBS)" => <<~SUBST
-              # -- Start of tebako patch --
-              LIBS = $(MAINLIBS) @LIBS@
-              # -- End of tebako patch --
-            SUBST
-          }
-          base_patch.merge!(mlibs_subst(ostype, deps_lib_dir, ruby_ver))
+          base_patch = TEMPLATE_MAKEFILE_IN_BASE_PATCH_ONE.merge(mlibs_subst(ostype, deps_lib_dir, ruby_ver))
           base_patch.merge!(template_makefile_in_patch_two(ruby_ver))
         end