feat: an option to remove GLIBC_PRIVATE reference on Linux Gnu

tamatebako · Sep 1, 2024 · 6a2b0db · 6a2b0db
1 parent 35314a1
commit 6a2b0db
Show file tree

Hide file tree

Showing 13 changed files with 267 additions and 133 deletions.
diff --git a/.github/workflows/gem-test-and-release.yml b/.github/workflows/gem-test-and-release.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023-2024 [Ribose Inc](https://www.ribose.com)
 # All rights reserved.
 # This file is a part of tamatebako
 #
@@ -49,7 +49,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CACHE_VER: 04
+  CACHE_VER: 05
   DEBIAN_FRONTEND: "noninteractive"
   TZ: "Etc/UTC"
 # show cmake output
@@ -258,7 +258,7 @@ jobs:
         gem: [ metanorma, fontist ]
         package_ruby_ver: [ '3.2.5' ]
         env:
-          - { CC: clang-12, CXX: clang++-12 }
+          - { CC: gcc-10, CXX: g++-10 }
     env: ${{ matrix.env }}
     steps:
     - name: Download actions
@@ -289,15 +289,52 @@ jobs:
       if: matrix.gem == 'fontist'
       uses: ./.github/actions/test-package-fontist
 
+    - name: Upload fontist test package
+      if: matrix.gem == 'fontist'
+      uses: actions/upload-artifact@v4
+      with:
+        name: fontist-package
+        retention-days: 1
+        path: |
+          fontist-package
+
     - name: Package metanorma
       if: matrix.gem == 'metanorma'
       uses: ./.github/actions/test-package-metanorma
 
+    - name: Upload metanorma test package
+      if: matrix.gem == 'metanorma'
+      uses: actions/upload-artifact@v4
+      with:
+        name: metanorma-package
+        retention-days: 1
+        path: |
+          metanorma-package
+
+  package-tests-ubuntu-newer:
+    name: cross-test on ${{ matrix.os }}
+    needs: [ package-tests-ubuntu ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-22.04', 'ubuntu-24.04' ]
+    steps:
+    - name: Download test packages
+      uses: actions/download-artifact@v4
+      with:
+        pattern: '*-package'
+
+    - name: Run test packages
+      run:  |
+          set -o errexit -o pipefail -o noclobber -o nounset
+          for TP in *-package; do chmod +x "$TP/$TP"; "$TP/$TP" help; done
+
 # ----- Release -----
 
   release:
     name: Release gem
-    needs: [ package-tests-windows, package-tests-ubuntu, package-tests-alpine, package-tests-macos ]
+    needs: [ package-tests-windows, package-tests-ubuntu-newer, package-tests-alpine, package-tests-macos ]
     runs-on: ubuntu-latest
     if: contains(github.ref, 'refs/tags/v') && !contains(github.ref, 'pre')
     steps:

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -129,10 +129,19 @@ jobs:
           cc: ${{ env.CC }}
           version: ${{ env.CACHE_VER }}
 
+      - name: Shall upload artifacts?
+        id: shall-upload
+        run: |
+          if [ "${{ matrix.package_ruby_ver }}" == "3.2.5" ] && [ "${{ matrix.env.CC }}" == "gcc-10" ]; then
+            echo "upload=true" >> $GITHUB_OUTPUT
+          else
+            echo "upload=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Run test set 1
         uses: ./.github/actions/test-set-1
         with:
-          upload: 'false'
+          upload: "${{ steps.shall-upload.outputs.upload }}"
 
   tests-2:
     needs: setup
@@ -174,17 +183,22 @@ jobs:
       - name: Run tebako tests (set no. 2)
         run: RUBY_VER=${{ matrix.package_ruby_ver }} ruby ${{github.workspace}}/tests-2/tebako-test.rb
 
-#  test-on-ubuntu-latest:
-#    needs: tests-1
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Download test packages
-#        uses: actions/download-artifact@v4
-#        with:
-#          name: test-packages
-#
-#      - name: Run test packages
-#        run:  |
-#          set -o errexit -o pipefail -o noclobber -o nounset
-#          chmod +x test-*-package
-#          for TP in test-*-package; do "./$TP"; done
+  test-on-ubuntu-newer:
+    name: cross-test on ${{ matrix.os }}
+    needs: tests-1
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-22.04', 'ubuntu-24.04' ]
+    steps:
+      - name: Download test packages
+        uses: actions/download-artifact@v4
+        with:
+          name: test-packages
+
+      - name: Run test packages
+        run:  |
+          set -o errexit -o pipefail -o noclobber -o nounset
+          chmod +x test-*-package
+          for TP in test-*-package; do "./$TP"; done
diff --git a/.tebako.yml b/.tebako.yml
@@ -1,3 +1,4 @@
 options:
   prefix: PWD
+  patchelf: true
 #  Ruby: 3.2.5
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -113,6 +113,7 @@ else(OSTYPE_RES EQUAL 0)
   message(FATAL_ERROR "Failed to detect OSTYPE: ${OSTYPE_TXT}")
 endif(OSTYPE_RES EQUAL 0)
 
+set(IS_GNU OFF)
 set(IS_MUSL OFF)
 set(IS_MSYS OFF)
 set(IS_DARWIN OFF)
@@ -121,8 +122,18 @@ set(RUBY_WITHOUT_EXT "dbm,win32,win32ole,-test-/*")
 set(RUBY_NAME "ruby")
 set(EXE_SUFFIX "")
 set(DWARFS_PRELOAD OFF)
-
-if("${OSTYPE_TXT}" MATCHES "^linux-musl.*")
+set(WITH_PATCHELF OFF)
+
+if("${OSTYPE_TXT}" MATCHES "^linux-gnu.*")
+  set(IS_GNU ON)
+  if(REMOVE_GLIBC_PRIVATE)
+    if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
+      set(WITH_PATCHELF ON)
+    else()
+      message(WARNING "Elf file patching is supported for GNU toolchain only. 'patchelf' setting ignored")
+    endif()
+  endif(REMOVE_GLIBC_PRIVATE)
+elseif("${OSTYPE_TXT}" MATCHES "^linux-musl.*")
   set(IS_MUSL ON)
 elseif("${OSTYPE_TXT}" MATCHES "^msys*")
   set(IS_MSYS ON)
@@ -180,9 +191,7 @@ set(DATA_RES_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/resources)
 # External projects
 
 if ("-${RUBY_VER}" STREQUAL "-" OR "-${RUBY_HASH}" STREQUAL "-")
-  set(RUBY_VER "3.1.4")
-  set(RUBY_HASH "a3d55879a0dfab1d7141fdf10d22a07dbf8e5cdc4415da1bde06127d5cc3c7b6")
-  message(STATUS "Using default Ruby version ${RUBY_VER}")
+  message(FATAL_ERROR "Ruby version is not specified")
 endif()
 
 set(RUBY_NAME ruby)
@@ -207,6 +216,8 @@ string(CONCAT RUBY_API_VER ${RUBY_VER_BASE} ".0")
 def_ext_prj_g(DWARFS_WR "v0.5.9")
 #endif(DWARFS_PRELOAD)
 
+def_ext_prj_g(PATCHELF "master")
+
 set(LIBYAML_RUBY_OPTION "")
 if(${RUBY_VER} VERSION_LESS "3.2.0")
   set(LIBYAML_RUBY_OPTION "--enable-bundled-libyaml")
@@ -266,6 +277,7 @@ message(STATUS "Target local directory: ${TLD}")
 message(STATUS "Target Gem directory:: ${TGD}")
 message(STATUS "FS_MOUNT_POINT: ${FS_MOUNT_POINT}")
 message(STATUS "Building for Win32 Ruby (RB_W32): ${RB_W32}")
+message(STATUS "Removing GLIBC_PRIVATE reference: ${WITH_PATCHELF}")
 
 # ...................................................................
 # Other options
@@ -307,7 +319,7 @@ if(DWARFS_PRELOAD)
   )
 else(DWARFS_PRELOAD)
   ExternalProject_Add(${DWARFS_WR_PRJ}
-      PREFIX ${DEPS}
+     PREFIX ${DEPS}
      GIT_REPOSITORY https://github.com/tamatebako/libdwarfs.git
      GIT_TAG ${DWARFS_WR_TAG}
      SOURCE_DIR ${DWARFS_WR_SOURCE_DIR}
@@ -328,6 +340,22 @@ else(DWARFS_PRELOAD)
   )
 endif(DWARFS_PRELOAD)
 
+if(IS_GNU)
+  ExternalProject_Add(${PATCHELF_PRJ}
+    PREFIX ${DEPS}
+    GIT_REPOSITORY https://github.com/chitao1234/patchelf.git
+    GIT_TAG ${PATCHELF_WR_TAG}
+    SOURCE_DIR ${PATCHELF_SOURCE_DIR}
+    BINARY_DIR ${PATCHELF_BINARY_DIR}
+    UPDATE_COMMAND ""
+    CONFIGURE_COMMAND ${CMAKE_COMMAND} -E chdir ${PATCHELF_SOURCE_DIR} ./bootstrap.sh
+              COMMAND ${PATCHELF_SOURCE_DIR}/configure
+                  --srcdir=${PATCHELF_SOURCE_DIR}
+                  --prefix=${DEPS}
+    TEST_COMMAND ""
+  )
+endif(IS_GNU)
+
 # ...................................................................
 # Ruby
 
@@ -396,6 +424,11 @@ ExternalProject_Add(${RUBY_PRJ}
 
 add_dependencies(${RUBY_PRJ} ${DWARFS_WR_PRJ} ${LIBDWARFS_WR_PRJ})
 
+if(IS_GNU)
+  add_dependencies(${RUBY_PRJ} ${PATCHELF_PRJ})
+endif(IS_GNU)
+
+
 if (${SETUP_MODE})
   add_custom_target(setup
     ${CMAKE_COMMAND} -E echo "Tebako setup has completed"
@@ -488,16 +521,13 @@ else (${SETUP_MODE})
     ${DEPS_INCLUDE_DIR}/tebako/tebako-version.h
   )
 
-  #et_target_properties(tebako-fs PROPERTIES POSITION_INDEPENDENT_CODE ON)
-  #target_compile_options(tebako-fs PRIVATE -fPIE)
-
   if(${RUBY_VER} VERSION_LESS "3.3.0" AND "${OSTYPE_TXT}" MATCHES "^msys*")
     target_compile_definitions(tebako-fs PUBLIC RB_W32_PRE_33)
   endif(${RUBY_VER} VERSION_LESS "3.3.0" AND "${OSTYPE_TXT}" MATCHES "^msys*")
 
   add_dependencies(tebako-fs packaged_filesystem)
 
-  add_custom_target(tebako COMMAND ruby ${EXE}/tebako-packager finalize ${OSTYPE_TXT} ${RUBY_SOURCE_DIR} ${APP_NAME} ${RUBY_VER})
+  add_custom_target(tebako COMMAND ruby ${EXE}/tebako-packager finalize ${OSTYPE_TXT} ${RUBY_SOURCE_DIR} ${APP_NAME} ${RUBY_VER} ${DEPS_BIN_DIR}/patchelf ${WITH_PATCHELF})
   add_dependencies(tebako setup tebako-fs)
 
 endif(${SETUP_MODE})
diff --git a/README.adoc b/README.adoc
@@ -657,6 +657,7 @@ tebako press \
   [-l|--log-level=<error|warn|debug|trace>] \
   [-c|--cwd=<package current working directory>]
   [-D|--devmode] \
+  [-P|--patchelf] \
   [-t|--tebafile=<path-to-tebafile>]
 ----
 
@@ -689,12 +690,17 @@ If not provided, the application will start within the current folder of the hos
 This option is required because it is not possible to change the directory to a memfs folder until the package is started, as opposed to any host folder
 that can be set as the current directory before Tebako package invocation.  Tebako saves original working directory in a global Ruby variable `$tebako_original_pwd`.
 
+`devmode`:: flag that activates development mode, in which Tebako's cache and
+packaging consistency checks are relaxed.
+
+`patchelf`::
+flag that removal a reference to GLIBC_PRIVATE version of libpthread from tebako package. This allows Linux Gnu packages to run against versions of
+libpthread that differ from the version used for packaging. For example, package created at Ubuntu 20 system can be used on Ubuntu 22. This option works on Gnu Linux with
+Gnu toolchain only (not for LLVM/clang). The feature is exeprimental, we may consider other approach in the future.
+
 `tebafile`::
 the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`).
 Please refer to the separate section below for tebafile description.
-
-`devmode`:: flag that activates development mode, in which Tebako's cache and
-packaging consistency checks are relaxed.
 +
 NOTES:
   * Development mode is *not intended for production use* and should only be used during development.

diff --git a/exe/tebako-packager b/exe/tebako-packager
@@ -105,12 +105,15 @@ begin
     #       ARGV[2] -- RUBY_SOURCE_DIR
     #       ARGV[3] -- APP_NAME
     #       ARGV[4] -- RUBY_VER
-    unless ARGV.length == 5
+    #       ARGV[5] -- patchelf executable
+    #       ARGV[6] -- WITH_PATHELF
+    unless ARGV.length == 7
       raise Tebako::Error,
-            "tebako-packager finalize command expects 5 arguments, #{ARGV.length} has been provided."
+            "tebako-packager finalize command expects 7 arguments, #{ARGV.length} has been provided."
     end
     ruby_ver = Tebako::RubyVersion.new(ARGV[4])
-    Tebako::Packager.finalize(ARGV[1], ARGV[2], ARGV[3], ruby_ver)
+    with_patchelf = ARGV[6].casecmp("ON").zero? || ARGV[6].casecmp("YES").zero?
+    Tebako::Packager.finalize(ARGV[1], ARGV[2], ARGV[3], ruby_ver, with_patchelf ? ARGV[5] : nil)
   else
     raise Tebako::Error, "tebako-packager cannot process #{ARGV[0]} command"
   end

diff --git a/lib/tebako/build_helpers.rb b/lib/tebako/build_helpers.rb
@@ -39,7 +39,7 @@ def ncores
           out, st = Open3.capture2e("nproc", "--all")
         end
 
-        if st.exitstatus.zero?
+        if !st.signaled? && st.exitstatus.zero?
           out.strip.to_i
         else
           4
@@ -49,7 +49,7 @@ def ncores
       def run_with_capture(args)
         puts "   ... @ #{args.join(" ")}"
         out, st = Open3.capture2e(*args)
-        raise Tebako::Error, "Failed to run #{args.join(" ")} (#{st}):\n #{out}" unless st.exitstatus.zero?
+        raise Tebako::Error, "Failed to run #{args.join(" ")} (#{st}):\n #{out}" if st.signaled? || !st.exitstatus.zero?
 
         out
       end

diff --git a/lib/tebako/cli.rb b/lib/tebako/cli.rb
@@ -46,7 +46,7 @@ class Cli < Thor
     package_name "Tebako"
     class_option :prefix, type: :string, aliases: "-p", required: false,
                           desc: "A path to tebako packaging environment, '~/.tebako' ('$HOME/.tebako') by default"
-    class_option :devmode, type: :boolean, aliases: "-D", required: false,
+    class_option :devmode, type: :boolean, aliases: "-D",
                            desc: "Developer mode, please do not use if unsure"
     class_option :tebafile, type: :string, aliases: "-t", required: false,
                             desc: "tebako configuration file 'tebafile', '$PWD/.tebako.yml' by default"
@@ -75,7 +75,13 @@ def hash
 
     CWD_DESCRIPTION = <<~DESC
       Current working directory for packaged application. This directory shall be specified relative to root.
-      #{" " * 62}# If this parameter is not set, the application will start in the current directory of the host file system.
+      #{" " * 65}# If this parameter is not set, the application will start in the current directory of the host file system.
+    DESC
+
+    RGP_DESCRIPTION = <<~DESC
+      Activates removal a reference to GLIBC_PRIVATE version of libpthread from tebako package. This allows Linux Gnu packages to run against versions of
+      #{" " * 65}# libpthread that differ from the version used for packaging. For example, package created at Ubuntu 20 system can be used on Ubuntu 22. This option works on Gnu Linux with
+      #{" " * 65}# Gnu toolchain only (not for LLVM/clang). The feature is exeprimental, we may consider other approach in the future.
     DESC
 
     desc "press", "Press tebako image"
@@ -91,6 +97,8 @@ def hash
     method_option :Ruby, type: :string, aliases: "-R", required: false,
                          enum: Tebako::CliRubies::RUBY_VERSIONS.keys,
                          desc: "Tebako package Ruby version, #{Tebako::CliRubies::DEFAULT_RUBY_VERSION} by default"
+    method_option :patchelf, aliases: "-P", type: :boolean,
+                             desc: RGP_DESCRIPTION
     def press
       version_cache_check unless options[:devmode]
 

diff --git a/lib/tebako/cli_helpers.rb b/lib/tebako/cli_helpers.rb
@@ -56,7 +56,7 @@ def cfg_options
       @cfg_options ||=
         "-DCMAKE_BUILD_TYPE=Release -DRUBY_VER:STRING=\"#{ruby_ver}\" -DRUBY_HASH:STRING=\"#{ruby_hash}\" " \
         "-DDEPS:STRING=\"#{deps}\" -G \"#{m_files}\" -B \"#{output_folder}\" -S \"#{source}\" " \
-        "-DTEBAKO_VERSION:STRING=\"#{Tebako::VERSION}\""
+        "-DREMOVE_GLIBC_PRIVATE=#{remove_glibc_private} -DTEBAKO_VERSION:STRING=\"#{Tebako::VERSION}\""
     end
 
     def clean_cache
@@ -174,6 +174,10 @@ def package
                    end
     end
 
+    def remove_glibc_private
+      @remove_glibc_private ||= options["patchelf"] ? "ON" : "OFF"
+    end
+
     def handle_nil_prefix
       env_prefix = ENV.fetch("TEBAKO_PREFIX", nil)
       if env_prefix.nil?