From 1c66f14ce5bec461112364943356c69e0b01bda1 Mon Sep 17 00:00:00 2001 From: nitin sanghi Date: Tue, 17 Sep 2024 10:32:47 +0530 Subject: [PATCH 1/5] Hab plan file for windows and linux to build hab pkg Signed-off-by: nitin sanghi --- Gemfile | 2 +- habitat/plan.ps1 | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ habitat/plan.sh | 63 +++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 habitat/plan.ps1 create mode 100644 habitat/plan.sh diff --git a/Gemfile b/Gemfile index 31584eccb..7ab9c3de4 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ gemspec # pull these gems from main of chef/chef so that we're testing against what we will release gem "chef-config", git: "https://github.com/chef/chef", branch: "main", glob: "chef-config/chef-config.gemspec" gem "chef-utils", git: "https://github.com/chef/chef", branch: "main", glob: "chef-utils/chef-utils.gemspec" - +gem "appbundler" # NOTE: do not submit PRs to add pry as a dep, add to your Gemfile.local group :development do gem "cookstyle", ">= 7.32.8" diff --git a/habitat/plan.ps1 b/habitat/plan.ps1 new file mode 100644 index 000000000..a93c74a47 --- /dev/null +++ b/habitat/plan.ps1 @@ -0,0 +1,92 @@ +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction']='Stop' + +$pkg_name="ohai" +$pkg_origin="chef" +$pkg_version=$(Get-Content "$PLAN_CONTEXT/../VERSION") +$pkg_maintainer="The Chef Maintainers " + +$pkg_deps=@( + "chef/ruby31-plus-devkit" + "core/git" +) +$pkg_bin_dirs=@("bin" + "vendor/bin") +$project_root= (Resolve-Path "$PLAN_CONTEXT/../").Path + +function pkg_version { + Get-Content "$SRC_PATH/VERSION" +} + +function Invoke-Before { + Set-PkgVersion +} +function Invoke-SetupEnvironment { + Push-RuntimeEnv -IsPath GEM_PATH "$pkg_prefix/vendor" + + Set-RuntimeEnv APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH + Set-RuntimeEnv FORCE_FFI_YAJL "ext" + Set-RuntimeEnv LANG "en_US.UTF-8" + Set-RuntimeEnv LC_CTYPE "en_US.UTF-8" +} + +function Invoke-Build { + try { + $env:Path += ";c:\\Program Files\\Git\\bin" + Push-Location $project_root + $env:GEM_HOME = "$HAB_CACHE_SRC_PATH/$pkg_dirname/vendor" + + Write-BuildLine " ** Configuring bundler for this build environment" + bundle config --local without integration deploy maintenance + bundle config --local jobs 4 + bundle config --local retry 5 + bundle config --local silence_root_warning 1 + Write-BuildLine " ** Using bundler to retrieve the Ruby dependencies" + bundle install + + gem build ohai.gemspec + Write-BuildLine " ** Using gem to install" + gem install ohai-*.gem --no-document + + + If ($lastexitcode -ne 0) { Exit $lastexitcode } + } finally { + Pop-Location + } +} + +function Invoke-Install { + Write-BuildLine "** Copy built & cached gems to install directory" + Copy-Item -Path "$HAB_CACHE_SRC_PATH/$pkg_dirname/*" -Destination $pkg_prefix -Recurse -Force -Exclude @("gem_make.out", "mkmf.log", "Makefile", + "*/latest", "latest", + "*/JSON-Schema-Test-Suite", "JSON-Schema-Test-Suite") + + try { + Push-Location $pkg_prefix + bundle config --local gemfile $project_root/Gemfile + Write-BuildLine "** generating binstubs for ohai with precise version pins" + Write-BuildLine "** generating binstubs for ohai with precise version pins $project_root $pkg_prefix/bin " + Invoke-Expression -Command "appbundler.bat $project_root $pkg_prefix/bin ohai" + If ($lastexitcode -ne 0) { Exit $lastexitcode } + Write-BuildLine " ** Running the ohai project's 'rake install' to install the path-based gems so they look like any other installed gem." + + If ($lastexitcode -ne 0) { Exit $lastexitcode } + } finally { + Pop-Location + } +} + +function Invoke-After { + # We don't need the cache of downloaded .gem files ... + Remove-Item $pkg_prefix/vendor/cache -Recurse -Force + # We don't need the gem docs. + Remove-Item $pkg_prefix/vendor/doc -Recurse -Force + # We don't need to ship the test suites for every gem dependency, + # only inspec's for package verification. + Get-ChildItem $pkg_prefix/vendor/gems -Filter "spec" -Directory -Recurse -Depth 1 ` + | Where-Object -FilterScript { $_.FullName -notlike "*ohai*" } ` + | Remove-Item -Recurse -Force + # Remove the byproducts of compiling gems with extensions + Get-ChildItem $pkg_prefix/vendor/gems -Include @("gem_make.out", "mkmf.log", "Makefile") -File -Recurse ` + | Remove-Item -Force +} \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh new file mode 100644 index 000000000..eb96a8730 --- /dev/null +++ b/habitat/plan.sh @@ -0,0 +1,63 @@ +pkg_name=ohai +pkg_origin=chef +ruby_pkg="core/ruby31" +pkg_deps=(${ruby_pkg} core/coreutils) +pkg_build_deps=( + core/make + core/sed + core/gcc + ) +pkg_bin_dirs=(bin) +do_setup_environment() { + build_line 'Setting GEM_HOME="$pkg_prefix/lib"' + export GEM_HOME="$pkg_prefix/lib" + + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" +} + +pkg_version() { + cat "$SRC_PATH/VERSION" +} +do_before() { + update_pkg_version +} +do_unpack() { + mkdir -pv "$HAB_CACHE_SRC_PATH/$pkg_dirname" + cp -RT "$PLAN_CONTEXT"/.. "$HAB_CACHE_SRC_PATH/$pkg_dirname/" +} +do_build() { + pushd "$HAB_CACHE_SRC_PATH/$pkg_dirname/" + gem build ohai.gemspec + popd +} +do_install() { + pushd "$HAB_CACHE_SRC_PATH/$pkg_dirname/" + gem install ohai-*.gem --no-document + popd + wrap_ruby_bin +} +wrap_ruby_bin() { + local bin="$pkg_prefix/bin/$pkg_name" + local real_bin="$GEM_HOME/gems/ohai-${pkg_version}/bin/ohai" + build_line "Adding wrapper $bin to $real_bin" + cat < "$bin" +#!$(pkg_path_for core/bash)/bin/bash +set -e + +# Set binary path that allows ohai to use non-Hab pkg binaries +export PATH="/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:\$PATH" + +# Set Ruby paths defined from 'do_setup_environment()' +export GEM_HOME="$GEM_HOME" +export GEM_PATH="$GEM_PATH" + +exec $(pkg_path_for core/ruby31)/bin/ruby $real_bin \$@ +EOF + chmod -v 755 "$bin" +} + + +do_strip() { + return 0 +} \ No newline at end of file From dbbdd5ba4dbd948e3dc7631c9079ea89a995c281 Mon Sep 17 00:00:00 2001 From: nitin sanghi Date: Wed, 18 Sep 2024 11:25:51 +0530 Subject: [PATCH 2/5] Linux plan change based on appbundler Signed-off-by: nitin sanghi --- .byebug_history | 1 + Gemfile | 35 +++++----- habitat/plan.ps1 | 4 +- habitat/plan.sh | 145 +++++++++++++++++++++++++++++------------ post-bundle-install.rb | 32 +++++++++ 5 files changed, 155 insertions(+), 62 deletions(-) create mode 100644 .byebug_history create mode 100644 post-bundle-install.rb diff --git a/.byebug_history b/.byebug_history new file mode 100644 index 000000000..5b6985884 --- /dev/null +++ b/.byebug_history @@ -0,0 +1 @@ +next diff --git a/Gemfile b/Gemfile index 7ab9c3de4..39c75defd 100644 --- a/Gemfile +++ b/Gemfile @@ -1,28 +1,29 @@ # frozen_string_literal: true -source "https://rubygems.org" + +source 'https://rubygems.org' gemspec # pull these gems from main of chef/chef so that we're testing against what we will release -gem "chef-config", git: "https://github.com/chef/chef", branch: "main", glob: "chef-config/chef-config.gemspec" -gem "chef-utils", git: "https://github.com/chef/chef", branch: "main", glob: "chef-utils/chef-utils.gemspec" -gem "appbundler" +gem 'appbundler' +gem 'chef-config', git: 'https://github.com/chef/chef', branch: 'main', glob: 'chef-config/chef-config.gemspec' +gem 'chef-utils', git: 'https://github.com/chef/chef', branch: 'main', glob: 'chef-utils/chef-utils.gemspec' # NOTE: do not submit PRs to add pry as a dep, add to your Gemfile.local group :development do - gem "cookstyle", ">= 7.32.8" - gem "ipaddr_extensions" - gem "rake", ">= 10.1.0" - gem "rspec-collection_matchers", "~> 1.0" - gem "rspec-core", "~> 3.0" - gem "rspec-expectations", "~> 3.0" - gem "rspec-mocks", "~> 3.0" - gem "rubocop-performance", "1.18.0" - gem "rubocop-rspec" + gem 'cookstyle', '>= 7.32.8' + gem 'ipaddr_extensions' + gem 'rake', '>= 10.1.0' + gem 'rspec-collection_matchers', '~> 1.0' + gem 'rspec-core', '~> 3.0' + gem 'rspec-expectations', '~> 3.0' + gem 'rspec-mocks', '~> 3.0' + gem 'rubocop-performance', '1.18.0' + gem 'rubocop-rspec' end group :debug do - gem "pry" - gem "pry-byebug" - gem "pry-stack_explorer" - gem "rb-readline" + gem 'pry' + gem 'pry-byebug' + gem 'pry-stack_explorer' + gem 'rb-readline' end diff --git a/habitat/plan.ps1 b/habitat/plan.ps1 index a93c74a47..d2d415129 100644 --- a/habitat/plan.ps1 +++ b/habitat/plan.ps1 @@ -45,8 +45,8 @@ function Invoke-Build { bundle install gem build ohai.gemspec - Write-BuildLine " ** Using gem to install" - gem install ohai-*.gem --no-document + Write-BuildLine " ** Using gem to install" + gem install ohai-*.gem --no-document If ($lastexitcode -ne 0) { Exit $lastexitcode } diff --git a/habitat/plan.sh b/habitat/plan.sh index eb96a8730..e01a51fbb 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -1,62 +1,121 @@ -pkg_name=ohai -pkg_origin=chef -ruby_pkg="core/ruby31" -pkg_deps=(${ruby_pkg} core/coreutils) +_chef_client_ruby="core/ruby31" +pkg_name="ohai" +pkg_origin="chef" +pkg_maintainer="The Chef Maintainers " +pkg_description="The Chef Ohai" +pkg_license=('Apache-2.0') +pkg_bin_dirs=( + bin + vendor/bin +) pkg_build_deps=( - core/make - core/sed - core/gcc - ) -pkg_bin_dirs=(bin) -do_setup_environment() { - build_line 'Setting GEM_HOME="$pkg_prefix/lib"' - export GEM_HOME="$pkg_prefix/lib" - - build_line "Setting GEM_PATH=$GEM_HOME" - export GEM_PATH="$GEM_HOME" -} + core/make + core/gcc + core/git +) +pkg_deps=( + $_chef_client_ruby + core/coreutils +) +pkg_svc_user=root pkg_version() { - cat "$SRC_PATH/VERSION" + cat "${SRC_PATH}/VERSION" } + do_before() { + do_default_before update_pkg_version + # We must wait until we update the pkg_version to use the pkg_version + pkg_filename="${pkg_name}-${pkg_version}.tar.gz" +} + +do_download() { + build_line "Locally creating archive of latest repository commit at ${HAB_CACHE_SRC_PATH}/${pkg_filename}" + # source is in this repo, so we're going to create an archive from the + # appropriate path within the repo and place the generated tarball in the + # location expected by do_unpack + ( cd "${SRC_PATH}" || exit_with "unable to enter hab-src directory" 1 + git archive --prefix="${pkg_name}-${pkg_version}/" --output="${HAB_CACHE_SRC_PATH}/${pkg_filename}" HEAD + ) } -do_unpack() { - mkdir -pv "$HAB_CACHE_SRC_PATH/$pkg_dirname" - cp -RT "$PLAN_CONTEXT"/.. "$HAB_CACHE_SRC_PATH/$pkg_dirname/" + +do_verify() { + build_line "Skipping checksum verification on the archive we just created." + return 0 } + +do_setup_environment() { + push_runtime_env GEM_PATH "${pkg_prefix}/vendor" + + set_runtime_env APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH + set_runtime_env LANG "en_US.UTF-8" + set_runtime_env LC_CTYPE "en_US.UTF-8" +} + +do_prepare() { + export GEM_HOME="${pkg_prefix}/vendor" + export CPPFLAGS="${CPPFLAGS} ${CFLAGS}" + + ( cd "$CACHE_PATH" + bundle config --local jobs "$(nproc)" + bundle config --local without server docgen maintenance pry travis integration ci + bundle config --local shebang "$(pkg_path_for "$_chef_client_ruby")/bin/ruby" + bundle config --local retry 5 + bundle config --local silence_root_warning 1 + ) + + build_line "Setting link for /usr/bin/env to 'coreutils'" + if [ ! -f /usr/bin/env ]; then + ln -s "$(pkg_interpreter_for core/coreutils bin/env)" /usr/bin/env + fi +} + do_build() { - pushd "$HAB_CACHE_SRC_PATH/$pkg_dirname/" - gem build ohai.gemspec - popd + ( cd "$CACHE_PATH" || exit_with "unable to enter hab-cache directory" 1 + build_line "Installing gem dependencies ..." + bundle install --jobs=3 --retry=3 + build_line "Installing gems from git repos properly ..." + ruby ./post-bundle-install.rb + build_line "Installing this project's gems ..." + bundle exec rake install:local + ) } + do_install() { - pushd "$HAB_CACHE_SRC_PATH/$pkg_dirname/" - gem install ohai-*.gem --no-document - popd - wrap_ruby_bin + ( cd "$pkg_prefix" || exit_with "unable to enter pkg prefix directory" 1 + export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile" + build_line "** fixing binstub shebangs" + fix_interpreter "${pkg_prefix}/vendor/bin/*" "$_chef_client_ruby" bin/ruby + export BUNDLE_GEMFILE="${CACHE_PATH}/Gemfile" + for gem in ohai; do + build_line "** generating binstubs for $gem with precise version pins" + appbundler $CACHE_PATH $pkg_prefix/bin $gem + done + ) } -wrap_ruby_bin() { - local bin="$pkg_prefix/bin/$pkg_name" - local real_bin="$GEM_HOME/gems/ohai-${pkg_version}/bin/ohai" - build_line "Adding wrapper $bin to $real_bin" - cat < "$bin" -#!$(pkg_path_for core/bash)/bin/bash -set -e -# Set binary path that allows ohai to use non-Hab pkg binaries -export PATH="/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:\$PATH" +do_after() { + build_line "Trimming the fat ..." -# Set Ruby paths defined from 'do_setup_environment()' -export GEM_HOME="$GEM_HOME" -export GEM_PATH="$GEM_PATH" - -exec $(pkg_path_for core/ruby31)/bin/ruby $real_bin \$@ -EOF - chmod -v 755 "$bin" + # We don't need the cache of downloaded .gem files ... + rm -r "$pkg_prefix/vendor/cache" + # ... or bundler's cache of git-ref'd gems + rm -r "$pkg_prefix/vendor/bundler" + # We don't need the gem docs. + rm -r "$pkg_prefix/vendor/doc" + # We don't need to ship the test suites for every gem dependency, + # only Chef's for package verification. + find "$pkg_prefix/vendor/gems" -name spec -type d | grep -v "ohai-${pkg_version}" \ + | while read spec_dir; do rm -r "$spec_dir"; done } +do_end() { + if [ "$(readlink /usr/bin/env)" = "$(pkg_interpreter_for core/coreutils bin/env)" ]; then + build_line "Removing the symlink we created for '/usr/bin/env'" + rm /usr/bin/env + fi +} do_strip() { return 0 diff --git a/post-bundle-install.rb b/post-bundle-install.rb new file mode 100644 index 000000000..85451e912 --- /dev/null +++ b/post-bundle-install.rb @@ -0,0 +1,32 @@ +#!/usr/bin/env ruby + +gem_home = Gem.paths.home + +puts "fixing bundle installed gems in #{gem_home}" + +# Install gems from git repos. This makes the assumption that there is a .gemspec and +# you can simply gem build + gem install the resulting gem, so nothing fancy. This does not use +# rake install since we need --conservative --minimal-deps in order to not install duplicate gems. +# +# +puts "gem path #{gem_home}" + +Dir["#{gem_home}/bundler/gems/*"].each do |gempath| + matches = File.basename(gempath).match(/.*-[A-Fa-f0-9]{12}/) + next unless matches + + %w[chef-utils chef-config].each do |gem_need_install| + dir_path = "#{gempath}/#{gem_need_install}" + gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, '.gemspec') + # FIXME: should strip any valid ruby platform off of the gem_name if it matches + + next unless gem_name + + puts "re-installing #{gem_name}..." + + Dir.chdir(dir_path) do + system("gem build #{gem_name}.gemspec") or raise 'gem build failed' + system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise 'gem install failed' + end + end +end From 192df7f418354c297669325d5a2409d731f5a17c Mon Sep 17 00:00:00 2001 From: nitin sanghi Date: Wed, 25 Sep 2024 13:32:35 +0530 Subject: [PATCH 3/5] Gemfile change on lint issue Signed-off-by: nitin sanghi --- Gemfile | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Gemfile b/Gemfile index 39c75defd..84d289a15 100644 --- a/Gemfile +++ b/Gemfile @@ -1,29 +1,29 @@ # frozen_string_literal: true -source 'https://rubygems.org' +source "https://rubygems.org" gemspec -# pull these gems from main of chef/chef so that we're testing against what we will release -gem 'appbundler' -gem 'chef-config', git: 'https://github.com/chef/chef', branch: 'main', glob: 'chef-config/chef-config.gemspec' -gem 'chef-utils', git: 'https://github.com/chef/chef', branch: 'main', glob: 'chef-utils/chef-utils.gemspec' +# pull these gems from main of chef/chef so that we"re testing against what we will release +gem "appbundler" +gem "chef-config", git: "https://github.com/chef/chef", branch: "main", glob: "chef-config/chef-config.gemspec" +gem "chef-utils", git: "https://github.com/chef/chef", branch: "main", glob: "chef-utils/chef-utils.gemspec" # NOTE: do not submit PRs to add pry as a dep, add to your Gemfile.local group :development do - gem 'cookstyle', '>= 7.32.8' - gem 'ipaddr_extensions' - gem 'rake', '>= 10.1.0' - gem 'rspec-collection_matchers', '~> 1.0' - gem 'rspec-core', '~> 3.0' - gem 'rspec-expectations', '~> 3.0' - gem 'rspec-mocks', '~> 3.0' - gem 'rubocop-performance', '1.18.0' - gem 'rubocop-rspec' + gem "cookstyle", ">= 7.32.8" + gem "ipaddr_extensions" + gem "rake", ">= 10.1.0" + gem "rspec-collection_matchers", "~> 1.0" + gem "rspec-core", "~> 3.0" + gem "rspec-expectations", "~> 3.0" + gem "rspec-mocks", "~> 3.0" + gem "rubocop-performance", "1.18.0" + gem "rubocop-rspec" end group :debug do - gem 'pry' - gem 'pry-byebug' - gem 'pry-stack_explorer' - gem 'rb-readline' + gem "pry" + gem "pry-byebug" + gem "pry-stack_explorer" + gem "rb-readline" end From 788b0d58819645634edc05af25e1fe43656142a9 Mon Sep 17 00:00:00 2001 From: nitin sanghi Date: Thu, 26 Sep 2024 17:31:41 +0530 Subject: [PATCH 4/5] Remove bybug file Signed-off-by: nitin sanghi --- .byebug_history | 1 - post-bundle-install.rb | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 .byebug_history diff --git a/.byebug_history b/.byebug_history deleted file mode 100644 index 5b6985884..000000000 --- a/.byebug_history +++ /dev/null @@ -1 +0,0 @@ -next diff --git a/post-bundle-install.rb b/post-bundle-install.rb index 85451e912..d35308c70 100644 --- a/post-bundle-install.rb +++ b/post-bundle-install.rb @@ -17,7 +17,7 @@ %w[chef-utils chef-config].each do |gem_need_install| dir_path = "#{gempath}/#{gem_need_install}" - gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, '.gemspec') + gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, ".gemspec") # FIXME: should strip any valid ruby platform off of the gem_name if it matches next unless gem_name @@ -25,8 +25,8 @@ puts "re-installing #{gem_name}..." Dir.chdir(dir_path) do - system("gem build #{gem_name}.gemspec") or raise 'gem build failed' - system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise 'gem install failed' + system("gem build #{gem_name}.gemspec") or raise "gem build failed" + system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise "gem install failed" end end end From 30ed8625b9f4f883fa68220bad8349c0838c07e8 Mon Sep 17 00:00:00 2001 From: nikhil2611 Date: Thu, 17 Oct 2024 12:57:35 +0530 Subject: [PATCH 5/5] fixinf chefstyle changes Signed-off-by: nikhil2611 --- post-bundle-install.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/post-bundle-install.rb b/post-bundle-install.rb index d35308c70..f20728b8a 100644 --- a/post-bundle-install.rb +++ b/post-bundle-install.rb @@ -15,9 +15,9 @@ matches = File.basename(gempath).match(/.*-[A-Fa-f0-9]{12}/) next unless matches - %w[chef-utils chef-config].each do |gem_need_install| + %w{chef-utils chef-config}.each do |gem_need_install| dir_path = "#{gempath}/#{gem_need_install}" - gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, ".gemspec") + gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, ".gemspec") # FIXME: should strip any valid ruby platform off of the gem_name if it matches next unless gem_name @@ -25,8 +25,8 @@ puts "re-installing #{gem_name}..." Dir.chdir(dir_path) do - system("gem build #{gem_name}.gemspec") or raise "gem build failed" - system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise "gem install failed" + system("gem build #{gem_name}.gemspec") or raise "gem build failed" + system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise "gem install failed" end end end