diff --git a/.expeditor/build.habitat.yml b/.expeditor/build.habitat.yml new file mode 100644 index 000000000..72ea2567a --- /dev/null +++ b/.expeditor/build.habitat.yml @@ -0,0 +1,9 @@ +--- +origin: chef + +expeditor: + defaults: + buildkite: + retry: + automatic: + limit: 1 diff --git a/.expeditor/buildkite/artifact.habitat.test.ps1 b/.expeditor/buildkite/artifact.habitat.test.ps1 new file mode 100755 index 000000000..61e95d222 --- /dev/null +++ b/.expeditor/buildkite/artifact.habitat.test.ps1 @@ -0,0 +1,86 @@ +#!/usr/bin/env powershell + +#Requires -Version 5 +# https://stackoverflow.com/questions/9948517 +# TODO: Set-StrictMode -Version Latest +$PSDefaultParameterValues['*:ErrorAction']='Stop' +$ErrorActionPreference = 'Stop' +$env:HAB_BLDR_CHANNEL = "LTS-2024" +$env:HAB_ORIGIN = 'ci' +$env:CHEF_LICENSE = 'accept-no-persist' +$env:HAB_LICENSE = 'accept-no-persist' +$Plan = 'ohai' + +Write-Host "--- system details" +$Properties = 'Caption', 'CSName', 'Version', 'BuildType', 'OSArchitecture' +Get-CimInstance Win32_OperatingSystem | Select-Object $Properties | Format-Table -AutoSize + +Write-Host "--- Installing the version of Habitat required" + +function Stop-HabProcess { + $habProcess = Get-Process hab -ErrorAction SilentlyContinue + if ($habProcess) { + Write-Host "Stopping hab process..." + Stop-Process -Name hab -Force + } +} + +# Installing Habitat +function Install-Habitat { + Write-Host "Downloading and installing Habitat..." + Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.ps1')) +} + +try { + hab --version +} +catch { + Set-ExecutionPolicy Bypass -Scope Process -Force + + Stop-HabProcess + + # Remove the existing hab.exe if it exists and if you have permissions + $habPath = "C:\ProgramData\Habitat\hab.exe" + if (Test-Path $habPath) { + Write-Host "Attempting to remove existing hab.exe..." + Remove-Item $habPath -Force -ErrorAction SilentlyContinue + if (Test-Path $habPath) { + Write-Host "Failed to remove hab.exe, re-running script with elevated permissions." + Start-Process powershell -Verb runAs -ArgumentList "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`"" + exit + } + } + + Install-Habitat +} +finally { + Write-Host ":habicat: I think I have the version I need to build." +} + + +Write-Host "--- Generating fake origin key" +hab origin key generate $env:HAB_ORIGIN + +Write-Host "--- Building $Plan" +$project_root = "$(git rev-parse --show-toplevel)" +Set-Location $project_root + +$env:DO_CHECK=$true; hab pkg build . + +. $project_root/results/last_build.ps1 + +Write-Host "--- Installing $pkg_ident/$pkg_artifact" +hab pkg install -b $project_root/results/$pkg_artifact + +Write-Host "+++ Testing $Plan" + +Push-Location $project_root + +try { + Write-Host "Running unit tests..." + habitat/tests/test.ps1 $pkg_ident +} +finally { + # Ensure we always return to the original directory + Pop-Location +} \ No newline at end of file diff --git a/.expeditor/buildkite/artifact.habitat.test.sh b/.expeditor/buildkite/artifact.habitat.test.sh new file mode 100755 index 000000000..021c715f3 --- /dev/null +++ b/.expeditor/buildkite/artifact.habitat.test.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -eo pipefail + +export HAB_ORIGIN='ci' +export PLAN='ohai' +export CHEF_LICENSE="accept-no-persist" +export HAB_LICENSE="accept-no-persist" +export HAB_BLDR_CHANNEL="LTS-2024" + +echo "--- checking if git is installed" +if ! command -v git &> /dev/null; then + echo "Git is not installed. Installing Git..." + sudo yum install -y git +else + echo "Git is already installed." + git --version +fi + +echo "--- add an exception for this directory since detected dubious ownership in repository at /workdir" +git config --global --add safe.directory /workdir + +echo "--- git status for this workdir" +git status + +echo "--- ruby version" +ruby -v + +export project_root="$(git rev-parse --show-toplevel)" +echo "The value for project_root is: $project_root" + +export HAB_NONINTERACTIVE=true +export HAB_NOCOLORING=true +export HAB_STUDIO_SECRET_HAB_NONINTERACTIVE=true + +echo "--- system details" +uname -a + +echo "--- Installing Habitat" +id -a +curl https://raw.githubusercontent.com/habitat-sh/habitat/main/components/hab/install.sh | bash + +echo "--- :key: Generating fake origin key" +hab origin key generate "$HAB_ORIGIN" + + +echo "--- Building $PLAN" +cd "$project_root" +DO_CHECK=true hab pkg build . + +echo "--- Sourcing 'results/last_build.sh'" +if [ -f ./results/last_build.env ]; then + cat ./results/last_build.env + . ./results/last_build.env + export pkg_artifact +fi +echo "+++ Installing ${pkg_ident:?is undefined}" +echo "++++" +echo $project_root +echo "+++" +hab pkg install -b "${project_root:?is undefined}/results/${pkg_artifact:?is undefined}" + +echo "+++ Testing $PLAN" + +PATH="$(hab pkg path ci/ohai)/bin:$PATH" +export PATH +echo "PATH is $PATH" + +echo "--- :mag_right: Testing $PLAN" +${project_root}/habitat/tests/test.sh "$pkg_ident" || error 'failures during test of executables' \ No newline at end of file diff --git a/.expeditor/config.yml b/.expeditor/config.yml index cebe00737..c6db7abfb 100644 --- a/.expeditor/config.yml +++ b/.expeditor/config.yml @@ -5,9 +5,12 @@ slack: notify_channel: chef-infra-notify + # This publish is triggered by the `built_in:publish_rubygems` artifact_action. rubygems: - ohai +artifact_channels: + - dev github: # This deletes the GitHub PR branch after successfully merged into the release branch @@ -21,6 +24,7 @@ github: major_bump_labels: - "Expeditor: Bump Version Major" + # Which Github branches to build Omnibus releases from, and what versions # (as determined by the value in the VERSION file) those branches are responsible # for building. @@ -55,8 +59,23 @@ subscriptions: - "Expeditor: Skip All" - built_in:build_gem: only_if: built_in:bump_version + - trigger_pipeline:habitat/publish: + only_if: built_in:bump_version + ignore_labels: + - "Expeditor: Skip Habitat" + - "Expeditor: Skip All" + - trigger_pipeline:habitat/build-test: + only_if: built_in:bump_version + ignore_labels: + - "Expeditor: Skip Habitat" + - "Expeditor: Skip All" - workload: project_promoted:{{agent_id}}:* actions: - built_in:rollover_changelog - built_in:publish_rubygems + - workload: buildkite_hab_build_group_published:{{agent_id}}:* + actions: + # when all of the hab package publish to the unstable channel, test and promote them + - trigger_pipeline:habitat/build-test + - built_in:promote_habitat_packages diff --git a/.expeditor/habitat-test.pipeline.yml b/.expeditor/habitat-test.pipeline.yml new file mode 100644 index 000000000..12a3d1922 --- /dev/null +++ b/.expeditor/habitat-test.pipeline.yml @@ -0,0 +1,36 @@ +--- +expeditor: + defaults: + buildkite: + timeout_in_minutes: 30 + retry: + automatic: + limit: 1 + + +steps: + + - label: ":linux: Validate Habitat Builds of ohai" + commands: + - .expeditor/buildkite/artifact.habitat.test.sh + expeditor: + executor: + docker: + image: ruby:3.1 + privileged: true + + - label: ":windows: Validate Habitat Builds of ohai" + commands: + - .expeditor/buildkite/artifact.habitat.test.ps1 + expeditor: + executor: + docker: + host_os: windows + shell: ["powershell", "-Command"] + image: rubydistros/windows-2019:3.1 + user: 'NT AUTHORITY\SYSTEM' + environment: + - FORCE_FFI_YAJL=ext + - EXPIRE_CACHE=true + - CHEF_LICENSE=accept-no-persist + - CHEF_LICENSE_SERVER=http://hosted-license-service-lb-8000-606952349.us-west-2.elb.amazonaws.com:8000/ diff --git a/Gemfile b/Gemfile index 31584eccb..84d289a15 100644 --- a/Gemfile +++ b/Gemfile @@ -1,12 +1,13 @@ # frozen_string_literal: true + source "https://rubygems.org" gemspec -# pull these gems from main of chef/chef so that we're testing against what we will release +# pull these gems from main of chef/chef so that we"re testing against what we will release +gem "appbundler" gem "chef-config", git: "https://github.com/chef/chef", branch: "main", glob: "chef-config/chef-config.gemspec" gem "chef-utils", git: "https://github.com/chef/chef", branch: "main", glob: "chef-utils/chef-utils.gemspec" - # NOTE: do not submit PRs to add pry as a dep, add to your Gemfile.local group :development do gem "cookstyle", ">= 7.32.8" diff --git a/habitat/plan.ps1 b/habitat/plan.ps1 new file mode 100644 index 000000000..d2d415129 --- /dev/null +++ b/habitat/plan.ps1 @@ -0,0 +1,92 @@ +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction']='Stop' + +$pkg_name="ohai" +$pkg_origin="chef" +$pkg_version=$(Get-Content "$PLAN_CONTEXT/../VERSION") +$pkg_maintainer="The Chef Maintainers " + +$pkg_deps=@( + "chef/ruby31-plus-devkit" + "core/git" +) +$pkg_bin_dirs=@("bin" + "vendor/bin") +$project_root= (Resolve-Path "$PLAN_CONTEXT/../").Path + +function pkg_version { + Get-Content "$SRC_PATH/VERSION" +} + +function Invoke-Before { + Set-PkgVersion +} +function Invoke-SetupEnvironment { + Push-RuntimeEnv -IsPath GEM_PATH "$pkg_prefix/vendor" + + Set-RuntimeEnv APPBUNDLER_ALLOW_RVM "true" # prevent appbundler from clearing out the carefully constructed runtime GEM_PATH + Set-RuntimeEnv FORCE_FFI_YAJL "ext" + Set-RuntimeEnv LANG "en_US.UTF-8" + Set-RuntimeEnv LC_CTYPE "en_US.UTF-8" +} + +function Invoke-Build { + try { + $env:Path += ";c:\\Program Files\\Git\\bin" + Push-Location $project_root + $env:GEM_HOME = "$HAB_CACHE_SRC_PATH/$pkg_dirname/vendor" + + Write-BuildLine " ** Configuring bundler for this build environment" + bundle config --local without integration deploy maintenance + bundle config --local jobs 4 + bundle config --local retry 5 + bundle config --local silence_root_warning 1 + Write-BuildLine " ** Using bundler to retrieve the Ruby dependencies" + bundle install + + gem build ohai.gemspec + Write-BuildLine " ** Using gem to install" + gem install ohai-*.gem --no-document + + + If ($lastexitcode -ne 0) { Exit $lastexitcode } + } finally { + Pop-Location + } +} + +function Invoke-Install { + Write-BuildLine "** Copy built & cached gems to install directory" + Copy-Item -Path "$HAB_CACHE_SRC_PATH/$pkg_dirname/*" -Destination $pkg_prefix -Recurse -Force -Exclude @("gem_make.out", "mkmf.log", "Makefile", + "*/latest", "latest", + "*/JSON-Schema-Test-Suite", "JSON-Schema-Test-Suite") + + try { + Push-Location $pkg_prefix + bundle config --local gemfile $project_root/Gemfile + Write-BuildLine "** generating binstubs for ohai with precise version pins" + Write-BuildLine "** generating binstubs for ohai with precise version pins $project_root $pkg_prefix/bin " + Invoke-Expression -Command "appbundler.bat $project_root $pkg_prefix/bin ohai" + If ($lastexitcode -ne 0) { Exit $lastexitcode } + Write-BuildLine " ** Running the ohai project's 'rake install' to install the path-based gems so they look like any other installed gem." + + If ($lastexitcode -ne 0) { Exit $lastexitcode } + } finally { + Pop-Location + } +} + +function Invoke-After { + # We don't need the cache of downloaded .gem files ... + Remove-Item $pkg_prefix/vendor/cache -Recurse -Force + # We don't need the gem docs. + Remove-Item $pkg_prefix/vendor/doc -Recurse -Force + # We don't need to ship the test suites for every gem dependency, + # only inspec's for package verification. + Get-ChildItem $pkg_prefix/vendor/gems -Filter "spec" -Directory -Recurse -Depth 1 ` + | Where-Object -FilterScript { $_.FullName -notlike "*ohai*" } ` + | Remove-Item -Recurse -Force + # Remove the byproducts of compiling gems with extensions + Get-ChildItem $pkg_prefix/vendor/gems -Include @("gem_make.out", "mkmf.log", "Makefile") -File -Recurse ` + | Remove-Item -Force +} \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh new file mode 100644 index 000000000..d184e3bc7 --- /dev/null +++ b/habitat/plan.sh @@ -0,0 +1,84 @@ +export HAB_BLDR_CHANNEL="LTS-2024" +ruby_pkg="core/ruby3_1" +pkg_name="ohai" +pkg_origin="chef" +pkg_maintainer="The Chef Maintainers " +pkg_description="The Chef Ohai" +pkg_license=('Apache-2.0') +pkg_bin_dirs=( + bin +) +pkg_build_deps=( + core/make + core/gcc + core/git +) +pkg_deps=(${ruby_pkg} core/coreutils) + +pkg_svc_user=root + +do_setup_environment() { + build_line 'Setting GEM_HOME="$pkg_prefix/vendor"' + export GEM_HOME="$pkg_prefix/vendor" + + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" +} + +pkg_version() { + cat "$SRC_PATH/VERSION" +} +do_before() { + update_pkg_version +} +do_unpack() { + mkdir -pv "$HAB_CACHE_SRC_PATH/$pkg_dirname" + cp -RT "$PLAN_CONTEXT"/.. "$HAB_CACHE_SRC_PATH/$pkg_dirname/" +} +do_build() { + + export GEM_HOME="$pkg_prefix/vendor" + + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" + bundle config --local without integration deploy maintenance + bundle config --local jobs 4 + bundle config --local retry 5 + bundle config --local silence_root_warning 1 + bundle install + ruby ./post-bundle-install.rb + gem build ohai.gemspec +} +do_install() { + export GEM_HOME="$pkg_prefix/vendor" + + build_line "Setting GEM_PATH=$GEM_HOME" + export GEM_PATH="$GEM_HOME" + gem install ohai-*.gem --no-document + set_runtime_env "GEM_PATH" "${pkg_prefix}/vendor" + wrap_ruby_bin +} +wrap_ruby_bin() { + local bin="$pkg_prefix/bin/$pkg_name" + local real_bin="$GEM_HOME/gems/ohai-${pkg_version}/bin/ohai" + build_line "Adding wrapper $bin to $real_bin" + cat < "$bin" +#!$(pkg_path_for core/bash)/bin/bash +set -e + +# Set binary path that allows Ohai to use non-Hab pkg binaries +export PATH="/sbin:/usr/sbin:/usr/local/sbin:/usr/local/bin:/usr/bin:/bin:\$PATH" + +# Set Ruby paths defined from 'do_setup_environment()' + export GEM_HOME="$pkg_prefix/vendor" + export GEM_PATH="$GEM_PATH" + +exec $(pkg_path_for ${ruby_pkg})/bin/ruby $real_bin \$@ +EOF + chmod -v 755 "$bin" +} + + +do_strip() { + return 0 +} \ No newline at end of file diff --git a/habitat/tests/test.ps1 b/habitat/tests/test.ps1 new file mode 100644 index 000000000..cce3cd444 --- /dev/null +++ b/habitat/tests/test.ps1 @@ -0,0 +1,21 @@ +param ( + [Parameter()] + [string]$PackageIdentifier = $(throw "Usage: test.ps1 [test_pkg_ident] e.g. test.ps1 ci/user-windows/1.0.0/20190812103929") +) + + +Write-Host "--- :fire: Smokish test" +# Pester the Package +$version=hab pkg exec "${pkg_ident}" ohai -v +$actual_version=[Regex]::Match($version,"([0-9]+.[0-9]+.[0-9]+)").Value +$package_version=$PackageIdentifier.split("/",4)[2] + +Write-Host "package_version $package_version actual version $actual_version" +if ($package_version -eq $actual_version) +{ + Write "ohai working fine" +} +else { + Write-Error "ohai version not met expected $package_version actual version $actual_version " + throw "ohai windows pipeline not working for hab pkg" +} \ No newline at end of file diff --git a/habitat/tests/test.sh b/habitat/tests/test.sh new file mode 100755 index 000000000..ed2519714 --- /dev/null +++ b/habitat/tests/test.sh @@ -0,0 +1,27 @@ + +set -euo pipefail + + +project_root="$(git rev-parse --show-toplevel)" +pkg_ident="$1" +# print error message followed by usage and exit +error () { + local message="$1" + + echo -e "\nERROR: ${message}\n" >&2 + + exit 1 +} + +[[ -n "$pkg_ident" ]] || error 'no hab package identity provided' + +package_version=$(awk -F / '{print $3}' <<<"$pkg_ident") + +cd "${project_root}" + +echo "--- :mag_right: Testing ${pkg_ident} executables" +actual_version=$(hab pkg exec "${pkg_ident}" ohai -v | sed -E 's/.*: ([0-9]+\.[0-9]+\.[0-9]+).*/\1/') +[[ "$package_version" = "$actual_version" ]] || error "ohai version is not the expected version. Expected '$package_version', got '$actual_version'" + + + diff --git a/post-bundle-install.rb b/post-bundle-install.rb new file mode 100644 index 000000000..f20728b8a --- /dev/null +++ b/post-bundle-install.rb @@ -0,0 +1,32 @@ +#!/usr/bin/env ruby + +gem_home = Gem.paths.home + +puts "fixing bundle installed gems in #{gem_home}" + +# Install gems from git repos. This makes the assumption that there is a .gemspec and +# you can simply gem build + gem install the resulting gem, so nothing fancy. This does not use +# rake install since we need --conservative --minimal-deps in order to not install duplicate gems. +# +# +puts "gem path #{gem_home}" + +Dir["#{gem_home}/bundler/gems/*"].each do |gempath| + matches = File.basename(gempath).match(/.*-[A-Fa-f0-9]{12}/) + next unless matches + + %w{chef-utils chef-config}.each do |gem_need_install| + dir_path = "#{gempath}/#{gem_need_install}" + gem_name = File.basename(Dir["#{dir_path}/*.gemspec"].first, ".gemspec") + # FIXME: should strip any valid ruby platform off of the gem_name if it matches + + next unless gem_name + + puts "re-installing #{gem_name}..." + + Dir.chdir(dir_path) do + system("gem build #{gem_name}.gemspec") or raise "gem build failed" + system("gem install #{gem_name}*.gem --conservative --minimal-deps --no-document") or raise "gem install failed" + end + end +end