Merge pull request #2884 from justinstoller/fix-pdb-integration-accep…

…tance-test

(maint) Fix pdb integration acceptance test

Gemfile

@@ -18,11 +18,11 @@ gem 'rake', :group => [:development, :test]
 
 group :test do
   gem 'rspec'
-  gem 'beaker', *location_for(ENV['BEAKER_VERSION'] || '~> 4.11')
+  gem 'beaker', *location_for(ENV['BEAKER_VERSION'] || '~> 6.0')
   gem "beaker-hostgenerator", *location_for(ENV['BEAKER_HOSTGENERATOR_VERSION'] || "~> 2.4")
-  gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || "~> 0.1")
+  gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || "~> 1.0")
   gem "beaker-vmpooler", *location_for(ENV['BEAKER_VMPOOLER_VERSION'] || "~> 1.3")
-  gem "beaker-puppet", *location_for(ENV['BEAKER_PUPPET_VERSION'] || ["~> 1.0", ">= 1.0.1"])
+  gem "beaker-puppet", *location_for(ENV['BEAKER_PUPPET_VERSION'] || "~> 4.0")
   gem 'uuidtools'
   gem 'httparty'
   gem 'master_manipulator'

acceptance/lib/helper.rb

@@ -63,13 +63,14 @@ def self.print_config
   def puppetdb_supported_platforms()
     [
       /debian-11/,
-      /debian-10/,
-      /el-7/,
+      /debian-12/,
       /el-8/,
+      /el-9/,
       /sles-12/,
       /sles-15/,
-      /ubuntu-18.04/,
-      /ubuntu-20.04/
+      /ubuntu-20.04/,
+      /ubuntu-22.04/,
+      /ubuntu-24.04/
     ]
   end
 
@@ -231,8 +232,7 @@ def get_defaults_file
   def get_defaults_var(host, varname)
     defaults_file = get_defaults_file
 
-    on(host, "source #{defaults_file}; echo -n $#{varname}")
-    stdout
+    on(host, "source #{defaults_file}; echo -n $#{varname}").stdout
   end
 
   # If we are getting the certificate for the first time, store it in the

acceptance/lib/puppetserver/acceptance/gem_utils.rb

@@ -7,8 +7,8 @@ def get_gem_list(host, gem_list_command)
   #
   gem_list_regex = Regexp.new('(?<package>[\w-]*) (?<version>.*)')
   array = []
-  on(host, "#{gem_list_command}") do
-    split_output = stdout.split
+  on(host, "#{gem_list_command}") do |result|
+    split_output = result.stdout.split
     split_output.each do |line|
       match = gem_list_regex.match(line)
       if match

acceptance/scripts/generic/testrun.sh

@@ -15,7 +15,6 @@ do_init()
   BEAKER_INIT="$BEAKER_INIT --keyfile $BEAKER_KEYFILE"
   BEAKER_INIT="$BEAKER_INIT --helper $BEAKER_HELPER"
   BEAKER_INIT="$BEAKER_INIT --options-file $BEAKER_OPTIONS"
-  BEAKER_INIT="$BEAKER_INIT --post-suite $BEAKER_POSTSUITE"
   BEAKER_INIT="$BEAKER_INIT --load-path $BEAKER_LOADPATH"
   BEAKER_INIT="$BEAKER_INIT --hosts $BEAKER_CONFIG"
   BEAKER_INIT="$BEAKER_INIT --pre-suite $BEAKER_PRESUITE"
@@ -39,10 +38,9 @@ do_init()
 set -x
 
 export GEM_SOURCE="https://artifactory.delivery.puppetlabs.net/artifactory/api/gems/rubygems/"
-export GENCONFIG_LAYOUT="${GENCONFIG_LAYOUT:-redhat8-64ma-debian11-64a}"
+export GENCONFIG_LAYOUT="${GENCONFIG_LAYOUT:-redhat9-64ma-debian12-64a}"
 export BEAKER_TESTSUITE="${BEAKER_TESTSUITE:-acceptance/suites/tests}"
 export BEAKER_PRESUITE="${BEAKER_PRESUITE:-acceptance/suites/pre_suite/foss}"
-export BEAKER_POSTSUITE="${BEAKER_POSTSUITE:-acceptance/suites/post_suite}"
 export BEAKER_OPTIONS="${BEAKER_OPTIONS:-acceptance/config/beaker/options.rb}"
 export BEAKER_CONFIG="${BEAKER_CONFIG:-acceptance/scripts/hosts.cfg}"
 export BEAKER_KEYFILE="${BEAKER_KEYFILE:-~/.ssh/id_rsa-acceptance}"

acceptance/suites/pre_suite/foss/95_install_pdb.rb

@@ -10,38 +10,19 @@
   on(master, "rm -f #{sitepp}")
 end
 
-# Puppet pulls in OpenSSL 3 which breaks ssl-cert < 1.1.1
-# Unfortunately we need jammy to bring a workable version of ssl-cert into bionic
-step 'Update Ubuntu 18 package repo' do
-  if master.platform =~ /ubuntu-18/
-    # There's a bunch of random crap that gets upgraded in our installs,
-    # just upgrade everything before we try to install postgres
-    on master, 'apt-get update'
-    on master, 'DEBIAN_FRONTEND=noninteractive apt-get upgrade --assume-yes --force-yes -o "DPkg::Options::=--force-confold"'
-    # Install jammy repos so we can pull in its ssl-cert
-    on master, "echo 'deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe multiverse' > /etc/apt/sources.list.d/jammy.list"
-    on master, "echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted universe multiverse' >> /etc/apt/sources.list.d/jammy.list"
-    on master, 'apt-get update'
-    on master, 'apt-get install -y -t jammy ssl-cert'
-    # Once we have jammy's ssl-cert get rid of jammy packages to avoid unintentially pulling in other packages
-    on master, 'rm /etc/apt/sources.list.d/jammy.list'
-    on master, 'apt-get update'
-
-    # bionic is EOL, so get postgresql from the archive
-    on master, 'echo "deb https://apt-archive.postgresql.org/pub/repos/apt bionic-pgdg main" >> /etc/apt/sources.list'
-    on master, 'curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -'
-    on master, 'apt update'
-  end
-end
-
 step 'Install Puppet nightly repo' do
   install_puppetlabs_release_repo_on(master, 'puppet8-nightly')
 end
 
-step 'Update EL 8 postgresql repos' do
-  if master.platform =~ /el-8/
-    # work around for testing on rhel8 and the repos on the image not finding the pg packages it needs
-    on master, "dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm"
+step 'Update EL postgresql repos' do
+  # work around for testing on rhel and the repos on the image not finding the pg packages it needs
+  if master.platform =~ /el-/
+    major_version = case master.platform
+      when /-8/ then 8
+      when /-9/ then 9
+      end
+
+    on master, "dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-#{major_version}-x86_64/pgdg-redhat-repo-latest.noarch.rpm"
     on master, "dnf -qy module disable postgresql"
   end
 end
@@ -55,12 +36,11 @@
 end
 
 step 'Configure PuppetDB via site.pp' do
-  manage_package_repo = ! master.platform.match?(/ubuntu-18/)
   create_remote_file(master, sitepp, <<SITEPP)
 node default {
   class { 'puppetdb':
     manage_firewall     => false,
-    manage_package_repo => #{manage_package_repo},
+    manage_package_repo => true,
     postgres_version    => '14',
   }
 

acceptance/suites/puppet3_tests/puppet3_version_test.rb

@@ -4,13 +4,13 @@
 legacy_agents = agents.reject { |agent| agent == master }
 
 step "Check that legacy agents have Puppet 3.x installed"
-on(legacy_agents, puppet("--version")) do
-  assert(stdout.start_with? "3.", "puppet --version does not start with major version 3.")
+on(legacy_agents, puppet("--version")) do |result|
+  assert(result.stdout.start_with? "3.", "puppet --version does not start with major version 3.")
 end
 
 step "Check that Puppet Server has Puppet 6.x installed"
-on(master, puppet("--version")) do
-  assert_match(/\A6/, stdout, "puppet --version does not start with major version 6.x")
+on(master, puppet("--version")) do |result|
+  assert_match(/\A6/, result.stdout, "puppet --version does not start with major version 6.x")
 end
 
 step "Check that the agent on the master runs against the master"

acceptance/suites/tests/00_smoke/puppetdb_integration.rb

@@ -101,8 +101,8 @@
     # ISO 8601 timestamp, with milliseconds and time zone. Local time is used
     # instead of UTC as both PuppetDB and Puppet Server log in local time.
     run_timestamp = Time.iso8601(on(master, 'date +"%Y-%m-%dT%H:%M:%S.%3N%:z"').stdout.chomp)
-    on(master, puppet_agent("--test"), :acceptable_exit_codes => [0,2]) do
-      assert_match(/Notice: #{random_string}/, stdout,
+    on(master, puppet_agent("--test"), :acceptable_exit_codes => [0,2]) do |result|
+      assert_match(/Notice: #{random_string}/, result.stdout,
                   'Puppet run collects exported Notify')
     end
   end
@@ -116,15 +116,26 @@
     # name passed to Puppet::Util::Profiler.profile over in the Ruby
     # terminus code of the PuppetDB project without realizing that is a
     # breaking change to metrics critical for measuring compiler performance.
-    %w[
-      facts_encode command_submit_replace_facts
-      catalog_munge command_submit_replace_catalog
-      report_convert_to_wire_format_hash command_submit_store_report
-      resource_search query
+    [
+      "query",
+      "resource_search",
+      "facts_find",
+      "catalog_save",
+      "facts_save",
+      "command_submit_replace_catalog",
+      "command_submit_replace_facts",
+      "report_process",
+      "command_submit_store_report",
+      "payload_format",
+      "facts_encode",
+      "catalog_munge",
+      "report_convert_to_wire_format_hash"
     ].each do |metric_name|
-      metric_data = pdb_metrics.find({}) {|m| m['metric'] == metric_name }
+      metric_data = pdb_metrics.find {|m| m['metric'] == metric_name } || {}
 
-      assert_operator(metric_data.fetch('count', 0), :>, 0,
+      metric_count = metric_data.fetch('count', 0)
+      logger.debug("PuppetDB metrics #{metric_name} recorded #{metric_count} times")
+      assert_operator(metric_count, :>, 0,
                       "PuppetDB metrics recorded for: #{metric_name}")
     end
   end

acceptance/suites/tests/010-puppetserver-cli/subcommand/SERVER-297_common_behavior.rb

@@ -5,8 +5,8 @@
 proxy_env_vars = "HTTP_PROXY=foo http_proxy=foo HTTPS_PROXY=foo https_proxy=foo NO_PROXY=foo no_proxy=foo"
 
 step "ruby: Check that PATH, HOME, GEM_HOME JARS_REQUIRE and JARS_NO_REQUIRE are present"
-on(master, "puppetserver ruby -rjson -e 'puts JSON.pretty_generate(ENV.to_hash)'") do
-  env = JSON.parse(stdout)
+on(master, "puppetserver ruby -rjson -e 'puts JSON.pretty_generate(ENV.to_hash)'") do |result|
+  env = JSON.parse(result.stdout)
   assert(env['PATH'], "PATH missing")
   assert(env['HOME'], "HOME missing")
   assert(env['GEM_HOME'], "GEM_HOME missing")
@@ -15,8 +15,8 @@
 end
 
 step "ruby: Check that proxy env-variables are present"
-on(master, "#{proxy_env_vars} puppetserver ruby -rjson -e 'puts JSON.pretty_generate(ENV.to_hash)'") do
-  env = JSON.parse(stdout)
+on(master, "#{proxy_env_vars} puppetserver ruby -rjson -e 'puts JSON.pretty_generate(ENV.to_hash)'") do |result|
+  env = JSON.parse(result.stdout)
   assert_equal(env['HTTP_PROXY'], "foo",
                "HTTP_PROXY is missing or has wrong value: '#{env['HTTP_PROXY']}'")
   assert_equal(env['http_proxy'], "foo",
@@ -32,20 +32,20 @@
 end
 
 step "irb: Check that PATH, HOME, GEM_HOME JARS_REQUIRE and JARS_NO_REQUIRE are present"
-on(master, "echo 'puts JSON.pretty_generate(ENV.to_hash)' | puppetserver irb -f -rjson") do
-  assert_match(/\bPATH\b/, stdout, "PATH missing")
-  assert_match(/\bHOME\b/, stdout, "HOME missing")
-  assert_match(/\bGEM_HOME\b/, stdout, "GEM_HOME missing")
-  assert_match(/\bJARS_REQUIRE\b/, stdout, "JARS_REQUIRE missing")
-  assert_match(/\bJARS_NO_REQUIRE\b/, stdout, "JARS_NO_REQUIRE missing")
+on(master, "echo 'puts JSON.pretty_generate(ENV.to_hash)' | puppetserver irb -f -rjson") do |result|
+  assert_match(/\bPATH\b/, result.stdout, "PATH missing")
+  assert_match(/\bHOME\b/, result.stdout, "HOME missing")
+  assert_match(/\bGEM_HOME\b/, result.stdout, "GEM_HOME missing")
+  assert_match(/\bJARS_REQUIRE\b/, result.stdout, "JARS_REQUIRE missing")
+  assert_match(/\bJARS_NO_REQUIRE\b/, result.stdout, "JARS_NO_REQUIRE missing")
 end
 
 step "irb: Check that proxy env-variables are present"
-on(master, "echo 'puts JSON.pretty_generate(ENV.to_hash)' | #{proxy_env_vars} puppetserver irb -f -rjson") do
-  assert_match(/\bHTTP_PROXY\b\W\W\s\W\bfoo\b/, stdout, "HTTP_PROXY missing or has wrong value")
-  assert_match(/\bhttp_proxy\b\W\W\s\W\bfoo\b/, stdout, "http_proxy missing or has wrong value")
-  assert_match(/\bHTTPS_PROXY\b\W\W\s\W\bfoo\b/, stdout, "HTTPS_PROXY missing or has wrong value")
-  assert_match(/\bhttps_proxy\b\W\W\s\W\bfoo\b/, stdout, "https_proxy missing or has wrong value")
-  assert_match(/\bNO_PROXY\b\W\W\s\W\bfoo\b/, stdout, "NO_PROXY missing or has wrong value")
-  assert_match(/\bno_proxy\b\W\W\s\W\bfoo\b/, stdout, "no_proxy missing or has wrong value")
+on(master, "echo 'puts JSON.pretty_generate(ENV.to_hash)' | #{proxy_env_vars} puppetserver irb -f -rjson") do |result|
+  assert_match(/\bHTTP_PROXY\b\W\W\s\W\bfoo\b/, result.stdout, "HTTP_PROXY missing or has wrong value")
+  assert_match(/\bhttp_proxy\b\W\W\s\W\bfoo\b/, result.stdout, "http_proxy missing or has wrong value")
+  assert_match(/\bHTTPS_PROXY\b\W\W\s\W\bfoo\b/, result.stdout, "HTTPS_PROXY missing or has wrong value")
+  assert_match(/\bhttps_proxy\b\W\W\s\W\bfoo\b/, result.stdout, "https_proxy missing or has wrong value")
+  assert_match(/\bNO_PROXY\b\W\W\s\W\bfoo\b/, result.stdout, "NO_PROXY missing or has wrong value")
+  assert_match(/\bno_proxy\b\W\W\s\W\bfoo\b/, result.stdout, "no_proxy missing or has wrong value")
 end

acceptance/suites/tests/010-puppetserver-cli/subcommand/foreground.rb

@@ -1,9 +1,5 @@
 test_name "Puppetserver 'foreground' subcommand tests."
 
-if (master['platform'] =~ /el-5/)
-  skip_test("RHEL5 is not supported by this test case. See SERVER-653")
-end
-
 cli = "puppetserver"
 service = options['puppetservice']
 
@@ -25,7 +21,7 @@
 
 step "Run #{cli} with foreground subcommand, wait for #{timout_length}"
 on(master, timeout_cmd, :acceptable_exit_codes => [124]) do |result|
-  assert_no_match(/error:/i, result.stderr, "Unexpected error running puppetserver!")
+  refute_match(/error:/i, result.stderr, "Unexpected error running puppetserver!")
 
   step "Check that #{cli} ran successfully and shutdown triggered"
   expected_messages.each do |message, explanation|

acceptance/suites/tests/010-puppetserver-cli/subcommand/gem.rb

@@ -43,8 +43,8 @@
   on(master, "#{gem_install} #{gem_name} -v #{gem_version}")
 
   step "Check that test gem is present."
-  on(master, "#{gem_list}") do
-    assert(/^#{gem_name}/.match(stdout), "#{gem_name} was not found after installation.")
+  on(master, "#{gem_list}") do |result|
+    assert(/^#{gem_name}/.match(result.stdout), "#{gem_name} was not found after installation.")
   end
 
   if gem_name == 'excon'
@@ -56,8 +56,8 @@
       end
       on(master, "su #{runuser} -s /bin/bash -c "\
         "'/opt/puppetlabs/bin/puppetserver ruby "\
-        "-rexcon -e \"puts Excon::VERSION\"'") do
-        assert_equal(gems['excon'], stdout.strip,
+        "-rexcon -e \"puts Excon::VERSION\"'") do |result|
+        assert_equal(gems['excon'], result.stdout.strip,
                      "Unexpected output for excon version")
       end
     end
@@ -67,8 +67,8 @@
   on(master, "#{gem_uninstall} #{gem_name}")
 
   step "Check that test gem is no longer present."
-  on(master, "#{gem_list}") do
-    assert_no_match(/^#{gem_name}/, stdout, "#{gem_name} was found after uninstallation.")
+  on(master, "#{gem_list}") do |result|
+    refute_match(/^#{gem_name}/, result.stdout, "#{gem_name} was found after uninstallation.")
   end
 end
 
@@ -88,14 +88,14 @@
 on(master, "#{cli} gem env", :acceptable_exit_codes => [0])
 
 step "Verify that Java cli args passed through to gem command"
-on(master, "JAVA_ARGS_CLI=-Djruby.cli.version=true #{cli} gem help") do
-  assert_match(/jruby \d\.\d\.\d.*$/, stdout,
+on(master, "JAVA_ARGS_CLI=-Djruby.cli.version=true #{cli} gem help") do |result|
+  assert_match(/jruby \d\.\d\.\d.*$/, result.stdout,
                'jruby version not included in gem command output')
 end
 
 step "(SERVER-1759) Verify that installing a non-existent gem produces a non-zero exit return value"
 
 gem_name = 'if-this-gem-exists-then-someone-has-a-cruel-sense-of-humor'
-on(master, "#{cli} gem install #{gem_name}", :acceptable_exit_codes => [2]) do
-  assert_match(/Could not find a valid gem/, stderr)
+on(master, "#{cli} gem install #{gem_name}", :acceptable_exit_codes => [2]) do |result|
+  assert_match(/Could not find a valid gem/, result.stderr)
 end

acceptance/suites/tests/010-puppetserver-cli/subcommand/irb.rb

@@ -10,28 +10,28 @@
 
 step "Check that GEM_HOME is managed"
 cmd = "echo 'puts ENV.to_hash.to_yaml' | GEM_HOME=UHOH #{cli} irb -ryaml -f"
-on(master, cmd) do
-  assert_match(/GEM_HOME/, stdout)
-  assert_no_match(/UHOH/, stdout)
+on(master, cmd) do |result|
+  assert_match(/GEM_HOME/, result.stdout)
+  refute_match(/UHOH/, result.stdout)
 end
 
 step "Check that FOO_DEBUG is filtered"
 cmd = "echo 'puts ENV[%{FOO_DEBUG}] || %{OK}' | FOO_DEBUG=BAD #{cli} irb -f"
-on(master, cmd) do
-  assert_match(/^OK$/, stdout)
-  assert_no_match(/^BAD$/, stdout, "FOO_DEBUG is not being filtered out")
+on(master, cmd) do |result|
+  assert_match(/^OK$/, result.stdout)
+  refute_match(/^BAD$/, result.stdout, "FOO_DEBUG is not being filtered out")
 end
 
 step "Check that puppet is loadable"
 cmd = "echo 'puts %{GOOD: } + Puppet.version' | #{cli} irb -rpuppet -f"
-on(master, cmd) do
-  assert_match(/GOOD:/, stdout)
-  assert_no_match(/error/i, stdout)
+on(master, cmd) do |result|
+  assert_match(/GOOD:/, result.stdout)
+  refute_match(/error/i, result.stdout)
 end
 
 step "Verify that Java cli args passed through to irb command"
-on(master, "echo '' | JAVA_ARGS_CLI=-Djruby.cli.version=true #{cli} irb -f") do
-  assert_match(/jruby \d\.\d\.\d.*$/, stdout,
+on(master, "echo '' | JAVA_ARGS_CLI=-Djruby.cli.version=true #{cli} irb -f") do |result|
+  assert_match(/jruby \d\.\d\.\d.*$/, result.stdout,
                'jruby version not included in irb command output')
 end