From db24a7ca17c14d004aca9ced0286dd1883efb088 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Thu, 11 Apr 2024 07:42:32 +0200 Subject: [PATCH 1/5] Fix replset acceptance testing --- .../provider/mongodb_database/mongodb.rb | 14 +- spec/acceptance/replset_spec.rb | 238 +++++++++++++++--- .../provider/mongodb_database/mongodb_spec.rb | 2 +- 3 files changed, 216 insertions(+), 38 deletions(-) diff --git a/lib/puppet/provider/mongodb_database/mongodb.rb b/lib/puppet/provider/mongodb_database/mongodb.rb index 1cf061d25..7b91895a7 100644 --- a/lib/puppet/provider/mongodb_database/mongodb.rb +++ b/lib/puppet/provider/mongodb_database/mongodb.rb @@ -9,12 +9,16 @@ def self.instances require 'json' - pre_cmd = 'db.getMongo().setReadPref("primaryPreferred")' - dbs = JSON.parse mongo_eval("#{pre_cmd};EJSON.stringify(db.getMongo().getDBs())") + if db_ismaster + dbs = JSON.parse mongo_eval('EJSON.stringify(db.getMongo().getDBs())') - dbs['databases'].map do |db| - new(name: db['name'], - ensure: :present) + dbs['databases'].map do |db| + new(name: db['name'], + ensure: :present) + end + else + Puppet.warning 'Database info is available only from master host' + [] end end diff --git a/spec/acceptance/replset_spec.rb b/spec/acceptance/replset_spec.rb index 68d0a3c37..4393ed56c 100644 --- a/spec/acceptance/replset_spec.rb +++ b/spec/acceptance/replset_spec.rb @@ -8,9 +8,6 @@ if hosts.length > 1 && supported_version?(default[:platform], repo_version) describe 'mongodb_replset resource' do after :all do - # Have to drop the DB to disable replsets for further testing - on hosts, %{mongosh local --verbose --eval 'db.dropDatabase()'} - pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} @@ -20,10 +17,8 @@ class { 'mongodb::globals': package_ensure => purged, service_ensure => stopped } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': - ensure => purged - } + -> class { 'mongodb::client': + ensure => purged } EOS @@ -36,12 +31,10 @@ class { 'mongodb::globals': #{repo_ver_param} } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], replset => 'test', } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } - } + -> class { 'mongodb::client': } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) @@ -59,11 +52,81 @@ class { 'mongodb::client': } expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} end + sleep(30) + on(hosts_as('slave'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end end it 'inserts data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'db.test.insertOne({name:"test1",value:"some value"})'} + end + + it 'checks the data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + expect(r.stdout).to match %r{some value} + end + end + + it 'checks the data on the slave' do + sleep(10) + on hosts_as('slave'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + expect(r.stdout).to match %r{some value} + end + end + end + + describe 'mongodb::server with replset_members' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + + it 'configures mongo on both nodes' do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + bind_ip => ['0.0.0.0'], + replset => 'test', + replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + -> class { 'mongodb::client': } + EOS + + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) + end + + it 'sets up the replset with puppet' do + on(hosts_as('master'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end sleep(30) - on hosts_as('master'), %{mongosh --verbose --eval 'db.test.save({name:"test1",value:"some value"})'} + on(hosts_as('slave'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end + end + + it 'inserts data on the master' do + on hosts_as('master'), %{mongosh --verbose --eval 'db.getMongo().setReadPref("primaryPreferred");db.test.insertOne({name:"test1",value:"some value"})'} end it 'checks the data on the master' do @@ -74,7 +137,7 @@ class { 'mongodb::client': } it 'checks the data on the slave' do sleep(10) - on hosts_as('slave'), %{mongosh --verbose --eval 'db.getMongo().setReadPref("primaryPreferred"); EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| + on hosts_as('slave'), %{mongosh --verbose --eval 'EJSON.stringify(db.test.findOne({name:"test1"}))'} do |r| expect(r.stdout).to match %r{some value} end end @@ -82,9 +145,6 @@ class { 'mongodb::client': } describe 'mongodb_replset resource with auth => true' do after :all do - # Have to drop the DB to disable replsets for further testing - on hosts, %{mongosh local --verbose --eval 'db.dropDatabase()'} - pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} @@ -94,28 +154,36 @@ class { 'mongodb::globals': package_ensure => purged, service_ensure => stopped } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': - ensure => purged - } + -> class { 'mongodb::client': + ensure => purged } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) end + let(:keyfile_path) do + os_family = fact('os.family') + if os_family == 'RedHat' + '/var/lib/mongo' + else + '/var/lib/mongodb' + end + end + it 'configures mongo on both nodes' do pp = <<~EOS class { 'mongodb::globals': #{repo_ver_param} - } -> - class { 'mongodb::server': + } + -> class { 'mongodb::server': admin_username => 'admin', admin_password => 'password', auth => true, - bind_ip => '0.0.0.0', + store_creds => true, + bind_ip => ['0.0.0.0'], replset => 'test', - keyfile => '/var/lib/mongodb/mongodb-keyfile', + keyfile => "#{keyfile_path}/mongodb-keyfile", key => '+dxlTrury7xtD0FRqFf3YWGnKqWAtlyauuemxuYuyz9POPUuX1Uj3chGU8MFMHa7 UxASqex7NLMALQXHL+Th4T3dyb6kMZD7KiMcJObO4M+JLiX9drcTiifsDEgGMi7G vYn3pWSm5TTDrHJw7RNWfMHw3sHk0muGQcO+0dWv3sDJ6SiU8yOKRtYcTEA15GbP @@ -131,11 +199,9 @@ class { 'mongodb::server': g+Bybk5qHv1b7M8Tv9/I/BRXcpLHeIkMICMY8sVPGmP8xzL1L3i0cws8p5h0zPBa YG/QX0BmltAni8owgymFuyJgvr/gaRX4WHbKFD+9nKpqJ3ocuVNuCDsxDqLsJEME nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG - YXIsJ0gYcu9XG3mx10LbdPJvxSMg' - } - if $::osfamily == 'RedHat' { - include mongodb::client + YXIsJ0gYcu9XG3mx10LbdPJvxSMg', } + -> class { 'mongodb::client': } EOS apply_manifest_on(hosts.reverse, pp, catch_failures: true) @@ -145,9 +211,16 @@ class { 'mongodb::server': it 'sets up the replset with puppet' do pp = <<~EOS mongodb_replset { 'test': - auth_enabled => true, - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - before => Mongodb_user['admin'] + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + -> mongodb_user { "User admin on db admin": + ensure => present, + password_hash => mongodb_password('admin', 'password'), + username => 'admin', + database => 'admin', + roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', + 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', + 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], } EOS apply_manifest_on(hosts_as('master'), pp, catch_failures: true) @@ -156,11 +229,112 @@ class { 'mongodb::server': expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} end + sleep(30) + on(hosts_as('slave'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end end it 'inserts data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insertOne({"created_by_puppet": 1})'} + end + + it 'checks the data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");EJSON.stringify(db.dummyData.findOne())'} do |r| + expect(r.stdout).to match %r{created_by_puppet} + end + end + + it 'checks the data on the slave' do + sleep(10) + on hosts_as('slave'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.getMongo().setReadPref("primaryPreferred");EJSON.stringify(db.dummyData.findOne())'} do |r| + expect(r.stdout).to match %r{created_by_puppet} + end + end + end + + describe 'mongodb::server with replset_members and auth => true' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + + let(:keyfile_path) do + os_family = fact('os.family') + if os_family == 'RedHat' + '/var/lib/mongo' + else + '/var/lib/mongodb' + end + end + + it 'configures mongo on both nodes' do + pp = <<~EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::client': } + -> class { 'mongodb::server': + admin_username => 'admin', + admin_password => 'password', + auth => true, + store_creds => true, + create_admin => true, + bind_ip => ['0.0.0.0'], + replset => 'test', + replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + keyfile => "#{keyfile_path}/mongodb-keyfile", + key => '+dxlTrury7xtD0FRqFf3YWGnKqWAtlyauuemxuYuyz9POPUuX1Uj3chGU8MFMHa7 + UxASqex7NLMALQXHL+Th4T3dyb6kMZD7KiMcJObO4M+JLiX9drcTiifsDEgGMi7G + vYn3pWSm5TTDrHJw7RNWfMHw3sHk0muGQcO+0dWv3sDJ6SiU8yOKRtYcTEA15GbP + ReDZuHFy1T1qhk5NIt6pTtPGsZKSm2wAWIOa2f2IXvpeQHhjxP8aDFb3fQaCAqOD + R7hrimqq0Nickfe8RLA89iPXyadr/YeNBB7w7rySatQBzwIbBUVGNNA5cxCkwyx9 + E5of3xi7GL9xNxhQ8l0JEpofd4H0y0TOfFDIEjc7cOnYlKAHzBgog4OcFSILgUaF + kHuTMtv0pj+MMkW2HkeXETNII9XE1+JiZgHY08G7yFEJy87ttUoeKmpbI6spFz5U + 4K0amj+N6SOwXaS8uwp6kCqay0ERJLnw+7dKNKZIZdGCrrBxcZ7wmR/wLYrxvHhZ + QpeXTxgD5ebwCR0cf3Xnb5ql5G/HHKZDq8LTFHwELNh23URGPY7K7uK+IF6jSEhq + V2H3HnWV9teuuJ5he9BB/pLnyfjft6KUUqE9HbaGlX0f3YBk/0T3S2ESN4jnfRTQ + ysAKvQ6NasXkzqXktu8X4fS5QNqrFyqKBZSWxttfJBKXnT0TxamCKLRx4AgQglYo + 3KRoyfxXx6G+AjP1frDJxFAFEIgEFqRk/FFuT/y9LpU+3cXYX1Gt6wEatgmnBM3K + g+Bybk5qHv1b7M8Tv9/I/BRXcpLHeIkMICMY8sVPGmP8xzL1L3i0cws8p5h0zPBa + YG/QX0BmltAni8owgymFuyJgvr/gaRX4WHbKFD+9nKpqJ3ocuVNuCDsxDqLsJEME + nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG + YXIsJ0gYcu9XG3mx10LbdPJvxSMg', + } + EOS + + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) + end + + it 'sets up the replset with puppet' do + on(hosts_as('master'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end sleep(30) - on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insert({"created_by_puppet": 1})'} + on(hosts_as('slave'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| + expect(r.stdout).to match %r{#{hosts[0]}:27017} + expect(r.stdout).to match %r{#{hosts[1]}:27017} + end + end + + it 'inserts data on the master' do + on hosts_as('master'), %{mongosh test --verbose --eval 'load("/root/.mongoshrc.js");db.dummyData.insertOne({"created_by_puppet": 1})'} end it 'checks the data on the master' do diff --git a/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb b/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb index 8f25fb384..c0a4825de 100644 --- a/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb +++ b/spec/unit/puppet/provider/mongodb_database/mongodb_spec.rb @@ -38,7 +38,7 @@ tmp = Tempfile.new('test') mongodconffile = tmp.path allow(provider.class).to receive(:mongod_conf_file).and_return(mongodconffile) - allow(provider.class).to receive(:mongo_eval).with('db.getMongo().setReadPref("primaryPreferred");EJSON.stringify(db.getMongo().getDBs())').and_return(raw_dbs) + allow(provider.class).to receive(:mongo_eval).with('EJSON.stringify(db.getMongo().getDBs())').and_return(raw_dbs) allow(provider.class).to receive(:db_ismaster).and_return(true) end From 69160cd3b96189facd9533fec85896259a2692d6 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Tue, 16 Apr 2024 11:16:50 +0200 Subject: [PATCH 2/5] Fix sharding acceptance testing --- lib/puppet/provider/mongodb_shard/mongo.rb | 62 +------------------ spec/acceptance/sharding_spec.rb | 62 +++++++++++++------ .../provider/mongodb_shard/mongodb_spec.rb | 32 +++++----- 3 files changed, 61 insertions(+), 95 deletions(-) diff --git a/lib/puppet/provider/mongodb_shard/mongo.rb b/lib/puppet/provider/mongodb_shard/mongo.rb index 6347bf59b..b30602973 100644 --- a/lib/puppet/provider/mongodb_shard/mongo.rb +++ b/lib/puppet/provider/mongodb_shard/mongo.rb @@ -102,7 +102,7 @@ def self.shard_collection_details(obj, shard_name) def self.shard_properties(shard) properties = {} - output = mongo_command('sh.status()') + output = mongo_command('sh.status()')['value'] output['shards'].each do |s| next unless s['_id'] == shard @@ -118,7 +118,7 @@ def self.shard_properties(shard) end def self.shards_properties - output = mongo_command('sh.status()') + output = mongo_command('sh.status()')['value'] properties = if output['shards'].empty? [] else @@ -163,64 +163,6 @@ def self.mongo_command(command, host = nil, _retries = 4) retry end - # NOTE: (spredzy) : sh.status() - # does not return a json stream - # we jsonify it so it is easier - # to parse and deal with it - if command == 'sh.status()' - myarr = output.split("\n") - myarr.shift - myarr.pop - myarr.pop - final_stream = [] - prev_line = nil - in_shard_list = 0 - in_chunk = 0 - myarr.each do |line| - line.gsub!(%r{sharding version:}, '{ "sharding version":') - line.gsub!(%r{shards:}, ',"shards":[') - line.gsub!(%r{databases:}, '], "databases":[') - line.gsub!(%r{"clusterId" : ObjectId\("(.*)"\)}, '"clusterId" : "ObjectId(\'\1\')"') - line.gsub!(%r{\{ "_id" :}, ',{ "_id" :') if %r{_id} =~ prev_line - # Modification for shard - line = '' if line =~ %r{on :.*Timestamp} - if line =~ %r{_id} && in_shard_list == 1 - in_shard_list = 0 - last_line = final_stream.pop.strip - proper_line = "#{last_line}]}," - final_stream << proper_line - end - if line =~ %r{shard key} && in_shard_list == 1 - shard_name = final_stream.pop.strip - proper_line = ",{\"#{shard_name}\":" - final_stream << proper_line - end - if line =~ %r{shard key} && in_shard_list.zero? - in_shard_list = 1 - shard_name = final_stream.pop.strip - id_line = "#{final_stream.pop[0..-2]}, \"shards\": " - proper_line = "[{\"#{shard_name}\":" - final_stream << id_line - final_stream << proper_line - end - if in_chunk == 1 - in_chunk = 0 - line = "\"#{line.strip}\"}}" - end - in_chunk = 1 if line =~ %r{chunks} && in_chunk.zero? - line.gsub!(%r{shard key}, '{"shard key"') - line.gsub!(%r{chunks}, ',"chunks"') - final_stream << line unless line.empty? - prev_line = line - end - final_stream << ' ] }' if in_shard_list == 1 - final_stream << ' ] }' - output = final_stream.join("\n") - end - - # Hack to avoid non-json empty sets - output = '{}' if output == "null\n" - output.gsub!(%r{\s*}, '') JSON.parse(output) end end diff --git a/spec/acceptance/sharding_spec.rb b/spec/acceptance/sharding_spec.rb index 2c826234a..66c7c34ac 100644 --- a/spec/acceptance/sharding_spec.rb +++ b/spec/acceptance/sharding_spec.rb @@ -7,21 +7,41 @@ if hosts.length > 1 && supported_version?(default[:platform], repo_version) describe 'mongodb_shard resource' do + after :all do + pp = <<-EOS + class { 'mongodb::globals': + #{repo_ver_param} + } + -> class { 'mongodb::mongos' : + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::server': + ensure => absent, + package_ensure => purged, + service_ensure => stopped + } + -> class { 'mongodb::client': + ensure => purged + } + EOS + + apply_manifest_on(hosts.reverse, pp, catch_failures: true) + end + it 'configures the shard server' do pp = <<-EOS class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], replset => 'foo', shardsvr => true, - }-> - mongodb_replset { 'foo' : - members => ["shard:27018"], } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } + -> mongodb_replset { 'foo' : + members => [#{hosts_as('shard').map { |x| "'#{x}:27018'" }.join(',')}], } EOS @@ -34,27 +54,29 @@ class { 'mongodb::client': } class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': - bind_ip => '0.0.0.0', + bind_ip => ['0.0.0.0'], + replset => 'conf', configsvr => true, - } -> - class { 'mongodb::mongos' : - configdb => ["router:27019"], - } -> - exec { '/bin/sleep 20' : - } -> - mongodb_shard { 'foo': - member => 'foo/shard:27018', - keys => [{'foo.toto' => {'name' => 1}}] } - if $::osfamily == 'RedHat' { - class { 'mongodb::client': } + -> mongodb_replset { 'conf' : + members => [#{hosts_as('router').map { |x| "'#{x}:27019'" }.join(',')}], + } + -> class { 'mongodb::mongos' : + configdb => ["conf/#{hosts_as('router').map { |x| "#{x}:27019" }.join(',')}"], + } + -> exec { '/bin/sleep 20' : + } + -> mongodb_shard { 'foo': + member => "foo/#{hosts_as('shard').map { |x| "#{x}:27018" }.join(',')}", + keys => [{'foo.toto' => {'name' => 1}}] } EOS - apply_manifest_on(hosts_as('router'), pp, catch_failures: true) + apply_manifest_on(hosts_as('router'), pp, catch_failures: true, trace: true) on(hosts_as('router'), 'mongosh --quiet --eval "EJSON.stringify(sh.status())"') do |r| - expect(r.stdout).to match %r{foo/shard:27018} + expect(r.stdout).to match %r{foo/#{hosts[0]}:27018} expect(r.stdout).to match %r{foo\.toto} end end diff --git a/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb b/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb index 865ed8d87..d749519cc 100644 --- a/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb +++ b/spec/unit/puppet/provider/mongodb_shard/mongodb_spec.rb @@ -20,21 +20,23 @@ let(:raw_shards) do { - 'sharding version' => { - '_id' => 1, - 'version' => 4, - 'minCompatibleVersion' => 4, - 'currentVersion' => 5, - 'clusterId' => "ObjectId('548e9110f3aca177c94c5e49')" - }, - 'shards' => [ - { '_id' => 'rs_test', 'host' => 'rs_test/mongo1:27018' } - ], - 'databases' => [ - { '_id' => 'admin', 'partitioned' => false, 'primary' => 'config' }, - { '_id' => 'test', 'partitioned' => false, 'primary' => 'rs_test' }, - { '_id' => 'rs_test', 'partitioned' => true, 'primary' => 'rs_test' } - ] + 'value' => { + 'sharding version' => { + '_id' => 1, + 'version' => 4, + 'minCompatibleVersion' => 4, + 'currentVersion' => 5, + 'clusterId' => "ObjectId('548e9110f3aca177c94c5e49')" + }, + 'shards' => [ + { '_id' => 'rs_test', 'host' => 'rs_test/mongo1:27018' } + ], + 'databases' => [ + { '_id' => 'admin', 'partitioned' => false, 'primary' => 'config' }, + { '_id' => 'test', 'partitioned' => false, 'primary' => 'rs_test' }, + { '_id' => 'rs_test', 'partitioned' => true, 'primary' => 'rs_test' } + ] + } } end From 97d2fd1df82bcbda5b9f4faf68ea08d86e885a40 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Wed, 17 Apr 2024 00:04:29 +0200 Subject: [PATCH 3/5] Use multinode setfile in ci workflow --- .github/workflows/ci.yml | 1 + .sync.yml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1b263c33..6c4d89c1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,4 +21,5 @@ jobs: uses: voxpupuli/gha-puppet/.github/workflows/beaker.yml@v2 with: pidfile_workaround: 'false' + beaker_hosts: 'host1:shard.ma;host2:slave,router.a' beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' diff --git a/.sync.yml b/.sync.yml index ee3be25a5..35d767165 100644 --- a/.sync.yml +++ b/.sync.yml @@ -2,4 +2,6 @@ appveyor.yml: delete: true .github/workflows/ci.yml: - beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' + with: + beaker_hosts: 'host1:shard.ma;host2:slave,router.a' + beaker_facter: 'mongodb_repo_version:MongoDB:4.4,5.0,6.0,7.0' From d203c12ca9f667717e8762bc8bec396704253265 Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Sun, 21 Apr 2024 00:58:14 +0200 Subject: [PATCH 4/5] Look at all active hosts to detect if replset init is successfully ended --- lib/puppet/provider/mongodb_replset/mongo.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/puppet/provider/mongodb_replset/mongo.rb b/lib/puppet/provider/mongodb_replset/mongo.rb index e9d134c6c..1a125fde1 100644 --- a/lib/puppet/provider/mongodb_replset/mongo.rb +++ b/lib/puppet/provider/mongodb_replset/mongo.rb @@ -284,14 +284,14 @@ def create_replica_set(alive_hosts) retry_sleep = 3 retry_limit.times do |n| - if db_ismaster(alive_hosts[0]['host'])['ismaster'] - Puppet.debug 'Replica set initialization has successfully ended' - return true - else - Puppet.debug "Waiting for replica initialization. Retry: #{n}" - sleep retry_sleep - next + alive_hosts.each do |alive_host| + if db_ismaster(alive_host['host'])['ismaster'] + Puppet.debug 'Replica set initialization has successfully ended' + return true + end end + Puppet.debug "Waiting for replica initialization. Retry: #{n}" + sleep retry_sleep end raise Puppet::Error, "rs.initiate() failed for replicaset #{name}" end From 9636c73871a8ba5ef12f9e37727fd2160afdaf0c Mon Sep 17 00:00:00 2001 From: Hugo Haakseth Date: Sun, 21 Apr 2024 22:44:00 +0200 Subject: [PATCH 5/5] replset integration tests: Manage server and replset in one puppet run. --- spec/acceptance/replset_spec.rb | 56 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/spec/acceptance/replset_spec.rb b/spec/acceptance/replset_spec.rb index 4393ed56c..bb1568354 100644 --- a/spec/acceptance/replset_spec.rb +++ b/spec/acceptance/replset_spec.rb @@ -30,24 +30,22 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': bind_ip => ['0.0.0.0'], replset => 'test', } - -> class { 'mongodb::client': } + -> mongodb_replset { 'test': + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + EOS - apply_manifest_on(hosts.reverse, pp, catch_failures: true) - apply_manifest_on(hosts.reverse, pp, catch_changes: true) + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) end it 'sets up the replset with puppet' do - pp = <<-EOS - mongodb_replset { 'test': - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - } - EOS - apply_manifest_on(hosts_as('master'), pp, catch_failures: true) on(hosts_as('master'), 'mongosh --quiet --eval "EJSON.stringify(rs.conf())"') do |r| expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017} @@ -101,12 +99,12 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': bind_ip => ['0.0.0.0'], replset => 'test', replset_members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], } - -> class { 'mongodb::client': } EOS apply_manifest_on(hosts, pp, catch_failures: true) @@ -176,6 +174,7 @@ class { 'mongodb::globals': class { 'mongodb::globals': #{repo_ver_param} } + -> class { 'mongodb::client': } -> class { 'mongodb::server': admin_username => 'admin', admin_password => 'password', @@ -201,30 +200,29 @@ class { 'mongodb::globals': nc1ohyB0lNt8lHf1U00mtgDSV3fwo5LkwhRi6d+bDBTL/C6MZETMLdyCqDlTdUWG YXIsJ0gYcu9XG3mx10LbdPJvxSMg', } - -> class { 'mongodb::client': } + -> mongodb_replset { 'test': + members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], + } + + unless $facts['mongodb_is_master'] == 'false' { # lint:ignore:quoted_booleans + mongodb_user { "User admin on db admin": + ensure => present, + password_hash => mongodb_password('admin', 'password'), + username => 'admin', + database => 'admin', + roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', + 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', + 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], + require => Mongodb_replset['test'], + } + } EOS - apply_manifest_on(hosts.reverse, pp, catch_failures: true) - apply_manifest_on(hosts.reverse, pp, catch_changes: true) + apply_manifest_on(hosts, pp, catch_failures: true) + apply_manifest_on(hosts, pp, catch_changes: true) end it 'sets up the replset with puppet' do - pp = <<~EOS - mongodb_replset { 'test': - members => [#{hosts.map { |x| "'#{x}:27017'" }.join(',')}], - } - -> mongodb_user { "User admin on db admin": - ensure => present, - password_hash => mongodb_password('admin', 'password'), - username => 'admin', - database => 'admin', - roles => ['userAdmin', 'readWrite', 'dbAdmin', 'dbAdminAnyDatabase', 'readAnyDatabase', - 'readWriteAnyDatabase', 'userAdminAnyDatabase', 'clusterAdmin', - 'clusterManager', 'clusterMonitor', 'hostManager', 'root', 'restore',], - } - EOS - apply_manifest_on(hosts_as('master'), pp, catch_failures: true) - apply_manifest_on(hosts_as('master'), pp, catch_changes: true) on(hosts_as('master'), 'mongosh --quiet --eval "load(\'/root/.mongoshrc.js\');EJSON.stringify(rs.conf())"') do |r| expect(r.stdout).to match %r{#{hosts[0]}:27017} expect(r.stdout).to match %r{#{hosts[1]}:27017}