From 2466a3f39e2559c682ee4432f90dc386a8ca3066 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 19 Oct 2022 02:31:59 -0700 Subject: [PATCH 01/55] feat: zfs-based rsnapshot backup --- modules/ocf_backups/files/backup-zfs.sh | 24 +++++++++++ modules/ocf_backups/manifests/rsnapshot.pp | 48 ++++++++-------------- 2 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 modules/ocf_backups/files/backup-zfs.sh diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh new file mode 100644 index 000000000..c0ec8d6a2 --- /dev/null +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +CURRENT_SNAPSHOT_FILE=/opt/share/backups/current-zfs-snapshot +CURRENT_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) +OFFSITE_HOST=$(cat /opt/share/backups/offsite-host) +echo $CURRENT_SNAPSHOT + +rsnapshot -V -c /opt/share/backups/rsnapshot-zfs.conf sync +rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync +rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-git.conf sync +rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync + +zfs-auto-snapshot --syslog --label=after-backup --keep=10 // | awk -F"," '{print $1}' | cut -c2- > $CURRENT_SNAPSHOT_FILE +NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) + +if [ -t 0 ]; then + echo $CURRENT_SNAPSHOT + echo $NEW_SNAPSHOT + zfs send -cRwi backup/encrypted/rsnapshot@$CURRENT_SNAPSHOT backup/encrypted/rsnapshot@$NEW_SNAPSHOT | pv | ssh $OFFSITE_HOST "zfs recv -d data1/ocfbackup" +else + echo $CURRENT_SNAPSHOT + echo $NEW_SNAPSHOT + zfs send -cRwi backup/encrypted/rsnapshot@$CURRENT_SNAPSHOT backup/encrypted/rsnapshot@$NEW_SNAPSHOT | ssh $OFFSITE_HOST "zfs recv -d data1/ocfbackup" +fi diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 8af41eb58..978220052 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -5,6 +5,21 @@ '/opt/share/backups/rsnapshot.conf': source => 'puppet:///modules/ocf_backups/rsnapshot.conf'; + '/opt/share/backups/rsnapshot-zfs.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs.conf'; + + '/opt/share/backups/rsnapshot-zfs-mysql.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-mysql.conf'; + '/opt/share/backups/rsnapshot-zfs-pgsql.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-pgsql.conf'; + '/opt/share/backups/rsnapshot-zfs-git.conf': + source => 'puppet:///modules/ocf_backups/rsnapshot-zfs-git.conf'; + + '/usr/local/sbin/backup-zfs.sh': + source => 'puppet:///modules/ocf_backups/backup-zfs.sh', + mode => '0755'; + + # TODO: update for ZFS '/opt/share/backups/check-rsnapshot-backups': source => 'puppet:///modules/ocf_backups/check-rsnapshot-backups', mode => '0755'; @@ -13,43 +28,16 @@ # TODO: update times listed here after move to remote backups - # Since we use sync_first, actual backups only happen at the most frequent - # ("smallest") backup level, i.e. daily. - # - # The other backup levels just promote a daily backup into a weekly/monthly - # one, so they are comparatively fast. - # - # As of 2015-03-29, it takes 30 minutes to do a promotion, and 4 hours to do - # a full backup. So we leave 2 hours for promotions and 8 hours for a full - # backup to be safe. - # - # It's important that jobs don't overlap, so our plan is: - # 10pm-12am monthly backup takes place (~30 minutes) - # 12am-2am: weekly backup takes place (~30 minutes) - # 2am-10am: daily backup takes place (~4 hours) - - $rsnapshot = 'rsnapshot -c /opt/share/backups/rsnapshot.conf' + $rsnapshot = '/usr/local/sbin/backup-zfs.sh' cron { default: user => root, minute => '0'; - # 10pm on 1st of month - 'rsnapshot-monthly': - command => "${rsnapshot} monthly", - hour => '22', - monthday => '1'; - - # 12am Saturday mornings - 'rsnapshot-weekly': - command => "${rsnapshot} weekly", - hour => '0', - weekday => '6'; - - # 2am daily + # ZFS 'rsnapshot-daily': - command => "${rsnapshot} sync && ${rsnapshot} daily", + command => "${rsnapshot}", hour => '2'; # check rsnapshot backups to ensure they're actually happening From db127fe2c577425e93507afc325062290b11fcc1 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 19 Oct 2022 02:38:55 -0700 Subject: [PATCH 02/55] more work --- hieradata/dummy_secrets.yaml | 1 + modules/ocf_backups/files/backup-zfs.sh | 0 .../ocf_backups/files/rsnapshot-zfs-git.conf | 45 +++++++++++ .../files/rsnapshot-zfs-mysql.conf | 45 +++++++++++ .../files/rsnapshot-zfs-pgsql.conf | 45 +++++++++++ modules/ocf_backups/files/rsnapshot-zfs.conf | 74 +++++++++++++++++++ modules/ocf_backups/manifests/init.pp | 6 ++ 7 files changed, 216 insertions(+) mode change 100644 => 100755 modules/ocf_backups/files/backup-zfs.sh create mode 100644 modules/ocf_backups/files/rsnapshot-zfs-git.conf create mode 100644 modules/ocf_backups/files/rsnapshot-zfs-mysql.conf create mode 100644 modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf create mode 100644 modules/ocf_backups/files/rsnapshot-zfs.conf diff --git a/hieradata/dummy_secrets.yaml b/hieradata/dummy_secrets.yaml index 77178ecc5..65aa0c278 100644 --- a/hieradata/dummy_secrets.yaml +++ b/hieradata/dummy_secrets.yaml @@ -15,6 +15,7 @@ ocfbackups::box: api_client_id: dummy_client_id api_client_secret: dummy_client_secret ocfbackups::mysql::password: dummypassword +ocfbackups::offsite_host: dummyhost sensu::redis::password: dummypassword diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh old mode 100644 new mode 100755 diff --git a/modules/ocf_backups/files/rsnapshot-zfs-git.conf b/modules/ocf_backups/files/rsnapshot-zfs-git.conf new file mode 100644 index 000000000..1d3617473 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-git.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/git/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-git . diff --git a/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf b/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf new file mode 100644 index 000000000..04124bbd9 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-mysql.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/mysql/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-mysql . diff --git a/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf b/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf new file mode 100644 index 000000000..3cdec78d5 --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs-pgsql.conf @@ -0,0 +1,45 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/pgsql/ + +retain daily 1 +# backup points/scripts +# nfs (homedirs, webdirs) + +# scripts +backup_script /opt/share/backups/backup-pgsql . diff --git a/modules/ocf_backups/files/rsnapshot-zfs.conf b/modules/ocf_backups/files/rsnapshot-zfs.conf new file mode 100644 index 000000000..166ff3f3b --- /dev/null +++ b/modules/ocf_backups/files/rsnapshot-zfs.conf @@ -0,0 +1,74 @@ +################################################# +# rsnapshot.conf - rsnapshot configuration file # +################################################# +# # +# PLEASE BE AWARE OF THE FOLLOWING RULES: # +# # +# This file requires tabs between elements # +# # +# Directories require a trailing slash: # +# right: /home/ # +# wrong: /home # +# # +################################################# + +config_version 1.2 + +cmd_cp /bin/cp +cmd_rm /bin/rm +cmd_rsync /usr/local/bin/rsync-no-vanished +cmd_ssh /usr/bin/ssh +cmd_logger /usr/bin/logger + +# remote backups require login as ocfbackups, then `sudo rsync-no-vanished' +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_postexec /usr/bin/kdestroy + +# default is "--delete --numeric-ids --relative --delete-excluded" +# we add the 'sudo rsync-no-vanished' bits +rsync_long_args --delete --numeric-ids --relative --delete-excluded --rsync-path="sudo ionice -c2 -n7 nice -n15 /usr/local/bin/rsync-no-vanished" + +no_create_root 1 +one_fs 1 +sync_first 1 + +lockfile /run/rsnapshot.pid + +# backup root directory +snapshot_root /backup/encrypted/rsnapshot/ + +retain daily 1 +# backup points/scripts +# scripts +# nfs (homedirs, webdirs) +backup ocfbackups@filehost:/opt/homes/ nfs/ + +# remote servers +backup ocfbackups@hal:/etc/libvirt/qemu/ servers/vm_xml/hal/ +backup ocfbackups@jaws:/etc/libvirt/qemu/ servers/vm_xml/jaws/ +backup ocfbackups@pandemic:/etc/libvirt/qemu/ servers/vm_xml/pandemic/ +backup ocfbackups@riptide:/etc/libvirt/qemu/ servers/vm_xml/riptide/ +backup ocfbackups@scurvy:/etc/libvirt/qemu/ servers/vm_xml/scurvy/ +backup ocfbackups@kerberos:/var/lib/heimdal-kdc/ servers/kerberos/ +backup ocfbackups@kerberos:/var/backups/kerberos/ servers/kerberos/ +backup ocfbackups@ldap:/var/lib/ldap/ servers/ldap/ +backup ocfbackups@ldap:/var/backups/ldap/ servers/ldap/ + +backup ocfbackups@puppet:/etc/puppetlabs/ servers/puppet/ +backup ocfbackups@puppet:/opt/puppetlabs/ servers/puppet/ + +backup ocfbackups@puppetdb:/etc/puppetlabs/puppet/ssl/ servers/puppetdb/ + +backup ocfbackups@munin:/var/lib/munin/ servers/munin/ + +backup ocfbackups@apt:/opt/apt/ servers/apt/ + +backup ocfbackups@jenkins:/var/lib/jenkins/ servers/jenkins/ + +backup ocfbackups@rancid:/var/lib/rancid/ servers/rancid/ + +backup ocfbackups@ns:/etc/bind/keys/ servers/ns/ + +backup ocfbackups@irc:/var/lib/znc/ servers/irc/ + +# vim: ts=16 sts=16 sw=16 noet diff --git a/modules/ocf_backups/manifests/init.pp b/modules/ocf_backups/manifests/init.pp index 57ff99b0b..148ec5b21 100644 --- a/modules/ocf_backups/manifests/init.pp +++ b/modules/ocf_backups/manifests/init.pp @@ -13,6 +13,12 @@ ensure => directory, group => ocfroot, mode => '0750'; + + '/opt/share/backups/offsite-host': + content => lookup('ocfbackups::offsite_host'), + owner => root, + group => root, + mode => '0400'; } # keytab for ocfbackups user, used to rsync from remote servers From d8bdaec1b0a785564e8d95f547d0ad8608d856bc Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 19 Oct 2022 02:44:12 -0700 Subject: [PATCH 03/55] lint & shellcheck --- modules/ocf_backups/files/backup-zfs.sh | 13 ++++++------- modules/ocf_backups/manifests/init.pp | 8 ++++---- modules/ocf_backups/manifests/rsnapshot.pp | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index c0ec8d6a2..a5888ed29 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -3,7 +3,7 @@ CURRENT_SNAPSHOT_FILE=/opt/share/backups/current-zfs-snapshot CURRENT_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) OFFSITE_HOST=$(cat /opt/share/backups/offsite-host) -echo $CURRENT_SNAPSHOT +echo "$CURRENT_SNAPSHOT" rsnapshot -V -c /opt/share/backups/rsnapshot-zfs.conf sync rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync @@ -13,12 +13,11 @@ rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync zfs-auto-snapshot --syslog --label=after-backup --keep=10 // | awk -F"," '{print $1}' | cut -c2- > $CURRENT_SNAPSHOT_FILE NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) +echo "$CURRENT_SNAPSHOT" +echo "$NEW_SNAPSHOT" + if [ -t 0 ]; then - echo $CURRENT_SNAPSHOT - echo $NEW_SNAPSHOT - zfs send -cRwi backup/encrypted/rsnapshot@$CURRENT_SNAPSHOT backup/encrypted/rsnapshot@$NEW_SNAPSHOT | pv | ssh $OFFSITE_HOST "zfs recv -d data1/ocfbackup" + zfs send -cRwi backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | pv | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" else - echo $CURRENT_SNAPSHOT - echo $NEW_SNAPSHOT - zfs send -cRwi backup/encrypted/rsnapshot@$CURRENT_SNAPSHOT backup/encrypted/rsnapshot@$NEW_SNAPSHOT | ssh $OFFSITE_HOST "zfs recv -d data1/ocfbackup" + zfs send -cRwi backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" fi diff --git a/modules/ocf_backups/manifests/init.pp b/modules/ocf_backups/manifests/init.pp index 148ec5b21..41c239865 100644 --- a/modules/ocf_backups/manifests/init.pp +++ b/modules/ocf_backups/manifests/init.pp @@ -15,10 +15,10 @@ mode => '0750'; '/opt/share/backups/offsite-host': - content => lookup('ocfbackups::offsite_host'), - owner => root, - group => root, - mode => '0400'; + content => lookup('ocfbackups::offsite_host'), + owner => root, + group => root, + mode => '0400'; } # keytab for ocfbackups user, used to rsync from remote servers diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 978220052..dca89d55d 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -37,7 +37,7 @@ # ZFS 'rsnapshot-daily': - command => "${rsnapshot}", + command => $rsnapshot, hour => '2'; # check rsnapshot backups to ensure they're actually happening From be1b32c3e2c2e24ae6717d050feac498767dea60 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 00:43:29 -0700 Subject: [PATCH 04/55] Update rsnapshot-zfs.conf --- modules/ocf_backups/files/rsnapshot-zfs.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/rsnapshot-zfs.conf b/modules/ocf_backups/files/rsnapshot-zfs.conf index 166ff3f3b..2d7fdc9f3 100644 --- a/modules/ocf_backups/files/rsnapshot-zfs.conf +++ b/modules/ocf_backups/files/rsnapshot-zfs.conf @@ -21,7 +21,7 @@ cmd_ssh /usr/bin/ssh cmd_logger /usr/bin/logger # remote backups require login as ocfbackups, then `sudo rsync-no-vanished' -cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_preexec /usr/bin/kinit -l 6h -t /opt/share/backups/ocfbackups.keytab ocfbackups cmd_postexec /usr/bin/kdestroy # default is "--delete --numeric-ids --relative --delete-excluded" From 3fe9d126c12c24f00418f89e31827b9f3a49470f Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 03:14:08 -0700 Subject: [PATCH 05/55] Revert "Update rsnapshot-zfs.conf" This reverts commit be1b32c3e2c2e24ae6717d050feac498767dea60. --- modules/ocf_backups/files/rsnapshot-zfs.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/rsnapshot-zfs.conf b/modules/ocf_backups/files/rsnapshot-zfs.conf index 2d7fdc9f3..166ff3f3b 100644 --- a/modules/ocf_backups/files/rsnapshot-zfs.conf +++ b/modules/ocf_backups/files/rsnapshot-zfs.conf @@ -21,7 +21,7 @@ cmd_ssh /usr/bin/ssh cmd_logger /usr/bin/logger # remote backups require login as ocfbackups, then `sudo rsync-no-vanished' -cmd_preexec /usr/bin/kinit -l 6h -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups cmd_postexec /usr/bin/kdestroy # default is "--delete --numeric-ids --relative --delete-excluded" From 556f8874b98d048e732e826ff36ececde6ac0f3c Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 22:43:04 -0700 Subject: [PATCH 06/55] remove old backup jobs --- modules/ocf_backups/manifests/rsnapshot.pp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index dca89d55d..4617838f7 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -38,11 +38,5 @@ # ZFS 'rsnapshot-daily': command => $rsnapshot, - hour => '2'; - - # check rsnapshot backups to ensure they're actually happening - 'check-rsnapshot-backups': - command => '/opt/share/backups/check-rsnapshot-backups', - hour => '10'; - } + hour => '23'; } From 35f8d532109b1f9d3f16a2e2384887ded759834a Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 22:44:03 -0700 Subject: [PATCH 07/55] fix: syntax --- modules/ocf_backups/manifests/rsnapshot.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 4617838f7..410575374 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -39,4 +39,5 @@ 'rsnapshot-daily': command => $rsnapshot, hour => '23'; + } } From cf4adcd7e9eca31ff58ebbe084585c9818dde2b9 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 25 Oct 2022 22:15:26 -0700 Subject: [PATCH 08/55] change time --- modules/ocf_backups/manifests/rsnapshot.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 410575374..47e24235f 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -38,6 +38,7 @@ # ZFS 'rsnapshot-daily': command => $rsnapshot, - hour => '23'; + hour => '22', + minute => '20'; } } From 4a4fd79b15eb3edef714eec4f0cce51bf620eb65 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sat, 29 Oct 2022 14:19:30 -0700 Subject: [PATCH 09/55] feat: use zfs native compression for sql --- modules/ocf_backups/files/backup-mysql | 2 +- modules/ocf_backups/files/backup-pgsql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ocf_backups/files/backup-mysql b/modules/ocf_backups/files/backup-mysql index 132629da0..024d7ff39 100755 --- a/modules/ocf_backups/files/backup-mysql +++ b/modules/ocf_backups/files/backup-mysql @@ -18,4 +18,4 @@ parallel -i \ --triggers \ --routines \ --single-transaction \ - --databases {} | pigz > "mysql-{}-$(date +%F).sql.gz"' -- $databases + --databases {} > "mysql-{}-$(date +%F).sql"' -- $databases diff --git a/modules/ocf_backups/files/backup-pgsql b/modules/ocf_backups/files/backup-pgsql index 6fd361f01..0f76953fc 100755 --- a/modules/ocf_backups/files/backup-pgsql +++ b/modules/ocf_backups/files/backup-pgsql @@ -3,4 +3,4 @@ set -euo pipefail # Dumps the entire PostgreSQL instance to one .sql file. # Requires that a valid ~/.pgpass file be available on the PostgreSQL host -ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost | pigz' > "pgsql-all-$(date +%F).sql.gz" +ssh -K ocfbackups@postgres 'pg_dumpall -U postgres -h localhost' > "pgsql-all-$(date +%F).sql" From 602122540ceace87db35c0d0ffc956ce09769dfb Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sat, 5 Nov 2022 20:09:20 -0700 Subject: [PATCH 10/55] fix: zfs send args --- modules/ocf_backups/files/backup-zfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index a5888ed29..f80f23e74 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -17,7 +17,7 @@ echo "$CURRENT_SNAPSHOT" echo "$NEW_SNAPSHOT" if [ -t 0 ]; then - zfs send -cRwi backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | pv | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" + zfs send -cRwI backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | pv | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" else - zfs send -cRwi backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" + zfs send -cRwI backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" fi From 823d271a57c2865fab7dda4bac3b1906f7453b9e Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sat, 5 Nov 2022 21:30:58 -0700 Subject: [PATCH 11/55] feat: syncoid instead of raw zfs send commands --- modules/ocf_backups/files/backup-zfs.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index f80f23e74..18799afac 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -16,8 +16,4 @@ NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) echo "$CURRENT_SNAPSHOT" echo "$NEW_SNAPSHOT" -if [ -t 0 ]; then - zfs send -cRwI backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | pv | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" -else - zfs send -cRwI backup/encrypted/rsnapshot@"$CURRENT_SNAPSHOT" backup/encrypted/rsnapshot@"$NEW_SNAPSHOT" | ssh "$OFFSITE_HOST" "zfs recv -d data1/ocfbackup" -fi +syncoid -r --no-sync-snap --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot From 44e512422fa52bca0167966bad0eb40a324f61ea Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 7 Nov 2022 02:35:59 -0800 Subject: [PATCH 12/55] feat: logging and logrotate --- modules/ocf_backups/files/backup-zfs-logrotate | 5 +++++ modules/ocf_backups/manifests/init.pp | 3 +++ modules/ocf_backups/manifests/rsnapshot.pp | 6 +++--- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 modules/ocf_backups/files/backup-zfs-logrotate diff --git a/modules/ocf_backups/files/backup-zfs-logrotate b/modules/ocf_backups/files/backup-zfs-logrotate new file mode 100644 index 000000000..b549cc46c --- /dev/null +++ b/modules/ocf_backups/files/backup-zfs-logrotate @@ -0,0 +1,5 @@ +/var/log/ocf-backup-zfs.log { + rotate 100 + daily + compress +} diff --git a/modules/ocf_backups/manifests/init.pp b/modules/ocf_backups/manifests/init.pp index 41c239865..fc9630db9 100644 --- a/modules/ocf_backups/manifests/init.pp +++ b/modules/ocf_backups/manifests/init.pp @@ -19,6 +19,9 @@ owner => root, group => root, mode => '0400'; + + '/etc/logrotate.d/backup-zfs': + source => 'puppet:///modules/ocf_backups/backup-zfs-logrotate'; } # keytab for ocfbackups user, used to rsync from remote servers diff --git a/modules/ocf_backups/manifests/rsnapshot.pp b/modules/ocf_backups/manifests/rsnapshot.pp index 47e24235f..c4330776d 100644 --- a/modules/ocf_backups/manifests/rsnapshot.pp +++ b/modules/ocf_backups/manifests/rsnapshot.pp @@ -28,7 +28,7 @@ # TODO: update times listed here after move to remote backups - $rsnapshot = '/usr/local/sbin/backup-zfs.sh' + $rsnapshot = '/usr/local/sbin/backup-zfs.sh | tee -a /var/log/zfs-backup.log' cron { default: @@ -38,7 +38,7 @@ # ZFS 'rsnapshot-daily': command => $rsnapshot, - hour => '22', - minute => '20'; + hour => '03', + minute => '00'; } } From 515e96f2fdcf5166a2396d81360b9a42eee5db80 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 21 Nov 2022 01:51:11 -0800 Subject: [PATCH 13/55] feat: reduce verbosity --- modules/ocf_backups/files/backup-zfs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index 18799afac..09a813740 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -5,10 +5,10 @@ CURRENT_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) OFFSITE_HOST=$(cat /opt/share/backups/offsite-host) echo "$CURRENT_SNAPSHOT" -rsnapshot -V -c /opt/share/backups/rsnapshot-zfs.conf sync -rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync -rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-git.conf sync -rsnapshot -V -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-mysql.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-git.conf sync +rsnapshot -c /opt/share/backups/rsnapshot-zfs-pgsql.conf sync zfs-auto-snapshot --syslog --label=after-backup --keep=10 // | awk -F"," '{print $1}' | cut -c2- > $CURRENT_SNAPSHOT_FILE NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) From c8f4f51176a7017c3aed2476f249fd2a8d6c0256 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sat, 29 Oct 2022 22:25:51 -0700 Subject: [PATCH 14/55] fix: disable healthcheck on videolan-ftp timestamp file on source is empty --- modules/ocf_mirrors/manifests/projects/videolan_ftp.pp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/ocf_mirrors/manifests/projects/videolan_ftp.pp b/modules/ocf_mirrors/manifests/projects/videolan_ftp.pp index cf44e201b..a4b40792e 100644 --- a/modules/ocf_mirrors/manifests/projects/videolan_ftp.pp +++ b/modules/ocf_mirrors/manifests/projects/videolan_ftp.pp @@ -9,14 +9,6 @@ recurse => true; } - ocf_mirrors::monitoring { 'videolan-ftp': - ensure => 'present', - type => 'unix_timestamp', - upstream_host => 'ftp.videolan.org', - upstream_path => '/videolan', - ts_path => 'trace'; - } - ocf_mirrors::timer { 'videolan-ftp': exec_start => '/opt/mirrors/project/videolan-ftp/sync-archive', From d37300d73b8f02393ed8bf5f32abf115cc219744 Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Sun, 30 Oct 2022 11:54:29 -0700 Subject: [PATCH 15/55] Update vcsrepo to 5.3.0 (#1340) vcsrepo 2.3.0 doesn't support Puppet 7 --- Puppetfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Puppetfile b/Puppetfile index a1a9b14d4..0f46ec2ea 100644 --- a/Puppetfile +++ b/Puppetfile @@ -31,6 +31,6 @@ mod 'puppetlabs-puppet_authorization', '0.4.0' mod 'puppetlabs-puppetdb', '6.0.2' mod 'puppetlabs-stdlib', '4.25.1' mod 'puppetlabs-tagmail', '2.4.0' -mod 'puppetlabs-vcsrepo', '2.3.0' +mod 'puppetlabs-vcsrepo', '5.3.0' mod 'thias-sysctl', '1.0.6' mod 'puppetlabs/translate', '2.0.0' # Dependency of puppetlabs-kubernetes From a5066c9b29bcf5ad8228adde8247964c51a472ee Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Sun, 30 Oct 2022 12:01:46 -0700 Subject: [PATCH 16/55] Add `*.new.o.b.e` to HAProxy (#1339) --- .../templates/master/loadbalancer/haproxy.cfg.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ocf_kubernetes/templates/master/loadbalancer/haproxy.cfg.erb b/modules/ocf_kubernetes/templates/master/loadbalancer/haproxy.cfg.erb index 8e4a8402a..8332f8fae 100644 --- a/modules/ocf_kubernetes/templates/master/loadbalancer/haproxy.cfg.erb +++ b/modules/ocf_kubernetes/templates/master/loadbalancer/haproxy.cfg.erb @@ -48,6 +48,9 @@ frontend lb-https # send *.code traffic into the cluster use_backend kubernetes if { hdr_end(host) -i .code.ocf.berkeley.edu .code.ocf.berkeley.edu:443 } + # send *.new traffic into the cluster + use_backend kubernetes if { hdr_end(host) -i .new.ocf.berkeley.edu .new.ocf.berkeley.edu:443 } + <%- @kubernetes_services.each do |service_name| -%> use_backend kubernetes if { hdr(host) -i <%= service_name %>.ocf.berkeley.edu <%= service_name %>.ocf.berkeley.edu:443 } <%- end -%> From 771e48480cd71fa5885e82a4202a7f7285eb5cfb Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 2 Nov 2022 01:40:18 -0700 Subject: [PATCH 17/55] update fw --- hieradata/common.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 6a1eeafe0..fe932916a 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -12,9 +12,7 @@ devices_ipv4_only: - bedbugs-mgmt - corruption-mgmt - dataloss-mgmt - - fallingrocks-mgmt - hal-mgmt - - jaws-mgmt - logjam - pagefault - pandemic-mgmt From b4d9ad6b459843330b993585c1449c499308b3e2 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 2 Nov 2022 01:46:44 -0700 Subject: [PATCH 18/55] foo --- hieradata/common.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index fe932916a..85e7ae3b7 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -24,6 +24,7 @@ devices_ipv4_only: devices_ipv46: - radiation + internal_zone_exceptions: - tsunami - vampires From 11b3ac957089651d88c61577acf53fb026c4c4e6 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 2 Nov 2022 02:30:31 -0700 Subject: [PATCH 19/55] fw update pt3 --- hieradata/common.yaml | 2 ++ modules/ocf/manifests/firewall/post.pp | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 85e7ae3b7..a01548e89 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -42,6 +42,8 @@ internal_zone_range_6: 2607:f140:8801::1:3-2607:f140:8801::1:90 desktop_src_range_6: 2607:f140:8801::1:100-2607:f140:8801::1:139 staffvm_src_range_6: 2607:f140:8801::1:200-2607:f140:8801::1:252 +special_range_4: 169.229.226.131-169.229.226.139 + death_ipv4: 169.229.226.23 death_ipv6: 2607:f140:8801::1:23 diff --git a/modules/ocf/manifests/firewall/post.pp b/modules/ocf/manifests/firewall/post.pp index b8a2188e7..4d8249b5f 100644 --- a/modules/ocf/manifests/firewall/post.pp +++ b/modules/ocf/manifests/firewall/post.pp @@ -79,6 +79,15 @@ before => undef, } + $special_range_4 = lookup('special_range_4') + firewall_multi { '999 reject output (special devices range)': + chain => 'PUPPET-OUTPUT', + proto => 'all', + action => 'reject', + dst_range => $special_range_4, + before => undef, + } + # reject from hosts in internal zone range but not actually internal if $ocf::firewall::reject_unrecognized_input { $reject_all = lookup('internal_zone_exceptions') From 4c1b8f09fd11256f12b49363f638f0fca28c2b64 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 2 Nov 2022 02:48:11 -0700 Subject: [PATCH 20/55] elts updates: use ocf mirrors and use backport kernel --- modules/ocf/manifests/apt.pp | 2 +- modules/ocf/manifests/packages.pp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/ocf/manifests/apt.pp b/modules/ocf/manifests/apt.pp index 837e2bbb6..e7a21f564 100644 --- a/modules/ocf/manifests/apt.pp +++ b/modules/ocf/manifests/apt.pp @@ -44,7 +44,7 @@ }; 'extended-lts': - location => 'http://deb.freexian.com/extended-lts', + location => 'https://mirrors.ocf.berkeley.edu/freexian/', release => $::lsbdistcodename, repos => $repos; diff --git a/modules/ocf/manifests/packages.pp b/modules/ocf/manifests/packages.pp index a7a619123..ac106e8f8 100644 --- a/modules/ocf/manifests/packages.pp +++ b/modules/ocf/manifests/packages.pp @@ -154,12 +154,15 @@ } # only install the python3.7 packages on stretch # python3 is python3.7 on buster and python3.9 on bullseye + + # install elts kernel on stretch if $::lsbdistcodename == 'stretch' { package { [ 'python3.7', 'python3.7-dev', 'python3.7-venv', + 'linux-image-5.10-amd64', ]:; } } From 0e54a74427da4298e96bf512d40336b3f38da199 Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Wed, 2 Nov 2022 19:50:44 -0700 Subject: [PATCH 21/55] Ignore DNS resolution timeout error (#1341) --- modules/ocf_www/files/lets-encrypt-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_www/files/lets-encrypt-update b/modules/ocf_www/files/lets-encrypt-update index add8b015a..61af892ac 100755 --- a/modules/ocf_www/files/lets-encrypt-update +++ b/modules/ocf_www/files/lets-encrypt-update @@ -63,7 +63,7 @@ def eligible_domains(domains, target_domain): def resolve(fqdn): try: answers = list(resolver.query(fqdn, 'A')) - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.exception.Timeout): return else: if answers: From 62aecdb1daf9a84e45e714b9faa2d7ecbb2d6e7d Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 8 Nov 2022 02:23:06 -0800 Subject: [PATCH 22/55] fix: chattr exec strings containing variables need to be double quoted --- modules/ocf/manifests/privatefile.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ocf/manifests/privatefile.pp b/modules/ocf/manifests/privatefile.pp index 32b388074..3f472a267 100644 --- a/modules/ocf/manifests/privatefile.pp +++ b/modules/ocf/manifests/privatefile.pp @@ -64,13 +64,13 @@ if $immutable { exec { 'chattr -immutable': before => File[$title], - command => 'chattr -i $title', + command => "chattr -i $title", refreshonly => true, } exec { 'chattr +immutable': subscribe => File[$title], - command => 'chattr +i $title', + command => "chattr +i $title", refreshonly => true, } } From 9987254faff6640a803cac4b516b87ae588d38a9 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 8 Nov 2022 02:25:08 -0800 Subject: [PATCH 23/55] fix: syntax rekt, variable not inside brackets --- modules/ocf/manifests/privatefile.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/ocf/manifests/privatefile.pp b/modules/ocf/manifests/privatefile.pp index 3f472a267..cf74d8d38 100644 --- a/modules/ocf/manifests/privatefile.pp +++ b/modules/ocf/manifests/privatefile.pp @@ -64,13 +64,13 @@ if $immutable { exec { 'chattr -immutable': before => File[$title], - command => "chattr -i $title", + command => "chattr -i ${title}", refreshonly => true, } exec { 'chattr +immutable': subscribe => File[$title], - command => "chattr +i $title", + command => "chattr +i ${title}", refreshonly => true, } } From 8ae3563ef4d3a2c23cfa8fb7462ac9db6b8de377 Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Tue, 8 Nov 2022 19:38:55 -0800 Subject: [PATCH 24/55] Revert social shorturl to regular form (#1342) --- modules/ocf_www/manifests/site/shorturl.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index 33a9716b9..43c63a7f1 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -132,7 +132,7 @@ {rewrite_rule => '^/sh$ https://ocf.io/staff-hours [R]'}, {rewrite_rule => '^/shs$ https://github.com/ocf/etc/blob/master/configs/staff_hours.yaml [R]'}, {rewrite_rule => '^/slack$ https://www.ocf.berkeley.edu/docs/contact/slack/ [R]'}, - {rewrite_rule => '^/social$ https://docs.google.com/forms/d/e/1FAIpQLSegWC0zsMDGBqOEojY3gy8yNbrtd-QbXnWlX0S3TdWKaSQzYg/viewform [R]'}, + {rewrite_rule => '^/social$ https://docs.google.com/forms/d/e/1FAIpQLSdLUhDYQug53NXdPEeJkLzB9JNUMUWJmXMoRSv18PkhEh4h4Q/viewform [R]'}, {rewrite_rule => '^/social-?rules$ https://www.recurse.com/social-rules [R]'}, {rewrite_rule => '^/socialbingo$ https://docs.google.com/forms/d/e/1FAIpQLSeU2iZf_SnephTu6KQ_VfeFBI-YMetTSioeibFKwgtoITlG-w/viewform [R]'}, {rewrite_rule => '^/ssh$ https://www.ocf.berkeley.edu/docs/services/shell/ [R]'}, From f0044976aea04182ce4653116b97e891c6d9f3e3 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 15 Nov 2022 02:59:27 -0500 Subject: [PATCH 25/55] cleanup --- modules/ocf_irc/manifests/services.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_irc/manifests/services.pp b/modules/ocf_irc/manifests/services.pp index 7144dfad7..50cd44d61 100644 --- a/modules/ocf_irc/manifests/services.pp +++ b/modules/ocf_irc/manifests/services.pp @@ -7,7 +7,7 @@ } $irc_creds = lookup('irc_creds') - $root_nicks = ['waf', 'nattofriends', 'ckuehl', 'jvperrin', 'mattmcal', 'abizer', 'dkessler', 'cooperc', 'jaw', 'kmo', 'njha', 'fydai', 'etw', 'rjz', 'ethanhs'] + $root_nicks = ['waf', 'nattofriends', 'ckuehl', 'jvperrin', 'mattmcal', 'abizer', 'dkessler', 'cooperc', 'rjz'] file { default: From 9f47a4cf49f70796b57220e74d263eddb6afb14a Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Thu, 8 Dec 2022 20:53:11 -0800 Subject: [PATCH 26/55] Add vaultwarden.o.b.e to load balancer (#1344) --- modules/ocf_kubernetes/manifests/master/loadbalancer.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ocf_kubernetes/manifests/master/loadbalancer.pp b/modules/ocf_kubernetes/manifests/master/loadbalancer.pp index 85e9fc37c..c607f2619 100644 --- a/modules/ocf_kubernetes/manifests/master/loadbalancer.pp +++ b/modules/ocf_kubernetes/manifests/master/loadbalancer.pp @@ -39,6 +39,7 @@ 'sourcegraph', 'static', 'templates', + 'vaultwarden', ] # redirects happen post-canonicalization, so needs fqdn From a691cdfeab53b67cb903efa960dbe59edbbff617 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Mon, 16 Jan 2023 20:53:42 -0800 Subject: [PATCH 27/55] doc: add comments to octocatalog-diff script This should have been in commit 4d980300 --- octocatalog-diff/scripts/git-extract.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octocatalog-diff/scripts/git-extract.py b/octocatalog-diff/scripts/git-extract.py index 6e8a45e73..c43da3c70 100644 --- a/octocatalog-diff/scripts/git-extract.py +++ b/octocatalog-diff/scripts/git-extract.py @@ -76,6 +76,7 @@ def get_commit_hash(ref: str, repo: Optional[Path] = None) -> str: needs_update = True eprint('adding worktree') if target_dir.exists(): + # the worktree was in a broken state, so reset it shutil.rmtree(target_dir) subprocess.run( ('git', 'worktree', 'add', str(target_dir), target_sha), From af1f2dbcfab841e8004acfb620a0ae8722be7a32 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Tue, 17 Jan 2023 00:32:42 -0800 Subject: [PATCH 28/55] fix(mirrors): reload nginx when certs change (#1345) * fix(mirrors): reload nginx when certs change Previously, nginx was not picking up renewed certs. * refactor(mirrors): move+use variable for ocfstats password Some history: the variable was introduced in #169 when stat tracking was added, but #925 changed the implementation and didn't use the variable any more (probably because it was so far away from where it was used). This commit moves it next to where it is used now. --- modules/ocf_mirrors/manifests/init.pp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/ocf_mirrors/manifests/init.pp b/modules/ocf_mirrors/manifests/init.pp index bcc2f2a61..a77df90d1 100644 --- a/modules/ocf_mirrors/manifests/init.pp +++ b/modules/ocf_mirrors/manifests/init.pp @@ -97,7 +97,9 @@ END } - $ocfstats_password = lookup('ocfstats::mysql::password') + + # Restart nginx if any cert changes occur + Class['ocf::ssl::default'] ~> Class['Nginx::Service'] file { ['/opt/mirrors', '/opt/mirrors/ftp', '/opt/mirrors/project', '/opt/mirrors/bin']: @@ -215,9 +217,10 @@ mode => '0755', } + $ocfstats_password = lookup('ocfstats::mysql::password') file { '/opt/ocfstats-password': - content => lookup('ocfstats::mysql::password'), + content => $ocfstats_password, mode => '0600', owner => 'root', group => 'root', From aa74f2ca48861836062e3ce48d9462f6e75cd4c5 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 18 Jan 2023 16:32:15 -0800 Subject: [PATCH 29/55] chore: add corruption-new --- hieradata/nodes/corruption-new.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 hieradata/nodes/corruption-new.yaml diff --git a/hieradata/nodes/corruption-new.yaml b/hieradata/nodes/corruption-new.yaml new file mode 100644 index 000000000..ae4a0e702 --- /dev/null +++ b/hieradata/nodes/corruption-new.yaml @@ -0,0 +1,13 @@ +classes: + - ocf_hpc::compute + + +ocf::networking::bond: true +ocf::networking::is_vlan: true +ocf::networking::vlan: '875' +ocf::networking::netmask: '255.255.255.240' +ocf::networking::gateway: '169.229.200.71' +ocf::networking::netmask6: '64' +ocf::networking::gateway6: '2607:f140:0:32::1' +# TODO: temporary, remove +ocf::firewall::reject_unrecognized_input: false From 17459ceb53175169bc7ea8754a0c2398bbe48070 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 18 Jan 2023 16:40:15 -0800 Subject: [PATCH 30/55] fix: add new hpc to firewall --- modules/ocf/manifests/firewall/post.pp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/ocf/manifests/firewall/post.pp b/modules/ocf/manifests/firewall/post.pp index 4d8249b5f..ed393874d 100644 --- a/modules/ocf/manifests/firewall/post.pp +++ b/modules/ocf/manifests/firewall/post.pp @@ -120,6 +120,13 @@ action => 'accept', before => undef; + '998 allow from internal zone (corruption, IPv4)': + chain => 'PUPPET-INPUT', + source => '169.229.200.71', + proto => 'all', + action => 'accept', + before => undef; + '998 allow from internal zone (IPv6)': provider => 'ip6tables', chain => 'PUPPET-INPUT', From a66dee0936b7f21d3618ba64cad186ea7c09654c Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 18 Jan 2023 18:35:48 -0800 Subject: [PATCH 31/55] fix: wrong gateway --- hieradata/nodes/corruption-new.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hieradata/nodes/corruption-new.yaml b/hieradata/nodes/corruption-new.yaml index ae4a0e702..a63830f1f 100644 --- a/hieradata/nodes/corruption-new.yaml +++ b/hieradata/nodes/corruption-new.yaml @@ -6,7 +6,7 @@ ocf::networking::bond: true ocf::networking::is_vlan: true ocf::networking::vlan: '875' ocf::networking::netmask: '255.255.255.240' -ocf::networking::gateway: '169.229.200.71' +ocf::networking::gateway: '169.229.200.65' ocf::networking::netmask6: '64' ocf::networking::gateway6: '2607:f140:0:32::1' # TODO: temporary, remove From 46e09bdfddc71d82dc817d087377f1689d52aca9 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Wed, 18 Jan 2023 20:23:45 -0800 Subject: [PATCH 32/55] ncograin root (#1346) --- modules/ocf_kerberos/files/kadmind.acl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ocf_kerberos/files/kadmind.acl b/modules/ocf_kerberos/files/kadmind.acl index 6133c51e3..7b52b89f9 100644 --- a/modules/ocf_kerberos/files/kadmind.acl +++ b/modules/ocf_kerberos/files/kadmind.acl @@ -22,6 +22,7 @@ longlian/admin@OCF.BERKELEY.EDU all,get-keys mcint/admin@OCF.BERKELEY.EDU all,get-keys mdcha/admin@OCF.BERKELEY.EDU all,get-keys minos/admin@OCF.BERKELEY.EDU all,get-keys +ncograin/admin@OCF.BERKELEY.EDU all,get-keys njha/admin@OCF.BERKELEY.EDU all,get-keys rjz/admin@OCF.BERKELEY.EDU all,get-keys rmxu/admin@OCF.BERKELEY.EDU all,get-keys @@ -54,6 +55,7 @@ longlian/root@OCF.BERKELEY.EDU change-password mcint/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU mdcha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU minos/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU +ncograin/admin@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU njha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU rjz/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU rmxu/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU From 8140ffa832947ea98104648c6bc87c574ce39712 Mon Sep 17 00:00:00 2001 From: dpark00 <79130840+dpark00@users.noreply.github.com> Date: Tue, 24 Jan 2023 19:35:23 -0800 Subject: [PATCH 33/55] changed apply to spring 2023 (#1348) --- modules/ocf_www/manifests/site/shorturl.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index 43c63a7f1..82dac0e98 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -25,7 +25,7 @@ {rewrite_rule => '^/accessibility$ https://docs.google.com/document/d/15_ZFfYG6yv0YNmPPKfbaCK4kPQpRnkCTfx7L9t3LiLE/edit#heading=h.y7iaxq6cdj4b [R]'}, {rewrite_rule => '^/apphost$ https://www.ocf.berkeley.edu/docs/services/webapps/ [R]'}, {rewrite_rule => '^/announce$ https://groups.google.com/a/lists.berkeley.edu/forum/#!forum/ocf/join [NE,R]'}, - {rewrite_rule => '^/apply$ https://forms.gle/NJKkJt9JRQBvpxhF9 [R]'}, + {rewrite_rule => '^/apply$ https://forms.gle/ACnWLyaEpqgV1rP29 [R]'}, {rewrite_rule => '^/apparel$ https://docs.google.com/forms/d/e/1FAIpQLSdJJeyRjkJhGVcH9frHtE6dtXHbgfD-fy9J5d2wRkzqSl5nPw/viewform [R]'}, {rewrite_rule => '^/banquet$ https://forms.gle/z2d8H8K4VoJEq1hw5 [R]'}, {rewrite_rule => '^/bjb$ https://jukebox.ocf.berkeley.edu/ [R]'}, From 8bf7f2cda290fc0299b12e8f22b77e7b7d0abe03 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 24 Jan 2023 22:50:29 -0800 Subject: [PATCH 34/55] add ewdc ip --- modules/ocf_mail/templates/postfix/main.cf.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ocf_mail/templates/postfix/main.cf.erb b/modules/ocf_mail/templates/postfix/main.cf.erb index 8290365df..36f2bc7a4 100644 --- a/modules/ocf_mail/templates/postfix/main.cf.erb +++ b/modules/ocf_mail/templates/postfix/main.cf.erb @@ -3,7 +3,8 @@ smtpd_banner = $myhostname ESMTP Greetings from 171 MLK. # ocf, google apps # Google's CIDR ranges are retrieved from their SPF records: # dig TXT _spf.google.com -mynetworks = 169.229.226.0/24 [2607:f140:8801::]/48 +mynetworks = 169.229.226.0/24 169.229.200.64/28 [2607:f140:8801::]/48 + [2607:f140:0:32::]/64 35.191.0.0/16 66.102.0.0/20 64.233.160.0/19 66.249.80.0/20 72.14.192.0/18 74.125.0.0/16 108.177.8.0/21 108.177.96.0/19 130.211.0.0/22 172.217.0.0/19 172.217.32.0/20 172.217.128.0/19 172.217.160.0/20 172.217.192.0/19 From 1321235f2da59846efabc29c10db431ad2357e1d Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 30 Jan 2023 11:36:04 -0800 Subject: [PATCH 35/55] fix: update chrome and electron apps automatically (#1349) --- modules/ocf/files/apt/50unattended-upgrades | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ocf/files/apt/50unattended-upgrades b/modules/ocf/files/apt/50unattended-upgrades index 436c52bee..4da1a750b 100644 --- a/modules/ocf/files/apt/50unattended-upgrades +++ b/modules/ocf/files/apt/50unattended-upgrades @@ -1,5 +1,8 @@ Unattended-Upgrade::Origins-Pattern { "origin=Debian,codename=${distro_codename},label=Debian-Security"; + "origin=Google LLC"; + "origin=vscode stable"; + "origin=riot.im"; }; Unattended-Upgrade::Mail "root"; From bf3c305c5eb736a59d0d256cfdbcea31c168a551 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 30 Jan 2023 11:53:34 -0800 Subject: [PATCH 36/55] fix: guess what they changed the codename in 11!! --- modules/ocf/files/apt/50unattended-upgrades | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ocf/files/apt/50unattended-upgrades b/modules/ocf/files/apt/50unattended-upgrades index 4da1a750b..e110d3d3d 100644 --- a/modules/ocf/files/apt/50unattended-upgrades +++ b/modules/ocf/files/apt/50unattended-upgrades @@ -1,5 +1,6 @@ Unattended-Upgrade::Origins-Pattern { "origin=Debian,codename=${distro_codename},label=Debian-Security"; + "origin=Debian,codename=${distro_codename}-security,label=Debian-Security"; "origin=Google LLC"; "origin=vscode stable"; "origin=riot.im"; From 310c0d13821ddf026fbcdaa2c8368634e08b1393 Mon Sep 17 00:00:00 2001 From: Open Computing Facility Date: Wed, 1 Feb 2023 19:37:01 -0800 Subject: [PATCH 37/55] automatically bump version to 1.5.0 (#1343) --- hieradata/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index a01548e89..0a83766fa 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -112,5 +112,5 @@ kube_dev::controller_nodes: - hozer-74 # Prometheus config -prometheus::node_exporter::version: 1.4.0 +prometheus::node_exporter::version: 1.5.0 prometheus::node_exporter::extra_options: '--collector.textfile.directory /srv/prometheus' From dfd6e6d0d5d85579e30c6440caa45916972a7c21 Mon Sep 17 00:00:00 2001 From: "Frank X. Blackburn" <6500159+axmmisaka@users.noreply.github.com> Date: Thu, 2 Feb 2023 16:32:06 -0800 Subject: [PATCH 38/55] remove rmxu from root staff (#1350) Co-authored-by: rmxu --- modules/ocf_kerberos/files/kadmind.acl | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ocf_kerberos/files/kadmind.acl b/modules/ocf_kerberos/files/kadmind.acl index 7b52b89f9..401572e6f 100644 --- a/modules/ocf_kerberos/files/kadmind.acl +++ b/modules/ocf_kerberos/files/kadmind.acl @@ -25,7 +25,6 @@ minos/admin@OCF.BERKELEY.EDU all,get-keys ncograin/admin@OCF.BERKELEY.EDU all,get-keys njha/admin@OCF.BERKELEY.EDU all,get-keys rjz/admin@OCF.BERKELEY.EDU all,get-keys -rmxu/admin@OCF.BERKELEY.EDU all,get-keys ronitnath/admin@OCF.BERKELEY.EDU all,get-keys shasan/admin@OCF.BERKELEY.EDU all,get-keys snarain/admin@OCF.BERKELEY.EDU all,get-keys @@ -58,7 +57,6 @@ minos/root@OCF.BERKELEY.EDU change-password ncograin/admin@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU njha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU rjz/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -rmxu/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU ronitnath/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU shasan/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU snarain/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU From 7994dcf7b3de7123399e55cb51c4a68a56c3c4f2 Mon Sep 17 00:00:00 2001 From: Nikhil Jha Date: Thu, 2 Feb 2023 17:24:33 -0800 Subject: [PATCH 39/55] sp23: trim root ACL The people removed were notified last semester, kmo requested an extension to the end of that semester, so that was done. --- modules/ocf_kerberos/files/kadmind.acl | 42 -------------------------- 1 file changed, 42 deletions(-) diff --git a/modules/ocf_kerberos/files/kadmind.acl b/modules/ocf_kerberos/files/kadmind.acl index 401572e6f..56a673a88 100644 --- a/modules/ocf_kerberos/files/kadmind.acl +++ b/modules/ocf_kerberos/files/kadmind.acl @@ -1,66 +1,24 @@ #principal [priv1,priv2,...] [glob-pattern] #admin principals -abizer/admin@OCF.BERKELEY.EDU all,get-keys -asai/admin@OCF.BERKELEY.EDU all,get-keys -baisang/admin@OCF.BERKELEY.EDU all,get-keys bencuan/admin@OCF.BERKELEY.EDU all,get-keys -bernardzhao/admin@OCF.BERKELEY.EDU all,get-keys -bzh/admin@OCF.BERKELEY.EDU all,get-keys -cooperc/admin@OCF.BERKELEY.EDU all,get-keys -dkessler/admin@OCF.BERKELEY.EDU all,get-keys -ethanhs/admin@OCF.BERKELEY.EDU all,get-keys etw/admin@OCF.BERKELEY.EDU all,get-keys -fydai/admin@OCF.BERKELEY.EDU all,get-keys -jaw/admin@OCF.BERKELEY.EDU all,get-keys -jvperrin/admin@OCF.BERKELEY.EDU all,get-keys kian/admin@OCF.BERKELEY.EDU all,get-keys -kmo/admin@OCF.BERKELEY.EDU all,get-keys -kpengboy/admin@OCF.BERKELEY.EDU all,get-keys -kuoh/admin@OCF.BERKELEY.EDU all,get-keys laksith/admin@OCF.BERKELEY.EDU all,get-keys longlian/admin@OCF.BERKELEY.EDU all,get-keys -mcint/admin@OCF.BERKELEY.EDU all,get-keys -mdcha/admin@OCF.BERKELEY.EDU all,get-keys -minos/admin@OCF.BERKELEY.EDU all,get-keys ncograin/admin@OCF.BERKELEY.EDU all,get-keys njha/admin@OCF.BERKELEY.EDU all,get-keys rjz/admin@OCF.BERKELEY.EDU all,get-keys -ronitnath/admin@OCF.BERKELEY.EDU all,get-keys -shasan/admin@OCF.BERKELEY.EDU all,get-keys -snarain/admin@OCF.BERKELEY.EDU all,get-keys -wporr/admin@OCF.BERKELEY.EDU all,get-keys #principal [priv1,priv2,...] [glob-pattern] #root principals -abizer/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -asai/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -baisang/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU bencuan/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -bernardzhao/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -bzh/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -cooperc/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -dkessler/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -ethanhs/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU etw/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -fydai/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -jaw/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -jvperrin/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU kian/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -kmo/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -kpengboy/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -kuoh/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU laksith/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU longlian/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -mcint/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -mdcha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -minos/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU ncograin/admin@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU njha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU rjz/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -ronitnath/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -shasan/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -snarain/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -wporr/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU # create create/admin@OCF.BERKELEY.EDU add,get,change-password *@OCF.BERKELEY.EDU From 62eec41f45343b7df045d42784e4bcb40cc89d90 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 14 Feb 2023 21:40:14 -0800 Subject: [PATCH 40/55] fix: proper root principal for ncograin --- modules/ocf_kerberos/files/kadmind.acl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_kerberos/files/kadmind.acl b/modules/ocf_kerberos/files/kadmind.acl index 56a673a88..77c4a7308 100644 --- a/modules/ocf_kerberos/files/kadmind.acl +++ b/modules/ocf_kerberos/files/kadmind.acl @@ -16,7 +16,7 @@ etw/root@OCF.BERKELEY.EDU change-password kian/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU laksith/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU longlian/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU -ncograin/admin@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU +ncograin/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU njha/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU rjz/root@OCF.BERKELEY.EDU change-password *@OCF.BERKELEY.EDU From 62ffaf295051011bf7c33b44b2474a4c9230ad55 Mon Sep 17 00:00:00 2001 From: Kian Date: Thu, 23 Feb 2023 11:20:17 -0800 Subject: [PATCH 41/55] Add SSL configuration to windshear (#1354) --- hieradata/nodes/windshear.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hieradata/nodes/windshear.yaml b/hieradata/nodes/windshear.yaml index f81abd08f..b83fa75ab 100644 --- a/hieradata/nodes/windshear.yaml +++ b/hieradata/nodes/windshear.yaml @@ -1 +1,4 @@ +classes: + - ocf::ssl::default + owner: kian From 922180177240c56c450fe312d5606fae18291d21 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 27 Feb 2023 19:02:59 -0800 Subject: [PATCH 42/55] fix: don't redirect on ocf.b.e (#1251) * fix: don't redirect on ocf.b.e * fix: it's actually here, see if it works * fix: it works * fix: remove server cors not necessary * fix: wrong one --- modules/ocf_www/manifests/site/www.pp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/ocf_www/manifests/site/www.pp b/modules/ocf_www/manifests/site/www.pp index a381620e2..e3efa269a 100644 --- a/modules/ocf_www/manifests/site/www.pp +++ b/modules/ocf_www/manifests/site/www.pp @@ -84,13 +84,6 @@ ], directories => [ - { - path => '/.well-known/matrix/server', - provider => 'location', - custom_fragment => ' - Header set Access-Control-Allow-Origin "*" - ', - }, { path => '/.well-known/matrix/client', provider => 'location', @@ -169,7 +162,8 @@ port => 80, docroot => '/var/www/html', redirectmatch_status => 301, - redirectmatch_regexp => '^(.*)', + # ugly exceptions + redirectmatch_regexp => '^((?!\/\.well-known\/matrix\/(client|server)).*)', redirectmatch_dest => $canonical_url; # redirect weird HTTPS -> canonical HTTPS @@ -180,6 +174,15 @@ 'secure.ocf.berkeley.edu', $::fqdn, ], + directories => [ + { + path => '/.well-known/matrix/client', + provider => 'location', + custom_fragment => ' + Header set Access-Control-Allow-Origin "*" + ', + }, + ], port => 443, docroot => '/var/www/html', redirectmatch_status => 301, From 41196fce1b5987c6e72f69a1cde5e6a020773155 Mon Sep 17 00:00:00 2001 From: Kian Date: Mon, 27 Feb 2023 19:18:22 -0800 Subject: [PATCH 43/55] Add slidingsync.matrix (#1355) * Add slidingsync.matrix to load balancer * Add sliding sync to matrix/client well-known --- modules/ocf_kubernetes/manifests/master/loadbalancer.pp | 1 + modules/ocf_www/files/matrix-client | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/ocf_kubernetes/manifests/master/loadbalancer.pp b/modules/ocf_kubernetes/manifests/master/loadbalancer.pp index c607f2619..5a40113c8 100644 --- a/modules/ocf_kubernetes/manifests/master/loadbalancer.pp +++ b/modules/ocf_kubernetes/manifests/master/loadbalancer.pp @@ -37,6 +37,7 @@ 'prometheus-kube', 'rt', 'sourcegraph', + 'slidingsync.matrix', 'static', 'templates', 'vaultwarden', diff --git a/modules/ocf_www/files/matrix-client b/modules/ocf_www/files/matrix-client index d890b4535..7df15d876 100644 --- a/modules/ocf_www/files/matrix-client +++ b/modules/ocf_www/files/matrix-client @@ -5,5 +5,8 @@ }, "m.identity_server": { "base_url": "https://vector.im" + }, + "org.matrix.msc3575.proxy": { + "url": "https://slidingsync.matrix.ocf.berkeley.edu" } } From 72a363027c2476968a88b1daa535384f50be8296 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 27 Feb 2023 20:42:21 -0800 Subject: [PATCH 44/55] Update www.pp --- modules/ocf_www/manifests/site/www.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/ocf_www/manifests/site/www.pp b/modules/ocf_www/manifests/site/www.pp index e3efa269a..31fecb89c 100644 --- a/modules/ocf_www/manifests/site/www.pp +++ b/modules/ocf_www/manifests/site/www.pp @@ -186,7 +186,8 @@ port => 443, docroot => '/var/www/html', redirectmatch_status => 301, - redirectmatch_regexp => '^(.*)', + # ugly exceptions + redirectmatch_regexp => '^((?!\/\.well-known\/matrix\/(client|server)).*)', redirectmatch_dest => $canonical_url, ssl => true, From e10847182d4cf7dba512a02c6a19a9c4b1165312 Mon Sep 17 00:00:00 2001 From: Kian Date: Wed, 1 Mar 2023 20:49:32 -0800 Subject: [PATCH 45/55] Add subdomains@ RT queue (#1356) --- modules/ocf_mail/files/site_ocf/aliases | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ocf_mail/files/site_ocf/aliases b/modules/ocf_mail/files/site_ocf/aliases index 658b26273..334e3d0d1 100644 --- a/modules/ocf_mail/files/site_ocf/aliases +++ b/modules/ocf_mail/files/site_ocf/aliases @@ -100,6 +100,10 @@ security: "|/usr/bin/rt-mailgate --queue 'security' --action correspond --url ht security-comment: "|/usr/bin/rt-mailgate --queue 'security' --action comment --url https://rt.ocf.berkeley.edu" owner-security: root +subdomains: "|/usr/bin/rt-mailgate --queue 'subdomains' --action correspond --url https://rt.ocf.berkeley.edu" +subdomains-comment: "|/usr/bin/rt-mailgate --queue 'subdomains' --action comment --url https://rt.ocf.berkeley.edu" +owner-subdomains: root + techtalks: "|/usr/bin/rt-mailgate --queue 'techtalks' --action correspond --url https://rt.ocf.berkeley.edu" techtalks-comment: "|/usr/bin/rt-mailgate --queue 'techtalks' --action comment --url https://rt.ocf.berkeley.edu" owner-techtalks: root From 4086720b50c6dece1a3cf855e9fe2ec3d16b7a86 Mon Sep 17 00:00:00 2001 From: Kian Date: Wed, 1 Mar 2023 23:47:36 -0800 Subject: [PATCH 46/55] Add subdomains shorturl (#1357) --- modules/ocf_www/manifests/site/shorturl.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index 82dac0e98..7c7d6c9f4 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -145,6 +145,7 @@ {rewrite_rule => '^/starter-?tasks$ https://www.ocf.berkeley.edu/docs/staff/startertasks/ [R]'}, {rewrite_rule => '^/stats(/.*)?$ https://www.ocf.berkeley.edu/stats$1 [R]'}, {rewrite_rule => '^/status$ https://status.ocf.berkeley.edu/ [R]'}, + {rewrite_rule => '^/subdomains$ https://www.ocf.berkeley.edu/announcements/2023-03-01/subdomains [R]'}, {rewrite_rule => '^/swag$ https://ocf.io/s/swag [R]'}, {rewrite_rule => '^/templates?$ https://templates.ocf.berkeley.edu/ [R]'}, {rewrite_rule => '^/buster$ https://www.ocf.berkeley.edu/docs/staff/backend/buster/ [R]'}, From e871503c3a93a07205b02d368cd8b52d0eeaeafd Mon Sep 17 00:00:00 2001 From: Kian Date: Wed, 5 Apr 2023 14:00:10 -0700 Subject: [PATCH 47/55] Update opstaff hiring form link (#1358) --- modules/ocf_www/manifests/site/shorturl.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index 7c7d6c9f4..7401f763f 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -72,7 +72,7 @@ {rewrite_rule => '^/guest$ https://goo.gl/forms/ImNfnZkrRrakZcIr1 [R]'}, {rewrite_rule => '^/halloween$ https://forms.gle/44kTL8Eo9Tr3GsH66 [R=301]'}, {rewrite_rule => '^/help(/.*)?$ https://www.ocf.berkeley.edu/docs$1 [R]'}, - {rewrite_rule => '^/hiring$ https://docs.google.com/forms/d/e/1FAIpQLSeuJmYPV75oRWfVKEsIPYpPod2FqRwSqSj2i2-nRnY0NDCG8A/viewform [R]'}, + {rewrite_rule => '^/hiring$ https://docs.google.com/forms/d/e/1FAIpQLSfAAZIScKfF2GPEDXx4lgxZofYFpt6JlQxM-duAKpl-1FAYww/viewform [R]'}, {rewrite_rule => '^/hiringinterest$ https://docs.google.com/forms/d/e/1FAIpQLSfuu2EN28bPksyA8h_BjRsHQvnzE8gk7gC5ySEAp50GExhUgg/viewform [R]'}, {rewrite_rule => '^/hosting$ https://www.ocf.berkeley.edu/docs/services/web/ [R]'}, {rewrite_rule => '^/hours$ https://ocf.io/lab#hours [R,NE]'}, From 7d483eed4103b1c03b6da04a02db3b9c2a7179cf Mon Sep 17 00:00:00 2001 From: Kian Date: Tue, 11 Apr 2023 08:17:21 -0700 Subject: [PATCH 48/55] Add matrix client well-known authentication issuer (#1359) --- modules/ocf_www/files/matrix-client | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/ocf_www/files/matrix-client b/modules/ocf_www/files/matrix-client index 7df15d876..69b63648e 100644 --- a/modules/ocf_www/files/matrix-client +++ b/modules/ocf_www/files/matrix-client @@ -6,6 +6,9 @@ "m.identity_server": { "base_url": "https://vector.im" }, + "m.authentication": { + "issuer": "https://auth.ocf.berkeley.edu/realms/ocf" + }, "org.matrix.msc3575.proxy": { "url": "https://slidingsync.matrix.ocf.berkeley.edu" } From 9d6bfed0481a6f3c990b3eddcc4110aeb7923be3 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 11 Apr 2023 13:24:42 -0700 Subject: [PATCH 49/55] fix: TLSv1.2 fix SSL config to support TLS 1.2, update cipher list --- manifests/site_ssl.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/site_ssl.pp b/manifests/site_ssl.pp index c0c573a62..727337005 100644 --- a/manifests/site_ssl.pp +++ b/manifests/site_ssl.pp @@ -7,11 +7,11 @@ # # This should be updated from time-to-time. if $::lsbdistcodename == 'bullseye' { - $ssl_ciphersuite = 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384' - $ssl_protocols = 'TLSv1.2 TLSv1.3' + $ssl_ciphersuite = 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256' + $ssl_protocols = '-all +TLSv1.2 +TLSv1.3' } else { $ssl_ciphersuite = 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS' - $ssl_protocols = 'TLSv1 TLSv1.1 TLSv1.2' + $ssl_protocols = '-all +TLSv1.2' } # default for Apache and Nginx vhosts From ed3adc87f5533b7eac1a7d9c5d8647f2b342929d Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Tue, 11 Apr 2023 13:58:36 -0700 Subject: [PATCH 50/55] revert emerg --- manifests/site_ssl.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/site_ssl.pp b/manifests/site_ssl.pp index 727337005..0d51e231f 100644 --- a/manifests/site_ssl.pp +++ b/manifests/site_ssl.pp @@ -8,10 +8,10 @@ # This should be updated from time-to-time. if $::lsbdistcodename == 'bullseye' { $ssl_ciphersuite = 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256' - $ssl_protocols = '-all +TLSv1.2 +TLSv1.3' + $ssl_protocols = 'TLSv1.2 TLSv1.3' } else { $ssl_ciphersuite = 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS' - $ssl_protocols = '-all +TLSv1.2' + $ssl_protocols = 'TLSv1.2' } # default for Apache and Nginx vhosts From c55c47e0461b01e312be04a711eddd94e541f482 Mon Sep 17 00:00:00 2001 From: Kian Date: Tue, 11 Apr 2023 14:31:26 -0700 Subject: [PATCH 51/55] Update GitHub SSH public key (#1360) --- modules/ocf_ldap/files/generate_gh_known_hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_ldap/files/generate_gh_known_hosts b/modules/ocf_ldap/files/generate_gh_known_hosts index b001e8f32..6bfc2a8bf 100755 --- a/modules/ocf_ldap/files/generate_gh_known_hosts +++ b/modules/ocf_ldap/files/generate_gh_known_hosts @@ -5,7 +5,7 @@ import textwrap import requests GITHUB_META = 'https://api.github.com/meta' -GITHUB_PUBLIC_KEY = 'AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==' # noqa: E501 +GITHUB_PUBLIC_KEY = 'AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=' # noqa: E501 def fetch_github_ip_range(key='git'): From 6e8d3f3c18eeb88e8a50d0b320820b3bc651d75c Mon Sep 17 00:00:00 2001 From: Kian Date: Tue, 11 Apr 2023 19:30:41 -0700 Subject: [PATCH 52/55] Fix matrix well-known typo --- modules/ocf_www/files/matrix-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_www/files/matrix-client b/modules/ocf_www/files/matrix-client index 69b63648e..4a6de663e 100644 --- a/modules/ocf_www/files/matrix-client +++ b/modules/ocf_www/files/matrix-client @@ -7,7 +7,7 @@ "base_url": "https://vector.im" }, "m.authentication": { - "issuer": "https://auth.ocf.berkeley.edu/realms/ocf" + "issuer": "https://auth.ocf.berkeley.edu/auth/realms/ocf" }, "org.matrix.msc3575.proxy": { "url": "https://slidingsync.matrix.ocf.berkeley.edu" From f360be9d896f70980758e4cb50d54b93106023f6 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sun, 16 Apr 2023 22:50:12 -0700 Subject: [PATCH 53/55] feat: smart monitoring (#1362) * feat: smart monitoring --- modules/ocf/files/smartmon.sh | 208 +++++++++++++++++++++++++++++++++ modules/ocf/manifests/init.pp | 1 + modules/ocf/manifests/smart.pp | 18 +++ 3 files changed, 227 insertions(+) create mode 100755 modules/ocf/files/smartmon.sh create mode 100644 modules/ocf/manifests/smart.pp diff --git a/modules/ocf/files/smartmon.sh b/modules/ocf/files/smartmon.sh new file mode 100755 index 000000000..1fec9e4a4 --- /dev/null +++ b/modules/ocf/files/smartmon.sh @@ -0,0 +1,208 @@ +#!/bin/bash +# Script informed by the collectd monitoring script for smartmontools (using smartctl) +# by Samuel B. (c) 2012 +# source at: http://devel.dob.sk/collectd-scripts/ + +# TODO: This probably needs to be a little more complex. The raw numbers can have more +# data in them than you'd think. +# http://arstechnica.com/civis/viewtopic.php?p=22062211 + +# Formatting done via shfmt -i 2 +# https://github.com/mvdan/sh + +# Ensure predictable numeric / date formats, etc. +export LC_ALL=C + +parse_smartctl_attributes_awk="$( + cat <<'SMARTCTLAWK' +$1 ~ /^ *[0-9]+$/ && $2 ~ /^[a-zA-Z0-9_-]+$/ { + gsub(/-/, "_"); + printf "%s_value{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $4 + printf "%s_worst{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $5 + printf "%s_threshold{%s,smart_id=\"%s\"} %d\n", $2, labels, $1, $6 + printf "%s_raw_value{%s,smart_id=\"%s\"} %e\n", $2, labels, $1, $10 +} +SMARTCTLAWK +)" + +smartmon_attrs="$( + cat <<'SMARTMONATTRS' +airflow_temperature_cel +command_timeout +current_pending_sector +ecc_error_rate +end_to_end_error +erase_fail_count +erase_fail_count_total +g_sense_error_rate +hardware_ecc_recovered +helium_level +host_reads_32mib +host_reads_mib +host_writes_32mib +host_writes_mib +load_cycle_count +media_wearout_indicator +nand_writes_1gib +offline_uncorrectable +por_recovery_count +power_cycle_count +power_on_hours +pressure_limit +program_fail_cnt_total +program_fail_count +raw_read_error_rate +reallocated_event_count +reallocated_sector_ct +reported_uncorrect +runtime_bad_block +sata_downshift_count +seek_error_rate +spin_retry_count +spin_up_time +start_stop_count +temperature_case +temperature_celsius +temperature_internal +total_lbas_read +total_lbas_written +udma_crc_error_count +uncorrectable_error_cnt +unsafe_shutdown_count +unused_rsvd_blk_cnt_tot +wear_leveling_count +workld_host_reads_perc +workld_media_wear_indic +workload_minutes +SMARTMONATTRS +)" +smartmon_attrs="$(echo "${smartmon_attrs}" | xargs | tr ' ' '|')" + +parse_smartctl_attributes() { + local disk="$1" + local disk_type="$2" + local labels="disk=\"${disk}\",type=\"${disk_type}\"" + sed 's/^ \+//g' | + awk -v labels="${labels}" "${parse_smartctl_attributes_awk}" 2>/dev/null | + tr '[:upper:]' '[:lower:]' | + grep -E "(${smartmon_attrs})" +} + +parse_smartctl_scsi_attributes() { + local disk="$1" + local disk_type="$2" + local labels="disk=\"${disk}\",type=\"${disk_type}\"" + while read -r line; do + attr_type="$(echo "${line}" | tr '=' ':' | cut -f1 -d: | sed 's/^ \+//g' | tr ' ' '_')" + attr_value="$(echo "${line}" | tr '=' ':' | cut -f2 -d: | sed 's/^ \+//g')" + case "${attr_type}" in + number_of_hours_powered_up_) power_on="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Current_Drive_Temperature) temp_cel="$(echo "${attr_value}" | cut -f1 -d' ' | awk '{ printf "%e\n", $1 }')" ;; + Blocks_sent_to_initiator_) lbas_read="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Blocks_received_from_initiator_) lbas_written="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Accumulated_start-stop_cycles) power_cycle="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + Elements_in_grown_defect_list) grown_defects="$(echo "${attr_value}" | awk '{ printf "%e\n", $1 }')" ;; + esac + done + [ -n "$power_on" ] && echo "power_on_hours_raw_value{${labels},smart_id=\"9\"} ${power_on}" + [ -n "$temp_cel" ] && echo "temperature_celsius_raw_value{${labels},smart_id=\"194\"} ${temp_cel}" + [ -n "$lbas_read" ] && echo "total_lbas_read_raw_value{${labels},smart_id=\"242\"} ${lbas_read}" + [ -n "$lbas_written" ] && echo "total_lbas_written_raw_value{${labels},smart_id=\"241\"} ${lbas_written}" + [ -n "$power_cycle" ] && echo "power_cycle_count_raw_value{${labels},smart_id=\"12\"} ${power_cycle}" + [ -n "$grown_defects" ] && echo "grown_defects_count_raw_value{${labels},smart_id=\"-1\"} ${grown_defects}" +} + +parse_smartctl_info() { + local -i smart_available=0 smart_enabled=0 smart_healthy= + local disk="$1" disk_type="$2" + local model_family='' device_model='' serial_number='' fw_version='' vendor='' product='' revision='' lun_id='' + while read -r line; do + info_type="$(echo "${line}" | cut -f1 -d: | tr ' ' '_')" + info_value="$(echo "${line}" | cut -f2- -d: | sed 's/^ \+//g' | sed 's/"/\\"/')" + case "${info_type}" in + Model_Family) model_family="${info_value}" ;; + Device_Model) device_model="${info_value}" ;; + Serial_Number) serial_number="${info_value}" ;; + Firmware_Version) fw_version="${info_value}" ;; + Vendor) vendor="${info_value}" ;; + Product) product="${info_value}" ;; + Revision) revision="${info_value}" ;; + Logical_Unit_id) lun_id="${info_value}" ;; + esac + if [[ "${info_type}" == 'SMART_support_is' ]]; then + case "${info_value:0:7}" in + Enabled) smart_available=1; smart_enabled=1 ;; + Availab) smart_available=1; smart_enabled=0 ;; + Unavail) smart_available=0; smart_enabled=0 ;; + esac + fi + if [[ "${info_type}" == 'SMART_overall-health_self-assessment_test_result' ]]; then + case "${info_value:0:6}" in + PASSED) smart_healthy=1 ;; + *) smart_healthy=0 ;; + esac + elif [[ "${info_type}" == 'SMART_Health_Status' ]]; then + case "${info_value:0:2}" in + OK) smart_healthy=1 ;; + *) smart_healthy=0 ;; + esac + fi + done + echo "device_info{disk=\"${disk}\",type=\"${disk_type}\",vendor=\"${vendor}\",product=\"${product}\",revision=\"${revision}\",lun_id=\"${lun_id}\",model_family=\"${model_family}\",device_model=\"${device_model}\",serial_number=\"${serial_number}\",firmware_version=\"${fw_version}\"} 1" + echo "device_smart_available{disk=\"${disk}\",type=\"${disk_type}\"} ${smart_available}" + echo "device_smart_enabled{disk=\"${disk}\",type=\"${disk_type}\"} ${smart_enabled}" + [[ "${smart_healthy}" != "" ]] && echo "device_smart_healthy{disk=\"${disk}\",type=\"${disk_type}\"} ${smart_healthy}" +} + +output_format_awk="$( + cat <<'OUTPUTAWK' +BEGIN { v = "" } +v != $1 { + print "# HELP smartmon_" $1 " SMART metric " $1; + print "# TYPE smartmon_" $1 " gauge"; + v = $1 +} +{print "smartmon_" $0} +OUTPUTAWK +)" + +format_output() { + sort | + awk -F'{' "${output_format_awk}" +} + +smartctl_version="$(/usr/sbin/smartctl -V | head -n1 | awk '$1 == "smartctl" {print $2}')" + +echo "smartctl_version{version=\"${smartctl_version}\"} 1" | format_output + +if [[ "$(expr "${smartctl_version}" : '\([0-9]*\)\..*')" -lt 6 ]]; then + exit +fi + +device_list="$(/usr/sbin/smartctl --scan-open | awk '/^\/dev/{print $1 "|" $3}')" + +for device in ${device_list}; do + disk="$(echo "${device}" | cut -f1 -d'|')" + type="$(echo "${device}" | cut -f2 -d'|')" + active=1 + echo "smartctl_run{disk=\"${disk}\",type=\"${type}\"}" "$(TZ=UTC date '+%s')" + # Check if the device is in a low-power mode + /usr/sbin/smartctl -n standby -d "${type}" "${disk}" > /dev/null || active=0 + echo "device_active{disk=\"${disk}\",type=\"${type}\"}" "${active}" + # Skip further metrics to prevent the disk from spinning up + test ${active} -eq 0 && continue + # Get the SMART information and health + /usr/sbin/smartctl -i -H -d "${type}" "${disk}" | parse_smartctl_info "${disk}" "${type}" + # Get the SMART attributes + case ${type} in + sat) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_attributes "${disk}" "${type}" ;; + sat+megaraid*) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_attributes "${disk}" "${type}" ;; + scsi) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_scsi_attributes "${disk}" "${type}" ;; + megaraid*) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_scsi_attributes "${disk}" "${type}" ;; + nvme*) /usr/sbin/smartctl -A -d "${type}" "${disk}" | parse_smartctl_scsi_attributes "${disk}" "${type}" ;; + *) + (>&2 echo "disk type is not sat, scsi, nvme or megaraid but ${type}") + exit + ;; + esac +done | format_output diff --git a/modules/ocf/manifests/init.pp b/modules/ocf/manifests/init.pp index 2856065df..6b6f8e7c9 100644 --- a/modules/ocf/manifests/init.pp +++ b/modules/ocf/manifests/init.pp @@ -19,6 +19,7 @@ include ocf::puppet include ocf::rootpw include ocf::serial_getty + include ocf::smart include ocf::systemd include ocf::utils include ocf::walldeny diff --git a/modules/ocf/manifests/smart.pp b/modules/ocf/manifests/smart.pp new file mode 100644 index 000000000..4f5e62d56 --- /dev/null +++ b/modules/ocf/manifests/smart.pp @@ -0,0 +1,18 @@ +class ocf::smart { + if !str2bool($::is_virtual) { + file { + '/usr/local/sbin/smartmon.sh': + source => 'puppet:///modules/ocf/smartmon.sh', + owner => root, + group => root, + mode => '0755', + require => Package['smartmontools']; + } + cron { + 'smartmon': + command => '/usr/local/sbin/smartmon.sh | sponge /srv/prometheus/smartmon.prom', + minute => '*/5', + require => [Package['smartmontools'], File['/usr/local/sbin/smartmon.sh']]; + } + } +} From a2035a115f6ef41867c6da616db3230a32217a2e Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Sun, 30 Apr 2023 02:25:25 -0700 Subject: [PATCH 54/55] fix: do not send intermediate snapshots --- modules/ocf_backups/files/backup-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index 09a813740..786f81171 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -16,4 +16,4 @@ NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) echo "$CURRENT_SNAPSHOT" echo "$NEW_SNAPSHOT" -syncoid -r --no-sync-snap --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot +syncoid -r --no-stream --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot From d66a8d1a011cadd9bb6a223a1d27acdcca9b258b Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Fri, 4 Aug 2023 05:28:44 -0700 Subject: [PATCH 55/55] Revert "fix: do not send intermediate snapshots" This reverts commit a2035a115f6ef41867c6da616db3230a32217a2e. --- modules/ocf_backups/files/backup-zfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/backup-zfs.sh b/modules/ocf_backups/files/backup-zfs.sh index 786f81171..09a813740 100755 --- a/modules/ocf_backups/files/backup-zfs.sh +++ b/modules/ocf_backups/files/backup-zfs.sh @@ -16,4 +16,4 @@ NEW_SNAPSHOT=$(cat $CURRENT_SNAPSHOT_FILE) echo "$CURRENT_SNAPSHOT" echo "$NEW_SNAPSHOT" -syncoid -r --no-stream --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot +syncoid -r --no-sync-snap --sendoptions "L w c" backup/encrypted/rsnapshot "$OFFSITE_HOST":data1/ocfbackup/encrypted/rsnapshot