SVCPLAN-4857 new script to run generic SQL on a remote PG server

remote_control replaces jiractl
update mk_test_server for confluence
rename scripts for confluence
update fix_network for confluence server
fix_app_config fixed for Confluence
add rcp remote-copy
clean_xml.sh
start/stop services; upgrade_app
fix_java_heap_size.sh
set_web_access
move get_installer to utils
add elapsed time in upgrade
systemd wants=postgresql
move confluence sql to its own cleanup foldeer
split cleanup into jira & wiki specific dirs
update versions for workflow actions
remove COPY cleanup from Dockerfile
add workflow env
use version main in env
deactivate user working
quiet grep & puppet resource disable
use lvmdevices to deldev
HeapDumpOnOutOfMemoryError
update for production PM
update .gitignore
Complete sso_bypass.sh
Add option to specify key and additional errors

SVCPLAN-4131 migrate to JSM

go_log_rotate.sh go_mk_atlassian_supportzip.sh
add check for cmdline action in go_upgrade_app
attachments tarpipe script
migrate attachments working
enable service after installing service file
update api functions to work with netrc
script to enable/disable mail delivery

.gitignore

@@ -7,10 +7,19 @@
 *.cnf
 
 # test files
-*.input
+*.csv
 *.ids
-output
+*.input
 *.sample
-*.csv
 *.tsv
+*.txt
+NOTES
+output
+
+# log files
 *.log
+
+# working files
+*.swp
+*.tar
+attachments/

Dockerfile

@@ -11,7 +11,7 @@ COPY requirements.txt ./
 COPY jira_cleanup jira_cleanup/
 COPY wiki_cleanup wiki_cleanup/
 RUN python -m pip install -r /srv/requirements.txt
-RUN ln -s /home/.netrc /root/.netrc
-RUN ln -s /home/.atlassian-tools-config.ini /root/.atlassian-tools-config.ini
+#RUN ln -s /home/.netrc /root/.netrc
+#RUN ln -s /home/.atlassian-tools-config.ini /root/.atlassian-tools-config.ini
 
 CMD ["bash"]

bin/fix_app_config.sh

@@ -67,6 +67,7 @@ fi
 # See also:
 # https://confluence.atlassian.com/confkb/how-to-adjust-the-session-timeout-for-confluence-126910597.html
 # https://confluence.atlassian.com/jirakb/change-the-default-session-timeout-to-avoid-user-logout-in-jira-server-604209887.html
+# TODO - Is this needed anymore after change to SSO via CiLogon?
 WEB_XML=''
 if [[ $APP_NAME == 'jira' ]] ; then
   WEB_XML="$APP_INSTALL_DIR"/atlassian-jira/WEB-INF/web.xml

bin/fix_vgs.sh

@@ -6,8 +6,9 @@ YES=0
 NO=1
 VERBOSE=$YES
 DEBUG=$YES
+APP_NAME=jira
 
-VGS=( vg_pgsql vg_backups vg_confluence )
+VGS=( vg_pgsql vg_backups vg_$APP_NAME )
 
 ECHO=
 [[ $DEBUG -eq $YES ]] && ECHO=echo

bin/go_log_rotate.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/bash
+
+# See also: https://jira.ncsa.illinois.edu/browse/SVC-24653
+
+
+###
+# Global Variables
+###
+YES=0
+NO=1
+LOG_DIR=/var/log
+ARCHIVE_DIR=/backups/old_logs
+MAX_ARCHIVE_AGE=30 #days
+DATE=$(date +%Y%m%d_%H%M%S)
+VERBOSE=$YES
+DEBUG=$NO
+#DEBUG=$YES
+
+
+###
+# Functions
+###
+
+die() {
+  echo "ERROR: ${*}" 1>&2
+  exit 99
+}
+
+
+force_rotate_logs() {
+  echo "${FUNCNAME[0]} ..."
+  local _vopts _dopts
+  [[ $VERBOSE -eq $YES ]] && {
+    set -x
+    _vopts+=( '-v' )
+  }
+  [[ $DEBUG -eq $YES ]] && _dopts+=( '-d' )
+  /usr/sbin/logrotate -f ${_vopts[@]} ${_dopts[@]} /etc/logrotate.conf
+  echo "${FUNCNAME[0]} OK"
+}
+
+
+archive_logs() {
+  echo "${FUNCNAME[0]} ..."
+  local _vopts
+  [[ $VERBOSE -eq $YES ]] && {
+    set -x
+    _vopts+=( '-v' )
+  }
+  [[ $DEBUG -eq $YES ]] && _dopts+=( '-d' )
+  local _tgz="${ARCHIVE_DIR}"/"${DATE}".tgz
+  local _tmp=$( mktemp -p "${ARCHIVE_DIR}" )
+  >"${_tmp}" find "${LOG_DIR}" -type f \
+    -name '*.[0-9]' \
+    -o -name '*.gz' \
+    -o -regextype egrep -regex '.*-[0-9]{8}'
+  if [[ $DEBUG -eq $YES ]] ; then
+    echo "DEBUG - files that would have been archived:"
+    cat "${_tmp}"
+  elif [[ -s "${_tmp}" ]] ; then
+    tar -zcf "${_tgz}" ${_vopts[@]} -T "${_tmp}" --remove-files
+  fi
+  rm "${_tmp}"
+  echo "${FUNCNAME[0]} OK"
+}
+
+
+clean_old_logs() {
+  echo "${FUNCNAME[0]} ..."
+  [[ $VERBOSE -eq $YES ]] && set -x
+  local _action='-delete'
+  [[ $DEBUG -eq $YES ]] && _action='-print'
+  find "${ARCHIVE_DIR}" -type f -mtime +${MAX_ARCHIVE_AGE} $_action
+  echo "${FUNCNAME[0]} OK"
+}
+
+
+###
+# MAIN
+###
+
+[[ $VERBOSE -eq $YES ]] && set -x
+
+force_rotate_logs
+
+archive_logs
+
+clean_old_logs

bin/go_mk_atlassian_supportzip.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/bash
+
+# See also: https://jira.ncsa.illinois.edu/browse/SVCPLAN-5454
+
+
+###
+# Global Variables
+###
+YES=0
+NO=1
+CHOME=/srv/confluence/home
+CAPP=/srv/confluence/app
+DATE=$(date +%Y%m%d_%H%M%S)
+SUPPORTZIP=/root/${DATE}.support.zip
+CPUZIP=/root/${DATE}.cpu_usage.zip
+VERBOSE=$NO
+
+
+###
+# Functions
+###
+
+die() {
+  echo "ERROR: ${*}" 1>&2
+  exit 99
+}
+
+
+get_confluence_pid() {
+  [[ $VERBOSE -eq $YES ]] && set -x
+  local _pid=$( systemctl show -p MainPID --value confluence )
+  local _cmd=$( ps -p $_pid -o comm= )
+  local _usr=$( ps -p $_pid -o user= )
+  [[ "$_cmd" != "java" ]] && die "Unknown command '$_cmd' for pid '$_pid' ... expected 'java'"
+  [[ "$_usr" != "confluence" ]] && die "Unknown user '$_usr' for pid '$_pid' ... expected 'confluence'"
+  echo "$_pid"
+}
+
+
+dump_cpu_threads() {
+  # Get Thread dumps and CPU usage information
+  [[ $VERBOSE -eq $YES ]] && set -x
+  echo "Dump CPU Threads (this will take a few minutes) ..."
+  local _tempdir=$( mktemp -d )
+  pushd "$_tempdir"
+  for i in $(seq 6); do
+    top -b -H -p $CONF_PID -n 1 > conf_cpu_usage.$(date +%s).txt
+    kill -3 $CONF_PID
+    sleep 10
+  done
+  echo "... Dump CPU Threads OK"
+
+  echo "Make CPU Threads zip ..."
+  zip -q $CPUZIP conf_cpu_usage.*.txt
+  echo "... Make CPU Threads zip OK"
+
+  popd
+  rm -rf "$_tempdir"
+}
+
+
+mk_support_zip() {
+  [[ $VERBOSE -eq $YES ]] && set -x
+  echo "Make support zip (this will also take a minute) ..."
+  zip -q $SUPPORTZIP \
+    ${CHOME}/confluence.cfg.xml \
+    ${CHOME}/logs/* \
+    ${CAPP}/logs/* \
+    ${CAPP}/conf/server.xml \
+    ${CAPP}/bin/setenv.sh
+  echo "... Make support zip OK"
+}
+
+###
+# MAIN
+###
+
+[[ $VERBOSE -eq $YES ]] && set -x
+
+CONF_PID=$( get_confluence_pid )
+
+dump_cpu_threads
+
+mk_support_zip
+
+echo
+echo "Atlassian support files:"
+ls -l /root/${DATE}*.zip
+echo

bin/go_upgrade_app.sh

@@ -22,13 +22,18 @@ ABLEMENT=''
 ###
 
 
-run_installer() {
+assert_app_installed() {
   [[ -d "${APP_INSTALL_DIR}"/bin ]] || die "app install dir '$APP_INSTALL_DIR/bin' not found"
+}
 
+assert_installer_exists() {
   [[ -z "$INSTALLER" ]] && INSTALLER=$( get_installer )
 
   [[ -f "$INSTALLER" ]] || die "Installer file not found; '$INSTALLER' "
+}
+
 
+run_installer() {
   /usr/bin/chmod +x "${INSTALLER}"
   "${INSTALLER}"
 }
@@ -40,11 +45,14 @@ print_usage() {
 ${_prg}
   Upgrade the Atlassian app specified in config file '$CFG'
 SYNOPSYS
-  ${_prg} [OPTIONS] <--start|--finish> [installer_file.bin]
-OPTIONS
-  -h --help Print this help and exit
+  ${_prg} [OPTIONS] <--start | --finish>
+
+USAGE
   --start  Start an upgrade (stop services, install upgrade, etc.)
   --finish Finish an upgrade (re-enable services, re-enable notifications, etc.)
+
+OPTIONS
+  -h --help Print this help and exit
 ENDHERE
 }
 
@@ -73,10 +81,12 @@ while [[ $# -gt 0 ]] && [[ $ENDWHILE -eq 0 ]] ; do
   shift
 done
 
-INSTALLER="$1"
-
 if [[ $ACTION == 'start' ]] ; then
 
+  INSTALLER=$( get_installer )
+  assert_app_installed
+  assert_installer_exists
+
   "${BIN}"/backup_config_files.sh
 
   "${BIN}"/set_services.sh $ABLEMENT
@@ -99,6 +109,10 @@ elif [[ $ACTION == 'finish' ]] ; then
 
   "${BIN}"/set_services.sh $ABLEMENT
 
+  "${BIN}"/set_web_access.sh $ABLEMENT
+
+else
+  die "Missing one of '--start' | '--finish'"
 fi
 
 echo "Elapsed time: '$SECONDS' seconds."

bin/mk_systemd.sh

@@ -47,8 +47,15 @@ ENDHERE
 }
 
 
+enable_service() {
+  systemctl enable ${APP_NAME}.service
+}
+
+
 ###
 # MAIN
 ###
 
 mk_service_file
+
+enable_service

bin/pg_run_sql.sh

@@ -39,8 +39,8 @@ get_files() {
 prep() {
   CONF=~/.pgpass.cnf.${HOST}
   [[ -f "${CONF}" ]] || die "config file not found '${CONF}'"
-  rsync "${CONF}" ${HOST}:.pgpass
-  ssh ${HOST} chmod 600 .pgpass
+  rsync "${CONF}" ${HOST}:.pgpass || die 'failed to rsync pgpass file'
+  ssh ${HOST} chmod 600 .pgpass || die 'failed to chmod remote pgpass file'
 }
 
 

bin/restore_config_files.sh

@@ -16,7 +16,7 @@ FILES=(
 for f in "${FILES[@]}" ; do
   tgt_dir=$( dirname "$f" )
   reference=$( find "$tgt_dir" -type f -print -quit )
-  cp "${BACKUP_DIR}/${f}" "${f}"
+  cp "${BACKUP_DIR}/${f}" "${f}"        #BACKUP_DIR defined in config.sh
   chmod --reference="$reference" "$f"
   chown --reference="$reference" "$f"
 done