Merge pull request github#238 from github/snh/benchmark-backups

Benchmark backups
Guidebook · Jul 26, 2016 · ef4c5ef · ef4c5ef
2 parents 405ad4b + cb51a79
commit ef4c5ef
Show file tree

Hide file tree

Showing 20 changed files with 97 additions and 0 deletions.
diff --git a/bin/ghe-backup b/bin/ghe-backup
@@ -128,17 +128,23 @@ ghe-backup-settings ||
 failures="$failures settings"
 
 echo "Backing up SSH authorized keys ..."
+bm_start "ghe-export-authorized-keys"
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json ||
 failures="$failures authorized-keys"
+bm_end "ghe-export-authorized-keys"
 
 echo "Backing up SSH host keys ..."
+bm_start "ghe-export-ssh-host-keys"
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
 failures="$failures ssh-host-keys"
+bm_end "ghe-export-ssh-host-keys"
 
 echo "Backing up MySQL database ..."
+bm_start "ghe-export-mysql"
 echo 'set -o pipefail; ghe-export-mysql | gzip' |
 ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz ||
 failures="$failures mysql"
+bm_end "ghe-export-mysql"
 
 echo "Backing up Redis database ..."
 if $cluster; then

diff --git a/share/github-backup-utils/ghe-backup-alambic-cluster b/share/github-backup-utils/ghe-backup-alambic-cluster
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root backup snapshot directory based on config
 host="$GHE_HOSTNAME"
 backup_dir="$GHE_SNAPSHOT_DIR/storage"
@@ -67,6 +69,7 @@ if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/st
 fi
 
 for hostname in $hostnames; do
+  bm_start "$(basename $0) - $hostname"
   echo 1>&3
   echo "* Starting backup for host: $hostname"
   # Sync all auxiliary repository data. This includes files and directories like
@@ -82,4 +85,7 @@ for hostname in $hostnames; do
       $link_dest \
       "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \
       "$GHE_SNAPSHOT_DIR/storage" 1>&3
+  bm_end "$(basename $0) - $hostname"
 done
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root elastic backup directory based on config
 host="$GHE_HOSTNAME"
 
@@ -39,3 +41,5 @@ for index in $indices; do
     ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:$es_port/$index\"" | gzip > $GHE_SNAPSHOT_DIR/audit-log/$index.gz
   fi
 done
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-es-hookshot b/share/github-backup-utils/ghe-backup-es-hookshot
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root elastic backup directory based on config
 host="$GHE_HOSTNAME"
 
@@ -29,3 +31,5 @@ for index in $indices; do
     ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json 'http://localhost:9201/$index'" | gzip > $GHE_SNAPSHOT_DIR/hookshot/$index.gz
   fi
 done
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root elastic backup directory based on config
 host="$GHE_HOSTNAME"
 
@@ -93,3 +95,5 @@ ghe-rsync -avz \
     --exclude='elasticsearch.yml' \
     "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
     "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-es-tarball b/share/github-backup-utils/ghe-backup-es-tarball
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Snapshot all Elasticsearch data or fake it when no /data/elasticsearch
 # directory exists.
 echo "
@@ -18,3 +20,5 @@ echo "
         tar cvf - --files-from /dev/null
     fi
 " | ghe-ssh "$GHE_HOSTNAME" /bin/sh > "$GHE_SNAPSHOT_DIR"/elasticsearch.tar
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-fsck b/share/github-backup-utils/ghe-backup-fsck
@@ -7,6 +7,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 echo "Running git fsck on repos..."
 
 # Verify git is available.
@@ -83,3 +85,5 @@ done
 echo "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s"
 
 rm -f $log
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-git-hooks-cluster b/share/github-backup-utils/ghe-backup-git-hooks-cluster
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Verify rsync is available.
 if ! rsync --version 1>/dev/null 2>&1; then
     echo "Error: rsync not found." 1>&2

diff --git a/share/github-backup-utils/ghe-backup-pages-cluster b/share/github-backup-utils/ghe-backup-pages-cluster
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root backup snapshot directory based on config
 host="$GHE_HOSTNAME"
 backup_dir="$GHE_SNAPSHOT_DIR/pages"
@@ -67,6 +69,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page
 fi
 
 for hostname in $hostnames; do
+  bm_start "$(basename $0) - $hostname"
   echo 1>&3
   echo "* Starting backup for host: $hostname"
   # Sync all auxiliary repository data. This includes files and directories like
@@ -82,4 +85,7 @@ for hostname in $hostnames; do
       $link_dest \
       "$hostname:$GHE_REMOTE_DATA_USER_DIR/pages/" \
       "$GHE_SNAPSHOT_DIR/pages" 1>&3
+  bm_end "$(basename $0) - $hostname"
 done
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-pages-rsync b/share/github-backup-utils/ghe-backup-pages-rsync
@@ -9,8 +9,12 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$GHE_SNAPSHOT_DIR/pages"
 
 # Use the common user data rsync backup utility.
 ghe-backup-userdata pages
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-pages-tarball b/share/github-backup-utils/ghe-backup-pages-tarball
@@ -9,6 +9,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Snapshot all Pages data or fake it when no /data/pages directory exists.
 echo '
     if [ -d /data/pages ]; then
@@ -17,3 +19,5 @@ echo '
         tar cvf - --files-from /dev/null
     fi
 ' | ghe-ssh "$GHE_HOSTNAME" /bin/sh > "$GHE_SNAPSHOT_DIR"/pages.tar
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-redis b/share/github-backup-utils/ghe-backup-redis
@@ -11,6 +11,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
 
@@ -26,3 +28,5 @@ ghe-ssh "$GHE_HOSTNAME" /bin/sh <<EOF
     done
     $sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb'
 EOF
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-redis-cluster b/share/github-backup-utils/ghe-backup-redis-cluster
@@ -11,6 +11,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$GHE_HOSTNAME"
 
@@ -34,3 +36,5 @@ ghe-ssh "$GHE_HOSTNAME" /bin/sh <<EOF
       $sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb'
     fi
 EOF
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-repositories-cluster b/share/github-backup-utils/ghe-backup-repositories-cluster
@@ -34,6 +34,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Split host:port into parts
 port=$(ssh_port_part "$GHE_HOSTNAME")
 host=$(ssh_host_part "$GHE_HOSTNAME")
@@ -129,6 +131,7 @@ rsync_repository_data () {
 
 
 for hostname in $hostnames; do
+  bm_start "$(basename $0) - $hostname"
   echo 1>&3
   echo "* Starting backup for host: $hostname"
   # Sync all auxiliary repository data. This includes files and directories like
@@ -286,4 +289,7 @@ RULES
 + /info/*
 RULES
   echo 1>&3
+  bm_end "$(basename $0) - $hostname"
 done
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-repositories-rsync b/share/github-backup-utils/ghe-backup-repositories-rsync
@@ -34,6 +34,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Set up remote host and root backup snapshot directory based on config
 host="$GHE_HOSTNAME"
 backup_dir="$GHE_SNAPSHOT_DIR/repositories"
@@ -240,3 +242,5 @@ rsync_repository_data <<RULES
 + /info/*
 RULES
 echo 1>&3
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-repositories-tarball b/share/github-backup-utils/ghe-backup-repositories-tarball
@@ -9,5 +9,9 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 # Snapshot all Git repository data
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-repositories' > "$GHE_SNAPSHOT_DIR"/repositories.tar
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings
@@ -9,6 +9,8 @@ set -e
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$host"
 
+bm_start "$(basename $0)"
+
 # Grab the host
 host="$GHE_HOSTNAME"
 
@@ -50,3 +52,5 @@ if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then
     exit 1
   fi
 fi
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-store-version b/share/github-backup-utils/ghe-backup-store-version
@@ -6,6 +6,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0)"
+
 if [ -d $GHE_BACKUP_ROOT/.git ]; then
   version_info="$BACKUP_UTILS_VERSION"
   ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true)
@@ -15,3 +17,5 @@ if [ -d $GHE_BACKUP_ROOT/.git ]; then
   echo "$version_info" |
   ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1"
 fi
+
+bm_end "$(basename $0)"
diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata
@@ -8,6 +8,8 @@ set -e
 # Bring in the backup configuration
 . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config
 
+bm_start "$(basename $0) - $1"
+
 # Verify rsync is available.
 if ! rsync --version 1>/dev/null 2>&1; then
     echo "Error: rsync not found." 1>&2
@@ -57,3 +59,5 @@ ghe-rsync -avz \
     $link_dest \
     "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \
     "$GHE_SNAPSHOT_DIR/$dirname" 1>&3
+
+bm_end "$(basename $0) - $1"
diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh
@@ -234,6 +234,21 @@ begin_test "ghe-backup subsequent snapshot"
 )
 end_test
 
+begin_test "ghe-backup logs the benchmark"
+(
+  set -e
+
+  # wait a second for snapshot timestamp
+  sleep 1
+
+  export BM_TIMESTAMP=foo
+
+  ghe-backup
+
+  [ $(grep took $GHE_DATA_DIR/current/benchmarks/benchmark.foo.log | wc -l) -gt 1 ]
+)
+end_test
+
 begin_test "ghe-backup with relative data dir path"
 (
     set -e