Merge branch 'master' of https://github.com/performancecopilot/pcp

build/rpm/fedora.spec

@@ -3351,7 +3351,8 @@ cd
 
 %changelog
 * Fri Jun 15 2018 Nathan Scott <[email protected]> - 4.1.0-1
-- Work-in-progress, see https://pcp.io/roadmap
+- Rapid compression of PCP log data and metadata (BZ 1293471)
+- Update to latest PCP sources.
 
 * Fri May 11 2018 Mark Goodwin <[email protected]> - 4.0.2-1
 - Propogate build flags throughout PCP (BZ 1538187)

qa/4751

@@ -1,6 +1,9 @@
 #!/bin/sh
 # PCP QA Test No. 4751
 # multi-threaded context-creation
+#
+# See also QA 953 for the non-helgrind version of the same test.
+#
 
 seq=`basename $0`
 echo "QA output created by $seq"
@@ -17,7 +20,7 @@ echo "QA output created by $seq"
 # Similarly on slow VMs we sometimes also see SIGALRM while calling
 # pthread_mutex_lock().  Skip these VMs also.
 #
-case `hostname`
+case `hostname -s`
 in
     vm00|vm01|vm02|vm03|vm05|vm07|vm18|vm28|vm30|vm34)
 	_notrun "multithread10 and helgrind are not happy campers on `hostname`"

qa/675

@@ -0,0 +1,40 @@
+#!/bin/sh
+# PCP QA Test No. 675
+# Test reserved by kenj on Fri 15 Jun 11:34:17 AEST 2018
+# [what am I here for?]
+#
+# Copyright (c) 2018 [who are you?].  All Rights Reserved.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+# get standard environment, filters and checks
+. ./common.product
+. ./common.filter
+. ./common.check
+
+# test for-some-thing || _notrun No support for some-thing
+
+_cleanup()
+{
+    cd $here
+    $sudo rm -rf $tmp $tmp.*
+}
+
+status=1	# failure is the default!
+$sudo rm -rf $tmp $tmp.* $seq.full
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# real QA test starts here
+
+# if error
+exit
+
+# optional stuff if your test has verbose output to help resolve problems
+#echo
+#echo "If failure, check $seq.full"
+
+# success, all done
+status=0
+exit

qa/675.out

@@ -0,0 +1,2 @@
+QA output created by 675
+[skeleton from now, replace me]

qa/713

@@ -75,6 +75,10 @@ yes | pminfo -h $hostname -f hinv.ncpu 2>&1 | tee -a $seq.full | nss_filter_pmin
 
 unset PMPROXY_HOST PCP_SECURE_SOCKETS
 
+echo >>$here/$seq.full
+echo "--- pmproxy log after all done ---" >>$here/$seq.full
+cat $tmp.log >>$here/$seq.full
+
 # success, all done
 status=0
 exit

qa/group

@@ -1046,6 +1046,7 @@ BAD
 672 valgrind libpcp pmval local
 673 pmdumplog local decompress-xz
 674 pcp local
+675:reserved pmlogrewrite
 677 pmprobe archive multi-archive decompress-xz local pmlogextract
 684 pmstat archive multi-archive decompress-xz local pmlogextract
 689 pmclient archive multi-archive decompress-xz local sanity pmlogextract python

qa/new

@@ -4,7 +4,7 @@
 #
 # Copyright (c) 1997-2002 Silicon Graphics, Inc.  All Rights Reserved.
 #
-# Usage: new [-n|-s|-v] [base-seq-no group ...]
+# Usage: new [-n|-r|-s|-v] [base-seq-no group ...]
 #
 # base-seq-no defaults to some random place in the range $min-$max
 #
@@ -18,20 +18,34 @@ max_probes=200
 
 _usage()
 {
-    echo >&2 "Usage: new [-n|-s|-v] [starting-seq-no [group ...]]"
+    echo >&2 "Usage: new [options] [starting-seq-no [group ...]]"
+    echo >&2
+    echo >&2 "Options:"
+    echo >&2 "-n   show-me, change nothing"
+    echo >&2 "-r   reserve (and skeletal test added)"
+    echo >&2 "-s   pick smallest unallocated test number"
+    echo >&2 "     [default and preferred is to choose at random]"
+    echo >&2 "-v   verbose"
+    echo >&2
+    echo >&2 "starting-seq-no changes where search starts, but maybe - to ignore"
+    echo >&2 "if group(s) specified on the command line"
+
     exit 1
 }
 
 showme=false
+reserve=false
 verbose=false
 smallest=false
 random=true
-while getopts 'nsv?' p
+while getopts 'nrsv?' p
 do
     case "$p"
     in
 	n)	showme=true
 		;;
+	r)	reserve=true
+		;;
 	s)	smallest=true
 		random=false
 		;;
@@ -120,20 +134,30 @@ then
     echo "Warning: ${MAKE:-make} failed -- some tests may be missing"
 fi
 
+try=""
 if [ $# -gt 0 ]
 then
-    try=`printf "%03d\n" $1`
-    random=false
+    if [ x"$1" != x- ]
+    then
+	try=`printf "%03d\n" $1`
+	random=false
+    fi
     shift
-elif $smallest
+fi
+if [ -z "$try" ]
 then
-    try=000
-else
-    # random in the range $min-$max based on user name convered from alpha
-    # to numeric, prefixed by current clock seconds
-    #
-    seed="`date +%S``echo $USER | tr 'a-z' '0-90-90-9'`"
-    try=`awk 'BEGIN { srand('"$seed"'); printf "%03d\n",int('$min'+rand()*('$max'-'$min')) }' </dev/null`
+    if $smallest
+    then
+	# as of 15 Jun 2018, we're all full up to 650 and a bit
+	#
+	try=650
+    else
+	# random in the range $min-$max based on user name convered from alpha
+	# to numeric, prefixed by current clock seconds
+	#
+	seed="`date +%S``echo $USER | tr 'a-z' '0-90-90-9'`"
+	try=`awk 'BEGIN { srand('"$seed"'); printf "%03d\n",int('$min'+rand()*('$max'-'$min')) }' </dev/null`
+    fi
 fi
 
 # first $try, then keep randomly probing the min-max space
@@ -174,7 +198,7 @@ then
     exit
 fi
 
-echo "Next test is $id"
+echo "Allocated test number $id"
 
 if [ -f $id ]
 then
@@ -183,11 +207,15 @@ then
     exit
 fi
 
-$PCP_ECHO_PROG $PCP_ECHO_N "Creating skeletal script for you to edit ...""$PCP_ECHO_C"
-
 cat <<End-of-File >$id
 #!/bin/sh
 # PCP QA Test No. $id
+End-of-File
+if $reserve
+then
+    echo "# Test reserved by $LOGNAME on `date`" >>$id
+fi
+cat <<End-of-File >>$id
 # [what am I here for?]
 #
 # Copyright (c) `date +%Y` [who are you?].  All Rights Reserved.
@@ -226,12 +254,19 @@ exit
 status=0
 exit
 End-of-File
-
-sleep 2		# latency to read messages to this point
-echo ""
-
 chmod 755 $id
-${EDITOR-vi} $id
+
+if $reserve
+then
+    echo "Creating skeletal $id script"
+    tag=":reserved"
+else
+    $PCP_ECHO_PROG $PCP_ECHO_N "Creating skeletal $id script for you to edit ...""$PCP_ECHO_C"
+    sleep 2		# latency to read messages to this point, before vi(1)
+    echo ""
+    ${EDITOR-vi} $id
+    tag=""
+fi
 
 if [ $# -eq 0 ]
 then
@@ -265,7 +300,7 @@ else
     #
     for g in $*
     do
-	if grep "^$g[ 	]" group >/dev/null
+	if sed -e 's/$/ /' <group | grep "^$g[ 	]" >/dev/null
 	then
 	    :
 	else
@@ -275,9 +310,8 @@ else
     ans="$*"
 fi
 
-$PCP_ECHO_PROG $PCP_ECHO_N "Adding $id to group index ...""$PCP_ECHO_C"
-echo "$id $ans" >>group
-echo " done."
+echo "Adding $id to group index"
+echo "$id$tag $ans" >>group
 
 # sort the tests numerically
 #
@@ -288,4 +322,17 @@ state == "head" && /^[0-9]/	{ state = "list" }
 sort -n $tmp.list >>$tmp.head
 cp $tmp.head group
 
+# make a $id.out placeholder if not already done
+#
+if [ ! -f $id.out ]
+then
+    echo "Creating skeletal output, $id.out"
+    echo "QA output created by $id" >$id.out
+    echo "[skeleton from now, replace me]" >>$id.out
+fi
+
+# and finally, don't forget the git work ...
+#
+git add $id $id.out
+
 exit 0

src/pcp/dstat/GNUmakefile

@@ -16,7 +16,7 @@ TOPDIR = ../../..
 include $(TOPDIR)/src/include/builddefs
 
 TARGET = pcp-dstat
-MAN_PAGES = $(TARGET).1 #$(TARGET).conf.5
+MAN_PAGES = $(TARGET).1 $(TARGET).5
 
 SUBDIRS = plugins
 

src/pcp/dstat/TODO

@@ -1,68 +1,66 @@
 Key:
 [x]  =  implemented
-[*]  =  implemented, with caveat(s)
 [-]  =  not implemented, no plan to
-[ ]  =  not implemented, but needed
+[ ]  =  not implemented, still needed
 [?]  =  desirable, not clear if needed
 
 
 General Functionality:
 [x] initial code welding pmrep/pmconfig onto original dstat
 [x] local context fallback when no local pmcd available
 [x] make use of PCP metric units for scaling
-[ ] implement interval argument
-[ ] implement #samples argument
-[-] implement --all-plugins option
+[x] implement interval argument
+[x] implement #samples argument
+[?] implement --all-plugins option
 [x] new socket6 plugin for IPv6 socket metrics
 [x] handle dynamic instances correctly (esp. headings)
 [x] handle explicit command line instances correctly (-C|-D|-I|-N|-S)
 [x] need to handle dstat 'total' columns better?  (see instances too)
 [x] figure out why --disk and --io not usable together (same labels?)
 [ ] improved handling of first counter sample (see ticks() code)
-[ ] improved handling of invalid configuration files
 [ ] fix terminal title writing to use metric values (see code)
 [ ] fix version output writing to use metric values (see code)
 [ ] csv output option (for dstat backward compatibility)
 [x] implement sub-second time(stamp) and epoch reporting support
 [ ] implement historical reporting support via PCP archives
 [ ] create pmlogconf(1) configuration file(s)
-[ ] write additional regression tests
-[ ] write pcp-dstat.conf(5) man page
+[ ] write regression tests with archives
+[x] write pcp-dstat(5) man page
 [x] write pcp-dstat(1) man page
 
 
 Internal Plugins:
-[x] aio		(needs new vfs.aio.* kernel metrics)
+[x] aio
 [x] cpu
 [x] cpu-adv
-[*] cpu-use	(needs instance fixes for per-CPU reporting)
+[x] cpu-use
 [-] cpu24
-[*] disk	(needs instance fixes for per-dev reporting)
+[x] disk
 [-] disk24
 [-] disk24-old
 [x] epoch
 [x] fs
 [x] int
 [-] int24
-[*] io		(needs instance fixes for per-dev reporting)
+[x] io
 [x] ipc
-[*] load	(needs instance fixes for per-avg reporting)
-[x] lock	(needs new vfs.locks.* kernel metrics)
+[x] load
+[x] lock
 [x] mem
 [x] mem-adv
-[*] net		(needs new network.physical.* kernel metrics)
+[x] net
 [x] page
 [-] page24
 [x] proc
-[x] raw		(needs new network.raw.* kernel metrics)
+[x] raw
 [x] socket
 [x] swap
 [-] swap-old
 [x] sys
-[x] tcp		(needs new network.tcpconn6.* metrics)
+[x] tcp
 [x] time
-[x] udp		(needs new network.udpconn[6].* metrics)
-[x] unix	(needs new network.unixconn.* metics)
+[x] udp
+[x] unix
 [x] vm
 [x] vm-adv
 [?] zones	(compound indom, tough for derived metrics)
@@ -97,7 +95,7 @@ External Plugins:
 [?] jvm_full
 [?] jvm_vm
 [ ] lustre
-[ ] md-status		(needs more MD driver metrics)
+[ ] md-status		(needs additional MD driver metrics)
 [x] memcache-hits
 [?] mongodb_conn	(needs new MongoDB PMDA)
 [?] mongodb_mem		(needs new MongoDB PMDA)
@@ -180,10 +178,10 @@ Missing metrics:
 
 Missing PMDAs:
 [ ] acpi
-[ ] mongodb
 [?] dbus
-[?] gpfs
+[ ] gpfs
 [?] ntp
+[ ] mongodb
 [?] squid
 [?] vmware
 [?] zfs (Linux version)