From 75d81524054470c60d52121ef9965869e1070d27 Mon Sep 17 00:00:00 2001 From: Mike Horwath Date: Wed, 6 Feb 2019 14:11:55 -0600 Subject: [PATCH 1/5] adding Sumo Logic upload capability --- bin/btmon.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/bin/btmon.py b/bin/btmon.py index 7b1b735..3f4d177 100755 --- a/bin/btmon.py +++ b/bin/btmon.py @@ -1,5 +1,5 @@ #!/usr/bin/python -u -__version__ = '3.2.0' +__version__ = '3.2.1' """Data collector/processor for Brultech monitoring devices. Collect data from Brultech ECM-1240, ECM-1220, and GEM power monitors. Print @@ -706,6 +706,20 @@ mqtt_upload_period=60 +Sumo Logic Configuration: + +1) log into sumologic.com interface +2) create HTTP hook as you will use it in configuration + +By default, all channels on all ECMs will be uploaded. + +For example, this configuration will upload all data from all ECMs. + +[sumologic] +sumo_out=true +sumo_url=xxx + + Upgrading: Please consider the following when upgrading from ecmread.py: @@ -731,6 +745,9 @@ Changelog: +- 3.2.1 06feb19 mike horwath +* added Sumo Logic support + - 3.2.0 07sep16 mwall * added MQTT support (thanks to mrguessed) @@ -1215,6 +1232,7 @@ import traceback import urllib import urllib2 +import requests import warnings warnings.filterwarnings('ignore', category=DeprecationWarning) # MySQLdb in 2.6 @@ -4149,6 +4167,63 @@ def process_calculated(self, packets): dbgmsg('MQTT: Nothing to send') +class SumoLogicProcessor(UploadProcessor): + + def __init__(self, url, period, timeout): + super(SumoLogicProcessor, self).__init__() + self.url = url + self.process_period = int(period) + self.timeout = int(timeout) + + infmsg('Sumo: upload period: %d' % self.process_period) + infmsg('Sumo: timeout: %d' % self.timeout) + infmsg('Sumo: url: %s' % self.url) + + def setup(self): + if not (self.url): + print 'Sumo Logic Error: Insufficient parameters' + if not self.url: + print ' A URL is required' + sys.exit(1) + + def process_calculated(self, packets): + for p in packets: + osn = obfuscate_serial(p['serial']) + data = [] + data.append('"%s":"%.1f"' % (mklabel(osn, 'volts'), p['volts'])) + if INCLUDE_CURRENT: + for idx, c, in enumerate(PACKET_FORMAT.channels(FILTER_CURRENT)): + data.append('"%s":"%.2f"' % (mklabel(osn, c), p[c])) + for idx, c in enumerate(PACKET_FORMAT.channels(FILTER_PE_LABELS)): + data.append('"%s_w":"%.2f"' % (mklabel(osn, c), p[c + '_w'])) + # do you need Wh? + # for idx, c in enumerate(PACKET_FORMAT.channels(FILTER_PE_LABELS)): + # data.append('"%s_wh":"%.2f"' % (mklabel(osn, c), p[c + '_wh'])) + # do you need pulse? + # for idx, c in enumerate(PACKET_FORMAT.channels(FILTER_PULSE)): + # data.append('"%s":"%d"' % (mklabel(osn, c), p[c])) + # do you need sensor? + # for idx, c in enumerate(PACKET_FORMAT.channels(FILTER_SENSOR)): + # data.append('"%s":"%.2f"' % (mklabel(osn, c), p[c])) + if len(data): + url = '%s' % (self.url) + payload = '{"time_created":"%s",%s}' % ( + p['time_created'], ','.join(data)) + dbgmsg('Sumo: uploading %d bytes' % + sys.getsizeof(json.dumps(payload))) + result = requests.put(url, payload) + result.raise_for_status() + + def _create_request(self, url): + req = super(SumoLogicProcessor, self)._create_request(url) + return req + + def _handle_urlopen_error(self, e, url, payload): + errmsg(''.join(['%s Error: %s' % (self.__class__.__name__, e), + '\n URL: ' + url, + '\n data: ' + payload])) + + if __name__ == '__main__': parser = optparse.OptionParser(version=__version__) @@ -4371,6 +4446,16 @@ def process_calculated(self, packets): group.add_option('--mqtt-upload-period', type='int', help='upload period in seconds', metavar='PERIOD') parser.add_option_group(group) + group = optparse.OptionGroup(parser, 'Sumo Logic options') + group.add_option('--sumo', action='store_true', dest='sumo_out', + default=False, help='upload data using Sumo Logic API') + group.add_option('--sumo-url', help='URL', metavar='URL') + group.add_option('--sumo-upload-period', + help='upload period in seconds', metavar='PERIOD') + group.add_option( + '--sumo-timeout', help='timeout period in seconds', metavar='TIMEOUT') + parser.add_option_group(group) + (options, args) = parser.parse_args() if options.quiet: @@ -4562,7 +4647,7 @@ def process_calculated(self, packets): options.enersave_out or options.bidgely_out or options.peoplepower_out or options.eragy_out or options.smartenergygroups_out or options.thingspeak_out or - options.pachube_out or options.oem_out or + options.pachube_out or options.oem_out or options.sumo_out or options.wattvision_out or options.pvo_out or options.mqtt_out): print 'Please specify one or more processing options (or \'-h\' for help):' print ' --print print to screen' @@ -4713,6 +4798,11 @@ def process_calculated(self, packets): options.mqtt_tls, options.mqtt_map or MQTT_MAP, options.mqtt_upload_period or MQTT_UPLOAD_PERIOD)) + if options.sumo_out: + procs.append(SumoLogicProcessor + (options.sumo_url or OEM_URL, + options.sumo_upload_period or OEM_UPLOAD_PERIOD, + options.sumo_timeout or OEM_TIMEOUT)) mon = Monitor(col, procs) mon.run() From 6baecc975bcdd3ee117273ff12bebd475788cd6a Mon Sep 17 00:00:00 2001 From: Mike Horwath Date: Wed, 6 Feb 2019 14:14:38 -0600 Subject: [PATCH 2/5] adding Sumo Logic to support section --- bin/btmon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/btmon.py b/bin/btmon.py index 3f4d177..dd09f36 100755 --- a/bin/btmon.py +++ b/bin/btmon.py @@ -9,7 +9,7 @@ * MyEnerSave * SmartEnergyGroups * xively * WattzOn * PlotWatt * PeoplePower * thingspeak * Eragy * emoncms * Wattvision * PVOutput * Bidgely - * MQTT + * MQTT * Sumo Logic Thanks to: Amit Snyderman From e2a855ebb24eefc81543ba6c81b397ed0460ffb4 Mon Sep 17 00:00:00 2001 From: Mike Horwath Date: Wed, 6 Feb 2019 14:30:28 -0600 Subject: [PATCH 3/5] adding example items --- bin/btmon.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/bin/btmon.py b/bin/btmon.py index dd09f36..0a6e044 100755 --- a/bin/btmon.py +++ b/bin/btmon.py @@ -709,9 +709,44 @@ Sumo Logic Configuration: 1) log into sumologic.com interface -2) create HTTP hook as you will use it in configuration - -By default, all channels on all ECMs will be uploaded. +2) create HTTP source as you will use it in configuration (example) + a) Name: Choose any name you wish + b) Source Host is optional but suggest giving a descriptive name + c) Source Category: brultech/gem + d) Time Zone: set if you need to - defaults to UTC for this type of endpoint + e) Time Zone Format: Suggest setting as such: + 1) Format: epoch + 2) Timestamp locator: + +This will output a URL similar to: + https://endpoint1.collection.sumologic.com/receiver/v1/http/7bf1fd3366e55b372c05d85fca6873ff5166e8b0d5a65d2a== + +Uploaded data will be JSON, example: +{ "time_created": "1549470367", "XXX732_volts": "123.9", "XXX732_ch1_w": "889.80", ... } + +To create a simple dashboard use the following search for 3 panels: + ## graph volts + _sourceCategory="brultech/gem" and _collector="collector-name" + | json "time_created" as timestamp + | json "XXX732_volts" as volts + | timeslice 1m + | avg(volts) as volts by _timeslice + + ## graph channel 1 watts + _sourceCategory="brultech/gem" and _collector="collector-name" + | json "time_created" as timestamp + | json "XXX732_ch1_w" as ch1_watts + | timeslice 1m + | avg(ch1_watts) as watts by _timeslice + + ## graph channel 1 amps + _sourceCategory="brultech/gem" and _collector="collector-name" + | json "time_created" as timestamp + | json "XXX732_volts" as volts + | json "XXX732_ch1_w" as ch1_watts + | ch1_watts/volts as ch1_amps + | timeslice 1m + | avg(ch1_amps) as amps by _timeslice For example, this configuration will upload all data from all ECMs. From cb10cd5711c3e14c8d070e71dff8faf362ad8bbc Mon Sep 17 00:00:00 2001 From: Mike Horwath Date: Wed, 6 Feb 2019 14:40:05 -0600 Subject: [PATCH 4/5] fixing via comments from sandeen --- bin/btmon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/btmon.py b/bin/btmon.py index 0a6e044..3f049a6 100755 --- a/bin/btmon.py +++ b/bin/btmon.py @@ -1255,6 +1255,12 @@ MQTT_UPLOAD_PERIOD = MINUTE +# sumo logic defaults +SUMO_URL = 'https://support.sumologic.com/hc/en-us' +SUMO_UPLOAD_PERIOD = MINUTE +SUMO_TIMEOUT = 15 # seconds + + import base64 import bisect import calendar @@ -4835,9 +4841,9 @@ def _handle_urlopen_error(self, e, url, payload): options.mqtt_upload_period or MQTT_UPLOAD_PERIOD)) if options.sumo_out: procs.append(SumoLogicProcessor - (options.sumo_url or OEM_URL, - options.sumo_upload_period or OEM_UPLOAD_PERIOD, - options.sumo_timeout or OEM_TIMEOUT)) + (options.sumo_url or SUMO_URL, + options.sumo_upload_period or SUMO_UPLOAD_PERIOD, + options.sumo_timeout or SUMO_TIMEOUT)) mon = Monitor(col, procs) mon.run() From aac296f74d5eebd7c3f62de97fed4499f191bfb1 Mon Sep 17 00:00:00 2001 From: Mike Horwath Date: Wed, 6 Feb 2019 14:57:03 -0600 Subject: [PATCH 5/5] sandeen said leave blank, so I be --- bin/btmon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/btmon.py b/bin/btmon.py index 3f049a6..36f0203 100755 --- a/bin/btmon.py +++ b/bin/btmon.py @@ -1256,7 +1256,7 @@ # sumo logic defaults -SUMO_URL = 'https://support.sumologic.com/hc/en-us' +SUMO_URL = '' SUMO_UPLOAD_PERIOD = MINUTE SUMO_TIMEOUT = 15 # seconds