agent-a0labs-newrelic

#!/usr/bin/python

import httplib, urllib, csv, urllib2, socket, os, time, json, datetime, requests, ConfigParser, sys, traceback

def	my_int(sth):
	if (sth == None or sth.isdigit() == False):
		return 0
	return int(sth)

class LogsType:
	ERROR = "ERR"
	WARNING = "WRG"

def singleton(cls):
	instance=cls()
	cls.__new__ = cls.__call__= lambda cls: instance
	cls.__init__ = lambda self: None
	return instance

@singleton
class Config():
	def get(self, section, item):
		return self.cp.get(section, item)
	
	def getInt(self, section, item):
		return self.cp.getint(section, item)

	def getBoolean(self, section, item):
		return self.cp.getboolean(section, item)

	def __init__(self):
		self.cp = ConfigParser.RawConfigParser()
		self.cp.read('/etc/newrelic/agent-a0labs.cfg')

class Agent:
	def update_postdata(self):
		self.post_data['components'] = []
		for component in self.components:
			self.post_data['components'].append(self.components[component])

	def log(self, errtype, message):
		self.logs_file.write("[" + errtype + "]" + ' ' + str(datetime.datetime.now()) + ' : ' + message +'\n')

	def populateMetrics(self, metrics, value_type, scale, value):
		label = "Component/" + value_type + "[" + scale + "]"
		if (value == 'Y'):
			value = 1
		if (value == None or value == ''):
			value = 0
		if label in metrics:
			yet = metrics[label]
			if type(yet) is dict:
				yet['min'] = (yet['min'] < value and yet['min'] or value)
				yet['max'] = (yet['max'] > value and yet['max'] or value)
				yet['total'] += value
				yet['count'] += 1
				yet['sum_of_squares'] += (value * value)
				metrics[label] = yet
			else:
				metrics[label] = {}
				metrics[label]['min'] = (yet < value and yet or value)
				metrics[label]['max'] = (yet > value and yet or value)
				metrics[label]['total'] = yet + value
				metrics[label]['count'] = 2
				metrics[label]['sum_of_squares'] = (yet * yet + value * value)
		else:
			metrics[label] = value

	def populateDeltaMetrics(self, name, metrics, value_type, scale, value):
		label = "Component/" + value_type + "[" + scale + "]"
		if (name not in self.old):
			self.old[name] = {}
		if (value == None or value == ''):
			value = 0
		delta = 0
		if label in self.old[name] and value > self.old[name][label]:
			delta = value - self.old[name][label]
		self.old[name][label] = value
		self.populateMetrics(metrics, value_type, scale, delta)
	
	def get_csv(self):
		response = requests.get(Config().get('Haproxy', 'csv_url'), auth=(Config().get('Haproxy', 'username'), Config().get('Haproxy', 'password')))
		if (response.status_code != 200):
			raise Exception('impossible to get CSV')
		page = response.content
		cr = csv.DictReader(page.splitlines())
		return cr

	def post_datas(self):
		post_json = json.dumps(self.post_data, sort_keys=True, indent=4)
		headers = {"Content-Type": "application/json",
					"Accept": "application/json","X-License-Key": self.newrelic_license}

		conn = httplib.HTTPSConnection(self.platform_api_ip, timeout=5)
		conn.request("POST", self.platform_api_url, post_json, headers)
		response = conn.getresponse()
		return (response.status)

	def reformat_add_numeric_value(self, row, label, value):
		if (not str(value).isdigit()):
			value = 0
		else:
			value = int(value)
		if (label not in row):
			row[label] = value
		else:
			row[label] += value	

	def reformat_average_numeric_value(self, row, label, value):
		if (not str(value).isdigit()):
			value = 0
		else:
			value = int(value)
		cnt_label = label + '_cnt'
		if (label not in row):
			row[label] = value
			row[cnt_label] = 1
		else:
			row[label] = ((row[label] * row[cnt_label]) + value) / (row[cnt_label] + 1)
			row[cnt_label] += 1
	
	def reformat_csv(self, csv):
		ret = {}
		for row in csv:
			pxname = self.rounddns == True and row['# pxname'] + " - " + socket.getfqdn() or row['# pxname']
			if (pxname not in ret):
				tmp = {}
				tmp['svname'] = pxname
				tmp['servers'] = {}
			else:
				tmp = ret[pxname]
			svname = row['svname']
			if (svname == 'FRONTEND'):
				self.reformat_add_numeric_value(tmp, 'front_connection', my_int(row['stot']))
				self.reformat_add_numeric_value(tmp, 'front_connection_limit', my_int(row['slim']))
				self.reformat_add_numeric_value(tmp, 'front_connection_rate', my_int(row['scur']))
				httpreq = my_int(row['hrsp_1xx']) + my_int(row['hrsp_2xx']) + my_int(row['hrsp_3xx']) + my_int(row['hrsp_4xx']) + my_int(row['hrsp_5xx']) + my_int(row['hrsp_other'])
				self.reformat_add_numeric_value(tmp, 'front_requests', httpreq)

			elif (svname == 'BACKEND') :
				self.reformat_add_numeric_value(tmp, 'back_sessions', my_int(row['stot']))
				httpreq = my_int(row['hrsp_1xx']) + my_int(row['hrsp_2xx']) + my_int(row['hrsp_3xx']) + my_int(row['hrsp_4xx']) + my_int(row['hrsp_5xx']) + my_int(row['hrsp_other'])
				self.reformat_add_numeric_value(tmp, 'back_requests', httpreq)
				
				self.reformat_add_numeric_value(tmp, 'back_err_requests', my_int(row['ereq']))
				self.reformat_add_numeric_value(tmp, 'back_err_connection', my_int(row['econ']))
				self.reformat_add_numeric_value(tmp, 'back_err_rsp', my_int(row['eresp']))
				errors = my_int(row['ereq']) + my_int(row['econ']) + my_int(row['eresp'])
				self.reformat_add_numeric_value(tmp, 'back_err_tot', errors)

				self.reformat_add_numeric_value(tmp, 'back_rsp_3xx', my_int(row['hrsp_3xx']))
				self.reformat_add_numeric_value(tmp, 'back_rsp_4xx', my_int(row['hrsp_4xx']))
				self.reformat_add_numeric_value(tmp, 'back_rsp_5xx', my_int(row['hrsp_5xx']))
				
				self.reformat_add_numeric_value(tmp, 'back_bytes_in', my_int(row['bin']) * 8)
				self.reformat_add_numeric_value(tmp, 'back_bytes_out', my_int(row['bout']) * 8)

				self.reformat_average_numeric_value(tmp, 'back_cx_time', my_int(row['ctime']))
				self.reformat_average_numeric_value(tmp, 'back_rsp_time', my_int(row['rtime']))

				self.reformat_add_numeric_value(tmp, 'back_srv_up', my_int(row['act']))
				self.reformat_add_numeric_value(tmp, 'back_srv_max', my_int(row['weight']))
			else:
				if (svname not in tmp['servers']):
					tmp['servers'][svname] = {}
				self.reformat_add_numeric_value(tmp['servers'][svname], 'back_bytes_in', my_int(row['bin']) * 8)
				self.reformat_add_numeric_value(tmp['servers'][svname], 'back_bytes_out', my_int(row['bout']) * 8)
				
				self.reformat_add_numeric_value(tmp['servers'][svname], 'back_sessions', my_int(row['stot']))
				self.reformat_add_numeric_value(tmp['servers'][svname], 'downtime', my_int(row['downtime']))
				self.reformat_add_numeric_value(tmp['servers'][svname], 'back_connection_rate', my_int(row['scur']))
				httpreq = my_int(row['hrsp_1xx']) + my_int(row['hrsp_2xx']) + my_int(row['hrsp_3xx']) + my_int(row['hrsp_4xx']) + my_int(row['hrsp_5xx']) + my_int(row['hrsp_other'])
				self.reformat_add_numeric_value(tmp['servers'][svname], 'back_requests', httpreq)

			ret[pxname] = tmp
			
		return ret

	def get_component_from_csv(self, csv):
		duration = int(time.time() - self.last_poll_time)
		for key in csv:
			row = csv[key]
			if (row['svname'] not in self.components):
				tmp_component = {}
				tmp_component['name'] = row['svname']
				tmp_component['guid'] = self.guid
				tmp_component['duration'] = duration
				self.components[row['svname']] = tmp_component
				metrics = {}
			else:
				metrics = self.components[row['svname']]['metrics']
			self.components[row['svname']]['duration'] = duration

			## Front
			if ('front_connection' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Connections/Current", 'Connection', row['front_connection'])
			if ('front_connection_limit' in row):
				self.populateMetrics(metrics, "Connections/Max", 'Connection', row['front_connection_limit'] * 60)
			if ('front_connection_rate' in row):
				self.populateMetrics(metrics, "SessionRate/Frontend", 'Sessions', row['front_connection_rate'])
			if ('front_requests' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "FrontRequestRate", 'Requests', row['front_requests'])

			if ('back_sessions' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Session/Total", 'Sessions', row['back_sessions'])
			if ('back_requests' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "RequestRate/Total", 'Requests', row['back_requests'])

			if ('back_err_requests' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Errors/Request", "Errors", row['back_err_requests'])
			if ('back_err_connection' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Errors/Connection", "Errors", row['back_err_connection'])
			if ('back_err_rsp' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Errors/Response", "Errors", row['back_err_rsp'])
			if ('back_err_tot' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "TotalErrors", "Errors", row['back_err_tot'])

			if ('back_bytes_in' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Bytes/Received", "Bits", row['back_bytes_in'])
			if ('back_bytes_out' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Bytes/Sent", "Bits", row['back_bytes_out'])

			if ('back_rsp_3xx' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Responses/3xx", "Error", row['back_rsp_3xx'])
			if ('back_rsp_4xx' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Responses/4xx", "Error", row['back_rsp_4xx'])
			if ('back_rsp_5xx' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Responses/5xx", "Error", row['back_rsp_5xx'])

			if ('back_requests' in row):
				self.populateDeltaMetrics(row['svname'], metrics, "Request/Total", "Requests", row['back_requests'])
			
			if ('back_cx_time' in row):
				self.populateMetrics(metrics, "Responses/ConnectionTime", "MS", row['back_cx_time'])
			if ('back_rsp_time' in row):
				self.populateMetrics(metrics, "Responses/ResponseTime", "MS", row['back_rsp_time'])

			if ('back_srv_up' in row):
				self.populateMetrics(metrics, "Servers/Active", "Servers", row['back_srv_up'])
			if ('back_srv_max' in row):
				self.populateMetrics(metrics, "Servers/Count", "Servers", row['back_srv_max'])

			for server in row['servers']:
				self.populateDeltaMetrics(row['svname'], metrics, "Details/" + server + "/Bytes/Received", "Bits", row['servers'][server]['back_bytes_in'])
				self.populateDeltaMetrics(row['svname'], metrics, "Details/" + server + "/Bytes/Sent", "Bits", row['servers'][server]['back_bytes_out'])
				self.populateDeltaMetrics(row['svname'], metrics, "Session/" + server, 'Sessions', row['servers'][server]['back_sessions'])
				self.populateMetrics(metrics, "SessionRate/Servers/" + server, 'Sessions', row['servers'][server]['back_connection_rate'])
				self.populateDeltaMetrics(row['svname'], metrics, "RequestRate/" + server, 'Requests', row['servers'][server]['back_requests'])
				self.populateDeltaMetrics(row['svname'], metrics, "Downtime/" + server, 'Seconds', row['servers'][server]['downtime'])
			self.components[row['svname']]['metrics'] = metrics			

	def run(self):
		run = True
		code = 200
		while (run):
			try:
				csv = self.get_csv()
				reformat = self.reformat_csv(csv)
				self.get_component_from_csv(reformat)
				self.update_postdata()
				code = self.post_datas()
				if (code == 200):
					# Clearing component cash
					self.components = {}
					self.last_poll_time = time.time()
				elif (code == 400):
					self.log(LogsType.ERROR, " 400 error encontered, request uncorrect")
					run = False
				elif (code == 403):
					self.log(LogsType.ERROR, " 403 acces forbidden, checkout your license key")
					run = False
				elif (code == 404):
					self.log(LogsType.ERROR, " 404 bad URL - checkout with developper of this application")
					run = False
				elif (code == 415):
					self.log(LogsType.ERROR, " 415 request incorrect")
					run = False
			except Exception as e:
				traceback.print_exc(file=sys.stdout)
				self.log(LogsType.ERROR, str(e))
				code = "THROWN"
			if (run):
				time.sleep( int(self.poll_cycle))
				
	def __init__(self):
		# Init NewRelic Variables
		self.platform_api_uri = Config().get('NewRelicAPI', 'uri')
		self.platform_api_url = Config().get('NewRelicAPI', 'url')
		self.platform_api_ip = Config().get('NewRelicAPI', 'ip')
		self.newrelic_license = Config().get('NewRelicAPI', 'license_key')
		self.poll_cycle = Config().getInt('NewRelicAPI', 'poll_cycle')

		# Haproxy round dns - boolean
		self.rounddns = Config().getBoolean('Haproxy', 'rounddns')
		# Init of agent informations
		self.guid = Config().get('Agent', 'guid')
		self.agent_host = socket.gethostname()
		self.agent_pid = os.getpid()
		self.version = Config().get('Agent', 'version')
		self.agent_hash = {'host' : self.agent_host, 'pid' : self.agent_pid, 'version' : str(self.version)}

		# Init old dict for delta computing values
		self.old = {}
		
		# Init time of program start
		self.last_poll_time = time.time() - float(self.poll_cycle)

		# Init of component
		self.components = {}

		# Init post_datas
		self.post_data = {'agent': self.agent_hash, 'components':[]}

		# Init Logs
		self.logs_enabled = Config().getBoolean('Logs', 'enable')
		if (self.logs_enabled):
			self.logs_file = open(Config().get('Logs', 'file'), 'a')
			if (self.logs_file == None):
				sys.stderr.write('Impossible to open log_file, no logs saved')
				self.logs_enabled  = False

if __name__ == '__main__':
	p = Agent()
	p.run()