add hadoop, yarn and hive yaml config sample , for this rules has bee…

…n tested passed on hadoop2.4.1 ,yarn2.7.7 and hive2.3.9

Signed-off-by: 吴蓬云 <[email protected]>

example_configs/hadoop_datanode.yml

@@ -0,0 +1,15 @@
+ssl: false
+lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+rules:
+  - pattern: 'Hadoop<service=DataNode, name=DataNodeInfo, key=(.*), key_=networkErrors><>DatanodeNetworkCounts: (\d\.\-Ee+)'
+    name: hadoop_datanode_network_errors
+    value: $2
+    help: "DataNode networkError counts every host"
+    type: COUNTER
+    labels:
+      "host": "$1"
+  - pattern: 'Hadoop<service=(\w+), name=(\w+)><>(\w+)'
+    name: Hadoop_$1_$2_$3
+    attrNameSnakeCase: true
+    help: "Common config, parse from jmx bean: {Hadoop:service=$1, name=$2}.$3"

example_configs/hadoop_namenode.yml

@@ -0,0 +1,26 @@
+lowercaseOutputName: true
+rules:
+  - pattern: "Hadoop<service=NameNode, name=NameNodeStatus><>State:.*?active.*?"
+    name: hadoop_namenode_status
+    value: 1
+    help: "NameNodeStatus status, means active and means standby"
+    type: COUNTER
+  - pattern: "Hadoop<service=NameNode, name=NameNodeStatus><>State:.*?standby.*?"
+    name: hadoop_namenode_status
+    value: 0
+    help: "NameNodeStatus status, means active and means standby"
+    type: COUNTER
+  - pattern: "Hadoop<service=NameNode, name=FSNamesystemState><>FSState:.*?Operational.*?"
+    name: hadoop_namenode_fsnamesystem_state
+    value: 1
+    help: "FSNameSystem state, means Operational and means Safemode"
+    type: COUNTER
+  - pattern: "Hadoop<service=NameNode, name=FSNamesystemState><>FSState:.*?Safemode.*?"
+    name: hadoop_namenode_fsnamesystem_state
+    value: 0
+    help: "FSNameSystem state, means Operational and means Safemode"
+    type: COUNTER
+  - pattern: "Hadoop<service=(\w+), name=(\w+)><>(\w+)"
+    name: hadoop_$1_$2_$3
+    attrNameSnakeCase: true
+    help: "Common config, parse from jmx bean: {Hadoop:service=$1, name=$2}.$3"

example_configs/hive-2.0.0.yml

@@ -0,0 +1,94 @@
+ssl: false
+lowercaseOutputName: true
+# the data of these rules comes from hive jmx url: 192.168.205.197:10002/jmx
+# Noted that, you must config hive params in hive-site.xml to enable hive monitor itself:
+# hive.metastore.metrics.enabled:true
+# hive.server2.metrics.enabled	:true
+# hive.service.metrics.reporter	:JMX
+rules:
+  - pattern: 'Hadoop<service=hive, name=MetricsSystem, sub=Stats>(\w+)'
+    name: hive_metrics_system_$1
+    attrNameSnakeCase: true
+    help: "hadoop hive metrics system status"
+  - pattern: 'Hadoop<service=hive, name=UgiMetrics>(\w+)'
+    name: hive_ugi_metrics_$1
+    attrNameSnakeCase: true
+    help: "hadoop hive metrics system status"
+  - pattern: 'metrics<name=buffers.mapped.count><>Value'
+    name: hive_metrics_buffers_mapped_count
+  - pattern: 'metrics<name=memory.non-heap.usage><>Value'
+    name: hive_metrics_memory_non_heap_usage
+  - pattern: 'metrics<name=gc.PS-Scavenge.time><>Value'
+    name: hive_metrics_gc_PS_Scavenge_time
+  - pattern: 'metrics<name=memory.non-heap.init><>Value'
+    name: hive_metrics_memory_non_heap_init
+  - pattern: 'metrics<name=memory.pools.Code-Cache.usage><>Value'
+    name: hive_metrics_memory_pools_Code_Cache_usage
+  - pattern: 'metrics<name=buffers.direct.used><>Value'
+    name: hive_metrics_buffers_direct_used
+  - pattern: 'metrics<name=memory.total.committed><>Value'
+    name: hive_metrics_memory_total_committed
+  - pattern: 'metrics<name=buffers.mapped.capacity><>Value'
+    name: hive_metrics_buffers_mapped_capacity
+  - pattern: 'metrics<name=memory.heap.max><>Value'
+    name: hive_metrics_memory_heap_max
+  - pattern: 'metrics<name=threads.timed_waiting.count><>Value'
+    name: hive_metrics_threads_timed_waiting_count
+  - pattern: 'metrics<name=memory.heap.usage><>Value'
+    name: hive_metrics_memory_heap_usage
+  - pattern: 'metrics<name=memory.non-heap.max><>Value'
+    name: hive_metrics_memory_non_heap_max
+  - pattern: 'metrics<name=threads.runnable.count><>Value'
+    name: hive_metrics_threads_runnable_count
+  - pattern: 'metrics<name=memory.pools.PS-Eden-Space.usage><>Value'
+    name: hive_metrics_memory_pools_PS_Eden_Space_usage
+  - pattern: 'metrics<name=memory.non-heap.used><>Value'
+    name: hive_metrics_memory_non_heap_used
+  - pattern: 'metrics<name=gc.PS-MarkSweep.count><>Value'
+    name: hive_metrics_gc_PS_Mark_Sweep_count
+  - pattern: 'metrics<name=buffers.mapped.used><>Value'
+    name: hive_metrics_buffers_mapped_used
+  - pattern: 'metrics<name=gc.PS-Scavenge.count><>Value'
+    name: hive_metrics_gc_PS_Scavenge_count
+  - pattern: 'metrics<name=threads.new.count><>Value'
+    name: hive_metrics_threads_new_count
+  - pattern: 'metrics<name=memory.heap.used><>Value'
+    name: hive_metrics_memory_heap_used
+  - pattern: 'metrics<name=memory.total.init><>Value'
+    name: hive_metrics_memory_total_init
+  - pattern: 'metrics<name=memory.pools.Metaspace.usage><>Value'
+    name: hive_metrics_memory_pools_Metaspace_usage
+  - pattern: 'metrics<name=memory.pools.PS-Old-Gen.usage><>Value'
+    name: hive_metrics_memory_pools_PS_Old_Gen_usage
+  - pattern: 'metrics<name=threads.daemon.count><>Value'
+    name: hive_metrics_threads_daemon_count
+  - pattern: 'metrics<name=memory.total.max><>Value'
+    name: hive_metrics_memory_total_max
+  - pattern: 'metrics<name=threads.count><>Value'
+    name: hive_metrics_threads_count
+  - pattern: 'metrics<name=threads.deadlock.count><>Value'
+    name: hive_metrics_threads_deadlock_count
+  - pattern: 'metrics<name=memory.heap.init><>Value'
+    name: hive_metrics_memory_heap_init
+  - pattern: 'metrics<name=threads.terminated.count><>Value'
+    name: hive_metrics_threads_terminated_count
+  - pattern: 'metrics<name=buffers.direct.count><>Value'
+    name: hive_metrics_buffers_direct_count
+  - pattern: 'metrics<name=memory.total.used><>Value'
+    name: hive_metrics_memory_total_used
+  - pattern: 'metrics<name=memory.non-heap.committed><>Value'
+    name: hive_metrics_memory_non_heap_committed
+  - pattern: 'metrics<name=memory.pools.PS-Survivor-Space.usage><>Value'
+    name: hive_metrics_memory_pools_PS_Survivor_Space_usage
+  - pattern: 'metrics<name=classLoading.loaded><>Value'
+    name: hive_metrics_class_loading_loaded
+  - pattern: 'metrics<name=threads.waiting.count><>Value'
+    name: hive_metrics_threads_waiting_count
+  - pattern: 'metrics<name=classLoading.unloaded><>Value'
+    name: hive_metrics_class_loading_unloaded
+  - pattern: 'metrics<name=memory.heap.committed><>Value'
+    name: hive_metrics_memory_heap_committed
+  - pattern: 'metrics<name=gc.PS-MarkSweep.time><>Value'
+    name: hive_metrics_gc_PS_Mark_Sweep_time
+
+

example_configs/yarn-nodemanager.yml

@@ -0,0 +1,14 @@
+ssl: false
+lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+rules:
+  - pattern: 'Hadoop<service=NodeManager, name=RpcActivityForPort(\d+)><>(\w+)'
+    name: hadoop_nodemanager_rpc_activity_for_port_$2
+    attrNameSnakeCase: true
+    labels:
+      "port": $1
+    help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=NodeManager, name=RpcActivityForPort$1}.$2"
+  - pattern: 'Hadoop<service=NodeManager, name=(\w+)><>(\w+)'
+    name: hadoop_nodemanager_$1_$2
+    attrNameSnakeCase: true
+    help: "Common config, parse from jmx bean: {Hadoop:service=NodeManager, name=$1}.$2"

example_configs/yarn-resourcemanager.yml

@@ -0,0 +1,63 @@
+ssl: false
+lowercaseOutputName: true
+lowercaseOutputLabelNames: true
+rules:
+  - pattern: 'Hadoop<service=ResourceManager, name=MetricsSystem, sub=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_metrics_system_$1_$2
+    attrNameSnakeCase: true
+    help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
+  - pattern: 'Hadoop<service=ResourceManager, name=RpcActivityForPort(\d+)><>(\w+)'
+    name: hadoop_resourcemanager_rpc_activity_for_port_$2
+    attrNameSnakeCase: true
+    labels:
+      "port": $1
+    help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
+  - pattern: 'Hadoop<service=ResourceManager, name=RpcDetailedActivityForPort(\d+)><>(\w+)'
+    name: hadoop_resourcemanager_rpc_activity_for_port_$2
+    attrNameSnakeCase: true
+    labels:
+      "port": $1
+    help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), q2=(\w+), user=(.*)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$5
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+      "q1": "$2"
+      "q2": "$3"
+      "user": "$4"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), q2=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$4
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+      "q1": "$2"
+      "q2": "$3"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), user=(.*)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$4
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+      "q1": "$2"
+      "user": "$3"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$3
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+      "q1": "$2"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), user=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$3
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+      "user": "$2"
+  - pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_queue_metrics_$2
+    attrNameSnakeCase: true
+    labels:
+      "q0": "$1"
+  - pattern: 'Hadoop<service=ResourceManager, name=(\w+)><>(\w+)'
+    name: hadoop_resourcemanager_$1_$2
+    attrNameSnakeCase: true
+    help: "Common config, parse from jmx bean: {Hadoop:service=ResourceManager, name=$1}.$2"