Merge pull request #7 from HoopadCorp/feature/fporbe_filebeat_support

Feature/fporbe filebeat support

client-services/fprobe/fprobe.conf

@@ -0,0 +1,9 @@
+#fprobe default configuration file
+
+INTERFACE="any"
+#### interfaces to be monitored
+#### SET TO "any" for export all the interfaces data
+FLOW_COLLECTOR="localhost:2055"
+
+#fprobe can't distinguish IP packet from other (e.g. ARP)
+OTHER_ARGS="-fip"

client-services/pnp-client/pnp-client.py

@@ -6,6 +6,21 @@
 import sys
 import json
 import configparser
+import yaml
+
+filebeat_data = {'filebeat.config': {'modules': {'path': '${path.config}/modules.d/*.yml', 'reload.enabled': False}},
+                 'filebeat.modules': [
+                     {'module': 'system', 'syslog': {'enabled': True, 'var.paths': ['/var/log/syslog*']},
+                      'auth': {'enabled': True, 'var.paths': ['/var/log/auth.log*']}}, {'module': 'iptables',
+                                                                                        'log': {'enabled': True,
+                                                                                                'var.paths': [
+                                                                                                    '/var/log/kern.log*'],
+                                                                                                'var.input': 'file'}},
+                     {'module': 'netflow',
+                      'log': {'enabled': True, 'var': {'netflow_host': '0.0.0.0', 'netflow_port': 2055}}}],
+                 'output.elasticsearch': {
+                     'ssl.certificate_authorities': ['/usr/local/share/ca-certificates/WANPAD.crt'], 'hosts': 'ENV_ME',
+                     'username': 'ENV_ME', 'password': 'ENV_ME'}}
 
 
 def get_interfaces():
@@ -28,26 +43,34 @@ def client_program():
                 "token": config('TOKEN'),
                 "dsf": dsf}
         url = config('URI')
-        request_tourl = requests.post(url, verify=False, data=data, timeout=6)
-        if request_tourl.status_code == 400:
-            print(request_tourl.text)
+        request_to_url = requests.post(url, verify=False, data=data, timeout=6)
+        if request_to_url.status_code == 400:
+            print(request_to_url.text)
             return sys.exit(1)
-        elif request_tourl.status_code == 200:
-            print(request_tourl.text)
-        elif request_tourl.status_code == 201:
-            response = request_tourl.json()
+        elif request_to_url.status_code == 200:
+            print(request_to_url.text)
+        elif request_to_url.status_code == 201:
+            response = request_to_url.json()
             public_key = response.get('cspu')
             os.system(f"sudo echo {public_key} > /home/hoopad/.ssh/authorized_keys")
             gateway = response.get('gateway')
 
+            filebeat = response.get('filebeat')
+            filebeat_data['output.elasticsearch']['hosts'] = filebeat.get('hosts')
+            filebeat_data['output.elasticsearch']['username'] = filebeat.get('username')
+            filebeat_data['output.elasticsearch']['password'] = filebeat.get('password')
+            filebeat_data['output.elasticsearch']['ssl.certificate_authorities'] = filebeat.get('ssl_crt')
+
+            create_file_beat(filebeat_data, filebeat.get('conf_address'))
+
             # if gateway exist in response then send a request to gateway and create wireguard.conf file
             if bool(gateway):
                 ip_address, port, token = gateway.values()
                 gateway_address = f"{ip_address}:{port}"
                 scheme = 'http' if url.startswith('http') else 'https'
-                #client_request_to_gateway(scheme, gateway_address, token, dsf)
+                client_request_to_gateway(scheme, gateway_address, token, dsf)
         else:
-            print("Error Code:", request_tourl.status_code)
+            print("Error Code:", request_to_url.status_code)
         return sys.exit(0)
 
 
@@ -82,5 +105,10 @@ def client_request_to_gateway(scheme, gateway_address, token, dsf):
         print("Error Code:", result.status_code)
 
 
+def create_file_beat(data, address):
+    with open(address, 'w+') as filebeat_config:
+        yaml.dump(data, filebeat_config, default_flow_style=False)
+
+
 if __name__ == "__main__":
     client_program()

installation/.env

@@ -1,5 +1,6 @@
 DEFAULT_USER=hoopad
 CLIENT_SERVICES_DIR="../client-services"
+TAR_FILES_DIR="../tar-files"
 PNP_SERVICE_DIR="${CLIENT_SERVICES_DIR}/pnp-client"
 ZTP_ENV_FILE="${PNP_SERVICE_DIR}/.env"
 PIP_DEPS_FILE="${PNP_SERVICE_DIR}/requirements.txt"

installation/.install-lib.sh

@@ -71,3 +71,18 @@ function set_fib_ip_no_pmtu_disc_1 () {
 	sysctl -w net.ipv4.ip_no_pmtu_disc=1
 	echo 'net.ipv4.ip_no_pmtu_disc = 1' | tee /etc/sysctl.d/10-ip-no-pmtu-disc.conf
 }
+
+function fprobe_conf () {
+
+	local service='fprobe'
+	cp "${CLIENT_SERVICES_DIR}/${service}/${service}.conf" "/etc/default/${service}"
+	systemctl restart fprobe.service || systemctl restart fprobe.service
+}
+
+function extract_filebeat () {
+
+	local service='filebeat'
+	tar xvf ${TAR_FILES_DIR}/${service}.tar.gz -C "${CLIENT_SERVICES_DIR}/"
+}
+
+

installation/apt_deps

@@ -1 +1 @@
-git openvpn python3-pip wireguard snmpd libqmi-utils udhcpc build-essential python3-dev strongswan strongswan-starter frr bird2 keepalived
+git openvpn python3-pip wireguard snmpd libqmi-utils udhcpc build-essential python3-dev strongswan strongswan-starter frr bird2 keepalived fprobe

installation/install.sh

@@ -10,9 +10,11 @@ create_hoopad_user || true
 install_apt_deps
 install_pip_deps
 birdwatcher_conf
+extract_filebeat
 enable_wanpad_services
 start_wanpad_services
 add_update_cron || true
 enable_ipv4_forward
 set_fib_multipath_hash_policy_1
 set_fib_ip_no_pmtu_disc_1
+fprobe_conf

tar-files/.gitattributes

@@ -0,0 +1 @@
+filebeat.tar.gz filter=lfs diff=lfs merge=lfs -text