Merge pull request #4 from MrMarioMichel/development

Readme

ArchivStats.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 date=`date +%d%m%y`
-achivesize=`du -sh ./Archiv/` > ./Log/BackupCheck/log$date.txt
+achivesize=`du -sh ./Archiv/`
 echo Current size : $achivesize

Checker.sh

@@ -6,8 +6,11 @@ ciscohostfilecount=`egrep -v "^\s*(#|$)" ./Devices/Cisco/Cisco-Devices.txt | gre
 dellhostfilecount=`egrep -v "^\s*(#|$)" ./Devices/DELL/DELL-Devices.txt | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" | wc -l`
 dirsinachriv=`find ./Archiv -maxdepth 1 -type d | wc -l`
 configsinachive=`find ./Archiv -maxdepth 2 -type f -mtime -1 | wc -l`
+minus1=1 # Caused by also counting the ./Archiv as a folder 
 
 total=`expr $fortinethostfilecount + $hphostfilecount + $ciscohostfilecount + $dellhostfilecount`
+dirsinachriv=`expr $dirsinachriv - $minus1`
+
 echo "------------------------------------------------"
 echo "Fortinet .. : $fortinethostfilecount Hosts in Host File"
 echo "HP ........ : $hphostfilecount Hosts in Host File"

Fortinet.sh

@@ -11,7 +11,7 @@ echo "Started Backup of Config's"
 
 for device in `cat ./Devices/Fortinet/Fortinet-Devices.txt | egrep -v "^\s*(#|$)" | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b"` # Will have a look in the file "Fortinet-Devices.txt" for all fortinet devices
 do
-    echo -e "Host found in hostfile \e[35m$device\e[39m"
+    echo -e "Host found in hostfile --> $device"
     scp -v -i ./SSH-Keys/Backup-SSH-Key $user@$device:sys_config ./BackupConfigFortinet # Will copy to all devices in "Fortinet-Devices.txt" and copy it secure localy
     name=`pv BackupConfigFortinet | grep -m1 'set hostname' | sed 's|["?]||g' | sed 's/\<set hostname\>//g' | sed 's/ //g' | tr -dc '[:print:]'` # Will search for the host name to set create a directory and a file named like the hostname of the network device 
     mkdir -v Archiv/$name

OldConfigsAchiver.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+#  Use gunzip -r Archiv/ to unzip all ziped .gz files
+date=`date +%d%m%y`

OldLogsAchiver.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+#  Use gunzip -r Log/ to unzip all ziped .gz files
+date=`date +%d%m%y`

README.md

@@ -1,2 +1,100 @@
 # Config-Backupper
-This script can backup the configs from firewalls and switches
+This script can backup the configs from firewalls and switches.
+
+
+### Packet dependencies
+Check the file **REQUIREMENTS** to see all packet dependencies.
+
+# How to install 
+Step 1. ``` git clone https://github.com/MrMarioMichel/Config-Backupper ```  --> this branch should work otherwise download one of  the [newest release](https://github.com/MrMarioMichel/Config-Backupper/releases)
+
+Step 2. ```cd ./Config-Backupper``` --> enter the downloaded repository
+
+Step 3. ```chmod 700 ./setup.sh``` --> make the setup.sh executable
+
+Step 4. ```./setup.sh``` --> start the setup.sh
+
+# How to setup
+
+#### Set days after a config gets compressed (.gz format) [2,5x-3,5x SMALLER] (Numbers only):
+
+Here you can define (in days) when the archived configs should get compressed. (Recommended : 14)
+
+If you don't want to compress the configs is set it to 9999. --> No compression for 27 Years long enough...
+
+#### Enter bit lenth (Numbers only):
+
+Here you can define the SSH Key used to authenticate at the devices. (Recommended : 4096 or higher)
+
+Use low bit key length (1024 - 2048bits) to speed up the authentication **BUT SECURITY SUFFERS FROM IT !!!**.
+
+#### Enter passphrase (empty for no passphrase):
+
+Just press enter otherwise the script would need everytime it connects to a devices the password. Still want to have a password you will neeed to enter the password in clear text in the ./Modules/Backup module ```sshpass -p "<PASSWORD>" scp -i ...```
+
+#### Enter same passphrase again:
+
+Again just enter.
+
+# How to configure
+
+## Server side
+
+In ```./Devices/<VENDOR>/<VENDOR>-Devices.txt``` you need to enter line by line all IP addresses. For more information have a look in any of these device .txt files.
+
+## Client side (Network device)
+
+Create or use an existing read only profile for a user named "backup" (All in small letters, all togther, no spaces) on the device. This user should get only read rights for highest security. Also add the ./SSH-Keys/Backup-SSH-Key.pub to the user that this key pair can be used to login. See table **Use SSH-Key for Authenictaion** how to do that for each vendor.
+
+### Use SSH-Key for Authenictaion
+| Vendor        | Link           | Info  |
+| :------------- |:-------------| :-----|
+| Fortinet      |[Authenticate a CLI administrator using SSH keys](https://kb.fortinet.com/kb/documentLink.do?externalID=11985)| Also see [Technical Note: How to download a FortiGate configuration file and upload firmware file using secure file copy (SCP)](https://kb.fortinet.com/kb/documentLink.do?externalID=FD43754) |
+| DELL       |       | Note : [SSH Key Auth on Dell PowerConnect Switches](https://eengstrom.github.io/musings/ssh-key-auth-powerconnect)  |
+| Cisco      |       |   |
+| HP         | [Configure the switch for SSH authentication](http://h22208.www2.hpe.com/eginfolib/networking/docs/switches/YA-YB/15-18/5998-8153_yayb_2530_asg/content/ch08s06.html#s_5Configuring_the_switch_for_SSH_authentication)   | Note : See step **Option B: Configuring the switch for client Public-Key SSH authentication** in order to use the client, so in our case the server sided SSH key. |
+
+# How to test 
+
+You can test the script (Recommended) befor you run it automaticaly to get the config file with ```scp -P <PORT> -vvv -i ./SSH-Keys/Backup-SSH-Key backup@><IP>:<CONFIGFILE>  ./TestConfigFile```. If that worked and the config file is now stored as **TestConfigFile**. You can delete the **TestConfgiFile** and run the script and see what happens.
+
+### Remote name of the config file
+
+| Name     | File  |  Note | 
+| :------------- |:------------- | :----- |
+| Fortinet Firewall | sys_config | [Backup over SCP](https://forum.fortinet.com/tm.aspx?m=114055) |
+| 2  |  |   |
+| 3 |   |   |
+
+# About 
+
+The script uses a <YOU-CHOOSE-IT> bit long SSH Key for authentication. That key must be added to all devices at the user backup in order for the script to get the configs. In the directory ./Log will be all output generated by the script for each day and each vendor.
+
+  # Limitaions
+  | LimitNr        | Limitation           | Reason  | Will get fixed | 
+| :-------------: |:------------- | :----- | :-----: |
+| 1  |  If you try to run the scrip more often than once a day the logs of the secound run will also be in the same log file  | Logging has not been designed for this  |
+| 2  |  |   |
+| 3 |   |   |
+
+# Facing Problems
+
+### General :
+
+| Problem     | Solution  |  Description | 
+| :------------- |:------------- | :----- |
+| ssh: connect to host <IP-ADDRESS> port 22: Connection timed out | Check if port 22 used for SSH  | If SSH do not use the port 22 (Default) you need to place the host inside the special module of the certain vendor (Problem caused by scp because it automaticaly uses port 22 if no other port is defined) |
+| 2  |  |   |
+| 3 |   |   |
+
+
+
+### Fortinet :
+
+| Problem     | Solution  |  Description | 
+| :------------- |:------------- | :----- |
+| Sink: 501-Permission Denied 501-Permission Denied | Check if enabled SCP on the Fortinet Device | The script can connect but has problems with the rights to copy the config file. If you don't enable SCP you can run into this problem. |
+| 2  |  |   |
+| 3 |   |   |
+
+

setup.sh

@@ -6,9 +6,9 @@
 echo "[i] : Setup stared"
 mkdir --verbose ./Archiv
 mkdir --verbose -p ./Devices/{Fortinet,HP,Cisco,DELL}
-mkdir --verbose -p ./Modules/{Archiv,Backup,Checker,Clean,Debug}
+mkdir --verbose -p ./Modules/{Archiv,Backup,Clean,Debug}
 mkdir --verbose ./SSH-Keys
-mkdir --verbose -p ./Log/{Fortinet,HP,Cisco,DELL,BackupCheck}
+mkdir --verbose -p ./Log/{Archv,Backup,Cisco,DELL,Fortinet,HP,Log}
 echo "[i] : Directories where created"
 
 # Phase 2 create all device list's
@@ -21,10 +21,12 @@ echo '#| |   | |__| | | \ \  | |   _| |_| |\  | |____   | |     | |__| |  __/\ V
 echo '#|_|    \____/|_|  \_\ |_|  |_____|_| \_|______|  |_|     |_____/ \___| \_/ |_|\___\___||___/' >> ./Devices/Fortinet/Fortinet-Devices.txt
 echo '# Layout Syntax' >> ./Devices/Fortinet/Fortinet-Devices.txt
 echo '# ###############################' >> ./Devices/Fortinet/Fortinet-Devices.txt
-echo '# <IP> --> <Hostname>' >> ./Devices/Fortinet/Fortinet-Devices.txt
-echo '# 1.1.1.1 --> CloudFlare' >> ./Devices/Fortinet/Fortinet-Devices.txt
-echo '# #8.8.8.8 --> Uncommented line' >> ./Devices/Fortinet/Fortinet-Devices.txt
-echo '# Use a "#" in front of a line to uncomment a line (This will be ignored from the BackupScript)' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# <IP> --> <Hostname> ### Comment ###' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# 1.1.1.1 --> CloudFlare-DNS ### Backup CloudFlare ###' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# #8.8.8.8 --> Google-DNS ### Uncommented line ###' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# Use a "#" in front of a line to uncomment a line (This will get ignored from the backup script)' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# Hostname and Comment are optional' >> ./Devices/Fortinet/Fortinet-Devices.txt
+echo '# Hostname will be obtained from the backup file directly' >> ./Devices/Fortinet/Fortinet-Devices.txt
 echo "├── Fortinet Done"
 echo '# _    _ _____     _____             _' >> ./Devices/HP/HP-Devices.txt
 echo '#| |  | |  __ \   |  __ \           (_)' >> ./Devices/HP/HP-Devices.txt
@@ -34,10 +36,12 @@ echo '#| |  | | |       | |__| |  __/\ V /| | (_|  __/\__ \' >> ./Devices/HP/HP-
 echo '#|_|  |_|_|       |_____/ \___| \_/ |_|\___\___||___/' >> ./Devices/HP/HP-Devices.txt
 echo '# Layout Syntax' >> ./Devices/Fortinet/Fortinet-Devices.txt
 echo '# ###############################' >> ./Devices/HP/HP-Devices.txt
-echo '# <IP> --> <Hostname>' >> ./Devices/HP/HP-Devices.txt
-echo '# 1.1.1.1 --> CloudFlare' >> ./Devices/HP/HP-Devices.txt
-echo '# #8.8.8.8 --> Uncommented line' >> ./Devices/HP/HP-Devices.txt
-echo '# Use a "#" in front of a line to uncomment a line (This will be ignored from the BackupScript)' >> ./Devices/HP/HP-Devices.txt
+echo '# <IP> --> <Hostname> ### Comment ###' >> ./Devices/HP/HP-Devices.txt
+echo '# 1.1.1.1 --> CloudFlare-DNS ### Backup CloudFlare ###' >> ./Devices/HP/HP-Devices.txt
+echo '# #8.8.8.8 --> Google-DNS ### Uncommented line ###' >> ./Devices/HP/HP-Devices.txt
+echo '# Use a "#" in front of a line to uncomment a line (This will get ignored from the backup script)' >> ./Devices/HP/HP-Devices.txt
+echo '# Hostname and Comment are optional' >> ./Devices/HP/HP-Devices.txt
+echo '# Hostname will be obtained from the backup file directly' >> ./Devices/HP/HP-Devices.txt
 echo "├── HP Done"
 echo '#   _____ _____  _____  _____ ____     _____             _ ' >> ./Devices/Cisco/Cisco-Devices.txt
 echo '#  / ____|_   _|/ ____|/ ____/ __ \   |  __ \           (_)' >> ./Devices/Cisco/Cisco-Devices.txt
@@ -48,9 +52,11 @@ echo '#  \_____|_____|_____/ \_____\____/   |_____/ \___| \_/ |_|\___\___||___/'
 echo '# Layout Syntax' >> ./Devices/Cisco/Cisco-Devices.txt
 echo '# ###############################' >> ./Devices/Cisco/Cisco-Devices.txt
 echo '# <IP> --> <Hostname>' >> ./Devices/Cisco/Cisco-Devices.txt
-echo '# 1.1.1.1 --> CloudFlare' >> ./Devices/Cisco/Cisco-Devices.txt
-echo '# #8.8.8.8 --> Uncommented line' >> ./Devices/Cisco/Cisco-Devices.txt
-echo '# Use a "#" in front of a line to uncomment a line (This will be ignored from the BackupScript)' >> ./Devices/Cisco/Cisco-Devices.txt
+echo '# 1.1.1.1 --> CloudFlare-DNS ### Backup CloudFlare ###' >> ./Devices/Cisco/Cisco-Devices.txt
+echo '# #8.8.8.8 --> Google-DNS ### Uncommented line ###' >> ./Devices/Cisco/Cisco-Devices.txt
+echo '# Use a "#" in front of a line to uncomment a line (This will get ignored from the backup script)' >> ./Devices/Cisco/Cisco-Devices.txt
+echo '# Hostname and Comment are optional' >> ./Devices/Cisco/Cisco-Devices.txt
+echo '# Hostname will be obtained from the backup file directly' >> ./Devices/Cisco/Cisco-Devices.txt
 echo "├── Cisco Done"
 echo '#  _____  ______ _      _         _____             _' >> ./Devices/DELL/DELL-Devices.txt
 echo '# |  __ \|  ____| |    | |       |  __ \           (_)' >> ./Devices/DELL/DELL-Devices.txt
@@ -61,9 +67,11 @@ echo '# |_____/|______|______|______|  |_____/ \___| \_/ |_|\___\___||___/' >> .
 echo '# Layout Syntax' >> ./Devices/DELL/DELL-Devices.txt
 echo '# ###############################' >> ./Devices/DELL/DELL-Devices.txt
 echo '# <IP> --> <Hostname>' >> ./Devices/DELL/DELL-Devices.txt
-echo '# 1.1.1.1 --> CloudFlare' >> ./Devices/DELL/DELL-Devices.txt
-echo '# #8.8.8.8 --> Uncommented line' >> ./Devices/DELL/DELL-Devices.txt
-echo '# Use a "#" in front of a line to uncomment a line (This will be ignored from the BackupScript)' >> ./Devices/DELL/DELL-Devices.txt
+echo '# 1.1.1.1 --> CloudFlare-DNS ### Backup CloudFlare ###' >> ./Devices/DELL/DELL-Devices.txt
+echo '# #8.8.8.8 --> Google-DNS ### Uncommented line ###' >> ./Devices/DELL/DELL-Devices.txt
+echo '# Use a "#" in front of a line to uncomment a line (This will get ignored from the backup script)' >> ./Devices/DELL/DELL-Devices.txt
+echo '# Hostname and Comment are optional' >> ./Devices/DELL/DELL-Devices.txt
+echo '# Hostname will be obtained from the backup file directly' >> ./Devices/DELL/DELL-Devices.txt
 echo "└── DELL Done"
 echo "[i] : Devices List's where created"
 
@@ -75,33 +83,42 @@ mv --verbose ./HP.sh ./Modules/Backup/HP.sh
 mv --verbose ./Cisco.sh ./Modules/Backup/Cisco.sh
 mv --verbose ./Checker.sh ./Modules/Archiv/Checker.sh
 mv --verbose ./Fastdebug.sh ./Modules/Debug/Fastdebug.sh
-mv --verbose ./OldConfigsMover.sh ./Modules/Archiv/OldConfigsMover.sh
+mv --verbose ./OldConfigsAchiver.sh ./Modules/Archiv/OldConfigsAchiver.sh
+mv --verbose ./OldLogsAchiver.sh ./Modules/Archiv/OldLogsAchiver.sh
 mv --verbose ./ArchivStats.sh ./Modules/Archiv/ArchivStats.sh
 mv --verbose ./BackupConfigsCleanUp.sh ./Modules/Clean/BackupConfigsCleanUp.sh
 mv --verbose ./LogCleanUp.sh ./Modules/Clean/LogCleanUp.sh
 echo "[i] : Modules where moved"
 
 # Phase 4 create the main lanucher for all modules
+echo "[i] : Searchinf for installation path ... This can take a few moments"
 installpath=`find / -name "*Config-Backupper" 2>/dev/null`
 echo "Installation path : $installpath"
 echo "[i] : Found installation path"
 echo '#!/bin/bash'  >> ./Main-Launcher.sh
 echo 'date=`date +%d%m%y`'  >> ./Main-Launcher.sh
-echo "" >> ./Main-Launcher.sh
+echo " " >> ./Main-Launcher.sh
 echo "cd $installpath" >> ./Main-Launcher.sh
-echo "" >> ./Main-Launcher.sh
+echo " " >> ./Main-Launcher.sh
 echo './Modules/Backup/Fortinet.sh &>> ./Log/Fortinet/log$date.txt' >> ./Main-Launcher.sh
 echo './Modules/Backup/DELL.sh &>> ./Log/DELL/log$date.txt' >> ./Main-Launcher.sh
 echo './Modules/Backup/HP.sh &>> ./Log/HP/log$date.txt' >> ./Main-Launcher.sh
 echo './Modules/Backup/Cisco.sh &>> ./Log/Cisco/log$date.txt' >> ./Main-Launcher.sh
-echo "" >> ./Main-Launcher.sh
+echo " " >> ./Main-Launcher.sh
 echo './Modules/Archiv/Checker.sh >> ./Log/BackupCheck/log$date.txt' >> ./Main-Launcher.sh
+echo './Modules/Archiv/OldConfigsArchiver.sh ./Log/Archiv/AchivCompress/log$date.txt' >> ./Main-Launcher.sh
+echo './Modules/Archiv/OldLogsArchiver.sh ./Log/Log/LogCompress/log$date.txt'  >> ./Main-Launcher.sh
 echo './Modules/Archiv/ArchivStats.sh >> ./Log/BackupCheck/log$date.txt' >> ./Main-Launcher.sh
 echo "--------------------------------------------------------------------------------"
-echo "Set days after a config gets commpressed (.gz format) [2,5x-3,5x SMALLER]"
-read achivetime
+echo "Set days after a config gets commpressed (.gz format) [2,5x-3,5x SMALLER] (Numbers only):"
+read achivetimearchiv
 echo "--------------------------------------------------------------------------------"
-echo "find ./Archiv -mtime +$achivetime -exec gzip {} +" >> ./Modules/Archiv/OldConfigsMover.sh
+echo "find ./Archiv -mtime +$achivetimearchiv -exec gzip {} +" >> ./Modules/Archiv/OldConfigsArchiver.sh
+echo "Set days after a logs gets commpressed (.gz format) [2,5x-3,5x SMALLER] (Numbers only):"
+read achivetimelogs
+echo "--------------------------------------------------------------------------------"
+echo "find ./Archiv -mtime +$achivetimelogs -exec gzip {} +" >> ./Modules/Archiv/OldLogsArchiver.sh
+
 echo "du -sh ./Archiv >> ./Log/BackupCheck/log$date.txt" >> ./Modules/Archiv/ArchivStats.sh
 echo "[i] : Main Launcher where created"
 
@@ -117,7 +134,8 @@ chmod --verbose 700 ./Modules/Debug/Fastdebug.sh
 chmod --verbose 700 ./Modules/Clean/BackupConfigsCleanUp.sh
 chmod --verbose 700 ./Modules/Clean/LogCleanUp.sh
 chmod --verbose 700 ./Modules/Archiv/ArchivStats.sh
-chmod --verbose 700 ./Modules/Archiv/OldConfigsMover.sh
+chmod --verbose 700 ./Modules/Archiv/OldConfigsAchiver.sh
+chmod --verbose 700 ./Modules/Archiv/OldLogsAchiver.sh
 echo "[i] : Modules & Lanucher where modified"
 
 # Phase 6 create SSH Key
@@ -128,23 +146,25 @@ echo "4096 bit - Will be fine"
 echo "8192 bit - Are you paranoid ?"
 echo "16384 bit - What are you transferring?"
 echo "----------------------------------------"
-echo "Enter bit lenth (only Numbers)"
+echo "Enter bit lenth (Numbers only):"
 read rsakeylenth
 ssh-keygen -t rsa -b $rsakeylenth -f ./SSH-Keys/Backup-SSH-Key
 
 # Phase 7 show the new created Public SSH-Key
 echo "-----BEGIN PUBLIC KEY-----"
-echo "$(cat ./SSH-Keys/Backup-SSH-Key.pub)" | awk '{print $2,$3}'
+echo "$(cat ./SSH-Keys/Backup-SSH-Key.pub)"
 echo "-----END PUBLIC KEY-----"
-echo ""
+echo " "
 
 # Phase 8 show e.g. for a crontab
 echo "--------------------------------------------------------------------------------"
 echo "Create a crontab to run the backup every day @ 2:00 enter this line in crontab"
 echo "0 2 * * * $installpath/Main-Launcher.sh"
 echo "--------------------------------------------------------------------------------"
 
-# Phase 8 remove setup.sh
-rm setup.sh
+# Phase 9 remove setup.sh & .git/
+rm -v .git -Rf
+echo "[i] : Removed ./.git"
+rm -v setup.sh
 echo "[i] : Removed ./setup.sh"
 echo "[i] : IT'S DONE !"