Merge branch 'release/2.1.0' into update-readme

.env

@@ -28,3 +28,10 @@ POSTGRES_HOST=db
 #
 MAX_CONCURRENCY=80
 MIN_CONCURRENCY=10
+
+#
+# Rengine web interface super user (for non-interactive install)
+#
+DJANGO_SUPERUSER_USERNAME=rengine
+DJANGO_SUPERUSER_EMAIL=[email protected]
+DJANGO_SUPERUSER_PASSWORD=Sm7IJG.IfHAFw9snSKv

.github/workflows/build-pr.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the git repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: Build Docker image
 
 on:
   push:
-    branches: [ master]
+    branches: [ master ]
   schedule:
     - cron: '0 18 * * 5'
 
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout the git repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9

.github/workflows/codeql-analysis.yml

@@ -20,16 +20,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

Makefile

@@ -1,3 +1,4 @@
+include .env
 .DEFAULT_GOAL:=help
 
 # Credits: https://github.com/sherifabdlnaby/elastdocker/
@@ -25,7 +26,14 @@ build:			## Build all services.
 	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} build ${SERVICES}
 
 username:		## Generate Username (Use only after make up).
+ifeq ($(isNonInteractive), true)
+	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser --username ${DJANGO_SUPERUSER_USERNAME} --email ${DJANGO_SUPERUSER_EMAIL} --noinput
+else
 	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser
+endif
+
+migrate:		## Apply migrations
+	${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate
 
 pull:			## Pull Docker images.
 	docker login docker.pkg.github.com

README.md

@@ -319,22 +319,37 @@ screenshot: {
     git clone https://github.com/yogeshojha/rengine && cd rengine
     ```
 
-1. Edit the dotenv file, **please make sure to change the password for postgresql `POSTGRES_PASSWORD`!**
+1. Edit the `.env` file, **please make sure to change the password for postgresql `POSTGRES_PASSWORD`!** 
 
     ```bash
     nano .env
     ```
 
+1. **Optional, only for non-interactive install**: In the `.env` file, **please make sure to change the super admin values!**
+
+    ```bash
+    DJANGO_SUPERUSER_USERNAME=yourUsername
+    [email protected]
+    DJANGO_SUPERUSER_PASSWORD=yourStrongPassword
+    ```
+    If you need to carry out a non-interactive installation, you can setup the login, email and password of the web interface admin directly from the .env file (instead of manually setting them from prompts during the installation process). This option can be interesting for automated installation (via ansible, vagrant, etc.).
+
+    `DJANGO_SUPERUSER_USERNAME`: web interface admin username (used to login to the web interface).
+
+    `DJANGO_SUPERUSER_EMAIL`: web interface admin email.
+
+    `DJANGO_SUPERUSER_PASSWORD`: web interface admin password (used to login to the web interface).
+
 1. In the dotenv file, you may also modify the Scaling Configurations
 
     ```bash
     MAX_CONCURRENCY=80
     MIN_CONCURRENCY=10
     ```
 
-    MAX_CONCURRENCY: This parameter specifies the maximum number of reNgine's concurrent Celery worker processes that can be spawned. In this case, it's set to 80, meaning that the application can utilize up to 80 concurrent worker processes to execute tasks concurrently. This is useful for handling a high volume of scans or when you want to scale up processing power during periods of high demand. If you have more CPU cores, you will need to increase this for maximised performance.
+    `MAX_CONCURRENCY`: This parameter specifies the maximum number of reNgine's concurrent Celery worker processes that can be spawned. In this case, it's set to 80, meaning that the application can utilize up to 80 concurrent worker processes to execute tasks concurrently. This is useful for handling a high volume of scans or when you want to scale up processing power during periods of high demand. If you have more CPU cores, you will need to increase this for maximised performance.
 
-    MIN_CONCURRENCY: On the other hand, MIN_CONCURRENCY specifies the minimum number of concurrent worker processes that should be maintained, even during periods of lower demand. In this example, it's set to 10, which means that even when there are fewer tasks to process, at least 10 worker processes will be kept running. This helps ensure that the application can respond promptly to incoming tasks without the overhead of repeatedly starting and stopping worker processes.
+    `MIN_CONCURRENCY`: On the other hand, MIN_CONCURRENCY specifies the minimum number of concurrent worker processes that should be maintained, even during periods of lower demand. In this example, it's set to 10, which means that even when there are fewer tasks to process, at least 10 worker processes will be kept running. This helps ensure that the application can respond promptly to incoming tasks without the overhead of repeatedly starting and stopping worker processes.
 
     These settings allow for dynamic scaling of Celery workers, ensuring that the application efficiently manages its workload by adjusting the number of concurrent workers based on the workload's size and complexity
 
@@ -344,6 +359,12 @@ screenshot: {
     sudo ./install.sh
     ```
 
+    Or for a non-interactive installation, use `-n` argument (make sure you've modified the `.env` file before launching the installation).
+
+    ```bash
+    sudo ./install.sh -n
+    ```
+
     If `install.sh` does not have install permission, please change it, `chmod +x install.sh`
 
 **reNgine can now be accessed from <https://127.0.0.1> or if you're on the VPS <https://your_vps_ip_address>**
@@ -359,12 +380,10 @@ Installation instructions can be found at [https://reNgine.wiki/install/detailed
 1. Updating is as simple as running the following command:
 
     ```bash
-    cd rengine && sudo ./update.sh
+    cd rengine &&  sudo ./update.sh
     ```
 
     If `update.sh` does not have execution permissions, please change it, `sudo chmod +x update.sh`
-  
-    **NOTE:** if you're updating from 1.3.6, and you're getting a 'password authentication failed' error, consider uninstalling 1.3.6 first, then install 2.x.x as you'd normally do.
 
 ### Changelog
 
@@ -414,6 +433,54 @@ You can also [join our Discord channel #development](https://discord.gg/JuhHdHTt
 
 ![-----------------------------------------------------](https://raw.githubusercontent.com/andreasbm/readme/master/assets/lines/aqua.png)
 
+### Submitting issues
+
+You can submit issues related to this project, but you should do it in a way that helps developers to resolve it as quickly as possible.
+
+For that, you need to add as much valuable information as possible.
+
+You can have this valuable information by following these steps:
+
+- Go to the root of the git cloned project
+- Edit `web/entrypoint.sh` and add `export DEBUG=1` at the top
+This should give you this result
+
+  ```python
+  #!/bin/bash
+
+  export DEBUG=1
+
+  python3 manage.py migrate
+  python3 manage.py runserver 0.0.0.0:8000
+
+  exec "$@"
+  ```
+- Restart the web container: `docker-compose restart web`
+- To deactivate, set **DEBUG** to **0** and restart the web container again
+
+Then, with **DEBUG** set to **1**, in the `make logs` output you could see the full stack trace to debug reNgine.
+
+Example with the tool arsenal version check API bug.
+
+```
+web_1          |   File "/usr/local/lib/python3.10/dist-packages/celery/app/task.py", line 411, in __call__
+web_1          |     return self.run(*args, **kwargs)
+web_1          | TypeError: run_command() got an unexpected keyword argument 'echo'
+```
+Now you know the real error is `TypeError: run_command() got an unexpected keyword argument 'echo'`
+
+And you can post the full stack trace to your newly created issue to help developers to track the root cause of the bug and correct the bug easily
+
+**Activating debug like this also give you the full stack trace in the browser** instead of an error 500 without any details.
+So don't forget to open the developer console and check for any XHR request with error 500.
+If there's any, check the response of this request to get your detailed error.
+
+<img src="https://user-images.githubusercontent.com/1230954/276260955-ed1e1168-7c8f-43a3-b54d-b6285d52b771.png">
+
+Happy issuing ;)
+
+![-----------------------------------------------------](https://raw.githubusercontent.com/andreasbm/readme/master/assets/lines/aqua.png)
+
 ### First-time Open Source contributors
 
 Please note that reNgine is beginner-friendly. If you have never done open-source before, we encourage you to do so. **We will be happy and proud of your first PR ever.**

docker-compose.yml

@@ -9,6 +9,8 @@ services:
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_PORT=${POSTGRES_PORT}
+    ports:
+      - "127.0.0.1:5432:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     networks:
@@ -94,6 +96,7 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_PORT=${POSTGRES_PORT}
       - POSTGRES_HOST=${POSTGRES_HOST}
+      - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD}
       # THIS IS A MUST FOR CHECKING UPDATE, EVERYTIME A COMMIT IS MERGED INTO
       # MASTER, UPDATE THIS!!! MAJOR.MINOR.PATCH https://semver.org/
       - RENGINE_CURRENT_VERSION='2.0.2'
@@ -107,7 +110,7 @@ services:
       - tool_config:/root/.config
       - static_volume:/usr/src/app/staticfiles/
     ports:
-      - "8000:8000"
+      - "127.0.0.1:8000:8000"
     depends_on:
       - db
       - celery

install.sh

@@ -1,21 +1,45 @@
 #!/bin/bash
 
+usageFunction()
+{
+  echo " "
+  tput setaf 2;
+  echo "Usage: $0 (-n) (-h)"
+  echo -e "\t-n Non-interactive installation (Optional)"
+  echo -e "\t-h Show usage"
+  exit 1
+}
+
 tput setaf 2;
 cat web/art/reNgine.txt
 
 tput setaf 1; echo "Before running this script, please make sure Docker is running and you have made changes to .env file."
 tput setaf 2; echo "Changing the postgres username & password from .env is highly recommended."
 
 tput setaf 4;
-read -p "Are you sure, you made changes to .env file (y/n)? " answer
-case ${answer:0:1} in
-    y|Y|yes|YES|Yes )
-      echo "Continiuing Installation!"
-    ;;
-    * )
-      nano .env
-    ;;
-esac
+
+isNonInteractive=false
+while getopts nh opt; do
+   case $opt in
+      n) isNonInteractive=true ;;
+      h) usageFunction ;;
+      ?) usageFunction ;;
+   esac
+done
+
+if [ $isNonInteractive = false ]; then
+    read -p "Are you sure, you made changes to .env file (y/n)? " answer
+    case ${answer:0:1} in
+        y|Y|yes|YES|Yes )
+          echo "Continiuing Installation!"
+        ;;
+        * )
+          nano .env
+        ;;
+    esac
+else
+  echo "Non-interactive installation parameter set. Installation begins."
+fi
 
 echo " "
 tput setaf 3;
@@ -26,7 +50,7 @@ echo "#########################################################################"
 
 echo " "
 tput setaf 4;
-echo "Installing reNgine and it's dependencies"
+echo "Installing reNgine and its dependencies"
 
 echo " "
 if [ "$EUID" -ne 0 ]
@@ -60,6 +84,7 @@ else
   tput setaf 2; echo "Docker installed!!!"
 fi
 
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -68,12 +93,13 @@ echo "#########################################################################"
 if [ -x "$(command -v docker-compose)" ]; then
   tput setaf 2; echo "docker-compose already installed, skipping."
 else
-  curl -L "https://github.com/docker/compose/releases/download/v2.23.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+  curl -L "https://github.com/docker/compose/releases/download/v2.5.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
   chmod +x /usr/local/bin/docker-compose
   ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
   tput setaf 2; echo "docker-compose installed!!!"
 fi
 
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -100,6 +126,8 @@ else
   exit 1
 fi
 
+
+
 echo " "
 tput setaf 4;
 echo "#########################################################################"
@@ -115,9 +143,10 @@ if [ "${failed}" -eq 0 ]; then
   echo "#########################################################################"
   echo "Creating an account"
   echo "#########################################################################"
-  make username
+  make username isNonInteractive=$isNonInteractive
 
   tput setaf 2 && printf "\n%s\n" "Thank you for installing reNgine, happy recon!!"
+  echo "In case you have unapplied migrations (see above in red), run 'make migrate'"
 else
   tput setaf 1 && printf "\n%s\n" "reNgine installation failed!!"
 fi

make.bat

@@ -15,6 +15,8 @@ if "%1" == "up" docker-compose %COMPOSE_ALL_FILES% up -d --build %SERVICES%
 if "%1" == "build" docker-compose %COMPOSE_ALL_FILES% build %SERVICES%
 :: Generate Username (Use only after make up).
 if "%1" == "username" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py createsuperuser
+:: Apply migrations
+if "%1" == "migrate" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py migrate
 :: Pull Docker images.
 if "%1" == "pull" docker login docker.pkg.github.com & docker-compose %COMPOSE_ALL_FILES% pull
 :: Down all services.

web/api/views.py

@@ -547,6 +547,7 @@ def post(self, request):
 		h1_team_handle = data.get('h1_team_handle')
 		description = data.get('description')
 		domain_name = data.get('domain_name')
+		organization_name = data.get('organization')
 		slug = data.get('slug')
 
 		# Validate domain name
@@ -563,6 +564,20 @@ def post(self, request):
 		if not domain.insert_date:
 			domain.insert_date = timezone.now()
 		domain.save()
+
+		# Create org object in DB
+		if organization_name:
+			organization_obj = None
+			organization_query = Organization.objects.filter(name=organization_name)
+			if organization_query.exists():
+				organization_obj = organization_query[0]
+			else:
+				organization_obj = Organization.objects.create(
+					name=organization_name,
+					project=project,
+					insert_date=timezone.now())
+			organization_obj.domains.add(domain)
+
 		return Response({
 			'status': True,
 			'message': 'Domain successfully added as target !',
@@ -712,6 +727,7 @@ def post(self, request):
 				task_ids = scan.celery_ids
 				scan.scan_status = ABORTED_TASK
 				scan.stop_scan_date = timezone.now()
+				scan.aborted_by = request.user
 				scan.save()
 				create_scan_activity(
 					scan.id,
@@ -958,7 +974,7 @@ def get(self, request):
 			return Response({'status': False, 'message': 'Not Found'})
 		elif not response:
 			return Response({'status': False, 'message': 'Not Found'})
-		
+
 		# only send latest release
 		response = response[0]