Merge branch 'main' into update-emba

.github/workflows/codeql-analysis.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/default-install.yml

@@ -27,7 +27,7 @@ jobs:
           android: true
           dotnet: true
           haskell: true
-          large-packages: true
+          large-packages: false
           swap-storage: true
       - name: EMBArk default install
         uses: Wandalen/wretry.action@master

.github/workflows/django-check.yml

@@ -9,20 +9,20 @@ jobs:
 
     steps:
     - name: Checkout the Code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Get Python 3.10
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.10"
     - name: Case try other python
       if: ${{ failure() }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: "3.9"
     - name: Install pipenv
       run: python -m pip install --upgrade pipenv
     - id: cache-pipenv
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       with:
         path: ~/.local/share/virtualenvs/
         key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}

.github/workflows/build.yml → .github/workflows/install-test-onpush.yml

@@ -1,6 +1,6 @@
 # This workflow will run build EMBArk in dev-mode
 
-name: Build-EMBArk-DEV 
+name: Build-EMBArk-DEV-test
 
 on: [push]
 
@@ -20,7 +20,7 @@ jobs:
           android: true
           dotnet: true
           haskell: true
-          large-packages: true
+          large-packages: false
           swap-storage: true
       - name: EMBArk dev install
         uses: Wandalen/wretry.action@master

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
     - name: Checkout Branch
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Install dependencies for linting
       run: |
         sudo apt-get install -y python3-dev npm pycodestyle python3-pylint-django bandit

.github/workflows/server-tests.yml

@@ -20,7 +20,7 @@ jobs:
           android: true
           dotnet: true
           haskell: true
-          large-packages: true
+          large-packages: false
           swap-storage: true
       - name: EMBArk default install
         uses: Wandalen/wretry.action@master

.github/workflows/shellcheck.yml

@@ -23,7 +23,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: ShellCheck
         # You may pin to the exact commit or the version.

.gitignore

@@ -21,3 +21,6 @@ safe/
 *.report
 *.service
 *.sql
+
+# migrations
+**/migrations/*_initial.py

.pylintrc

@@ -367,4 +367,4 @@ exclude-protected=
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

README.md

@@ -18,7 +18,7 @@
 *EMBArk* is a tool for centralized firmware analyzing.
 Scan, Identify, Track, Report.
 Built to be simple but powerful.
-It's the web-based enterprise interface for the firmware security scanner *EMBA*. It is developed to provide the firmware security analyzer *[EMBA](https://github.com/e-m-b-a/emba)* as a service with feature-packed UI and to imporove accessibility to the firmware scanning backend *EMBA* regardless of the system and operating system.
+It's the web-based enterprise interface for the firmware security scanner *EMBA*. It is developed to provide the firmware security analyzer *[EMBA](https://github.com/e-m-b-a/emba)* as a service with feature-packed UI and to improve accessibility to the firmware scanning backend *EMBA* regardless of the system and operating system.
 Furthermore, *EMBArk* improves the data provision by aggregating the various scanning results in an [aggregated management dashboard](https://github.com/e-m-b-a/embark/wiki/Web-interface#main-dashboard).
 
 [![Watch EMBArk](https://raw.githubusercontent.com/wiki/e-m-b-a/embark/static/images/EMBArk-YouTube.png)](https://youtu.be/qSHuPWbfhmI "Watch EMBArk")

dev-tools/debug-server-start.sh

@@ -95,13 +95,7 @@ if ! (cd ./emba && ./emba -d 1) ; then
   exit 1
 fi
 
-echo -e "\n$GREEN""$BOLD""Setup mysql and redis docker images""$NC"
-if docker-compose -f ./docker-compose.yml up -d ; then
-  echo -e "$GREEN""$BOLD""Finished setup mysql and redis docker images""$NC"
-else
-  echo -e "$ORANGE""$BOLD""Failed setup mysql and redis docker images""$NC"
-  exit 1
-fi
+check_db
 
 if ! [[ -d "$PWD"/logs ]]; then
   mkdir logs

dev-tools/prepare.sh

@@ -83,13 +83,7 @@ if grep -q -i wsl /proc/version; then
   WSL=1
 fi
 
-echo -e "\n$GREEN""$BOLD""Setup mysql and redis docker images""$NC"
-if docker-compose -f ./docker-compose.yml up -d ; then
-  echo -e "$GREEN""$BOLD""Finished setup mysql and redis docker images""$NC"
-else
-  echo -e "$ORANGE""$BOLD""Failed setup mysql and redis docker images""$NC"
-  exit 1
-fi
+check_db
 
 if ! [[ -d "$PWD"/logs ]]; then
   mkdir logs

dev-tools/test.sh

@@ -17,7 +17,9 @@ ORANGE='\033[0;33m'
 # BLUE='\033[0;34m'
 BOLD='\033[1m'
 NC='\033[0m' # no color
+HELP_DIR=./helper
 
+export HELP_DIR='helper'
 export DJANGO_SETTINGS_MODULE=embark.settings.dev
 export EMBARK_DEBUG=True
 export PIPENV_VENV_IN_PROJECT="True"
@@ -28,34 +30,45 @@ cleaner() {
   fi
 
   # killall -9 -q "*daphne*"
-  docker container stop embark_db_dev
-  docker container stop embark_redis_dev
+  docker container stop embark_db
+  docker container stop embark_redis
 
   docker container prune -f --filter "label=flag"
 
   fuser -k "$PORT"/tcp
   exit 1
 }
 
+import_helper()
+{
+  local HELPERS=()
+  local HELPER_COUNT=0
+  local HELPER_FILE=""
+  mapfile -d '' HELPERS < <(find "$HELP_DIR" -iname "helper_embark_*.sh" -print0 2> /dev/null)
+  for HELPER_FILE in "${HELPERS[@]}" ; do
+    if ( file "$HELPER_FILE" | grep -q "shell script" ) && ! [[ "$HELPER_FILE" =~ \ |\' ]] ; then
+      # https://github.com/koalaman/shellcheck/wiki/SC1090
+      # shellcheck source=/dev/null
+      source "$HELPER_FILE"
+      (( HELPER_COUNT+=1 ))
+    fi
+  done
+  echo -e "\\n""==> ""$GREEN""Imported ""$HELPER_COUNT"" necessary files""$NC\\n"
+}
+
 set -a
 trap cleaner INT
 
 cd "$(dirname "$0")" || exit 1
 cd .. || exit 1
-
+import_helper
 echo -e "\n$GREEN""$BOLD""Configuring Embark""$NC"
 
 # shellcheck disable=SC1091
 source ./.venv/bin/activate || exit 1
 
-echo -e "\n$GREEN""$BOLD""Setup mysql and redis docker images""$NC"
-docker-compose -f ./docker-compose.yml up -d
-DU_RETURN=$?
-if [[ $DU_RETURN -eq 0 ]] ; then
-  echo -e "$GREEN""$BOLD""Finished setup mysql and redis docker images""$NC"
-else
-  echo -e "$ORANGE""$BOLD""Failed setup mysql and redis docker images""$NC"
-fi
+#start and check db
+check_db
 
 if ! [[ -d ./logs ]]; then
   mkdir ./logs
@@ -69,15 +82,20 @@ pipenv run ./embark/manage.py migrate | tee -a ./logs/migration.log
 # superuser
 pipenv run ./embark/manage.py createsuperuser --noinput
 
+# add privs
+# echo -e "$BLUE""$BOLD""[+] Adding permissions for testing""$NC\\n"
+# mysql --host="$(grep DATABASE_HOST ./.env | sed 's/DATABASE\_HOST\=//')" --user=root --password="$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON test_db.* TO 'embark'@'%';"
+
 echo -e "\n[""$BLUE JOB""$NC""] Redis logs are copied to ./embark/logs/redis.log""$NC"
 docker container logs embark_redis -f > ./logs/redis.log &
 echo -e "\n[""$BLUE JOB""$NC""] DB logs are copied to ./embark/logs/mysql.log""$NC"
 docker container logs embark_db -f > ./logs/mysql.log &
 
 ##
 echo -e "\n[""$BLUE JOB""$NC""] Testing""$NC"
+pipenv run ./embark/manage.py test
 pipenv run ./embark/manage.py test embark.test_logreader
-pipenv run ./embark/manage.py test users.tests.SeleniumTests.test_register
-pipenv run ./embark/manage.py test users.tests.SeleniumTests.test_login
-
+# pipenv run ./embark/manage.py test users.tests.SeleniumTests.test_register
+# pipenv run ./embark/manage.py test users.tests.SeleniumTests.test_login
+pipenv run ./embark/manage.py test porter.tests.TestImport
 echo -e "\n$ORANGE""$BOLD""Done. To clean-up use the clean-setup script""$NC"

docker-compose.yml

@@ -17,6 +17,7 @@ services:
     restart: always
     volumes:
       - ./mysql.cnf:/etc/my.cnf
+      - ./mysql-init:/docker-entrypoint-initdb.d
       - ./embark_db:/var/lib/mysql
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro

embark/dashboard/models.py

@@ -8,7 +8,7 @@ class Vulnerability(models.Model):
     """
     Many-to-Many object for CVEs
     """
-    cve = models.CharField(max_length=13, validators=[MinLengthValidator(17)], help_text='CVE-XXXX-XXXXXXX')
+    cve = models.CharField(max_length=18, validators=[MinLengthValidator(13)], help_text='CVE-XXXX-XXXXXXX')
     info = models.JSONField(null=True)
 
 
@@ -64,5 +64,6 @@ class Result(models.Model):
 
     bins_checked = models.IntegerField(default=0, help_text='')
     strcpy_bin = models.TextField(default='{}')
+    system_bin = models.TextField(default='{}')
 
     vulnerability = models.ManyToManyField(Vulnerability, help_text='CVE/Vulnerability', related_query_name='CVE', editable=True, blank=True)

embark/dashboard/urls.py

@@ -6,9 +6,9 @@
 # view routing
 urlpatterns = [
     path('dashboard/main/', views.main_dashboard, name='embark-MainDashboard'),
-    # TODO add un-auth view for main dashboard
     path('dashboard/service/', views.service_dashboard, name='embark-dashboard-service'),
     path('dashboard/report/', views.report_dashboard, name='embark-ReportDashboard'),
     path('dashboard/individualReport/<uuid:analysis_id>', views.individual_report_dashboard, name='embark-IndividualReportDashboard'),
-    path('dashboard/stop/', views.stop_analysis, name='embark-stop-analysis')
+    path('dashboard/stop/', views.stop_analysis, name='embark-stop-analysis'),
+    path('dashboard/log/<uuid:analysis_id>', views.show_log, name='embark-show-log')
 ]

embark/dashboard/views.py

@@ -1,11 +1,12 @@
 import builtins
 import logging
 import os
+from pathlib import Path
 import signal
 
 from django.conf import settings
 from django.shortcuts import render
-from django.http import HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect, HttpResponseServerError
+from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect, HttpResponseServerError
 from django.contrib.auth.decorators import login_required
 from django.views.decorators.http import require_http_methods
 from tracker.forms import AssociateForm
@@ -22,7 +23,7 @@
 @login_required(login_url='/' + settings.LOGIN_URL)
 def main_dashboard(request):
     if request.user.is_authenticated:
-        if Result.objects.all().count() > 0:
+        if FirmwareAnalysis.objects.filter(finished=True, failed=False).count() > 0 and Result.objects.all().count() > 0:
             return render(request, 'dashboard/mainDashboard.html', {'nav_switch': True, 'username': request.user.username})
         return HttpResponseRedirect('../../uploader/')
     return HttpResponseForbidden
@@ -68,8 +69,6 @@ def service_dashboard(request):
     :params request: req
     :return httpresp: html servicedashboard
     """
-    # TODO send logreader update on refresh!!!
-    # if FirmwareAnalysis.objects.all().count() > 0:
     form = StopAnalysisForm()
     form.fields['analysis'].queryset = FirmwareAnalysis.objects.filter(finished=False)
     return render(request, 'dashboard/serviceDashboard.html', {'username': request.user.username, 'form': form, 'success_message': False})
@@ -101,3 +100,25 @@ def individual_report_dashboard(request, analysis_id):
     logger.info("individual_dashboard - analyze_id: %s", analysis_id)
     form = AssociateForm()
     return render(request, 'dashboard/individualReportDashboard.html', {'username': request.user.username, 'analysis_id': analysis_id, 'form': form})
+
+
+@require_http_methods(["GET"])
+@login_required(login_url='/' + settings.LOGIN_URL)
+def show_log(request, analysis_id):
+    """
+    renders emba_run.log
+
+    :params request: HTTP request
+
+    :return: rendered emba_run.log
+    """
+    logger.info("showing log for analyze_id: %s", analysis_id)
+    firmware = FirmwareAnalysis.objects.get(id=analysis_id)
+    # get the file path
+    log_file_path_ = f"{Path(firmware.path_to_logs).parent}/emba_run.log"
+    logger.debug("Taking file at %s and render it", log_file_path_)
+    try:
+        with open(log_file_path_, 'r', encoding='utf-8') as log_file_:
+            return HttpResponse(content=log_file_, content_type="text/plain")
+    except FileNotFoundError:
+        return HttpResponseServerError(content="File is not yet available")

embark/embark/logreader.py

@@ -22,10 +22,10 @@
 logger = logging.getLogger(__name__)
 
 # EMBAs module count
-EMBA_S_MOD_CNT = 39
-EMBA_P_MOD_CNT = 20
-EMBA_F_MOD_CNT = 4
-EMBA_L_MOD_CNT = 7
+EMBA_S_MOD_CNT = settings.EMBA_S_MOD_CNT
+EMBA_P_MOD_CNT = settings.EMBA_P_MOD_CNT
+EMBA_F_MOD_CNT = settings.EMBA_F_MOD_CNT
+EMBA_L_MOD_CNT = settings.EMBA_L_MOD_CNT
 EMBA_MODULE_CNT = EMBA_S_MOD_CNT + EMBA_P_MOD_CNT + EMBA_F_MOD_CNT + EMBA_L_MOD_CNT
 
 EMBA_PHASE_CNT = 4  # P, S, L, F modules
@@ -148,7 +148,9 @@ def update_status(self, stream_item_list):
 
         # set attributes of current message
         self.status_msg["module"] = stream_item_list[0]
-        self.status_msg["percentage"] = percentage
+        # ignore all Q-modules for percentage calc
+        if not re.match(".*Q[0-9][0-9]", stream_item_list[0]):
+            self.status_msg["percentage"] = percentage
 
         # get copy of the current status message
         self.save_status()

embark/embark/settings/deploy.py

@@ -166,7 +166,7 @@
         },
         'uploader': {
             'handlers': ['debug_handler', 'info_handler', 'console_handler'],
-            'level': 'DEBUG',   # TODO change to INFO
+            'level': 'INFO',
         },
         'dashboard': {
             'handlers': ['debug_handler', 'info_handler', 'console_handler'],
@@ -292,3 +292,29 @@
 # SSL stuff
 SECURE_HSTS_SECONDS = 0
 SECURE_SSL_REDIRECT = False
+
+
+def count_emba_modules(emba_dir_path):
+    s_module_cnt, p_module_cnt, q_module_cnt, l_module_cnt, f_module_cnt = 0, 0, 0, 0, 0
+    for mod_file_ in os.listdir(f"{emba_dir_path}/modules"):
+        if mod_file_.startswith('S'):
+            s_module_cnt += 1
+        elif mod_file_.startswith('P'):
+            p_module_cnt += 1
+        elif mod_file_.startswith('F'):
+            f_module_cnt += 1
+        elif mod_file_.startswith('L'):
+            l_module_cnt += 1
+        elif mod_file_.startswith('Q'):
+            q_module_cnt += 1
+    return s_module_cnt, p_module_cnt, f_module_cnt, l_module_cnt, q_module_cnt
+
+
+try:
+    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_F_MOD_CNT, EMBA_L_MOD_CNT, EMBA_Q_MOD_CNT = count_emba_modules(EMBA_ROOT)
+except FileNotFoundError as file_error:
+    print("[Warning] Installation is missing the EMBA submodule")
+    EMBA_S_MOD_CNT = 44
+    EMBA_P_MOD_CNT = 18
+    EMBA_F_MOD_CNT = 4
+    EMBA_L_MOD_CNT = 8