DOC: Changes documentation to mkdocs.

.github/workflows/build-docs.yml

@@ -1,39 +1,61 @@
-#Github Workflow to run test documentation built
-#
-#SPDX-FileCopyrightText: 2020 IntelMQ Team <[email protected]>
-#SPDX-License-Identifier: AGPL-3.0-or-later
-#
-name: "Build the documentation with sphinx"
+# SPDX-FileCopyrightText: 2023 Filip Pokorný
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: "Build and publish documentation"
+
 on:
   push:
-    branches: [develop, maintenance, master]
-    paths-ignore:
-      - '.github/**'
+    branches:
+      - develop
+      - maintenance
+      - mkdocs
+
   pull_request:
-    branches: [develop, maintenance]
-    paths-ignore:
-      - '.github/**'
+    branches:
+      - develop
+      - maintenance
+      - mkdocs
+
+  release:
+    types:
+      - published
+
+permissions:
+  contents: write
 
 jobs:
-  documentationbuild:
+  build:
     runs-on: ubuntu-latest
-    name: Build the documentation
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
-
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: "Checkout repository"
+        uses: actions/checkout@v3
+
+      - name: "Setup python"
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+
+      - name: "Install build dependencies"
+        run: |
+          pip install mkdocs-material mike lunr pygments mkdocstrings[python] mkdocs-material mkdocs-glightbox mkdocs-redirects mkdocs-minify-plugin
+
+      - name: "Prepare git"
+        run: |
+          git fetch origin gh-pages --depth=1
+          git config user.name intelmq-bot
+          git config user.email intelmq-bot
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
+      - name: "Build docs without publishing"
+        if: github.event_name == 'pull_request'
+        run: |
+          mkdocs build
 
-    - name: Install documentation dependencies
-      run: pip install -r docs/requirements.txt
+      - name: "Build docs with version tag and publish"
+        if: github.event_name == 'release'
+        run: |
+          mike deploy --push --force --update-aliases ${{ github.ref_name }} latest
 
-    - name: Build documentation
-      run: make -C docs html
+      - name: "Build docs with branch tag and publish"
+        if: github.event_name == 'push'
+        run: |
+          mike deploy --push --force ${{ github.ref_name }}

.github/workflows/codespell.excludelines

@@ -26,14 +26,8 @@ The correct name for the parameter "delimeter" is "delimiter". Please fix your c
 # intelmq/tests/bots/collectors/fireeye/first_request.json "attch"
 {"alert": [{"explanation": {"malwareDetected": {"malware": [{"md5Sum": "21232f297a57a5a743894a0e4a801fc3", "sha256": "8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918", "name": "Downloader.Emotet"}]}, "osChanges": []}, "src": {"smtpMailFrom": "[email protected]"}, "alertUrl": "https://127.0.0.1/emps/eanalysis?e_id=161862&type=attch", "action": "blocked", "occurred": "2021-01-05 14:19:06 +0100", "dst": {"smtpTo": "[email protected]"}, "smtpMessage": {"subject": "Online Streaming am 30.06.2020 1800 Uhr REMINDER"}, "applianceId": "16EV1C1A6K94", "id": 1454270, "rootInfection": 7113664, "sensorIp": "127.0.0.1", "name": "MALWARE_OBJECT", "severity": "MAJR", "uuid": "1591de22-4926-4124-b3ed-ffff96766295", "ack": "no", "product": "EMAIL_MPS", "sensor": "mail", "vlan": 0, "malicious": "yes", "scVersion": "0000.000"}], "appliance": "CMS", "version": "CMS (CMS) 0.0.0.000000", "msg": "concise", "alertsCount": 1}
 
-# ./docs/user/universe.rst:55: bund ==> bind, bound
-Developed and maintained by `Intevation <https://intevation.de>`_, initially funded by `BSI <http://bsi.bund.de/>`_.
-
-# ./docs/dev/harmonization-fields.rst:27: compromized ==> compromised
-Destination                 destination.local_hostname          :ref:`string`                 Some sources report a internal hostname within a NAT related to the name configured for a compromized system
-
-# ./docs/dev/harmonization-fields.rst:28: compromized ==> compromised
-Destination                 destination.local_ip                :ref:`ipaddress`              Some sources report a internal (NATed) IP address related a compromized system. N.B. RFC1918 IPs are OK here.
+# ./docs/overview.md:60: bund ==> bind, bound
+Developed and maintained by [Intevation](https://intevation.de), initially funded by [BSI](https://bsi.bund.de/).
 
 # ./intelmq/tests/bots/parsers/shodan/test_parser.py:36: ALLO ==> ALLOW
                          '   ALLO   MLST   MLSD   SITE   P@SW   STRU   CLNT   MFMT\n'

.gitignore

@@ -27,13 +27,11 @@ src/
 .eggs
 .secrets
 .venv/
+venv/
 .env
 
-# sphinx
-docs/source
-docs/_build
-docs/user/feeds.rst
-docs/dev/harmonization-fields.rst
+# mkdocs
+docs_build
 
 # Debian build filed
 debian/files

CHANGELOG.md

@@ -3,8 +3,8 @@
    SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
-CHANGELOG
-==========
+# CHANGELOG
+
 
 
 3.2.2 (unreleased)
@@ -70,6 +70,7 @@ CHANGELOG
 - Add a readthedocs configuration file to fix the build fail (PR#2403 by Sebastian Wagner).
 - Update/fix/improve the stuff related to the STOMP bots and integration with the *n6*'s
   Stream API (PR#2408 by Jan Kaliszewski).
+- Complete documentation overhaul. Change to markdown format. Uses the mkdocs-material (PR#2419 by Filip Pokorný).
 
 ### Packaging
 

Makefile

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: 2023 Filip Pokorný
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+.PHONY: coverage requirements requirements-dev docs clean
+
+coverage:
+	python -m pytest --cov=intelmq -v
+
+# requires pip-tools python module
+requirements:
+	pip-compile --no-strip-extras --quiet --output-file=requirements.txt setup.py
+
+# requires pip-tools python module
+requirements-dev:
+	pip-compile --no-strip-extras --quiet --extra=development --output-file=requirements-dev.txt setup.py
+
+# requires git and pip-tools python module
+requirements-all:
+	rm intelmq/bots/collectors/blueliv/REQUIREMENTS.txt # blueliv-sdk has conflicting 'requests' requirements
+	pip-compile --no-strip-extras --quiet setup.py intelmq/bots/*/*/REQUIREMENTS.txt --output-file requirements-all.txt
+	echo 'blueliv-python-sdk @ git+https://github.com/Blueliv/api-python-sdk' >> requirements-all.txt
+	git restore intelmq/bots/collectors/blueliv/REQUIREMENTS.txt
+
+docs: mkdocs.yml docs/* intelmq/etc/feeds.yaml intelmq/etc/harmonization.conf intelmq/lib/harmonization.py
+	python3 scripts/generate-feeds-docs.py
+	python3 scripts/generate-event-docs.py
+	mkdocs build
+
+clean:
+	rm -rf docs_build .mypy_cache .coverage .pytest_cache dist

README.md

@@ -0,0 +1 @@
+docs/index.md

SECURITY.md

@@ -1,14 +1,10 @@
 <!-- comment
-   SPDX-FileCopyrightText: 2015-2021 Sebastian Wagner
+   SPDX-FileCopyrightText: 2015-2023 Sebastian Wagner, Filip Pokorný
    SPDX-License-Identifier: AGPL-3.0-or-later
 -->
 
-IntelMQ Security Notes
-======================
+# Found a security issue?
 
-Found a security issue?
------------------------
-
-In case you find security-relevant bugs in IntelMQ, please contact [email protected].
+In case you find security-relevant bugs in IntelMQ, please contact [[email protected]](mailto:[email protected]).
 More information including the PGP key can be found on [CERT.at's website](https://www.cert.at/about/contact/contact_en.html).
 

debian/rules

@@ -30,12 +30,12 @@ build:
 
 override_dh_auto_build:
 	dh_auto_build
-	$(MAKE) -C docs html
+	# $(MAKE) docs
 
 override_dh_auto_install: $(BOTDOCS)
 	rm intelmq/bin/rewrite_config_files.py
 	sed -i -e '/#!\/usr\/bin\//d' intelmq/bin/*.py
-	sed -i -f debian/sedfile intelmq/etc/* docs/user/intelmqctl.rst docs/user/bots.rst setup.py contrib/logrotate/intelmq contrib/logcheck/logcheck.logfiles
+	sed -i -f debian/sedfile intelmq/etc/* setup.py contrib/logrotate/intelmq contrib/logcheck/logcheck.logfiles
 	dh_auto_install
 	# create directories
 	mkdir -p debian/intelmq/var/log/intelmq