Merge branch 'development' of github.com:ipa-lab/hackingBuddyGPT into…

… log_infrastructure

.devcontainer/devcontainer.json

@@ -1,3 +1,3 @@
 {
-    "onCreateCommand": "./codespaces_create_and_start_containers.sh"
+	"onCreateCommand": "./scripts/codespaces_create_and_start_containers.sh"
 }

.gitignore

@@ -16,11 +16,12 @@ src/hackingBuddyGPT/usecases/web_api_testing/openapi_spec/
 src/hackingBuddyGPT/usecases/web_api_testing/converted_files/
 /src/hackingBuddyGPT/usecases/web_api_testing/documentation/openapi_spec/
 /src/hackingBuddyGPT/usecases/web_api_testing/documentation/reports/
-codespaces_ansible.cfg
-codespaces_ansible_hosts.ini
-codespaces_ansible_id_rsa
-codespaces_ansible_id_rsa.pub
-mac_ansible.cfg
-mac_ansible_hosts.ini
-mac_ansible_id_rsa
-mac_ansible_id_rsa.pub
+scripts/codespaces_ansible.cfg
+scripts/codespaces_ansible_hosts.ini
+scripts/codespaces_ansible_id_rsa
+scripts/codespaces_ansible_id_rsa.pub
+scripts/mac_ansible.cfg
+scripts/mac_ansible_hosts.ini
+scripts/mac_ansible_id_rsa
+scripts/mac_ansible_id_rsa.pub
+.aider*

MAC.md

@@ -14,7 +14,15 @@ There are bugs in Docker Desktop on Mac that prevent creation of a custom Docker
 
 Therefore, localhost TCP port 49152 (or higher) dynamic port number is used for an ansible-ready-ubuntu container
 
-http://localhost:8080 is genmini-openai-proxy
+http://localhost:8080 is gemini-openai-proxy
+
+gpt-4 maps to gemini-1.5-flash-latest
+
+Hence use gpt-4 below in --llm.model=gpt-4
+
+Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day
+
+Hence --max_turns 999999999 will exceed the daily limit
 
 For example:
 
@@ -23,7 +31,7 @@ export GEMINI_API_KEY=
 
 export PORT=49152
 
-wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gemini-1.5-flash-latest --llm.context_size=1000000 --conn.host=localhost --conn.port $PORT --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999
+wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gpt-4 --llm.context_size=1000000 --conn.host=localhost --conn.port $PORT --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999
 ```
 
 The above example is consolidated into shell scripts with prerequisites as follows:
@@ -40,7 +48,7 @@ The above example is consolidated into shell scripts with prerequisites as follo
 brew install bash
 ```
 
-Bash version 4 or higher is needed for `mac_create_and_start_containers.sh`
+Bash version 4 or higher is needed for `scripts/mac_create_and_start_containers.sh`
 
 Homebrew provides GNU Bash version 5 via license GPLv3+
 
@@ -49,7 +57,7 @@ Whereas Mac provides Bash version 3 via license GPLv2
 **Create and start containers:**
 
 ```zsh
-./mac_create_and_start_containers.sh
+./scripts/mac_create_and_start_containers.sh
 ```
 
 **Start hackingBuddyGPT against a container:**
@@ -59,7 +67,7 @@ export GEMINI_API_KEY=
 ```
 
 ```zsh
-./mac_start_hackingbuddygpt_against_a_container.sh
+./scripts/mac_start_hackingbuddygpt_against_a_container.sh
 ```
 
 **Troubleshooting:**

README.md

@@ -231,7 +231,7 @@ In the Command Palette, type `>` and `Terminal: Create New Terminal` and press t
 
 Type the following to manually run:
 ```bash
-./codespaces_start_hackingbuddygpt_against_a_container.sh
+./scripts/codespaces_start_hackingbuddygpt_against_a_container.sh
 ```
 7. Eventually, you should see:
 
@@ -300,6 +300,12 @@ $ python src/hackingBuddyGPT/cli/wintermute.py LinuxPrivesc --llm.api_key=sk...C
 $ pip install '.[testing]'
 ```
 
+## Use Cases
+
+Mac, Docker Desktop and Gemini-OpenAI-Proxy:
+
+* See https://github.com/ipa-lab/hackingBuddyGPT/blob/main/MAC.md
+
 ## Publications about hackingBuddyGPT
 
 Given our background in academia, we have authored papers that lay the groundwork and report on our efforts:

publish_notes.md

@@ -0,0 +1,34 @@
+# how to publish to pypi
+
+## start with testing if the project builds and tag the version
+
+```bash
+python -m venv venv
+source venv/bin/activate
+pip install -e .
+pytest
+git tag v0.3.0
+git push origin v0.3.0
+```
+
+## build and new package
+
+(according to https://packaging.python.org/en/latest/tutorials/packaging-projects/)
+
+```bash
+pip install build twine
+python3 -m build
+vi ~/.pypirc
+twine check dist/*
+```
+
+Now, for next time.. test install the package in a new vanilla environment, then..
+
+```bash
+twine upload dist/*
+```
+
+## repo todos
+
+- rebase development upon main
+- bump the pyproject version number to a new `-dev` 

pyproject.toml

@@ -4,22 +4,26 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hackingBuddyGPT"
-authors = [{ name = "Andreas Happe", email = "[email protected]" }]
+# original author was Andreas Happe, for an up-to-date list see
+# https://github.com/ipa-lab/hackingBuddyGPT/graphs/contributors
+authors = [
+	{ name = "HackingBuddyGPT maintainers", email = "[email protected]" }
+]
 maintainers = [
-    { name = "Andreas Happe", email = "[email protected]" },
-    { name = "Juergen Cito", email = "juergen.cito@tuwiena.c.at" },
+	{ name = "Andreas Happe", email = "[email protected]" },
+	{ name = "Juergen Cito", email = "juergen.cito@tuwien.ac.at" }
 ]
 description = "Helping Ethical Hackers use LLMs in 50 lines of code"
 readme = "README.md"
 keywords = ["hacking", "pen-testing", "LLM", "AI", "agent"]
 requires-python = ">=3.10"
-version = "0.3.1"
+version = "0.4.0-dev"
 license = { file = "LICENSE" }
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
-    "Development Status :: 4 - Beta",
+	"Development Status :: 4 - Beta",
 ]
 dependencies = [
     'fabric == 3.2.2',
@@ -61,8 +65,18 @@ pythonpath = "src"
 addopts = ["--import-mode=importlib"]
 [project.optional-dependencies]
 testing = ['pytest', 'pytest-mock']
+dev = [
+	'ruff',
+]
 
 [project.scripts]
 wintermute = "hackingBuddyGPT.cli.wintermute:main"
 hackingBuddyGPT = "hackingBuddyGPT.cli.wintermute:main"
 wintermuteViewer = "hackingBuddyGPT.cli.viewer:main"
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "B", "I"]
+ignore = ["E501", "F401", "F403"]

codespaces_create_and_start_containers.sh → scripts/codespaces_create_and_start_containers.sh

@@ -2,14 +2,23 @@
 
 # Purpose: In GitHub Codespaces, automates the setup of Docker containers,
 # preparation of Ansible inventory, and modification of tasks for testing.
-# Usage: ./codespaces_create_and_start_containers.sh
+# Usage: ./scripts/codespaces_create_and_start_containers.sh
 
 # Enable strict error handling for better script robustness
 set -e  # Exit immediately if a command exits with a non-zero status
 set -u  # Treat unset variables as an error and exit immediately
 set -o pipefail  # Return the exit status of the last command in a pipeline that failed
 set -x  # Print each command before executing it (useful for debugging)
 
+cd $(dirname $0)
+
+bash_version=$(/bin/bash --version | head -n 1 | awk '{print $4}' | cut -d. -f1)
+
+if (( bash_version < 4 )); then
+  echo 'Error: Requires Bash version 4 or higher.'
+  exit 1
+fi
+
 # Step 1: Initialization
 
 if [ ! -f hosts.ini ]; then

codespaces_start_hackingbuddygpt_against_a_container.sh → scripts/codespaces_start_hackingbuddygpt_against_a_container.sh

@@ -1,17 +1,27 @@
 #!/bin/bash
 
 # Purpose: In GitHub Codespaces, start hackingBuddyGPT against a container
-# Usage: ./codespaces_start_hackingbuddygpt_against_a_container.sh
+# Usage: ./scripts/codespaces_start_hackingbuddygpt_against_a_container.sh
 
 # Enable strict error handling for better script robustness
 set -e  # Exit immediately if a command exits with a non-zero status
 set -u  # Treat unset variables as an error and exit immediately
 set -o pipefail  # Return the exit status of the last command in a pipeline that failed
 set -x  # Print each command before executing it (useful for debugging)
 
+cd $(dirname $0)
+
+bash_version=$(/bin/bash --version | head -n 1 | awk '{print $4}' | cut -d. -f1)
+
+if (( bash_version < 4 )); then
+  echo 'Error: Requires Bash version 4 or higher.'
+  exit 1
+fi
+
 # Step 1: Install prerequisites
 
 # setup virtual python environment
+cd ..
 python -m venv venv
 source ./venv/bin/activate
 
@@ -35,3 +45,21 @@ echo "Starting hackingBuddyGPT against a container..."
 echo
 
 wintermute LinuxPrivesc --llm.api_key=$OPENAI_API_KEY --llm.model=gpt-4-turbo --llm.context_size=8192 --conn.host=192.168.122.151 --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1
+
+# Alternatively, the following comments demonstrate using gemini-openai-proxy and Gemini
+
+# http://localhost:8080 is gemini-openai-proxy
+
+# gpt-4 maps to gemini-1.5-flash-latest
+
+# Hence use gpt-4 below in --llm.model=gpt-4
+
+# Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day
+
+# Hence --max_turns 999999999 will exceed the daily limit
+
+# docker run --restart=unless-stopped -it -d -p 8080:8080 --name gemini zhu327/gemini-openai-proxy:latest
+
+# export GEMINI_API_KEY=
+
+# wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gpt-4 --llm.context_size=1000000 --conn.host=192.168.122.151 --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999

mac_create_and_start_containers.sh → scripts/mac_create_and_start_containers.sh

@@ -1,13 +1,22 @@
 #!/opt/homebrew/bin/bash
 
-# Purpose: Automates the setup of docker containers for local testing on Mac.
-# Usage: ./mac_create_and_start_containers.sh
+# Purpose: Automates the setup of docker containers for local testing on Mac
+# Usage: ./scripts/mac_create_and_start_containers.sh
 
-# Enable strict error handling
-set -e
-set -u
-set -o pipefail
-set -x
+# Enable strict error handling for better script robustness
+set -e  # Exit immediately if a command exits with a non-zero status
+set -u  # Treat unset variables as an error and exit immediately
+set -o pipefail  # Return the exit status of the last command in a pipeline that failed
+set -x  # Print each command before executing it (useful for debugging)
+
+cd $(dirname $0)
+
+bash_version=$(/opt/homebrew/bin/bash --version | head -n 1 | awk '{print $4}' | cut -d. -f1)
+
+if (( bash_version < 4 )); then
+  echo 'Error: Requires Bash version 4 or higher.'
+  exit 1
+fi
 
 # Step 1: Initialization
 
@@ -21,9 +30,6 @@ if [ ! -f tasks.yaml ]; then
     exit 1
 fi
 
-# Default value for base port
-# BASE_PORT=${BASE_PORT:-49152}
-
 # Default values for network and base port, can be overridden by environment variables
 DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME:-192_168_65_0_24}
 DOCKER_NETWORK_SUBNET="192.168.65.0/24"
@@ -251,6 +257,6 @@ docker --debug run --restart=unless-stopped -it -d -p 8080:8080 --name gemini-op
 
 # Step 14: Ready to run hackingBuddyGPT
 
-echo "You can now run ./mac_start_hackingbuddygpt_against_a_container.sh"
+echo "You can now run ./scripts/mac_start_hackingbuddygpt_against_a_container.sh"
 
 exit 0

mac_start_hackingbuddygpt_against_a_container.sh → scripts/mac_start_hackingbuddygpt_against_a_container.sh

@@ -1,17 +1,27 @@
 #!/bin/bash
 
 # Purpose: On a Mac, start hackingBuddyGPT against a container
-# Usage: ./mac_start_hackingbuddygpt_against_a_container.sh
+# Usage: ./scripts/mac_start_hackingbuddygpt_against_a_container.sh
 
 # Enable strict error handling for better script robustness
 set -e  # Exit immediately if a command exits with a non-zero status
 set -u  # Treat unset variables as an error and exit immediately
 set -o pipefail  # Return the exit status of the last command in a pipeline that failed
 set -x  # Print each command before executing it (useful for debugging)
 
+cd $(dirname $0)
+
+bash_version=$(/bin/bash --version | head -n 1 | awk '{print $4}' | cut -d. -f1)
+
+if (( bash_version < 3 )); then
+  echo 'Error: Requires Bash version 3 or higher.'
+  exit 1
+fi
+
 # Step 1: Install prerequisites
 
 # setup virtual python environment
+cd ..
 python -m venv venv
 source ./venv/bin/activate
 
@@ -44,9 +54,23 @@ echo
 
 PORT=$(docker ps | grep ansible-ready-ubuntu | cut -d ':' -f2 | cut -d '-' -f1)
 
+# http://localhost:8080 is gemini-openai-proxy
+
+# gpt-4 maps to gemini-1.5-flash-latest
+
+# https://github.com/zhu327/gemini-openai-proxy/blob/559085101f0ce5e8c98a94fb75fefd6c7a63d26d/README.md?plain=1#L146
+
+#    | gpt-4 | gemini-1.5-flash-latest |
+
+# https://github.com/zhu327/gemini-openai-proxy/blob/559085101f0ce5e8c98a94fb75fefd6c7a63d26d/pkg/adapter/models.go#L60-L61
+
+# 	case strings.HasPrefix(openAiModelName, openai.GPT4):
+# 		return Gemini1Dot5Flash
+
+# Hence use gpt-4 below in --llm.model=gpt-4
+
 # Gemini free tier has a limit of 15 requests per minute, and 1500 requests per day
-# Hence --max_turns 999999999 will exceed the daily limit
 
-# http://localhost:8080 is genmini-openai-proxy
+# Hence --max_turns 999999999 will exceed the daily limit
 
-wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gemini-1.5-flash-latest --llm.context_size=1000000 --conn.host=localhost --conn.port $PORT --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999
+wintermute LinuxPrivesc --llm.api_key=$GEMINI_API_KEY --llm.model=gpt-4 --llm.context_size=1000000 --conn.host=localhost --conn.port $PORT --conn.username=lowpriv --conn.password=trustno1 --conn.hostname=test1 --llm.api_url=http://localhost:8080 --llm.api_backoff=60 --max_turns 999999999

src/hackingBuddyGPT/capabilities/__init__.py

@@ -1,5 +1,13 @@
 from .capability import Capability
-from .psexec_test_credential import PSExecTestCredential
 from .psexec_run_command import PSExecRunCommand
+from .psexec_test_credential import PSExecTestCredential
 from .ssh_run_command import SSHRunCommand
-from .ssh_test_credential import SSHTestCredential
+from .ssh_test_credential import SSHTestCredential
+
+__all__ = [
+    "Capability",
+    "PSExecRunCommand",
+    "PSExecTestCredential",
+    "SSHRunCommand",
+    "SSHTestCredential",
+]