From 17eeabed27d6b4dab81ec48fded1a4bed2a2e828 Mon Sep 17 00:00:00 2001 From: Connor Nelson Date: Tue, 13 Sep 2022 00:56:11 -0700 Subject: [PATCH 1/2] Setup basic EFK stack --- Dockerfile | 2 ++ docker-compose.yml | 25 +++++++++++++++++++++++++ etc/docker/daemon.json | 7 +++++++ fluentd/Dockerfile | 12 ++++++++++++ fluentd/etc/fluent.conf | 26 ++++++++++++++++++++++++++ run.sh | 1 + 6 files changed, 73 insertions(+) create mode 100644 etc/docker/daemon.json create mode 100644 fluentd/Dockerfile create mode 100644 fluentd/etc/fluent.conf diff --git a/Dockerfile b/Dockerfile index 91078994..43b77ba9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ ADD script /opt/pwn.college/script ADD ssh /opt/pwn.college/ssh ADD logging /opt/pwn.college/logging ADD nginx-proxy /opt/pwn.college/nginx-proxy +ADD fluentd /opt/pwn.college/fluentd ADD challenge /opt/pwn.college/challenge ADD dojo_plugin /opt/CTFd/CTFd/plugins/dojo_plugin ADD dojo_theme /opt/CTFd/CTFd/themes/dojo_theme @@ -36,6 +37,7 @@ ADD data_example /opt/pwn.college/data_example ADD docker-compose.yml /opt/pwn.college/docker-compose.yml ADD docker-entrypoint.sh /opt/pwn.college/docker-entrypoint.sh +ADD etc/docker /etc/docker ADD etc/ssh/sshd_config /etc/ssh/sshd_config ADD etc/systemd/system/pwn.college.service /etc/systemd/system/pwn.college.service ADD etc/systemd/system/pwn.college.logging.service /etc/systemd/system/pwn.college.logging.service diff --git a/docker-compose.yml b/docker-compose.yml index 7b024869..411dfd03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,6 +112,31 @@ services: - acme:/etc/acme.sh - /var/run/docker.sock:/var/run/docker.sock:ro + fluentd: + container_name: fluentd + build: fluentd + restart: always + volumes: + - ./fluentd/etc/fluent.conf:/fluentd/etc/fluent.conf + ports: + - "24224:24224" + - "24224:24224/udp" + + elasticsearch: + container_name: elasticsearch + image: elasticsearch:8.1.2 + restart: always + environment: + - xpack.security.enabled=false + - discovery.type=single-node + + kibana: + container_name: kibana + image: kibana:8.1.2 + restart: always + ports: + - "5601:5601" + volumes: conf: html: diff --git a/etc/docker/daemon.json b/etc/docker/daemon.json new file mode 100644 index 00000000..eec170a6 --- /dev/null +++ b/etc/docker/daemon.json @@ -0,0 +1,7 @@ +{ + "log-driver": "fluentd", + "log-opts": { + "fluentd-address": "localhost:24224", + "tag": "docker.{{.Name}}.{{.ID}}" + } +} diff --git a/fluentd/Dockerfile b/fluentd/Dockerfile new file mode 100644 index 00000000..21dab52a --- /dev/null +++ b/fluentd/Dockerfile @@ -0,0 +1,12 @@ +FROM fluent/fluentd:v1.15.2-debian-1.0 +USER root +RUN gem install fluent-plugin-elasticsearch --no-document --version 5.2.3 \ + && gem uninstall --ignore-dependencies faraday \ + && gem install faraday --no-document --version 1.10.0 \ + && gem sources --clear-all \ + && apt-get purge -y --auto-remove \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem +USER fluent + +# TODO: https://github.com/uken/fluent-plugin-elasticsearch/issues/984#issuecomment-1239739636 diff --git a/fluentd/etc/fluent.conf b/fluentd/etc/fluent.conf new file mode 100644 index 00000000..42259ead --- /dev/null +++ b/fluentd/etc/fluent.conf @@ -0,0 +1,26 @@ + + @type forward + port 24224 + bind 0.0.0.0 + + + + @type copy + + + @type elasticsearch + host elasticsearch + port 9200 + logstash_format true + logstash_prefix fluentd + logstash_dateformat %Y%m%d + include_tag_key true + type_name access_log + tag_key @log_name + flush_interval 1s + + + + @type stdout + + diff --git a/run.sh b/run.sh index 0043d01a..e283bfde 100755 --- a/run.sh +++ b/run.sh @@ -15,6 +15,7 @@ docker run \ --publish ${SSH_PORT:-22}:22 \ --publish ${HTTP_PORT:-80}:80 \ --publish ${HTTPS_PORT:-443}:443 \ + --publish 5601:5601 \ --env SETUP_HOSTNAME="$SETUP_HOSTNAME" \ --hostname dojo \ --name pwn.college \ From 259de3d6ecd1f7bd857c74461af9fe19429945fd Mon Sep 17 00:00:00 2001 From: Connor Nelson Date: Tue, 13 Sep 2022 13:00:28 -0700 Subject: [PATCH 2/2] Wait for fluentd --- docker-compose.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 411dfd03..71743ba7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,7 @@ services: - /opt/CTFd:/opt/CTFd:ro - /var/run/docker.sock:/var/run/docker.sock:ro depends_on: + - fluentd - db db: @@ -59,6 +60,8 @@ services: - ./data/mysql:/var/lib/mysql # This command is required to set important mariadb defaults command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --wait_timeout=28800, --log-warnings=0] + depends_on: + - fluentd cache: container_name: ctfd_cache @@ -66,6 +69,8 @@ services: restart: always volumes: - ./data/redis:/data + depends_on: + - fluentd mailserver: container_name: mailserver @@ -80,6 +85,8 @@ services: - ./data/dms/mail-state:/var/mail-state - ./data/dms/mail-logs:/var/log/mail - ./data/dms/config:/tmp/docker-mailserver + depends_on: + - fluentd nginx-proxy: container_name: nginx_proxy @@ -100,6 +107,8 @@ services: - ./nginx-proxy/etc/passwd:/etc/passwd - ./data/homes:/var/homes:shared - /var/run/docker.sock:/tmp/${DOCKER_PSLR}/docker.sock:ro + depends_on: + - fluentd nginx-proxy-acme: container_name: nginx_proxy_acme @@ -111,6 +120,8 @@ services: - certs:/etc/nginx/certs:rw - acme:/etc/acme.sh - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - fluentd fluentd: container_name: fluentd @@ -121,6 +132,11 @@ services: ports: - "24224:24224" - "24224:24224/udp" + logging: + driver: "json-file" + options: + max-size: "200k" + max-file: "10" elasticsearch: container_name: elasticsearch @@ -129,6 +145,8 @@ services: environment: - xpack.security.enabled=false - discovery.type=single-node + depends_on: + - fluentd kibana: container_name: kibana @@ -136,6 +154,8 @@ services: restart: always ports: - "5601:5601" + depends_on: + - fluentd volumes: conf: