Skip to content

Commit

Permalink
docker entrypoint/configuration fixes + refactoring (#1959)
Browse files Browse the repository at this point in the history
  • Loading branch information
mmetc authored Jan 3, 2023
1 parent 3ab6429 commit ff6ade7
Show file tree
Hide file tree
Showing 6 changed files with 119 additions and 210 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# We include .git in the build context because excluding it would break the
# "make release" target, which uses git to retrieve the build version and tag.
#.git
/tests/ansible
84 changes: 8 additions & 76 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,105 +4,37 @@ ARG GOVERSION=1.19

FROM golang:${GOVERSION}-alpine AS build

RUN go install github.com/mikefarah/yq/[email protected]

WORKDIR /go/src/crowdsec

COPY . .

# wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
SYSTEM="docker" make clean release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists

FROM alpine:latest as build-slim

RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata yq bash && \
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data \
yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
mkdir -p /var/lib/crowdsec/data

COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml

# NOTE: setting default values here would overwrite the ones set in config.yaml
# every time the container is started. We set the default in docker/config.yaml
# and document them in docker/README.md, but keep the variables empty here.

ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
ENV USE_WAL=

# register to app.crowdsec.net

ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=

# log verbosity

ENV LEVEL_TRACE=
ENV LEVEL_DEBUG=
ENV LEVEL_INFO=

# TLS setup ----------------------------------- #

ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=

# TLS setup ----------------------------------- #

ENV USE_TLS=false
ENV INSECURE_SKIP_VERIFY=

ENV CACERT_FILE=

ENV LAPI_CERT_FILE=
ENV LAPI_KEY_FILE=

ENV CLIENT_CERT_FILE=
ENV CLIENT_KEY_FILE=

# deprecated in favor of LAPI_*
ENV CERT_FILE=
ENV KEY_FILE=

# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=

# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=

# Install the following hub items --------------#

ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=

# Uninstall the following hub items ------------#

ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=

ENV METRICS_PORT=
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml

ENTRYPOINT /bin/bash docker_start.sh

Expand Down
90 changes: 12 additions & 78 deletions Dockerfile.debian
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ ARG GOVERSION=1.19

FROM golang:${GOVERSION}-bullseye AS build

RUN go install github.com/mikefarah/yq/[email protected]

WORKDIR /go/src/crowdsec

COPY . .
Expand All @@ -14,17 +16,20 @@ ENV DEBCONF_NOWARNINGS="yes"
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
SYSTEM="docker" make release && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
SYSTEM="docker" make clean release && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/[email protected]
cscli parsers install crowdsecurity/whitelists

FROM debian:bullseye-slim as build-slim

ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"

RUN apt-get update && \
apt-get install -y -q --install-recommends --no-install-suggests \
procps \
Expand All @@ -35,88 +40,17 @@ RUN apt-get update && \
tzdata && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data \
yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
mkdir -p /var/lib/crowdsec/data

COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml

# NOTE: setting default values here would overwrite the ones set in config.yaml
# every time the container is started. We set the default in docker/config.yaml
# and document them in docker/README.md, but keep the variables empty here.

ENV CONFIG_FILE=/etc/crowdsec/config.yaml
ENV LOCAL_API_URL=
ENV CUSTOM_HOSTNAME=localhost
ENV PLUGIN_DIR=
ENV DISABLE_AGENT=false
ENV DISABLE_LOCAL_API=false
ENV DISABLE_ONLINE_API=false
ENV DSN=
ENV TYPE=
ENV TEST_MODE=false
ENV USE_WAL=

# register to app.crowdsec.net

ENV ENROLL_INSTANCE_NAME=
ENV ENROLL_KEY=
ENV ENROLL_TAGS=

# log verbosity

ENV LEVEL_TRACE=
ENV LEVEL_DEBUG=
ENV LEVEL_INFO=

# TLS setup ----------------------------------- #

ENV AGENT_USERNAME=
ENV AGENT_PASSWORD=

# TLS setup ----------------------------------- #

ENV USE_TLS=false
ENV INSECURE_SKIP_VERIFY=

ENV CACERT_FILE=

ENV LAPI_CERT_FILE=
ENV LAPI_KEY_FILE=

ENV CLIENT_CERT_FILE=
ENV CLIENT_KEY_FILE=

# deprecated in favor of LAPI_*
ENV CERT_FILE=
ENV KEY_FILE=

# comma-separated list of allowed OU values for TLS bouncer certificates
ENV BOUNCERS_ALLOWED_OU=

# comma-separated list of allowed OU values for TLS agent certificates
ENV AGENTS_ALLOWED_OU=

# Install the following hub items --------------#

ENV COLLECTIONS=
ENV PARSERS=
ENV SCENARIOS=
ENV POSTOVERFLOWS=

# Uninstall the following hub items ------------#

ENV DISABLE_COLLECTIONS=
ENV DISABLE_PARSERS=
ENV DISABLE_SCENARIOS=
ENV DISABLE_POSTOVERFLOWS=
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml

ENV METRICS_PORT=

ENTRYPOINT /bin/bash docker_start.sh

Expand Down
43 changes: 30 additions & 13 deletions docker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -198,22 +198,33 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
# Reference
## Environment Variables

Note for persistent configurations (i.e. bind mount or volumes): when a
variable is set, its value may be written to the appropriate file (usually
config.yaml) each time the container is run.


| Variable | Default | Description |
| ----------------------- | ------------------------- | ----------- |
| `CONFIG_FILE` | `/etc/crowdsec/config.yaml` | Configuration file location |
| `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
| `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
| `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
| `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
| `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
| `DISABLE_AGENT` | false | Disable the agent, run a LAPI-only container |
| `DISABLE_LOCAL_API` | false | Disable LAPI, run an agent-only container |
| `DISABLE_ONLINE_API` | false | Disable online API registration for signal sharing |
| `CUSTOM_HOSTNAME` | localhost | Custom hostname for LAPI registration (with agent and LAPI on the same container) |
| `TEST_MODE` | false | Don't run the service, only test the configuration: `-e TEST_MODE=true` |
| `TZ` | | Set the [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to ensure the logs have a local timestamp. |
| `LOCAL_API_URL` | `http://0.0.0.0:8080` | The LAPI URL, you need to change this when `DISABLE_LOCAL_API` is true: `-e LOCAL_API_URL="http://lapi-address:8080"` |
| `PLUGIN_DIR` | `/usr/local/lib/crowdsec/plugins/` | Directory for plugins: `-e PLUGIN_DIR="<path>"` |
| `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
| `METRICS_PORT` | 6060 | Port to expose Prometheus metrics |
| | | |
| __LAPI__ | | (useless with DISABLE_LOCAL_API) |
| `USE_WAL` | false | Enable Write-Ahead Logging with SQLite |
| `CUSTOM_HOSTNAME` | localhost | Name for the local agent (running in the container with LAPI) |
| | | |
| __Agent__ | | (these don't work with DISABLE_AGENT) |
| `TYPE` | | [`Labels.type`](https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine: `-e TYPE="<type>"` |
| `DSN` | | Process a single source in time-machine: `-e DSN="file:///var/log/toto.log"` or `-e DSN="cloudwatch:///your/group/path:stream_name?profile=dev&backlog=16h"` or `-e DSN="journalctl://filters=_SYSTEMD_UNIT=ssh.service"` |
| | | |
| __Bouncers__ | | |
| `BOUNCER_KEY_<name>` | | Register a bouncer with the name `<name>` and a key equal to the value of the environment variable. |
| | | |
| __Console__ | | |
| `ENROLL_KEY` | | Enroll key retrieved from [the console](https://app.crowdsec.net/) to enroll the instance. |
Expand All @@ -224,14 +235,16 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
| `AGENT_USERNAME` | | Agent username (to register if is LAPI or to use if it's an agent): `-e AGENT_USERNAME="machine_id"` |
| `AGENT_PASSWORD` | | Agent password (to register if is LAPI or to use if it's an agent): `-e AGENT_PASSWORD="machine_password"` |
| | | |
| __TLS Auth/encryption | | |
| `USE_TLS` | false | Enable TLS on the LAPI |
| `CACERT_FILE` | | CA certificate bundle (optional) |
| __TLS Encryption__ | | |
| `USE_TLS` | false | Enable TLS encryption (either as a LAPI or agent) |
| `CACERT_FILE` | | CA certificate bundle (for self-signed certificates) |
| `INSECURE_SKIP_VERIFY` | | Skip LAPI certificate validation |
| `CLIENT_CERT_FILE` | | Client TLS Certificate path (enables TLS auth) |
| `LAPI_CERT_FILE` | | LAPI TLS Certificate path |
| `LAPI_KEY_FILE` | | LAPI TLS Key path |
| | | |
| __TLS Authentication__ | | (these require USE_TLS=true) |
| `CLIENT_CERT_FILE` | | Client TLS Certificate path (enable TLS authentication) |
| `CLIENT_KEY_FILE` | | Client TLS Key path |
| `LAPI_CERT_FILE` | | LAPI TLS Certificate path (required if USE_TLS) |
| `LAPI_KEY_FILE` | | LAPI TLS Key path (required if USE_TLS) |
| `AGENTS_ALLOWED_OU` | agent-ou | OU values allowed for agents, separated by comma |
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
| | | |
Expand All @@ -249,6 +262,10 @@ Using binds rather than named volumes ([complete explanation here](https://docs.
| `LEVEL_INFO` | false | Force INFO level for the container log |
| `LEVEL_DEBUG` | false | Force DEBUG level for the container log |
| `LEVEL_TRACE` | false | Force TRACE level (VERY verbose) for the container log |
| | | |
| __Developer options__ | | |
| `CI_TESTING` | false | Used during functional tests |
| `DEBUG` | false | Trace the entrypoint |

## Volumes

Expand Down
2 changes: 1 addition & 1 deletion docker/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ api:
- 127.0.0.1
- ::1
online_client: # Central API credentials (to push signals and receive bad IPs)
#credentials_path: /etc/crowdsec/online_api_credentials.yaml
#credentials_path: /etc/crowdsec/online_api_credentials.yaml
tls:
agents_allowed_ou:
- agent-ou
Expand Down
Loading

0 comments on commit ff6ade7

Please sign in to comment.