Skip to content

Commit deb3e13

Browse files
committedDec 13, 2021
Update for 3.1.0 release
1 parent a02e6d5 commit deb3e13

25 files changed

+636
-461
lines changed
 

‎LICENSE.txt

+362-284
Large diffs are not rendered by default.

‎VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.0.5
1+
3.1.0

‎setup/auto_install_allinone.sh

+3-12
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,6 @@ cat << "__EOF__"
4242
UUU UUU UUU
4343
__EOF__
4444

45-
RAM_TOTAL=$(free -m | grep Mem | awk '{print $2}')
46-
if [ $RAM_TOTAL -lt 7200 ]; then
47-
echo
48-
echo "*** ERROR: Sandfly Server requires 8GB or more of RAM. Please increase"
49-
echo " this system's RAM and log in again to start the Sandfly Server"
50-
echo " installation process."
51-
exit 1
52-
fi
53-
5445
echo
5546
echo "******************************************************************************"
5647
echo "***** Sandfly Automated Single-VM Setup **************************************"
@@ -134,14 +125,14 @@ done
134125
echo
135126
echo "******************************************************************************"
136127
echo "Waiting for Sandfly Server to configure and start. This will take about"
137-
echo "60 seconds."
128+
echo "20 seconds."
138129
echo "******************************************************************************"
139130
echo
140131
$SUDO ./start_server.sh >/dev/null 2>&1
141132
# Wait a maximum of 2 minutes, double what we should need
142133
TIMER=120
143134
while true; do
144-
docker logs sandfly-server 2>&1 | grep "started and is ready" > /dev/null
135+
docker logs sandfly-server 2>&1 | grep "Starting Sandfly API service version" > /dev/null
145136
if [ $? -eq 0 ]; then
146137
echo
147138
break
@@ -189,7 +180,7 @@ echo "** SANDFLY INSTALLATION COMPLETE
189180
echo "** **"
190181
echo "** Use the URL and login information printed below to log in to your **"
191182
echo "** server. The initial admin password is stored on this server in **"
192-
echo "** the setup_data directory; we recommend you change your intial **"
183+
echo "** the setup_data directory; we recommend you change your initial **"
193184
echo "** password after logging in. **"
194185
echo "** **"
195186
echo "******************************************************************************"

‎setup/install.sh

+10-57
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
# This script will install the Sandfly server. By default, it will run
66
# through an interactive setup process that is appropriate for users wishing
7-
# to control the location of Elasticsearch, Rabbit, etc.
7+
# to control the location of Rabbit, etc.
88
#
99
# The script is also capable of performing a non-interactive automated all-
1010
# in-one single-system setup. To perform the automated setup, set the
@@ -19,10 +19,11 @@
1919

2020
# Make sure we run from the correct directory so relative paths work
2121
cd "$( dirname "${BASH_SOURCE[0]}" )"
22+
SETUP_DATA_DIR=./setup_data
2223

2324
VERSION=${SANDFLY_SETUP_VERSION:-$(cat ../VERSION)}
2425
DOCKER_BASE=${SANDFLY_SETUP_DOCKER_BASE:-quay.io/sandfly}
25-
export SANDFLY_MGMT_DOCKER_IMAGE="$DOCKER_BASE/sandfly-server-mgmt:$VERSION"
26+
export SANDFLY_MGMT_DOCKER_IMAGE="$DOCKER_BASE/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2627

2728
# Is this an automated install?
2829
[ -n "$SANDFLY_SETUP_AUTO_HOSTNAME" ] && export SANDFLY_AUTO=YES
@@ -49,63 +50,15 @@ Sandfly Management Image: $SANDFLY_MGMT_DOCKER_IMAGE
4950
5051
EOF
5152

52-
if [ -z "$SANDFLY_AUTO" ]; then
53-
cat << EOF
54-
55-
******************************************************************************
56-
Elasticsearch Database Setup
57-
58-
If you want to use an external Elasticsearch cluster, please fill in the field
59-
below with the URL. Otherwise, just hit enter and we'll use the default URL.
60-
61-
The default URL is internally routed only with the Sandfly server and is not
62-
reachable over the network.
63-
64-
External Elasticsearch clusters will need to be secured according to your
65-
network policies. If you are using a username/password and SSL for an external
66-
Elasticsearch cluster then the URL should be the format:
67-
68-
https://username:password@elastic.example.com:9200
69-
70-
Where username is the username for Elasticsearch (default "elastic") and
71-
password is the password for the login you configured.
72-
73-
After setup is completed, you can copy over a certificate for the SSL
74-
connection for the Elasticsearch cluster. Please see the documentation for
75-
more details on how to do this.
76-
******************************************************************************
77-
78-
EOF
79-
80-
read -p "Optional Elasticsearch URL (Default: http://elasticsearch:9200): " ELASTIC_SERVER_URL
81-
fi
82-
83-
if [[ ! "$ELASTIC_SERVER_URL" ]]; then
84-
echo "No Elasticsearch URL provided. Using default."
85-
else
86-
echo "Setting Elasticsearch URL to: $ELASTIC_SERVER_URL"
87-
export ELASTIC_SERVER_URL
88-
fi
89-
9053
docker network create sandfly-net 2>/dev/null
9154
docker rm sandfly-server-mgmt 2>/dev/null
9255

93-
# Use standard Elasticsearch image unless overriden.
94-
if [[ -z "${ELASTIC_SERVER_URL}" ]]
95-
then
96-
echo "Starting default Elasticsearch database. Please wait a bit."
97-
../start_scripts/start_elastic.sh
98-
temp_cnt=30
99-
while [[ ${temp_cnt} -gt 0 ]];
100-
do
101-
printf "\rWaiting %2d second(s) for Elasticsearch to start and settle down." ${temp_cnt}
102-
sleep 1
103-
((temp_cnt--))
104-
done
105-
else
106-
echo "Using remote Elasticsearch URL for database: $ELASTIC_SERVER_URL"
107-
fi
108-
56+
# The first time we start Postgres, we need to assign a superuser password.
57+
POSTGRES_ADMIN_PASSWORD=$(< /dev/urandom tr -dc A-Za-z0-9 | head -c40)
58+
echo "$POSTGRES_ADMIN_PASSWORD" > $SETUP_DATA_DIR/postgres.admin.password.txt
59+
echo "Starting Postgres database."
60+
../start_scripts/start_postgres.sh
61+
sleep 5
10962

11063
./setup_scripts/setup_server.sh
11164
if [[ $? -eq 1 ]]
@@ -169,7 +122,7 @@ server:
169122
170123
./start_sandfly.sh
171124
172-
Your randomly generated password for the admin account is is located under:
125+
Your randomly generated password for the admin account is located under:
173126
174127
$PWD/setup_data/admin.password.txt
175128
******************************************************************************

‎setup/setup_data/templates/config.server.template.json

+7-11
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"config_version": 2,
23
"server": {
34
"keys": {
45
"server": {
@@ -7,12 +8,12 @@
78
},
89
"node": {
910
"public_key": "server_keys_node_public_key_b64"
10-
}
11+
},
12+
"jwt": "server_keys_jwt"
1113
},
1214
"ssl": {
1315
"server": {
1416
"cacert": "server_ssl_server_cacert_b64",
15-
"dhparam": "server_ssl_server_dhparam_b64",
1617
"cert": "server_ssl_server_cert_b64",
1718
"cert_signed": "server_ssl_server_cert_signed_b64",
1819
"private_key": "server_ssl_server_key_b64",
@@ -32,15 +33,10 @@
3233
"password": "api_password"
3334
},
3435
"db": {
35-
"elastic": {
36-
"main": {
37-
"url": "db_elastic_main_url",
38-
"cacert": "db_elastic_main_cacert_b64"
39-
},
40-
"replication": {
41-
"url": "db_elastic_replication_url",
42-
"cacert": "db_elastic_replication_cacert_b64"
43-
}
36+
"postgres": {
37+
"connstr": "db_postgres_connstr",
38+
"postgres_password": "db_postgres_postgres_password",
39+
"sandfly_password": "db_postgres_sandfly_password"
4440
}
4541
},
4642
"options": {

‎setup/setup_scripts/setup_config_json.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ EOF
2020
# Use standard docker image unless overriden.
2121
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
2222
VERSION=$(cat ../VERSION)
23-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
23+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2424
fi
2525

2626
docker network create sandfly-net 2>/dev/null

‎setup/setup_scripts/setup_demo_license.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ docker version >/dev/null 2>&1 || { echo "This script must be run as root or as
1111
# Use standard docker image unless overriden.
1212
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
1313
VERSION=$(cat ../VERSION)
14-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
14+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
1515
fi
1616

1717
CONFIG_JSON=$(cat setup_data/config.server.json) \

‎setup/setup_scripts/setup_keys.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ EOF
2020
# Use standard docker image unless overriden.
2121
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
2222
VERSION=$(cat ../VERSION)
23-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
23+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2424
fi
2525

2626
# Sets up PGP keys pair for server and node.

‎setup/setup_scripts/setup_server.sh

+1-2
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ EOF
2020
# Use standard docker image unless overriden.
2121
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
2222
VERSION=$(cat ../VERSION)
23-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
23+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2424
fi
2525

2626
docker network create sandfly-net 2>/dev/null
@@ -33,6 +33,5 @@ docker run -v /dev/urandom:/dev/random:ro \
3333
-v $PWD/setup_data:/usr/local/sandfly/install/setup_data \
3434
--name sandfly-server-mgmt \
3535
--network sandfly-net \
36-
-e ELASTIC_SERVER_URL \
3736
-e SANDFLY_SETUP_AUTO_HOSTNAME \
3837
$DOCKER_INTERACTIVE $SANDFLY_MGMT_DOCKER_IMAGE /usr/local/sandfly/install/install_server.sh

‎setup/setup_scripts/setup_ssl.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ EOF
2020
# Use standard docker image unless overriden.
2121
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
2222
VERSION=$(cat ../VERSION)
23-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
23+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2424
fi
2525

2626
# Generates initial SSL keys for the Sandfly Server.

‎setup/setup_scripts/setup_ssl_renew_cert.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ EOF
2121
# Use standard docker image unless overriden.
2222
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
2323
VERSION=$(cat ../VERSION)
24-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
24+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
2525
fi
2626

2727
# Calls EFF Certbot to get a signed key for the Sandfly Server.

‎setup/setup_scripts/setup_ssl_signed.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ EOF
2929
# Use standard docker image unless overriden.
3030
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
3131
VERSION=$(cat ../VERSION)
32-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
32+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
3333
fi
3434

3535
# Calls EFF Certbot to get a signed key for the Sandfly Server.

‎setup/setup_scripts/setup_ssl_signed_aio.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ fi
6666
# Use standard docker image unless overriden.
6767
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
6868
VERSION=$(cat ../VERSION)
69-
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt:$VERSION"
69+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
7070
fi
7171

7272
docker network create sandfly-net 2>/dev/null

‎setup/upgrade.sh

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env bash
2+
# Sandfly Security LTD www.sandflysecurity.com
3+
# Copyright (c) 2016-2021 Sandfly Security LTD, All Rights Reserved.
4+
5+
# Make sure we run from the correct directory so relative paths work
6+
cd "$( dirname "${BASH_SOURCE[0]}" )"
7+
8+
SETUP_DATA=../setup/setup_data
9+
VERSION=${SANDFLY_VERSION:-$(cat ../VERSION)}
10+
IMAGE_BASE=${SANDFLY_IMAGE_BASE:-quay.io/sandfly}
11+
12+
# Use standard docker image unless overriden.
13+
if [[ -z "${SANDFLY_MGMT_DOCKER_IMAGE}" ]]; then
14+
SANDFLY_MGMT_DOCKER_IMAGE="quay.io/sandfly/sandfly-server-mgmt${IMAGE_SUFFIX}:$VERSION"
15+
fi
16+
17+
18+
# See if we can run Docker
19+
which docker >/dev/null 2>&1 || { echo "Unable to locate docker binary; please install Docker."; exit 1; }
20+
docker version >/dev/null 2>&1 || { echo "This script must be run as root or as a user with access to the Docker daemon."; exit 1; }
21+
22+
docker network create sandfly-net 2>/dev/null
23+
docker rm sandfly-server-mgmt 2>/dev/null
24+
25+
# We can only upgrade if Sandfly is already installed and configured.
26+
if [ ! -f $SETUP_DATA/config.server.json ]; then
27+
echo ""
28+
echo "********************************** ERROR **********************************"
29+
echo "* *"
30+
echo "* Sandfly does not appear to be configured. Please use install.sh to *"
31+
echo "* install Sandfly on this host, not upgrade.sh. *"
32+
echo "* *"
33+
echo "********************************** ERROR **********************************"
34+
echo ""
35+
exit 1
36+
fi
37+
38+
# Don't upgrade if currently running
39+
server_result=$(docker inspect --format="{{.State.Running}}" sandfly-server 2> /dev/null)
40+
rabbit_result=$(docker inspect --format="{{.State.Running}}" sandfly-rabbit 2> /dev/null)
41+
if [ "${server_result}z" == "truez" -o "${rabbit_result}z" == "truez" ]; then
42+
echo ""
43+
echo "********************************** ERROR **********************************"
44+
echo "* *"
45+
echo "* Sandfly is currently running, so cannot be upgraded. Please stop all *"
46+
echo "* Sandfly containers (e.g. \`docker ls\` to get list, then for each name, *"
47+
echo "* \`docker stop <name>\`). *"
48+
echo "* *"
49+
echo "********************************** ERROR **********************************"
50+
echo ""
51+
exit 1
52+
fi
53+
54+
# jq might not be available on the outer Docker host, so we'll do a simple grep
55+
# to make sure the config version isn't set yet (and thus needs to be upgraded)
56+
grep -q \"config_version\":\ 2, $SETUP_DATA/config.server.json
57+
if [ $? == 0 ]; then
58+
echo ""
59+
echo "********************************** ERROR **********************************"
60+
echo "* *"
61+
echo "* Sandfly appears to already be upgraded to the correct version. *"
62+
echo "* *"
63+
echo "********************************** ERROR **********************************"
64+
echo ""
65+
exit 1
66+
fi
67+
68+
# Get the old configuration.
69+
CONFIG_JSON=$(cat $SETUP_DATA/config.server.json)
70+
export CONFIG_JSON
71+
72+
# Back up the old config
73+
mkdir -p $SETUP_DATA/backup
74+
cp $SETUP_DATA/*.json $SETUP_DATA/backup
75+
76+
# Start the Postgres server
77+
# The first time we start Postgres, we need to assign a superuser password.
78+
if [ ! -f $SETUP_DATA_DIR/postgres.admin.password.txt ]; then
79+
POSTGRES_ADMIN_PASSWORD=$(< /dev/urandom tr -dc A-Za-z0-9 | head -c40)
80+
echo "$POSTGRES_ADMIN_PASSWORD" > $SETUP_DATA/postgres.admin.password.txt
81+
fi
82+
echo "*** Starting Postgres database."
83+
../start_scripts/start_postgres.sh
84+
if [ $? -ne 0 ]; then
85+
echo "*** ERROR: Error starting Postgres container; cannot proceed."
86+
exit 1
87+
fi
88+
sleep 5
89+
90+
### Start ElasticSearch if not already running
91+
esresult=$(docker inspect --format="{{.State.Running}}" elasticsearch 2> /dev/null)
92+
if [ "${esresult}z" != "truez" ]; then
93+
echo "*** Starting ElasticSearch."
94+
../start_scripts/start_elastic.sh
95+
if [ $? -ne 0 ]; then
96+
echo "*** ERROR: Error starting ElasticSearch container; cannot proceed."
97+
exit 2
98+
fi
99+
temp_cnt=30
100+
while [[ ${temp_cnt} -gt 0 ]];
101+
do
102+
printf "\rWaiting %2d second(s) for Elasticsearch to start and settle down." ${temp_cnt}
103+
sleep 1
104+
((temp_cnt--))
105+
done
106+
echo ""
107+
else
108+
echo "*** ElasticSearch container already running."
109+
fi
110+
111+
docker run \
112+
-v $PWD/setup_data:/usr/local/sandfly/install/setup_data \
113+
--name sandfly-server-mgmt \
114+
--network sandfly-net \
115+
$SANDFLY_MGMT_DOCKER_IMAGE /usr/local/sandfly/install/upgrade.sh
116+
117+
if [ $? != 0 ]; then
118+
echo "*** ERROR: Upgrade process failed. See above messages for details."
119+
exit 1
120+
fi
121+
122+
echo "Stopping Elasticsearch"
123+
docker stop elasticsearch
124+
docker rm elasticsearch
125+
126+
echo ""
127+
echo "*********************************** INFO ***********************************"
128+
echo "* *"
129+
echo "* The upgrade to Sandfly 3.1 is complete. Users, credentials, hosts, *"
130+
echo "* schedules, and other configuration data has been migrated to the 3.1 *"
131+
echo "* Postgres database. You will need to run new scans (or wait for scheduled *"
132+
echo "* scans) for results to start re-populating. *"
133+
echo "* *"
134+
echo "* Sandfly no longer uses Elasticsearch for local data storage. When *"
135+
echo "* starting Sandfly 3.1, use the start_postgres.sh start script instead of *"
136+
echo "* the old start_elastic.sh start script. (Or use the start_sandfly.sh *"
137+
echo "* script which starts all necessary server components automatically.) *"
138+
echo "* *"
139+
echo "* Your Sandfly 3.0 Elasticsearch database Docker volume is still available *"
140+
echo "* if you need to roll back the upgrade. The Sandfly 3.0 configuration *"
141+
echo "* files have been backed up to the setup_data/backup directory. *"
142+
echo "* *"
143+
echo "* When you are satisfied with the Sandfly 3.1 upgrade, you may permanently *"
144+
echo "* delete your Sandfly 3.0 Elasticsearch database with the command: *"
145+
echo "* docker volume rm sandfly-elastic-db-vol *"
146+
echo "* *"
147+
echo "* Sandfly support is available at https://support.sandflysecurity.com/ *"
148+
echo "* *"
149+
echo "*********************************** INFO ***********************************"
150+
echo ""

‎setup/util_scripts/dump_custom_sandflies.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ fi
4141

4242
echo "Pulling custom sandfly data from: $HOSTNAME"
4343

44-
ACCESS_TOKEN=$(curl -s -k --request POST --header "Content-Type: application/json" --url https://"$HOSTNAME"/v3/auth/login \
44+
ACCESS_TOKEN=$(curl -s -k --request POST --header "Content-Type: application/json" --url https://"$HOSTNAME"/v4/auth/login \
4545
--data "{\"username\":\"admin\",\"password\":\"$PASSWORD\"}" | jq -r ".access_token")
4646
if [[ "$ACCESS_TOKEN" == "null" ]]; then
4747
echo "Couldn't get access token for REST API. Check hostname and credentials and try again."
@@ -50,7 +50,7 @@ fi
5050
echo "Password OK. Dumping custom sandfly data."
5151

5252
SANDFLY_JSON=$(curl -s -k --request GET --header "Content-Type: application/json" --header "Authorization: Bearer $ACCESS_TOKEN" \
53-
--url https://"$HOSTNAME"/v3/sandflies/custom/backup | jq ".")
53+
--url https://"$HOSTNAME"/v4/sandflies/backup | jq ".")
5454
if [[ "$SANDFLY_JSON" == "null" ]]; then
5555
echo "Custom sandfly list appears empty. Nothing to dump."
5656
exit 1

‎setup/util_scripts/dump_hosts.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ fi
4343

4444
echo "Pulling host data from: $HOSTNAME"
4545

46-
ACCESS_TOKEN=$(curl -s -k --request POST --header "Content-Type: application/json" --url https://"$HOSTNAME"/v3/auth/login \
46+
ACCESS_TOKEN=$(curl -s -k --request POST --header "Content-Type: application/json" --url https://"$HOSTNAME"/v4/auth/login \
4747
--data "{\"username\":\"admin\",\"password\":\"$PASSWORD\"}" | jq -r ".access_token")
4848

4949
if [[ "$ACCESS_TOKEN" == "null" ]]; then
@@ -53,7 +53,7 @@ fi
5353
echo "Password OK. Dumping hosts."
5454

5555
HOST_JSON=$(curl -s -k --request GET --header "Content-Type: application/json" --header "Authorization: Bearer $ACCESS_TOKEN" \
56-
--url https://"$HOSTNAME"/v3/hosts | jq ".")
56+
--url https://"$HOSTNAME"/v4/hosts | jq ".")
5757
if [[ "$HOST_JSON" == "null" ]]; then
5858
echo "Host list appears empty. Nothing to dump."
5959
exit 1
@@ -63,6 +63,6 @@ echo "Saving host JSON to ./sandfly.hosts.json"
6363
echo "$HOST_JSON" > sandfly.hosts.json
6464

6565
echo "Saving hostname and credential ID to ./sandfly.hosts.csv"
66-
echo $HOST_JSON | jq -r '.hits.hits[]._source | "\(.hostname), \(.credentials_id)"' > sandfly.hosts.csv
66+
echo $HOST_JSON | jq -r '.data[] | "\(.hostname), \(.credentials_id)"' > sandfly.hosts.csv
6767

6868
echo "Done!"

‎setup/util_scripts/reset_admin_password.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ VERSION=${SANDFLY_VERSION:-$(cat ../../VERSION)}
1111
IMAGE_BASE=${SANDFLY_IMAGE_BASE:-quay.io/sandfly}
1212

1313
# Populate env variables.
14-
CONFIG_JSON=$(cat $SETUP_DATA/config.node.json)
14+
CONFIG_JSON=$(cat $SETUP_DATA/config.server.json)
1515
export CONFIG_JSON
1616

1717

@@ -21,5 +21,5 @@ docker rm sandfly-server-mgmt
2121
docker run --name sandfly-server-mgmt \
2222
--network sandfly-net \
2323
-e CONFIG_JSON \
24-
-it $IMAGE_BASE/sandfly-server-mgmt:"$VERSION" /usr/local/sandfly/utils/reset_admin_password.sh
24+
-it $IMAGE_BASE/sandfly-server-mgmt${IMAGE_SUFFIX}:"$VERSION" /usr/local/sandfly/utils/reset_admin_password.sh
2525

‎setup/util_scripts/reset_db_data.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ VERSION=${SANDFLY_VERSION:-$(cat ../../VERSION)}
1414
IMAGE_BASE=${SANDFLY_IMAGE_BASE:-quay.io/sandfly}
1515

1616
# Populate env variables.
17-
CONFIG_JSON=$(cat $SETUP_DATA/config.node.json)
17+
CONFIG_JSON=$(cat $SETUP_DATA/config.server.json)
1818
export CONFIG_JSON
1919

2020
docker network create sandfly-net 2>/dev/null
@@ -23,7 +23,7 @@ docker rm sandfly-server-mgmt
2323
docker run --name sandfly-server-mgmt \
2424
--network sandfly-net \
2525
-e CONFIG_JSON \
26-
-it $IMAGE_BASE/sandfly-server-mgmt:"$VERSION" /usr/local/sandfly/utils/init_data_db.sh
26+
-it $IMAGE_BASE/sandfly-server-mgmt${IMAGE_SUFFIX}:"$VERSION" /usr/local/sandfly/utils/init_data_db.sh
2727

2828

2929

‎setup/util_scripts/reset_system_password.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ VERSION=${SANDFLY_VERSION:-$(cat ../../VERSION)}
1313
IMAGE_BASE=${SANDFLY_IMAGE_BASE:-quay.io/sandfly}
1414

1515
# Populate env variables.
16-
CONFIG_JSON=$(cat $SETUP_DATA/config.node.json)
16+
CONFIG_JSON=$(cat $SETUP_DATA/config.server.json)
1717
export CONFIG_JSON
1818

1919
docker network create sandfly-net 2>/dev/null
@@ -22,6 +22,6 @@ docker rm sandfly-server-mgmt
2222
docker run --name sandfly-server-mgmt \
2323
--network sandfly-net \
2424
-e CONFIG_JSON \
25-
-it $IMAGE_BASE/sandfly-server-mgmt:"$VERSION" /usr/local/sandfly/utils/reset_system_password.sh
25+
-it $IMAGE_BASE/sandfly-server-mgmt${IMAGE_SUFFIX}:"$VERSION" /usr/local/sandfly/utils/reset_system_password.sh
2626

2727

‎start_scripts/start_elastic.sh

-51
This file was deleted.

‎start_scripts/start_node.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,4 @@ docker run -v /dev/urandom:/dev/random:ro \
2929
--disable-content-trust \
3030
--restart on-failure:5 \
3131
--security-opt="no-new-privileges:true" \
32-
-d $IMAGE_BASE/sandfly-node:"$VERSION" /usr/local/sandfly/start_node.sh
32+
-d $IMAGE_BASE/sandfly-node${IMAGE_SUFFIX}:"$VERSION" /usr/local/sandfly/start_node.sh

‎start_scripts/start_postgres.sh

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
3+
# Sandfly Security LTD www.sandflysecurity.com
4+
# Copyright (c) 2021 Sandfly Security LTD, All Rights Reserved.
5+
6+
# Make sure we run from the correct directory so relative paths work
7+
cd "$( dirname "${BASH_SOURCE[0]}" )"
8+
9+
# After the first time Postgres starts, the admin password will be set in the
10+
# database in the Docker volume we use, and setting the password through the
11+
# docker run command will have no effect (e.g. it doesn't try to change it if
12+
# a database already exists). If this is an initial install, the password we
13+
# want to use should be in setup_data courtesy of the install.sh script.
14+
POSTGRES_ADMIN_PASSWORD="unknown"
15+
if [ -f ../setup/setup_data/postgres.admin.password.txt ]; then
16+
POSTGRES_ADMIN_PASSWORD=$(cat ../setup/setup_data/postgres.admin.password.txt)
17+
fi
18+
19+
docker network create sandfly-net 2>/dev/null
20+
docker rm sandfly-postgres 2>/dev/null
21+
22+
docker run \
23+
--mount source=sandfly-pg14-db-vol,target=/var/lib/postgresql/data/pgdata \
24+
-d \
25+
-e POSTGRES_PASSWORD="$POSTGRES_ADMIN_PASSWORD" \
26+
-e PGDATA=/var/lib/postgresql/data/pgdata \
27+
--shm-size=1g \
28+
--restart on-failure:5 \
29+
--security-opt="no-new-privileges:true" \
30+
--network sandfly-net \
31+
--name sandfly-postgres \
32+
-t \
33+
postgres:14.0 \
34+
-c shared_buffers=375MB \
35+
-c effective_cache_size=1125MB \
36+
-c maintenance_work_mem=96000kB \
37+
-c checkpoint_completion_target=0.9 \
38+
-c wal_buffers=11520kB \
39+
-c default_statistics_target=100 \
40+
-c random_page_cost=1.1 \
41+
-c effective_io_concurrency=200 \
42+
-c work_mem=4800kB \
43+
-c min_wal_size=1GB \
44+
-c max_wal_size=4GB \
45+
-c max_worker_processes=2 \
46+
-c max_parallel_workers_per_gather=1 \
47+
-c max_parallel_workers=2 \
48+
-c max_parallel_maintenance_workers=1

‎start_scripts/start_rabbit.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,4 @@ docker run -v /dev/urandom:/dev/random:ro \
2727
--name sandfly-rabbit \
2828
--security-opt="no-new-privileges:true" \
2929
--publish 5673:5673 \
30-
-t $IMAGE_BASE/sandfly-rabbit:"$VERSION"
30+
-t $IMAGE_BASE/sandfly-rabbit${IMAGE_SUFFIX}:"$VERSION"

‎start_scripts/start_sandfly.sh

+6-14
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,17 @@ if [ $(id -u) -ne 0 ]; then
1414
fi
1515
fi
1616

17-
### Start ElasticSearch if not already running
18-
esresult=$($SUDO docker inspect --format="{{.State.Running}}" elasticsearch 2> /dev/null)
17+
### Start Postgres if not already running
18+
esresult=$($SUDO docker inspect --format="{{.State.Running}}" sandfly-postgres 2> /dev/null)
1919
if [ "${esresult}z" != "truez" ]; then
20-
echo "*** Starting ElasticSearch."
21-
$SUDO ./start_elastic.sh
20+
echo "*** Starting Postgres."
21+
$SUDO ./start_postgres.sh
2222
if [ $? -ne 0 ]; then
23-
echo "*** ERROR: Error starting ElasticSearch container; cannot proceed."
23+
echo "*** ERROR: Error starting Postgres container; cannot proceed."
2424
exit 2
2525
fi
26-
temp_cnt=30
27-
while [[ ${temp_cnt} -gt 0 ]];
28-
do
29-
printf "\rWaiting %2d second(s) for Elasticsearch to start and settle down." ${temp_cnt}
30-
sleep 1
31-
((temp_cnt--))
32-
done
33-
echo ""
3426
else
35-
echo "*** ElasticSearch container already running."
27+
echo "*** Postgres container already running."
3628
fi
3729

3830
### Start RabbitMQ if not already running

‎start_scripts/start_server.sh

+28-9
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,42 @@ if [ -e $SETUP_DATA/allinone ]; then
1414
fi
1515

1616
if [ -f $SETUP_DATA/config.node.json -a "$IGNORE_NODE_DATA_WARNING" != "YES" ]; then
17-
echo "********* WARNING ***********"
1817
echo ""
19-
echo "The node config data ($SETUP_DATA/config.node.json) is present on the server."
20-
echo "This file must be deleted from the server to fully protect the SSH keys stored"
21-
echo "in the database. It should only be on the nodes."
22-
echo ""
23-
echo "********* WARNING ***********"
18+
echo "********************************* WARNING *********************************"
19+
echo "* *"
20+
echo "* The node config data file at: *"
21+
printf "* %-67s *\n" "$SETUP_DATA/config.node.json"
22+
echo "* is present on the server. *"
23+
echo "* *"
24+
echo "* This file must be deleted from the server to fully protect the SSH keys *"
25+
echo "* stored in the database. It should only be on the nodes. *"
26+
echo "* *"
27+
echo "********************************* WARNING *********************************"
2428
echo ""
2529
echo "Are you sure you want to start the server with the node config data present?"
26-
read -p "Type YES if you're sure. (NO): " RESPONSE
30+
read -p "Type YES if you're sure. [NO]: " RESPONSE
2731
if [ "$RESPONSE" != "YES" ]; then
2832
echo "Halting server start."
2933
exit 1
3034
fi
3135
fi
3236

37+
# jq might not be available on the outer Docker host, so we'll do a simple grep
38+
# to make sure the config version is correct for this server version.
39+
grep -q \"config_version\":\ 2, $SETUP_DATA/config.server.json
40+
if [ $? != 0 ]; then
41+
echo ""
42+
echo "****************************** ERROR ******************************"
43+
echo "* *"
44+
echo "* The version of the server configuration file does not match *"
45+
echo "* this version of the Sandfly server. Please perform the upgrade *"
46+
echo "* procedure before starting Sandfly. *"
47+
echo "* *"
48+
echo "*******************************************************************"
49+
echo ""
50+
exit 1
51+
fi
52+
3353
# Populate env variables.
3454
CONFIG_JSON=$(cat $SETUP_DATA/config.server.json)
3555
export CONFIG_JSON
@@ -39,12 +59,11 @@ docker rm sandfly-server 2>/dev/null
3959

4060
docker run -v /dev/urandom:/dev/random:ro \
4161
-e CONFIG_JSON \
42-
--sysctl net.core.somaxconn=15000 \
4362
--disable-content-trust \
4463
--restart on-failure:5 \
4564
--security-opt="no-new-privileges:true" \
4665
--network sandfly-net \
4766
--name sandfly-server \
4867
--publish 443:8443 \
4968
--publish 80:8000 \
50-
-d $IMAGE_BASE/sandfly-server:"$VERSION" /usr/local/sandfly/start_api.sh
69+
-d $IMAGE_BASE/sandfly-server${IMAGE_SUFFIX}:"$VERSION" /opt/sandfly/start_api.sh

0 commit comments

Comments
 (0)
Please sign in to comment.