Merge branch 'release-2.0.0' into develop

cisagov · Nov 6, 2024 · fdabeb0 · fdabeb0
2 parents f8a41a2 + 93e9b19
commit fdabeb0
Show file tree

Hide file tree

Showing 46 changed files with 1,657 additions and 388 deletions.
diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml
@@ -1,10 +1,31 @@
 name: Cluster Run - Minimega
 
 on:
+  pull_request:
+    # branches:
+    #   - '*'
   workflow_dispatch:
-  # pull_request:
-  #  branches:
-  #    - '*'
+    inputs:
+      azure_region:
+        description: 'Azure region to deploy resources'
+        required: true
+        default: 'centralus'
+        type: choice
+        options:
+          - centralus
+          - eastus
+          - eastus2
+          - westus
+          - westus2
+          - westus3
+          - northcentralus
+          - southcentralus
+          - canadacentral
+          - canadaeast
+          - uksouth
+          - ukwest
+          - northeurope
+          - westeurope
 
 jobs:
   build-and-test-cluster:
@@ -84,7 +105,7 @@ jobs:
                 -g pipe-${{ env.UNIQUE_ID }} \
                 -s ${{ env.IP_ADDRESS }}/32 \
                 -vs Standard_D8_v4 \
-                -l centralus \
+                -l ${{ inputs.azure_region || 'centralus' }} \
                 -ast 23:00 \
                 -y
             "
@@ -199,7 +220,7 @@ jobs:
           # Retrieve policy ID
           POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
             ssh lme-user@${{ env.AZURE_IP }} '
-              curl -k -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
+              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
                 -H \"kbn-xsrf: true\" \
                 -H \"Content-Type: application/json\" |
               jq -r '.items[0].id'
@@ -210,7 +231,7 @@ jobs:
           # Retrieve enrollment token using the policy ID
           ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
             ssh lme-user@${{ env.AZURE_IP }} '
-              curl -k -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
+              curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
                 -H \"kbn-xsrf: true\" \
                 -H \"Content-Type: application/json\" \
                 -d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" |

diff --git a/README.md b/README.md
@@ -234,7 +234,7 @@ SERVICE_NAME=lme-elasticsearch.service
 sudo -i journalctl -xu $SERVICE_NAME
 ```
 
-If somehting is broken try restarting the services and making sure failed services reset before starting:
+If something is broken try restarting the services and making sure failed services reset before starting:
 ```bash
 #try resetting failed: 
 sudo -i systemctl  reset-failed lme*

diff --git a/ansible/post_install_local.yml b/ansible/post_install_local.yml
@@ -7,7 +7,7 @@
 
   vars:
     headers:
-      kbn-version: "8.12.2"
+      kbn-version: "8.15.3"
       kbn-xsrf: "kibana"
       Content-Type: "application/json"
     max_retries: 60
@@ -465,7 +465,7 @@
       register: dashboards
 
     - name: Upload dashboards to Kibana
-      shell: 'curl -X POST -k --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
+      shell: 'curl -X POST -kL --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
       loop: "{{ dashboards.files | map(attribute='path') | list }}"
       args:
         warn: false
@@ -610,7 +610,7 @@
       register: dashboards
 
     - name: Upload dashboards to Kibana
-      shell: 'curl -X POST -k --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
+      shell: 'curl -X POST -kL --user "{{ elastic_username }}":"{{ elastic_password }}" -H "kbn-xsrf: true" -F file=@"{{ item }}"  "{{ kibana_url }}"'
       args:
         warn: false
       loop: "{{ dashboards.files | map(attribute='path') | list }}"
@@ -901,7 +901,7 @@
       shell: >
         curl -X POST "{{ local_es_url }}/_security/user/readonly_user"
         -u "{{ elastic_username }}:{{ elastic_password }}"
-        -k
+        -kL
         -H "Content-Type: application/json"
         -d '{
               "password": "{{ read_only_password.stdout }}",

diff --git a/ansible/set_fleet.yml b/ansible/set_fleet.yml
@@ -6,7 +6,7 @@
 
   vars:
     headers:
-      kbn-version: "8.12.2"
+      kbn-version: "8.15.3"
       kbn-xsrf: "kibana"
       Content-Type: "application/json"
     max_retries: 60
@@ -109,7 +109,7 @@
         max_attempts=30
         delay=10
         while [ $attempt -lt $max_attempts ]; do
-          response=$(curl -s -o /dev/null -w "%{http_code}" -k -u elastic:{{ elastic_password }} {{ local_kbn_url }}/api/fleet/agents/setup)
+          response=$(curl -s -o /dev/null -w "%{http_code}" -kL -u elastic:{{ elastic_password }} {{ local_kbn_url }}/api/fleet/agents/setup)
           if [ "$response" = "200" ]; then
             echo "Fleet API is ready. Proceeding with configuration..."
             exit 0

diff --git a/config/containers.txt b/config/containers.txt
@@ -1,6 +1,6 @@
 docker.io/caddy:2-alpine
-docker.elastic.co/elasticsearch/elasticsearch:8.12.2
-docker.elastic.co/beats/elastic-agent:8.12.2
-docker.elastic.co/kibana/kibana:8.12.2
-docker.io/wazuh/wazuh-manager:4.7.5
+docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+docker.elastic.co/beats/elastic-agent:8.15.3
+docker.elastic.co/kibana/kibana:8.15.3
+docker.io/wazuh/wazuh-manager:4.9.1
 docker.io/jertel/elastalert2:2.20.0
diff --git a/config/elastalert2/misc/smtp_auth.yml b/config/elastalert2/misc/smtp_auth.yml
@@ -0,0 +1,2 @@
+user: "[email protected]"
+password: "giyq caym zqiw chje" #this is your app password if using gmail
diff --git a/config/elastalert2/rules/example-email-rule.yml b/config/elastalert2/rules/example-email-rule.yml
@@ -0,0 +1,21 @@
+name: EMAIL
+type: frequency
+index: wazuh-*
+num_events: 1
+timeframe: 
+  minutes: 1
+filter:
+- query:
+    match_phrase:
+      agent.ip: "10.1.0.4"
+alert: email
+alert_text: "ASDFASDF"
+alert_text_type: alert_text_only
+email:
+  - "[email protected]"
+smtp_ssl: true
+smtp_port: 465
+smtp_host: "smtp.gmail.com"
+from_addr: "[email protected]"
+smtp_auth_file: /opt/elastalert/misc/smtp_auth.yml
+
diff --git a/config/example.env b/config/example.env
@@ -23,7 +23,7 @@ LOCAL_ES_URL=https://127.0.0.1:9200
 #################
 
 # Version of Elastic products
-STACK_VERSION=8.12.2
+STACK_VERSION=8.15.3
 # Testing pre-releases? Use the SNAPSHOT option below:
 # STACK_VERSION=8.11.0-SNAPSHOT
 #

diff --git a/config/setup/acct-init.sh b/config/setup/acct-init.sh
@@ -15,7 +15,7 @@ if [ ! -f "${CERTS_DIR}/ACCOUNTS_CREATED" ]; then
   until curl -s --cacert config/certs/ca/ca.crt https://lme-elasticsearch:9200 | grep -q "missing authentication credentials"; do echo "WAITING"; sleep 30; done;
 
   echo "Setting kibana_system password";
-  until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://lme-elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 2; done;
+  until curl -L -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://lme-elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 2; done;
 
   echo "All done!" | tee "${CERTS_DIR}/ACCOUNTS_CREATED" ;
 fi

diff --git a/config/wazuh_cluster/wazuh_manager.conf b/config/wazuh_cluster/wazuh_manager.conf
@@ -101,91 +101,26 @@
     <skip_nfs>yes</skip_nfs>
   </sca>
 
-  <vulnerability-detector>
-    <enabled>yes</enabled>
-    <interval>5m</interval>
-    <min_full_scan_interval>6h</min_full_scan_interval>
-    <run_on_start>yes</run_on_start>
-
-    <!-- Ubuntu OS vulnerabilities -->
-    <provider name="canonical">
-      <enabled>yes</enabled>
-      <os>trusty</os>
-      <os>xenial</os>
-      <os>bionic</os>
-      <os>focal</os>
-      <os>jammy</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Debian OS vulnerabilities -->
-    <provider name="debian">
-      <enabled>no</enabled>
-      <os>buster</os>
-      <os>bullseye</os>
-      <os>bookworm</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- RedHat OS vulnerabilities -->
-    <provider name="redhat">
-      <enabled>no</enabled>
-      <os>5</os>
-      <os>6</os>
-      <os>7</os>
-      <os>8</os>
-      <os>9</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Amazon Linux OS vulnerabilities -->
-    <provider name="alas">
-      <enabled>no</enabled>
-      <os>amazon-linux</os>
-      <os>amazon-linux-2</os>
-      <os>amazon-linux-2022</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- SUSE OS vulnerabilities -->
-    <provider name="suse">
-      <enabled>no</enabled>
-      <os>11-server</os>
-      <os>11-desktop</os>
-      <os>12-server</os>
-      <os>12-desktop</os>
-      <os>15-server</os>
-      <os>15-desktop</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Arch OS vulnerabilities -->
-    <provider name="arch">
-      <enabled>no</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Alma Linux OS vulnerabilities -->
-    <provider name="almalinux">
-      <enabled>no</enabled>
-      <os>8</os>
-      <os>9</os>
-      <update_interval>1h</update_interval>
-    </provider>
-
-    <!-- Windows OS vulnerabilities -->
-    <provider name="msu">
-      <enabled>yes</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
+  <vulnerability-detection>
+     <enabled>yes</enabled>
+     <index-status>yes</index-status>
+     <feed-update-interval>60m</feed-update-interval>
+  </vulnerability-detection>
 
-    <!-- Aggregate vulnerabilities -->
-    <provider name="nvd">
-      <enabled>yes</enabled>
-      <update_interval>1h</update_interval>
-    </provider>
-
-  </vulnerability-detector>
+  <indexer>
+    <enabled>yes</enabled>
+    <hosts>
+      <!-- TODO make this use the official one we set in environment variables -->
+      <host>https://lme-elasticsearch:9200</host>
+    </hosts>
+    <ssl>
+      <certificate_authorities>
+        <ca>/etc/wazuh-manager/certs/ca/ca.crt</ca>
+      </certificate_authorities>
+      <certificate>/etc/wazuh-manager/certs/wazuh-manager/wazuh-manager.crt</certificate>
+      <key>/etc/wazuh-manager/certs/wazuh-manager/wazuh-manager.key</key>
+    </ssl>
+  </indexer>
 
   <!-- File integrity monitoring -->
   <syscheck>

diff --git a/dashboards/Readme.md b/dashboards/Readme.md
@@ -2,14 +2,16 @@
 
 ## Wazuh Dashboards: 
 For more info on these dashboards see wazuh's documentation: [LINK](https://documentation.wazuh.com/current/integrations-guide/elastic-stack/index.html)
-This is the dashboard URL: 
+This is the dashboard URL that inspired the current Wazuh dashboards: 
 ```bash
 https://packages.wazuh.com/integrations/elastic/4.x-8.x/dashboards/wz-es-4.x-8.x-dashboards.ndjson
 ```
 
 ## How to update dashboards 
 Currently you need to run `ansible-playbook post_install_local.yml` to upload the current LME dashboards.
 
+If you need to reupload them, you can delete the `INSTALLED` file in the appropriate `/opt/lme/dashboards` directory and re-run the `post install` script. 
+
 ## Updating to new dashboards and removing old ones (Starting with 1.1.0)
 Browse to `Kibana->Stack Management` then select `Saved Objects`.
 On the Saved Objects page, you can filter by dashboards.
@@ -20,7 +22,7 @@ Select the filter `Type` and select `dashboard`.
 You can delete all of the dashboards before importing the new ones. 
 
 
-### Exporting dashboards: *TODO test this*
+### Exporting dashboards: 
 It is recommended that you export your dashboards before updating them, especially if you have customized them or created new ones. 
 To export the dashboards use the `export_dashboards.py`.
 It is easiest to export them from the ubuntu machine where you have installed the ELK stack because the 
@@ -30,16 +32,24 @@ on your initial install.
 ##### The files will be exported to `./exported`
 
 #### Running on Ubuntu
-
+To get your password you can run: 
+```bash
+cd ~/LME #OR YOUR CLONE DIRECTORY
+source ./scripts/extract_secrets
 ```
-./export_dashboards.py -u elastic -p YOURUNIQUEPASS
+
+Then you can use the following command to export dashboards:
+```bash
+./export_dashboards.py -u elastic -p "$elastic"
 ```
 
 The modules should already be installed on Ubuntu, but If the script complains about missing modules:
-```
+```bash
 pip install -r requirements.txt 
 ```
 
+The dashboards will be exported to: `~/LME/dashboards/exported`
+
 #### Running on Windows
 You must have python and the modules installed. (You can install python 3 from the Microsoft Store). Then install the requirements: 
 ```