From 7bacc71d2fcbbc36547359ac2c30e94df70d16a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 11 Sep 2024 16:42:33 +0200 Subject: [PATCH 1/8] Add stateless index template definition Event generator is pending --- ecs/alerts/event-generator/event_generator.py | 114 ++++ ecs/alerts/fields/mapping-settings.json | 4 + ecs/alerts/fields/subset.yml | 596 ++++++++++++++++++ .../fields/template-settings-legacy.json | 18 + ecs/alerts/fields/template-settings.json | 18 + ecs/generate.sh | 10 +- 6 files changed, 759 insertions(+), 1 deletion(-) create mode 100644 ecs/alerts/event-generator/event_generator.py create mode 100644 ecs/alerts/fields/mapping-settings.json create mode 100644 ecs/alerts/fields/subset.yml create mode 100644 ecs/alerts/fields/template-settings-legacy.json create mode 100644 ecs/alerts/fields/template-settings.json diff --git a/ecs/alerts/event-generator/event_generator.py b/ecs/alerts/event-generator/event_generator.py new file mode 100644 index 0000000000000..f676f0176d444 --- /dev/null +++ b/ecs/alerts/event-generator/event_generator.py @@ -0,0 +1,114 @@ +#!/bin/python3 + +import datetime +import random +import json +import requests +import warnings +import logging + +# Constants and Configuration +LOG_FILE = 'generate_data.log' +GENERATED_DATA_FILE = 'generatedData.json' +DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +# Configure logging +logging.basicConfig(filename=LOG_FILE, level=logging.INFO) + +# Suppress warnings +warnings.filterwarnings("ignore") + + +def generate_random_date(): + start_date = datetime.datetime.now() + end_date = start_date - datetime.timedelta(days=10) + random_date = start_date + (end_date - start_date) * random.random() + return random_date.strftime(DATE_FORMAT) + + +def generate_random_agent(): + agent = { + 'id': f'agent{random.randint(0, 99)}', + 'name': f'Agent{random.randint(0, 99)}', + 'type': random.choice(['filebeat', 'windows', 'linux', 'macos']), + 'version': f'v{random.randint(0, 9)}-stable', + 'is_connected': random.choice([True, False]), + 'last_login': generate_random_date(), + 'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'], + 'key': f'key{random.randint(0, 999)}' + } + return agent + + +def generate_random_host(): + family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL']) + version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' + host = { + 'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}', + 'os': { + 'full': f'{family} {version}', + } + } + return host + + +def generate_random_data(number): + data = [] + for _ in range(number): + event_data = { + 'agent': generate_random_agent(), + 'host': generate_random_host(), + } + data.append(event_data) + return data + + +def inject_events(ip, port, index, username, password, data): + url = f'https://{ip}:{port}/{index}/_doc' + session = requests.Session() + session.auth = (username, password) + session.verify = False + headers = {'Content-Type': 'application/json'} + + try: + for event_data in data: + response = session.post(url, json=event_data, headers=headers) + if response.status_code != 201: + logging.error(f'Error: {response.status_code}') + logging.error(response.text) + break + logging.info('Data injection completed successfully.') + except Exception as e: + logging.error(f'Error: {str(e)}') + + +def main(): + try: + number = int(input("How many events do you want to generate? ")) + except ValueError: + logging.error("Invalid input. Please enter a valid number.") + return + + logging.info(f"Generating {number} events...") + data = generate_random_data(number) + + with open(GENERATED_DATA_FILE, 'a') as outfile: + for event_data in data: + json.dump(event_data, outfile) + outfile.write('\n') + + logging.info('Data generation completed.') + + inject = input( + "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() + if inject == 'y': + ip = input("Enter the IP of your Indexer: ") + port = input("Enter the port of your Indexer: ") + index = input("Enter the index name: ") + username = input("Username: ") + password = input("Password: ") + inject_events(ip, port, index, username, password, data) + + +if __name__ == "__main__": + main() diff --git a/ecs/alerts/fields/mapping-settings.json b/ecs/alerts/fields/mapping-settings.json new file mode 100644 index 0000000000000..0ad2b48fcc1be --- /dev/null +++ b/ecs/alerts/fields/mapping-settings.json @@ -0,0 +1,4 @@ +{ + "dynamic": "strict", + "date_detection": false +} \ No newline at end of file diff --git a/ecs/alerts/fields/subset.yml b/ecs/alerts/fields/subset.yml new file mode 100644 index 0000000000000..fa784b9806d6c --- /dev/null +++ b/ecs/alerts/fields/subset.yml @@ -0,0 +1,596 @@ +--- +name: main +fields: + base: + fields: "*" + agent: + fields: "*" + as: + fields: "*" + client: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + cloud: + fields: "*" + code_signature: + fields: "*" + container: + fields: "*" + data_stream: + fields: "*" + destination: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + device: + fields: "*" + dll: + fields: "*" + dns: + fields: "*" + ecs: + fields: "*" + elf: + fields: "*" + email: + fields: "*" + error: + fields: "*" + event: + fields: "*" + faas: + fields: "*" + file: + fields: "*" + geo: + fields: "*" + group: + fields: "*" + hash: + fields: "*" + host: + fields: "*" + http: + fields: "*" + interface: + fields: "*" + log: + fields: "*" + macho: + fields: "*" + network: + fields: "*" + observer: + fields: "*" + orchestrator: + fields: "*" + organization: + fields: "*" + os: + fields: "*" + package: + fields: "*" + pe: + fields: "*" + process: + fields: + args: {} + args_count: {} + code_signature: + fields: "*" + command_line: {} + elf: + fields: "*" + end: {} + entity_id: {} + entry_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + entry_meta: + fields: + type: {} + source: + fields: + ip: {} + executable: {} + interactive: {} + name: {} + parent: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + session_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + attested_user: + fields: + id: {} + name: {} + attested_groups: + fields: + name: {} + entry_meta: + fields: + type: + docs_only: True + env_vars: {} + executable: {} + exit_code: {} + group_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + executable: {} + interactive: {} + name: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + hash: + fields: "*" + interactive: {} + io: + fields: "*" + macho: + fields: "*" + name: {} + parent: + fields: + args: {} + args_count: {} + code_signature: + fields: "*" + command_line: {} + elf: + fields: "*" + end: {} + entity_id: {} + executable: {} + exit_code: {} + group_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + hash: + fields: "*" + interactive: {} + macho: + fields: "*" + name: {} + pe: + fields: "*" + pgid: {} + pid: {} + vpid: {} + start: {} + thread: + fields: + id: {} + name: {} + capabilities: + fields: + effective: {} + permitted: {} + title: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + uptime: {} + working_directory: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + pe: + fields: "*" + pgid: {} + pid: {} + vpid: {} + previous: + fields: + args: {} + args_count: {} + executable: {} + real_group: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + same_as_process: + docs_only: True + saved_group: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + start: {} + supplemental_groups: + fields: + id: {} + name: {} + session_leader: + fields: + args: {} + args_count: {} + command_line: {} + entity_id: {} + executable: {} + interactive: {} + name: {} + pid: {} + vpid: {} + same_as_process: {} + start: {} + tty: + fields: + char_device: + fields: + major: {} + minor: {} + working_directory: {} + parent: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + session_leader: + fields: + entity_id: {} + pid: {} + vpid: {} + start: {} + user: + fields: + id: {} + name: {} + real_user: + fields: + id: {} + name: {} + saved_user: + fields: + id: {} + name: {} + group: + fields: + id: {} + name: {} + real_group: + fields: + id: {} + name: {} + saved_group: + fields: + id: {} + name: {} + supplemental_groups: + fields: + id: {} + name: {} + thread: + fields: + id: {} + name: {} + capabilities: + fields: + effective: {} + permitted: {} + title: {} + tty: + fields: "*" + uptime: {} + user: + fields: + id: {} + name: {} + working_directory: {} + registry: + fields: "*" + related: + fields: "*" + risk: + fields: "*" + rule: + fields: "*" + server: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + service: + fields: "*" + source: + fields: + address: {} + as: + fields: "*" + bytes: {} + domain: {} + geo: + fields: "*" + ip: {} + mac: {} + nat: + fields: + ip: {} + port: {} + packets: {} + port: {} + subdomain: {} + registered_domain: {} + top_level_domain: {} + user: + fields: + domain: {} + email: {} + full_name: {} + group: + fields: "*" + hash: {} + id: {} + name: {} + roles: {} + threat: + fields: "*" + tls: + fields: "*" + tracing: + fields: "*" + url: + fields: "*" + user_agent: + fields: "*" + user: + fields: + changes: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + domain: {} + effective: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + risk: + fields: "*" + roles: {} + target: + fields: + domain: {} + email: {} + group: + fields: "*" + full_name: {} + hash: {} + id: {} + name: {} + roles: {} + vlan: + fields: "*" + vulnerability: + fields: "*" + x509: + fields: "*" \ No newline at end of file diff --git a/ecs/alerts/fields/template-settings-legacy.json b/ecs/alerts/fields/template-settings-legacy.json new file mode 100644 index 0000000000000..54aac2ceaf55c --- /dev/null +++ b/ecs/alerts/fields/template-settings-legacy.json @@ -0,0 +1,18 @@ +{ + "index_patterns": [ + "wazuh-alerts-5.x-*" + ], + "order": 1, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "mapping": { + "total_fields": { + "limit": 2500 + } + } + } + } +} \ No newline at end of file diff --git a/ecs/alerts/fields/template-settings.json b/ecs/alerts/fields/template-settings.json new file mode 100644 index 0000000000000..9982494c55ca2 --- /dev/null +++ b/ecs/alerts/fields/template-settings.json @@ -0,0 +1,18 @@ +{ + "index_patterns": [ + "wazuh-alerts-5.x-*" + ], + "priority": 1, + "template": { + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 2500 + } + }, + "refresh_interval": "5s" + } + } + } +} \ No newline at end of file diff --git a/ecs/generate.sh b/ecs/generate.sh index 3d96dd446284c..52b74fc9b8cb6 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -47,6 +47,14 @@ generate_mappings() { echo "Replacing \"match_only_text\" type with \"text\"" find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \; + # Replace "constant_keyword" type (not supported by OpenSearch) with "keyword" + echo "Replacing \"constant_keyword\" type with \"keyword\"" + find "$OUT_DIR" -type f -exec sed -i 's/constant_keyword/keyword/g' {} \; + + # Replace "flattened" type (not supported by OpenSearch) with "flat_object" + echo "Replacing \"flattened\" type with \"flat_object\"" + find "$OUT_DIR" -type f -exec sed -i 's/flattened/flat_object/g' {} \; + local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json" local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json" @@ -105,4 +113,4 @@ UPLOAD="${4:-false}" URL="${5:-https://localhost:9200}" # Generate mappings -generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" \ No newline at end of file +generate_mappings "$ECS_VERSION" "$INDEXER_SRC" "$MODULE" "$UPLOAD" "$URL" From 005336c5c0831983c39bd8069697b68d00da18a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 11 Sep 2024 17:49:40 +0200 Subject: [PATCH 2/8] Update to 8.11.0 --- ecs/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ecs/README.md b/ecs/README.md index d4897318585e2..a3818271a5e53 100644 --- a/ecs/README.md +++ b/ecs/README.md @@ -45,25 +45,25 @@ files to generate the mappings. These are the inputs for the ECS generator. * INDEXER_SRC: Path to the wazuh-indexer repository * MODULE: Module to generate mappings for * --upload : Upload generated index template to the OpenSearch cluster. Defaults to https://localhost:9200 - Example: generate.sh v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200 + Example: generate.sh v8.11.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200 ``` 3. Use the `generate.sh` script to generate the mappings for a module. The script takes 3 arguments, plus 2 optional arguments to upload the mappings to the `wazuh-indexer`. Both, composable and legacy mappings are generated. For example, to generate the mappings for the `vulnerability-detector` module using the - ECS version `v8.10.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`: + ECS version `v8.11.0` and assuming that path of this repository is `~/wazuh/wazuh-indexer`: ```bash - ./generate.sh v8.10.0 ~/wazuh/wazuh-indexer vulnerability-detector + ./generate.sh v8.11.0 ~/wazuh/wazuh-indexer vulnerability-detector ``` The tool will output the folder where they have been generated. ```console - Loading schemas from git ref v8.10.0 - Running generator. ECS version 8.10.0 + Loading schemas from git ref v8.11.0 + Running generator. ECS version 8.11.0 Replacing "match_only_text" type with "text" - Mappings saved to ~/wazuh/wazuh-indexer/ecs/vulnerability-detector/mappings/v8.10.0 + Mappings saved to ~/wazuh/wazuh-indexer/ecs/vulnerability-detector/mappings/v8.11.0 ``` 4. When you are done. Exit the virtual environment. From 4796e62b86be7946e576454287ba91fd569ec978 Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Wed, 11 Sep 2024 17:16:07 -0300 Subject: [PATCH 3/8] Adding template mappings and settings for states-inventory-packages index --- .../event-generator/event_generator.py | 114 ++++++++++++++++++ .../fields/custom/wazuh-agent.yml | 12 ++ .../fields/mapping-settings.json | 4 + .../fields/subset.yml | 21 ++++ .../fields/template-settings-legacy.json | 20 +++ .../fields/template-settings.json | 21 ++++ 6 files changed, 192 insertions(+) create mode 100644 ecs/states-inventory-packages/event-generator/event_generator.py create mode 100644 ecs/states-inventory-packages/fields/custom/wazuh-agent.yml create mode 100644 ecs/states-inventory-packages/fields/mapping-settings.json create mode 100644 ecs/states-inventory-packages/fields/subset.yml create mode 100644 ecs/states-inventory-packages/fields/template-settings-legacy.json create mode 100644 ecs/states-inventory-packages/fields/template-settings.json diff --git a/ecs/states-inventory-packages/event-generator/event_generator.py b/ecs/states-inventory-packages/event-generator/event_generator.py new file mode 100644 index 0000000000000..f676f0176d444 --- /dev/null +++ b/ecs/states-inventory-packages/event-generator/event_generator.py @@ -0,0 +1,114 @@ +#!/bin/python3 + +import datetime +import random +import json +import requests +import warnings +import logging + +# Constants and Configuration +LOG_FILE = 'generate_data.log' +GENERATED_DATA_FILE = 'generatedData.json' +DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + +# Configure logging +logging.basicConfig(filename=LOG_FILE, level=logging.INFO) + +# Suppress warnings +warnings.filterwarnings("ignore") + + +def generate_random_date(): + start_date = datetime.datetime.now() + end_date = start_date - datetime.timedelta(days=10) + random_date = start_date + (end_date - start_date) * random.random() + return random_date.strftime(DATE_FORMAT) + + +def generate_random_agent(): + agent = { + 'id': f'agent{random.randint(0, 99)}', + 'name': f'Agent{random.randint(0, 99)}', + 'type': random.choice(['filebeat', 'windows', 'linux', 'macos']), + 'version': f'v{random.randint(0, 9)}-stable', + 'is_connected': random.choice([True, False]), + 'last_login': generate_random_date(), + 'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'], + 'key': f'key{random.randint(0, 999)}' + } + return agent + + +def generate_random_host(): + family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL']) + version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' + host = { + 'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}', + 'os': { + 'full': f'{family} {version}', + } + } + return host + + +def generate_random_data(number): + data = [] + for _ in range(number): + event_data = { + 'agent': generate_random_agent(), + 'host': generate_random_host(), + } + data.append(event_data) + return data + + +def inject_events(ip, port, index, username, password, data): + url = f'https://{ip}:{port}/{index}/_doc' + session = requests.Session() + session.auth = (username, password) + session.verify = False + headers = {'Content-Type': 'application/json'} + + try: + for event_data in data: + response = session.post(url, json=event_data, headers=headers) + if response.status_code != 201: + logging.error(f'Error: {response.status_code}') + logging.error(response.text) + break + logging.info('Data injection completed successfully.') + except Exception as e: + logging.error(f'Error: {str(e)}') + + +def main(): + try: + number = int(input("How many events do you want to generate? ")) + except ValueError: + logging.error("Invalid input. Please enter a valid number.") + return + + logging.info(f"Generating {number} events...") + data = generate_random_data(number) + + with open(GENERATED_DATA_FILE, 'a') as outfile: + for event_data in data: + json.dump(event_data, outfile) + outfile.write('\n') + + logging.info('Data generation completed.') + + inject = input( + "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() + if inject == 'y': + ip = input("Enter the IP of your Indexer: ") + port = input("Enter the port of your Indexer: ") + index = input("Enter the index name: ") + username = input("Username: ") + password = input("Password: ") + inject_events(ip, port, index, username, password, data) + + +if __name__ == "__main__": + main() diff --git a/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml b/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml new file mode 100644 index 0000000000000..3482123af637a --- /dev/null +++ b/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml @@ -0,0 +1,12 @@ +--- +- name: agent + title: Wazuh Agents + short: Wazuh Inc. custom fields. + type: group + group: 2 + fields: + - name: groups + type: keyword + level: custom + description: > + The groups the agent belongs to. diff --git a/ecs/states-inventory-packages/fields/mapping-settings.json b/ecs/states-inventory-packages/fields/mapping-settings.json new file mode 100644 index 0000000000000..0ad2b48fcc1be --- /dev/null +++ b/ecs/states-inventory-packages/fields/mapping-settings.json @@ -0,0 +1,4 @@ +{ + "dynamic": "strict", + "date_detection": false +} \ No newline at end of file diff --git a/ecs/states-inventory-packages/fields/subset.yml b/ecs/states-inventory-packages/fields/subset.yml new file mode 100644 index 0000000000000..315aff5f01138 --- /dev/null +++ b/ecs/states-inventory-packages/fields/subset.yml @@ -0,0 +1,21 @@ +--- +name: wazuh-states-inventory-packages +fields: + base: + fields: + "@timestamp": {} + tags: [] + agent: + fields: + id: {} + groups: {} + package: + fields: + architecture: "" + description: "" + installed: {} + name: "" + path: "" + size: {} + type: "" + version: "" diff --git a/ecs/states-inventory-packages/fields/template-settings-legacy.json b/ecs/states-inventory-packages/fields/template-settings-legacy.json new file mode 100644 index 0000000000000..a21a57636d439 --- /dev/null +++ b/ecs/states-inventory-packages/fields/template-settings-legacy.json @@ -0,0 +1,20 @@ +{ + "index_patterns": ["wazuh-states-inventory-packages*"], + "order": 1, + "settings": { + "index": { + "hidden": true, + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "query.default_field": [ + "agent.id", + "agent.groups", + "package.architecture", + "package.name", + "package.version", + "package.type" + ] + } + } +} diff --git a/ecs/states-inventory-packages/fields/template-settings.json b/ecs/states-inventory-packages/fields/template-settings.json new file mode 100644 index 0000000000000..e6cd3078a8325 --- /dev/null +++ b/ecs/states-inventory-packages/fields/template-settings.json @@ -0,0 +1,21 @@ +{ + "index_patterns": ["wazuh-states-inventory-packages*"], + "priority": 1, + "template": { + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "refresh_interval": "5s", + "query.default_field": [ + "agent.id", + "agent.groups", + "package.architecture", + "package.name", + "package.version", + "package.type" + ] + } + } + } +} From b068e142e6f99f512ad4bfdcb6b26d261e021f4d Mon Sep 17 00:00:00 2001 From: Fede Tux Date: Thu, 12 Sep 2024 08:59:34 -0300 Subject: [PATCH 4/8] Fix indentation issue in subset.yml --- .../fields/subset.yml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/ecs/states-inventory-packages/fields/subset.yml b/ecs/states-inventory-packages/fields/subset.yml index 315aff5f01138..49028288fea80 100644 --- a/ecs/states-inventory-packages/fields/subset.yml +++ b/ecs/states-inventory-packages/fields/subset.yml @@ -5,17 +5,17 @@ fields: fields: "@timestamp": {} tags: [] - agent: - fields: - id: {} - groups: {} - package: - fields: - architecture: "" - description: "" - installed: {} - name: "" - path: "" - size: {} - type: "" - version: "" + agent: + fields: + id: {} + groups: {} + package: + fields: + architecture: "" + description: "" + installed: {} + name: "" + path: "" + size: {} + type: "" + version: "" From 43467cbd8fe5d5b92f6893f366c9d11d52df7501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 12 Sep 2024 17:53:24 +0200 Subject: [PATCH 5/8] Remove event generators --- ecs/alerts/event-generator/event_generator.py | 114 ------------------ .../event-generator/event_generator.py | 114 ------------------ .../fields/custom/wazuh-agent.yml | 12 -- 3 files changed, 240 deletions(-) delete mode 100644 ecs/alerts/event-generator/event_generator.py delete mode 100644 ecs/states-inventory-packages/event-generator/event_generator.py delete mode 100644 ecs/states-inventory-packages/fields/custom/wazuh-agent.yml diff --git a/ecs/alerts/event-generator/event_generator.py b/ecs/alerts/event-generator/event_generator.py deleted file mode 100644 index f676f0176d444..0000000000000 --- a/ecs/alerts/event-generator/event_generator.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/python3 - -import datetime -import random -import json -import requests -import warnings -import logging - -# Constants and Configuration -LOG_FILE = 'generate_data.log' -GENERATED_DATA_FILE = 'generatedData.json' -DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - -# Configure logging -logging.basicConfig(filename=LOG_FILE, level=logging.INFO) - -# Suppress warnings -warnings.filterwarnings("ignore") - - -def generate_random_date(): - start_date = datetime.datetime.now() - end_date = start_date - datetime.timedelta(days=10) - random_date = start_date + (end_date - start_date) * random.random() - return random_date.strftime(DATE_FORMAT) - - -def generate_random_agent(): - agent = { - 'id': f'agent{random.randint(0, 99)}', - 'name': f'Agent{random.randint(0, 99)}', - 'type': random.choice(['filebeat', 'windows', 'linux', 'macos']), - 'version': f'v{random.randint(0, 9)}-stable', - 'is_connected': random.choice([True, False]), - 'last_login': generate_random_date(), - 'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'], - 'key': f'key{random.randint(0, 999)}' - } - return agent - - -def generate_random_host(): - family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL']) - version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' - host = { - 'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}', - 'os': { - 'full': f'{family} {version}', - } - } - return host - - -def generate_random_data(number): - data = [] - for _ in range(number): - event_data = { - 'agent': generate_random_agent(), - 'host': generate_random_host(), - } - data.append(event_data) - return data - - -def inject_events(ip, port, index, username, password, data): - url = f'https://{ip}:{port}/{index}/_doc' - session = requests.Session() - session.auth = (username, password) - session.verify = False - headers = {'Content-Type': 'application/json'} - - try: - for event_data in data: - response = session.post(url, json=event_data, headers=headers) - if response.status_code != 201: - logging.error(f'Error: {response.status_code}') - logging.error(response.text) - break - logging.info('Data injection completed successfully.') - except Exception as e: - logging.error(f'Error: {str(e)}') - - -def main(): - try: - number = int(input("How many events do you want to generate? ")) - except ValueError: - logging.error("Invalid input. Please enter a valid number.") - return - - logging.info(f"Generating {number} events...") - data = generate_random_data(number) - - with open(GENERATED_DATA_FILE, 'a') as outfile: - for event_data in data: - json.dump(event_data, outfile) - outfile.write('\n') - - logging.info('Data generation completed.') - - inject = input( - "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() - if inject == 'y': - ip = input("Enter the IP of your Indexer: ") - port = input("Enter the port of your Indexer: ") - index = input("Enter the index name: ") - username = input("Username: ") - password = input("Password: ") - inject_events(ip, port, index, username, password, data) - - -if __name__ == "__main__": - main() diff --git a/ecs/states-inventory-packages/event-generator/event_generator.py b/ecs/states-inventory-packages/event-generator/event_generator.py deleted file mode 100644 index f676f0176d444..0000000000000 --- a/ecs/states-inventory-packages/event-generator/event_generator.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/python3 - -import datetime -import random -import json -import requests -import warnings -import logging - -# Constants and Configuration -LOG_FILE = 'generate_data.log' -GENERATED_DATA_FILE = 'generatedData.json' -DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" - -# Configure logging -logging.basicConfig(filename=LOG_FILE, level=logging.INFO) - -# Suppress warnings -warnings.filterwarnings("ignore") - - -def generate_random_date(): - start_date = datetime.datetime.now() - end_date = start_date - datetime.timedelta(days=10) - random_date = start_date + (end_date - start_date) * random.random() - return random_date.strftime(DATE_FORMAT) - - -def generate_random_agent(): - agent = { - 'id': f'agent{random.randint(0, 99)}', - 'name': f'Agent{random.randint(0, 99)}', - 'type': random.choice(['filebeat', 'windows', 'linux', 'macos']), - 'version': f'v{random.randint(0, 9)}-stable', - 'is_connected': random.choice([True, False]), - 'last_login': generate_random_date(), - 'groups': [f'group{random.randint(0, 99)}', f'group{random.randint(0, 99)}'], - 'key': f'key{random.randint(0, 999)}' - } - return agent - - -def generate_random_host(): - family = random.choice(['debian', 'ubuntu', 'macos', 'ios', 'android', 'RHEL']) - version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' - host = { - 'ip': f'{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}', - 'os': { - 'full': f'{family} {version}', - } - } - return host - - -def generate_random_data(number): - data = [] - for _ in range(number): - event_data = { - 'agent': generate_random_agent(), - 'host': generate_random_host(), - } - data.append(event_data) - return data - - -def inject_events(ip, port, index, username, password, data): - url = f'https://{ip}:{port}/{index}/_doc' - session = requests.Session() - session.auth = (username, password) - session.verify = False - headers = {'Content-Type': 'application/json'} - - try: - for event_data in data: - response = session.post(url, json=event_data, headers=headers) - if response.status_code != 201: - logging.error(f'Error: {response.status_code}') - logging.error(response.text) - break - logging.info('Data injection completed successfully.') - except Exception as e: - logging.error(f'Error: {str(e)}') - - -def main(): - try: - number = int(input("How many events do you want to generate? ")) - except ValueError: - logging.error("Invalid input. Please enter a valid number.") - return - - logging.info(f"Generating {number} events...") - data = generate_random_data(number) - - with open(GENERATED_DATA_FILE, 'a') as outfile: - for event_data in data: - json.dump(event_data, outfile) - outfile.write('\n') - - logging.info('Data generation completed.') - - inject = input( - "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() - if inject == 'y': - ip = input("Enter the IP of your Indexer: ") - port = input("Enter the port of your Indexer: ") - index = input("Enter the index name: ") - username = input("Username: ") - password = input("Password: ") - inject_events(ip, port, index, username, password, data) - - -if __name__ == "__main__": - main() diff --git a/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml b/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml deleted file mode 100644 index 3482123af637a..0000000000000 --- a/ecs/states-inventory-packages/fields/custom/wazuh-agent.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: agent - title: Wazuh Agents - short: Wazuh Inc. custom fields. - type: group - group: 2 - fields: - - name: groups - type: keyword - level: custom - description: > - The groups the agent belongs to. From cc8ae0f8b3b0eddfa309e6265820335a0a0ca6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 12 Sep 2024 17:54:49 +0200 Subject: [PATCH 6/8] Remove duplicated code con ECS generator --- ecs/generate.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ecs/generate.sh b/ecs/generate.sh index 1409404d4e6f5..7b860256f0936 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -57,14 +57,6 @@ generate_mappings() { echo "Removing scaling_factor lines" find "$OUT_DIR" -type f -exec sed -i '/scaling_factor/d' {} \; - # Replace "constant_keyword" type (not supported by OpenSearch) with "keyword" - echo "Replacing \"constant_keyword\" type with \"keyword\"" - find "$OUT_DIR" -type f -exec sed -i 's/constant_keyword/keyword/g' {} \; - - # Replace "flattened" type (not supported by OpenSearch) with "flat_object" - echo "Replacing \"flattened\" type with \"flat_object\"" - find "$OUT_DIR" -type f -exec sed -i 's/flattened/flat_object/g' {} \; - local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json" local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json" From 033c3013d8181eb915269d38a7f0d5a85dbafb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 12 Sep 2024 17:55:58 +0200 Subject: [PATCH 7/8] Add custom fields for states-inventory-packages --- .../fields/custom/agent.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ecs/states-inventory-packages/fields/custom/agent.yml diff --git a/ecs/states-inventory-packages/fields/custom/agent.yml b/ecs/states-inventory-packages/fields/custom/agent.yml new file mode 100644 index 0000000000000..3482123af637a --- /dev/null +++ b/ecs/states-inventory-packages/fields/custom/agent.yml @@ -0,0 +1,12 @@ +--- +- name: agent + title: Wazuh Agents + short: Wazuh Inc. custom fields. + type: group + group: 2 + fields: + - name: groups + type: keyword + level: custom + description: > + The groups the agent belongs to. From 325555ef6c09057345314bde71c15ff45e78a3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Thu, 12 Sep 2024 17:59:58 +0200 Subject: [PATCH 8/8] Remove hidden flag on index template --- .../fields/template-settings-legacy.json | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/states-inventory-packages/fields/template-settings-legacy.json b/ecs/states-inventory-packages/fields/template-settings-legacy.json index a21a57636d439..ca085a0dad45d 100644 --- a/ecs/states-inventory-packages/fields/template-settings-legacy.json +++ b/ecs/states-inventory-packages/fields/template-settings-legacy.json @@ -3,7 +3,6 @@ "order": 1, "settings": { "index": { - "hidden": true, "number_of_shards": "1", "number_of_replicas": "0", "refresh_interval": "5s",