Skip to content

Commit a20f5aa

Browse files
chaitanyaphalakchaitanyaphalak
authored
Merge pull request #216 from splunk/data-enrichment-test
Data enrichment test
2 parents 695b372 + 65688a0 commit a20f5aa

File tree

10 files changed

+245
-78
lines changed

10 files changed

+245
-78
lines changed

‎.circleci/config.yml

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
1515
CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
1616
CI_SPLUNK_PASSWORD: helloworld
17-
CI_INDEX_EVENTS: circleci_events
17+
CI_INDEX_EVENTS: main
1818
CI_DATAGEN_IMAGE: rock1017/log-generator:latest
1919
CI_KAFKA_VERSION: 2.4.0
2020
CI_KAFKA_FILENAME: kafka_2.13-2.4.0.tgz
@@ -74,7 +74,6 @@ jobs:
7474
# Disable SSL for HEC new-token
7575
#sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -enable-ssl 0 -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
7676
# Setup Indexes
77-
sudo /opt/splunk/bin/splunk add index $CI_INDEX_EVENTS -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
7877
sudo /opt/splunk/bin/splunk http-event-collector update -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -name splunk_hec_token -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -index $CI_INDEX_EVENTS -indexes $CI_INDEX_EVENTS
7978
sudo /opt/splunk/bin/splunk restart --accept-license --answer-yes --no-prompt
8079
- run:
@@ -121,12 +120,10 @@ jobs:
121120
sudo mkdir -p /usr/local/share/kafka/plugins/
122121
sudo cp /home/circleci/repo/target/splunk-kafka-connect*.jar /usr/local/share/kafka/plugins/
123122
sed -i 's/plugin\.path\=connectors\//plugin\.path\=\/usr\/local\/share\/kafka\/plugins\//' /home/circleci/repo/config/connect-distributed-quickstart.properties
123+
sed -i 's/key\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/key\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
124+
sed -i 's/value\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/value\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
124125
sudo /usr/local/kafka/bin/connect-distributed.sh /home/circleci/repo/config/connect-distributed-quickstart.properties
125126
background: true
126-
127-
- run:
128-
name: Generate data
129-
command: echo "TODO"
130127
- run:
131128
name: Run Functional tests
132129
command: |
@@ -137,13 +134,7 @@ jobs:
137134
pip install -r test/requirements.txt
138135
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
139136
echo "Running functional tests....."
140-
python -m pytest \
141-
--splunk-url https://$CI_SPLUNK_HOST:8089 \
142-
--splunk-user $CI_SPLUNK_USERNAME \
143-
--splunk-password $CI_SPLUNK_PASSWORD \
144-
--splunk-token $CI_SPLUNK_HEC_TOKEN \
145-
--kafka-topic $CI_KAFKA_TOPIC \
146-
-p no:warnings -s
137+
python -m pytest -p no:warnings -s
147138
148139
workflows:
149140
version: 2

‎.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,7 @@ pom.xml.versionsBackup
3030
.classpath
3131
.project
3232
*.iml
33-
.settings/
33+
.settings/
34+
35+
**/__pycache__/*
36+
venv/

‎test/README.md

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,35 +7,35 @@
77
`virtualenv --python=python3.7 venv`
88
`source venv/bin/activate`
99
1. Install the dependencies
10-
`pip install -r requirements.txt`
11-
2. Start the test with the required options configured
12-
`python -m pytest <options>`
13-
10+
`pip install -r test/requirements.txt`
11+
2. Provided required options at `test/config.yaml`
1412
**Options are:**
15-
--splunkd-url
13+
splunkd_url
1614
* Description: splunkd url used to send test data to. Eg: https://localhost:8089
17-
* Default: https://localhost:8089
1815

19-
--splunk-user
20-
* Description: splunk username
21-
* Default: admin
16+
splunk_hec_url
17+
* Description: splunk HTTP Event Collector's address and port. Eg: https://127.0.0.1:8088
2218

23-
--splunk-password
24-
* Description: splunk user password
25-
* Default: changeme
19+
splunk_user
20+
* Description: splunk username
2621

27-
--splunk-token
22+
splunk_token
2823
* Description: splunk hec token
29-
* Default: No default value
24+
25+
splunk_token_ack
26+
* Description: splunk hec token with ack enabled
27+
28+
splunk_index
29+
* Description: splunk index to ingest test data
3030

31-
--splunk-index
32-
* Description: splunk index
33-
* Default: main
31+
kafka_broker_url
32+
* address of kafka broker. Eg: 127.0.0.1:9092
3433

35-
--kafka-connect-url
34+
kafka_connect_url
3635
* Description: url used to interact with kafka connect
37-
* Default: http://localhost:8083
38-
39-
--kafka-topic
36+
37+
kafka-topic
4038
* Description: kafka topic used to get data with kafka connect
41-
* Default: No default value
39+
40+
3. Start the test
41+
`python -m pytest`

‎test/commonkafka.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,7 @@ def delete_kafka_connector(setup, params):
4949
if response.status_code == 204:
5050
logger.info("Deleted connector successfully - " + json.dumps(params))
5151
return True
52-
53-
return False
52+
else:
53+
logger.error("failed to create connector", param)
54+
logger.error(response.status_code)
55+
return False

‎test/config.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
splunkd_url: https://127.0.0.1:8089
2+
splunk_hec_url: https://127.0.0.1:8088
3+
splunk_user: admin
4+
splunk_password: helloworld
5+
splunk_index: main
6+
splunk_token: a6b5e77f-d5f6-415a-bd43-930cecb12959
7+
splunk_token_ack: a6b5e77f-d5f6-415a-bd43-930cecb12950
8+
kafka_broker_url: 127.0.0.1:9092
9+
kafka_topic: test-datagen
10+
kafka_connect_url: http://127.0.0.1:8083

‎test/conftest.py

Lines changed: 43 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
you may not use this file except in compliance with the License.
66
You may obtain a copy of the License at
77
8-
http://www.apache.org/licenses/LICENSE-2.0
8+
http://www.apache.org/licenses/LICENSE-2.0
99
1010
Unless required by applicable law or agreed to in writing, software
1111
distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,40 +15,46 @@
1515
"""
1616

1717
import pytest
18-
19-
def pytest_addoption(parser):
20-
parser.addoption("--splunk-url",
21-
help="splunk url used to send test data to. \
22-
Eg: https://localhost:8089",
23-
default="https://localhost:8089")
24-
parser.addoption("--splunk-user",
25-
help="splunk username",
26-
default="admin")
27-
parser.addoption("--splunk-password",
28-
help="splunk user password",
29-
default="password")
30-
parser.addoption("--splunk-token",
31-
help="splunk hec token")
32-
parser.addoption("--splunk-index",
33-
help="splunk index",
34-
default="main")
35-
parser.addoption("--kafka-connect-url",
36-
help="url used to interact with kafka connect. \
37-
Eg: http://localhost:8083",
38-
default="http://localhost:8083")
39-
parser.addoption("--kafka-topic",
40-
help="kafka topic used to get data with kafka connect")
41-
42-
43-
@pytest.fixture(scope="function")
18+
import sys
19+
import os
20+
import json
21+
import logging
22+
import requests
23+
import yaml
24+
import json
25+
import time
26+
from kafka import KafkaProducer
27+
from .connect_params import connect_params
28+
from .commonkafka import create_kafka_connector, delete_kafka_connector
29+
30+
logger = logging.getLogger(__name__)
31+
logger.setLevel(logging.INFO)
32+
formatter = logging.Formatter('%(message)s')
33+
handler = logging.StreamHandler(sys.stdout)
34+
handler.setFormatter(formatter)
35+
logger.addHandler(handler)
36+
37+
with open('test/config.yaml', 'r') as yaml_file:
38+
config = yaml.load(yaml_file)
39+
40+
@pytest.fixture()
4441
def setup(request):
45-
config = {}
46-
config["splunk_url"] = request.config.getoption("--splunk-url")
47-
config["splunk_user"] = request.config.getoption("--splunk-user")
48-
config["splunk_password"] = request.config.getoption("--splunk-password")
49-
config["splunk_token"] = request.config.getoption("--splunk-token")
50-
config["splunk_index"] = request.config.getoption("--splunk-index")
51-
config["kafka_connect_url"] = request.config.getoption("--kafka-connect-url")
52-
config["kafka_topic"] = request.config.getoption("--kafka-topic")
53-
54-
return config
42+
return config
43+
44+
def pytest_configure():
45+
# Generate data
46+
producer = KafkaProducer(bootstrap_servers=config["kafka_broker_url"], value_serializer=lambda v: json.dumps(v).encode('utf-8'))
47+
msg = {'foo': 'bar'}
48+
producer.send(config["kafka_topic"], msg)
49+
producer.flush()
50+
51+
# Launch all connectors for tests
52+
for param in connect_params:
53+
create_kafka_connector(config, param)
54+
# wait for data to be ingested to Splunk
55+
time.sleep(60)
56+
57+
def pytest_unconfigure():
58+
# Delete launched connectors
59+
for param in connect_params:
60+
delete_kafka_connector(config, param)

‎test/connect_params.py

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import yaml
2+
3+
with open('test/config.yaml', 'r') as yaml_file:
4+
config = yaml.load(yaml_file)
5+
6+
connect_params = []
7+
data_enrichment_connect_params=[]
8+
data_onboarding_connect_params=[]
9+
10+
param = {
11+
"name": "data-enrichment-latin1-sup",
12+
"config": {
13+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
14+
"tasks.max": "1",
15+
"topics": config["kafka_topic"],
16+
"splunk.indexes": config["splunk_index"],
17+
"splunk.hec.uri": config["splunk_hec_url"],
18+
"splunk.hec.token": config["splunk_token"],
19+
"splunk.hec.raw": "false",
20+
"splunk.hec.ack.enabled": "false",
21+
"splunk.hec.ssl.validate.certs": "false",
22+
"splunk.hec.json.event.enrichment" : "test=supplement,chars=¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
23+
}
24+
}
25+
data_enrichment_connect_params.append(param)
26+
param = {
27+
"name": "data-enrichment-latin1-A",
28+
"config": {
29+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
30+
"tasks.max": "1",
31+
"topics": config["kafka_topic"],
32+
"splunk.indexes": config["splunk_index"],
33+
"splunk.hec.uri": config["splunk_hec_url"],
34+
"splunk.hec.token": config["splunk_token"],
35+
"splunk.hec.raw": "false",
36+
"splunk.hec.ack.enabled": "false",
37+
"splunk.hec.ssl.validate.certs": "false",
38+
"splunk.hec.json.event.enrichment" : "test=extendedA,chars=ĀāĂ㥹ĆćĈĉĊċČčĎďĐđĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıIJijĴĵĶķĸĹĺĻļĽľĿŀŁłŃńŅņŇňʼnŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽžſ"
39+
}
40+
}
41+
data_enrichment_connect_params.append(param)
42+
param = {
43+
"name": "data-enrichment-latin1-B",
44+
"config": {
45+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
46+
"tasks.max": "1",
47+
"topics": config["kafka_topic"],
48+
"splunk.indexes": config["splunk_index"],
49+
"splunk.hec.uri": config["splunk_hec_url"],
50+
"splunk.hec.token": config["splunk_token"],
51+
"splunk.hec.raw": "false",
52+
"splunk.hec.ack.enabled": "false",
53+
"splunk.hec.ssl.validate.certs": "false",
54+
"splunk.hec.json.event.enrichment" : "test=extendedB,chars=ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢƣƤƥƦƧƨƩƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƼƽƾƿǀǁǂǃDŽDždžLJLjljNJNjnjǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳȴȵȶȷȸȹȺȻȼȽȾȿɀɁɂɃɄɅɆɇɈɉɊɋɌɍɎɏ"
55+
}
56+
}
57+
data_enrichment_connect_params.append(param)
58+
param = {
59+
"name": "data-enrichment-latin1-spaceModifier",
60+
"config": {
61+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
62+
"tasks.max": "1",
63+
"topics": config["kafka_topic"],
64+
"splunk.indexes": config["splunk_index"],
65+
"splunk.hec.uri": config["splunk_hec_url"],
66+
"splunk.hec.token": config["splunk_token"],
67+
"splunk.hec.raw": "false",
68+
"splunk.hec.ack.enabled": "false",
69+
"splunk.hec.ssl.validate.certs": "false",
70+
"splunk.hec.json.event.enrichment" : "test=spaceModifier,chars=ʰʱʲʳʴʵʶʷʸʹʺʻʼʽʾʿˀˁ˂˃˄˅ˆˇˈˉˊˋˌˍˎˏːˑ˒˓˔˕˖˗˘˙˚˛˜˝˞˟ˠˡˢˣˤ˥˦˧˨˩˪˫ˬ˭ˮ˯˰˱˲˳˴˵˶˷˸˹˺˻˼˽˾˿"
71+
}
72+
}
73+
data_enrichment_connect_params.append(param)
74+
param = {
75+
"name": "data-enrichment-latin1-IPA",
76+
"config": {
77+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
78+
"tasks.max": "1",
79+
"topics": config["kafka_topic"],
80+
"splunk.indexes": config["splunk_index"],
81+
"splunk.hec.uri": config["splunk_hec_url"],
82+
"splunk.hec.token": config["splunk_token"],
83+
"splunk.hec.raw": "false",
84+
"splunk.hec.ack.enabled": "false",
85+
"splunk.hec.ssl.validate.certs": "false",
86+
"splunk.hec.json.event.enrichment" : "test=IPAextensions,chars=ɐɑɒɓɔɕɖɗɘəɚɛɜɝɞɟɠɡɢɣɤɥɦɧɨɩɪɫɬɭɮɯɰɱɲɳɴɵɶɷɸɹɺɻɼɽɾɿʀʁʂʃʄʅʆʇʈʉʊʋʌʍʎʏʐʑʒʓʔʕʖʗʘʙʚʛʜʝʞʟʠʡʢʣʤʥʦʧʨʩʪʫʬʭʮʯ"
87+
}
88+
}
89+
data_enrichment_connect_params.append(param)
90+
param = {
91+
"name": "data-enrichment-latin1-greek",
92+
"config": {
93+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
94+
"tasks.max": "1",
95+
"topics": config["kafka_topic"],
96+
"splunk.indexes": config["splunk_index"],
97+
"splunk.hec.uri": config["splunk_hec_url"],
98+
"splunk.hec.token": config["splunk_token"],
99+
"splunk.hec.raw": "false",
100+
"splunk.hec.ack.enabled": "false",
101+
"splunk.hec.ssl.validate.certs": "false",
102+
"splunk.hec.json.event.enrichment" : "test=greek,chars=ͰͱͲͳʹ͵Ͷͷ͸͹ͺͻͼͽ;Ϳ΀΁΂΃΄΅Ά·ΈΉΊ΋Ό΍ΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡ΢ΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώϏϐϑϒϓϔϕϖϗϘϙϚϛϜϝϞϟϠϡϢϣϤϥϦϧϨϩϪϫϬϭϮϯϰϱϲϳϴϵ϶ϷϸϹϺϻϼϽϾϿ"
103+
}
104+
}
105+
data_enrichment_connect_params.append(param)
106+
param = {
107+
"name": "data-enrichment-latin1-diacriticalMarks",
108+
"config": {
109+
"connector.class": "com.splunk.kafka.connect.SplunkSinkConnector",
110+
"tasks.max": "1",
111+
"topics": config["kafka_topic"],
112+
"splunk.indexes": config["splunk_index"],
113+
"splunk.hec.uri": config["splunk_hec_url"],
114+
"splunk.hec.token": config["splunk_token"],
115+
"splunk.hec.raw": "false",
116+
"splunk.hec.ack.enabled": "false",
117+
"splunk.hec.ssl.validate.certs": "false",
118+
"splunk.hec.json.event.enrichment" : "test=diacriticalMarks,chars= ́ ̂ ̃ ̄ ̅ ̆ ̇ ̈ ̉ ̊ ̋ ̌ ̍ ̎ ̏ ̐ ̑ ̒ ̓ ̔ ̕ ̖ ̗ ̘ ̙ ̚ ̛ ̜ ̝ ̞ ̟ ̠ ̡ ̢ ̣ ̤ ̥ ̦ ̧ ̨ ̩ ̪ ̫ ̬ ̭ ̮ ̯ ̰ ̱ ̲ ̳ ̴ ̵ ̶ ̷ ̸ ̹ ̺ ̻ ̼ ̽ ̾ ̿ ̀ ́ ͂ ̓ ̈́ ͅ ͆ ͇ ͈ ͉ ͊ ͋ ͌ ͍ ͎ ͏ ͐ ͑ ͒ ͓ ͔ ͕ ͖ ͗ ͘ ͙ ͚ ͛ ͜ ͝ ͞ ͟ ͠ ͡ ͢ ͣ ͤ ͥ ͦ ͧ ͨ ͩ ͪ ͫ ͬ ͭ ͮ ͯ"
119+
}
120+
}
121+
data_enrichment_connect_params.append(param)
122+
123+
connect_params.extend(data_enrichment_connect_params)
124+
connect_params.extend(data_onboarding_connect_params)

0 commit comments

Comments
 (0)