-
Notifications
You must be signed in to change notification settings - Fork 4
/
readme.txt
108 lines (83 loc) · 3.01 KB
/
readme.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
Logstash
bin/logstash -f logstash-simple.conf --config.test_and_exit
bin/logstash -f logstash-simple.conf --config.reload.automatic
nohup bin/logstash -f logstash-simple.conf > my_logstash.log &
https://www.elastic.co/guide/en/logstash/5.5/advanced-pipeline.html#configuring-filebeat
Kafka
Start zookeper
nohup bin/zookeeper-server-start.sh config/zookeeper.properties > my_zookeepper.log &
Start kafka server
nohup bin/kafka-server-start.sh config/server.properties > my_kafkaserver.log &
nohup bin/kafka-server-start.sh config/server-1.properties > my_kafkaserver1.log &
Create a topic
bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 2 --partitions 1 --topic my-replicated-topic
Take a look
bin/kafka-topics.sh --list --zookeeper localhost:2181
bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic my-replicated-topic
send msg from stdin
bin/kafka-console-producer.sh --broker-list localhost:9092 --topic my-replicated-topic
Start a consumer
bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic my-replicated-topic --from-beginning
elasticsearch
start esearch
nohup bin/elasticsearch > my_esearch.log &
---------------
append below text to /etc/sysctl.conf
vm.max_map_count=262144
sudo sysctl -p
append below text to /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
root soft nofile 65536
root hard nofile 65536
/etc/pam.d/common-session
session required pam_limits.so
/etc/pam.d/common-session-noninteractive
session required pam_limits.so
-------------
elasticsearch_head_plugin
git clone git://github.com/mobz/elasticsearch-head.git
cd elasticsearch-head
npm install
npm start
Run chmod +x ./startup.sh && ./startup.sh to start this java program.
Setting JAVA and MAVEN env variables manually may be necessary for you to run this program.
curl -XGET 'localhost:9200/_template/logstash_1?pretty'
curl -XDELETE 'localhost:9200/_template/logstash_1?pretty'
curl -XPUT 'localhost:9200/_template/logstash_1?pretty' -H 'Content-Type: application/json' -d'
{
"template": "logstash-*",
"order": 1,
"mappings": {
"_default_": {
"_all": {
"enabled": true,
"omit_norms": true
},
"dynamic_templates": [
{
"message_field": {
"path_match": "message",
"mapping": {
"norms": false,
"type": "text"
},
"match_mapping_type": "string"
}
},
{
"string_fields": {
"mapping": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
},
"match_mapping_type": "string",
"match": "*"
}
}
]
}
}
}
'