forked from cloudera-labs/envelope
-
Notifications
You must be signed in to change notification settings - Fork 0
/
nav-morphline.conf
88 lines (85 loc) · 2.36 KB
/
nav-morphline.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
SOLR_LOCATOR : {
# Name of solr collection
collection : nav-audit
# ZooKeeper ensemble
zkHost : "zk1.example.com:2181/solr,zk2.example.com:2181/solr,zk3.example.com:2181/solr"
# The maximum number of documents to send to Solr per network batch (throughput knob)
# batchSize : 100
}
morphlines : [
{
id: nav-json-input
importCommands : ["org.kitesdk.**"]
commands: [
{readJson {}}
{extractJsonPaths {
flatten : false
paths : {
additionalInfo : /additionalInfo
allowed : /allowed
collectionName : /collectionName
databaseName : /databaseName
db : /db
DELEGATION_TOKEN_ID : /DELEGATION_TOKEN_ID
dst : /dst
entityId : /entityId
family : /family
impersonator : /impersonator
ip : /ip
name : /name
objectType : /objectType
objType : /objType
objUsageType : /objUsageType
operationParams : /operationParams
operationText : /operationText
op : /op
opText : /opText
path : /path
perms : /perms
privilege : /privilege
qualifier : /qualifier
QUERY_ID : /QUERY_ID
resourcePath : /resourcePath
service : /service
SESSION_ID : /SESSION_ID
solrVersion : /solrVersion
src : /src
status : /status
subOperation : /subOperation
tableName : /tableName
table : /table
time : /time
timeMillis : /time
type : /type
url : /url
user : /user
}
}
}
{convertTimestamp {
field : time
inputFormats : ["unixTimeInMillis"]
inputTimezone : UTC
outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
}}
{logDebug {format: "nav-json-input output: {}", args: ["@{}"]}}
]
},
{
id: nav-load-solr
importCommands : ["org.kitesdk.**", "org.apache.solr.**"]
commands: [
{logDebug {format: "nav-load-solr input: {}", args: ["@{}"]}}
{setValues {
id : "@{timeMillis}_@{service}_@{ip}"
}}
{sanitizeUnknownSolrFields {
solrLocator : ${SOLR_LOCATOR}
}}
{logDebug {format: "nav-load-solr output: {}", args: ["@{}"]}}
{loadSolr {
solrLocator:${SOLR_LOCATOR}
}}
]
}
]