diff --git a/namespaces/_default_.yml b/namespaces/_default_.yml index d3623cb..a4f786c 100644 --- a/namespaces/_default_.yml +++ b/namespaces/_default_.yml @@ -8,8 +8,9 @@ field_defaults: _default_: type: group description: > - Contains common fields, this is what populates the actual the mappings of the _default_. - Oritinated from the mapping for rsyslog clients, but can be used with other clients as well. + Contains common fields and generic pattern-matched field descriptions; this + is what populates the actual mappings of _default_ in the template's mapping + section. fields: - name: "@timestamp" type: date @@ -60,8 +61,9 @@ _default_: type: string example: info description: > - Loging level as provided by: rsyslog(severitytext property), python's logging module, etc. - Possible values: info, warning, notice, err, debug + Logging level as provided by: rsyslog(severitytext property), python's + logging module, etc. + Possible values: trace, crit, alert, emerg - name: message type: string @@ -69,19 +71,22 @@ _default_: doc_values: false example: TODO description: > - Typical log entry message, or payload, possibly stripped of metadata pulled out of it by collector/normalizer, UTF-8 encoded. + Typical log entry message, or payload, possibly stripped of metadata + pulled out of it by collector/normalizer, UTF-8 encoded. norms: enabled: false - name: pid type: string description: > - PID... look into details on systemd stuff + This is the process ID of the logging entity, if available. - name: service type: string description: > - name of the service. syslog's APP-NAME and programname are renamed to service field as well. + Name of the service associated with the logging entity, if available. + For example, syslog's APP-NAME and rsyslog's programname property are + mapped to the service field. - name: tags type: string @@ -89,15 +94,17 @@ _default_: index: analyzed analyzer: whitespace description: > - Field contains whitespace-delimited tags. Tagging is configured on normalizers/collectors. - Please note that this field is analyzed and not an array since rsyslog doesn't play well with JSON lists + Optionally provided operator defined list of tags placed on each log + by the collector or normalizer. The payload can be a string with + whitespace-delimited string tokens, or a JSON list of string tokens. - name: file type: string index: analyzed doc_values: false description: > - Full local path. + Optional path to the file containing the log entry local to the + collector TODO: analyzer for file paths norms: enabled: True @@ -109,4 +116,8 @@ _default_: - name: offset type: long description: > - Offset in the log file. + The offset value can represent bytes to the start of the log line in the + file (zero or one based), or log line numbers (zero or one based), so + long as the values are strictly monotonically increasing in the context + of a single log file. They values are allowed to wrap, representing a + new version of the log file (rotation). diff --git a/namespaces/ci_job.yml b/namespaces/ci_job.yml new file mode 100644 index 0000000..e02ecff --- /dev/null +++ b/namespaces/ci_job.yml @@ -0,0 +1,70 @@ + +namespace: + name: ci_job + type: group + description: > + Namespace for CI-specific metadata. f.e. Jenkins job run metadata. + The namespace contains the details for the metadata of individual job run. + fields: + - name: master + type: string + description: > + FQDN of the CI Master server. + + - name: name + type: string + description: > + Name of the CI job. + + - name: build_id + type: integer + description: > + Build ID of the CI job. + + - name: build_timestamp + type: date + format: yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime + description: > + Time of the build completion if different from field "@timestamp". + + - name: duration + type: float + description: > + Build duration in seconds. + + - name: phase + type: string + description: > + Phase of the CI job. + + - name: result + type: string + description: > + CI job result. + + - name: url + type: string + description: > + Full URL to the CI job on the CI master. + + - name: artifacts + type: group + description: > + List of the artifacts of the build. + The console output in case of Jenkins is considered an artifact with name + `consoleText`. + fields: + - name: name + type: string + description: > + file name of the artifact + + - name: url + type: string + description: > + URL to the html log of the CI job on cold storage. + + - name: original_url + type: string + description: > + URL to the html log of the CI job on the CI server. diff --git a/namespaces/docker.yml b/namespaces/docker.yml index 74def00..ef39316 100644 --- a/namespaces/docker.yml +++ b/namespaces/docker.yml @@ -3,13 +3,9 @@ namespace: name: docker type: group description: > - Docker container group + Namespace for docker container-specific metadata fields: - name: container_id type: string - index: not_analyzed description: > Docker container id - -doc_sections: - - ["pod_log", "Openshift logging metadata"] diff --git a/namespaces/kubernetes.yml b/namespaces/kubernetes.yml index 1895141..363bca5 100644 --- a/namespaces/kubernetes.yml +++ b/namespaces/kubernetes.yml @@ -3,15 +3,12 @@ namespace: name: kubernetes type: group description: > - Kubernetes-specific metadata. + Namespace for kubernetes-specific metadata fields: - name: pod_name type: string - index: not_analyzed description: > The name of the pod - norms: - enabled: True - name: labels type: group @@ -23,35 +20,23 @@ namespace: - name: pod_id type: string - index: not_analyzed description: > - TODO - norms: - enabled: True + Kubernetes ID of the pod. - name: namespace_name type: string - index: not_analyzed description: > - The name of the namespace - norms: - enabled: True + The name of the namespace in Kubernetes. - name: namespace_id type: string - index: not_analyzed description: > - The name of the namespace - norms: - enabled: True + ID of the namespace in Kubernetes. - name: host type: string - index: not_analyzed description: > Kubernetes node name - norms: - enabled: True doc_sections: - ["pod_log", "Openshift logging metadata"] diff --git a/namespaces/pipeline_metadata.yml b/namespaces/pipeline_metadata.yml index 9ac6ccc..839d398 100644 --- a/namespaces/pipeline_metadata.yml +++ b/namespaces/pipeline_metadata.yml @@ -7,6 +7,13 @@ namespace: Everything about log collector, normalizers, mappings goes here. Data in this subgroup is stored for troublehsooting purposes mostly. fields: + - name: "@version" + type: string + example: TODO + description: > + Version of “com.redhat.viaq” mapping the document is intended to adhere by the normalizer. It must be set by the normalizer. + The value must correspond to the [_meta][version]. + - name: collector type: group description: > @@ -123,14 +130,6 @@ namespace: type: string ignore_above: 256 - - name: "@version" - type: string - example: TODO - description: > - Version of “com.redhat.viaq” mapping the document is intended to adhere by the normalizer. It must be set by the normalizer. - The value must correspond to the [_meta][version]. - - - name: trace type: string doc_values: false diff --git a/namespaces/rsyslog.yml b/namespaces/rsyslog.yml index 213b36f..a7408a9 100644 --- a/namespaces/rsyslog.yml +++ b/namespaces/rsyslog.yml @@ -3,7 +3,7 @@ namespace: name: rsyslog type: group description: > - rsyslog/syslog-specific metadata, excluding systemd metadata + RFC5424 based metadata. fields: - name: facility type: string @@ -25,7 +25,7 @@ namespace: - name: msgid type: long description: > - rsyslog protocol version + syslog's msgid field - name: appname type: string diff --git a/namespaces/systemd.yml b/namespaces/systemd.yml index 3941749..997e46f 100644 --- a/namespaces/systemd.yml +++ b/namespaces/systemd.yml @@ -3,14 +3,10 @@ namespace: name: systemd type: group description: > - Contains common fields specific to systemd journal + Contains common fields specific to systemd journal. + For more information please see + https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html fields: - - name: CEE - type: group - product_subgroup: True - description: > - Previously a CEE structure as it comes from rsyslog. - TODO: come up with a good name as CEE is very specific to syslog - name: k type: group description: > diff --git a/namespaces/testcase.yml b/namespaces/testcase.yml new file mode 100644 index 0000000..4458ec1 --- /dev/null +++ b/namespaces/testcase.yml @@ -0,0 +1,119 @@ + +namespace: + name: testcase + type: group + description: > + Namespace for testcase-specific metadata. It generally follows XUNIT schema. + Testcase is considered 1st class citizen, Testsuites are also covered here, + they are nested under testsuite group. + fields: + - name: name + type: string + description: > + Name of the testcase. + + - name: id + type: string + description: > + ID of the testcase. + + - name: time + type: float + description: > + Duration of testcase execution in seconds. + + - name: status + type: string + description: > + Result of the testcase. Possible values are: passed, skipped, failed. + + - name: classname + type: string + description: > + Xunit classname of the testcase. + + - name: line + type: integer + description: > + TODO. Line number of testcase(?). + + - name: file + type: string + description: > + File name that contains this testcase. + + - name: system-out + type: string + description: > + Content of tag in the xunit testcase. + + - name: type + type: string + description: > + TODO. + + - name: details + type: string + index: analyzed + doc_values: false + description: > + Xunit text from the body of testcase tag. + norms: + enabled: True + + - name: properties + type: group + description: > + Testcase-level properties. Xunit-style key-value properties are stored + as Hash. + + - name: testsuite + type: group + description: > + testsuite-specific metadata. + fields: + - name: name + type: string + description: > + Name of the testsuite + + - name: id + type: string + description: > + id of the testsuite + + - name: errors + type: integer + description: > + Number of testcases that resulted in errors in this testsuite. + + - name: failures + type: integer + description: > + Number of testcases that resulted in failures in this testsuite. + + - name: skips + type: integer + description: > + Number of testcases that resulted in skips in this testsuite. + + - name: tests + type: integer + description: > + Total number of testcases that were executed in this testsuite. + + - name: time + type: float + description: > + Duration of testsuite execution in seconds. + + - name: properties + type: group + description: > + Testsuite-level properties. Xunit-style key-value properties are stored + as Hash. + fields: + - name: name + type: string + description: > + Name of the testsuite diff --git a/templates/ci_model/Makefile b/templates/ci_model/Makefile new file mode 100644 index 0000000..88fb4f7 --- /dev/null +++ b/templates/ci_model/Makefile @@ -0,0 +1,9 @@ +TEMPLATE_NAME = com.redhat.viaq-ci + +.PHONY: all clean + +all: + python ../../scripts/generate_template.py template.yml ../../namespaces/ + +clean: + rm *.template.json diff --git a/templates/ci_model/README.md b/templates/ci_model/README.md new file mode 100644 index 0000000..84420e0 --- /dev/null +++ b/templates/ci_model/README.md @@ -0,0 +1,37 @@ +viaq CI index template for Elasticsearch +================================= + +The template files are automatically generated. +Please _do not edit_ the `.json` files directly. + +In order to edit the template please modify [template.yml](template.yml) and +the respective namespace files referenced there. + +To rebuild the template, run: +> make + +For details about the mapping please see [Elasticsearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) + +skeleton.json +------------- +This file contains the skeleton of the template without the mappings section. + +by sections: +`aliases`: Aliases for the indices produces from the template. Aliases will be automatically generated along with the indices in future. +`mappings`: mappings section + `_default_`: default mapping skeleton. It is the only mapping present in the skeleton. Other mappings from [fields.yml](fields.yml) will copy this skeleton. + `date_detection`: we force no date detection in the unknown fields. + `dynamic_templates`: describes the mapping for autocreated fields. + `properties`: empty section that is populated with the content from [fields.yml](fields.yml) +`order`: order of the template. lower order templates are applied first. +`settings`: various settings +`template`: the pattern of indices that will be matched by this template + +template.yml +---------- +This is the file that contains all the settings information and pointers to the specific mappings. +* `skeleton_path`: The path to the `skeleton.json` file that contain the initial JSON structure of the template. +* `elasticsearch_template`: This section defines the parameters common for the entire template, they are explicitly overwritten in the final template file. +** `name`: index pattern matched. +** `order`: template order. Lower order is applied first. [details](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html#multiple-templates) +* `namespaces`: filenames of various document object types to be included in the template diff --git a/templates/ci_model/com.redhat.viaq-ci.template.json b/templates/ci_model/com.redhat.viaq-ci.template.json new file mode 100644 index 0000000..c10065b --- /dev/null +++ b/templates/ci_model/com.redhat.viaq-ci.template.json @@ -0,0 +1,457 @@ +{ + "aliases": {}, + "mappings": { + "_default_": { + "_meta": { + "version": "2016.10.12.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "message_field": { + "mapping": { + "index": "analyzed", + "omit_norms": true, + "type": "string" + }, + "match": "message", + "match_mapping_type": "string" + } + }, + { + "string_fields": { + "mapping": { + "fields": { + "raw": { + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "analyzed", + "omit_norms": true, + "type": "string" + }, + "match": "*", + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "doc_values": true, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime", + "index": "not_analyzed", + "type": "date" + }, + "ci_job": { + "properties": { + "artifacts": { + "properties": { + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "original_url": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "url": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + }, + "build_id": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "build_timestamp": { + "doc_values": true, + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime", + "index": "not_analyzed", + "type": "date" + }, + "duration": { + "doc_values": "true", + "type": "float" + }, + "master": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "phase": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "result": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "url": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + }, + "file": { + "doc_values": false, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "analyzed", + "norms": { + "enabled": true + }, + "type": "string" + }, + "geoip": { + "dynamic": true, + "properties": { + "location": { + "type": "geo_point" + } + }, + "type": "object" + }, + "hostname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "ipaddr4": { + "doc_values": true, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "not_analyzed", + "norms": { + "enabled": false + }, + "type": "ip" + }, + "ipaddr6": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "level": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "message": { + "doc_values": false, + "index": "analyzed", + "norms": { + "enabled": false + }, + "type": "string" + }, + "offset": { + "doc_values": true, + "index": "not_analyzed", + "type": "long" + }, + "pid": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "pipeline_metadata": { + "properties": { + "@version": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "collector": { + "properties": { + "hostname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "inputname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "ipaddr4": { + "doc_values": true, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "not_analyzed", + "norms": { + "enabled": false + }, + "type": "ip" + }, + "ipaddr6": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "original_raw_message": { + "doc_values": false, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "analyzed", + "type": "string" + }, + "received_at": { + "doc_values": true, + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime", + "index": "not_analyzed", + "type": "date" + }, + "version": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + }, + "normalizer": { + "properties": { + "hostname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "inputname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "ipaddr4": { + "doc_values": true, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "not_analyzed", + "norms": { + "enabled": false + }, + "type": "ip" + }, + "ipaddr6": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "original_raw_message": { + "doc_values": false, + "fields": { + "raw": { + "doc_values": true, + "ignore_above": 256, + "index": "not_analyzed", + "type": "string" + } + }, + "index": "analyzed", + "type": "string" + }, + "received_at": { + "doc_values": true, + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime", + "index": "not_analyzed", + "type": "date" + }, + "version": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + }, + "trace": { + "analyzer": "whitespace", + "doc_values": false, + "index": "analyzed", + "type": "string" + } + } + }, + "service": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "tags": { + "analyzer": "whitespace", + "doc_values": false, + "index": "analyzed", + "type": "string" + }, + "testcase": { + "properties": { + "classname": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "details": { + "doc_values": false, + "index": "analyzed", + "norms": { + "enabled": true + }, + "type": "string" + }, + "file": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "id": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "line": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "status": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "system-out": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "testsuite": { + "properties": { + "errors": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "failures": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "id": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + }, + "properties": { + "properties": { + "name": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + }, + "skips": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "tests": { + "doc_values": true, + "index": "not_analyzed", + "type": "integer" + }, + "time": { + "doc_values": "true", + "type": "float" + } + } + }, + "time": { + "doc_values": "true", + "type": "float" + }, + "type": { + "doc_values": true, + "index": "not_analyzed", + "type": "string" + } + } + } + } + } + }, + "order": 10, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "viaq-ci-*" +} \ No newline at end of file diff --git a/templates/ci_model/template.yml b/templates/ci_model/template.yml new file mode 100644 index 0000000..fd68cba --- /dev/null +++ b/templates/ci_model/template.yml @@ -0,0 +1,11 @@ +skeleton_path: ../skeleton.json + +elasticsearch_template: + name: com.redhat.viaq-ci + index_pattern: "viaq-ci-*" + order: 10 + +namespaces: + - ci_job.yml + - testcase.yml + - pipeline_metadata.yml diff --git a/templates/openshift/com.redhat.viaq-openshift.template.json b/templates/openshift/com.redhat.viaq-openshift.template.json index f2c21d6..ef55eae 100644 --- a/templates/openshift/com.redhat.viaq-openshift.template.json +++ b/templates/openshift/com.redhat.viaq-openshift.template.json @@ -117,41 +117,26 @@ "host": { "doc_values": true, "index": "not_analyzed", - "norms": { - "enabled": true - }, "type": "string" }, "namespace_id": { "doc_values": true, "index": "not_analyzed", - "norms": { - "enabled": true - }, "type": "string" }, "namespace_name": { "doc_values": true, "index": "not_analyzed", - "norms": { - "enabled": true - }, "type": "string" }, "pod_id": { "doc_values": true, "index": "not_analyzed", - "norms": { - "enabled": true - }, "type": "string" }, "pod_name": { "doc_values": true, "index": "not_analyzed", - "norms": { - "enabled": true - }, "type": "string" } }