diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index b46ee2c925..6ba655c14b 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -16,6 +16,8 @@ Thanks, you're awesome :-) --> #### Added +* Add email SMTP transmission fields for "MAIL FROM", "RCPT TO". #2413 + #### Improvements * Define base encoding of `x509.serial_number`. #2383 diff --git a/docs/fields/field-details.asciidoc b/docs/fields/field-details.asciidoc index 6d5ba6a780..307d3f6160 100644 --- a/docs/fields/field-details.asciidoc +++ b/docs/fields/field-details.asciidoc @@ -3102,6 +3102,28 @@ example: `c26dbea0-80d5-463b-b93c-4e8b708219ce` // =============================================================== +| +[[field-email-mailfrom-address]] +<> + +a| beta:[ This field is beta and subject to change. ] + +The address from the SMTP `MAIL` command associated with the email, at the observed email relay host. + +This is commonly called `MAIL FROM` value of the SMTP envelope. + +For example, if the SMTP command is `MAIL FROM:`, this field has a value of `Smith@bar.com` + +type: keyword + + + +example: `userx@y.foo.org` + +| extended + +// =============================================================== + | [[field-email-message-id]] <> @@ -3134,6 +3156,31 @@ example: `2020-11-10T22:12:34.8196921Z` // =============================================================== +| +[[field-email-recipient-addresses]] +<> + +a| beta:[ This field is beta and subject to change. ] + +The email address(es) from SMTP `RCPT` command(s) associated with the email message. + +This is commonly called `RCPT TO` value of the SMTP envelope. + +As there may be multiple `RCPT` commands during a single message transaction, this may contain all email address from the entire transaction. + +type: keyword + + +Note: this field should contain an array of values. + + + +example: `["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]` + +| extended + +// =============================================================== + | [[field-email-reply-to-address]] <> diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 3cc3a5847b..3d9c1e4222 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1914,6 +1914,19 @@ Identifier is not persistent across hops.' example: c26dbea0-80d5-463b-b93c-4e8b708219ce default_field: false + - name: mailfrom.address + level: extended + type: keyword + ignore_above: 1024 + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field + has a value of `Smith@bar.com`' + example: userx@y.foo.org + default_field: false - name: message_id level: extended type: wildcard @@ -1928,6 +1941,19 @@ will fill in this value automatically when the message is sent by a user. example: '2020-11-10T22:12:34.8196921Z' default_field: false + - name: recipient.addresses + level: extended + type: keyword + ignore_above: 1024 + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + default_field: false - name: reply_to.address level: extended type: keyword diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 2f73b45091..bfb37cd549 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -229,8 +229,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 9.0.0-dev+exp,true,email,email.direction,keyword,extended,,inbound,Direction of the message. 9.0.0-dev+exp,true,email,email.from.address,keyword,extended,array,sender@example.com,The sender's email address. 9.0.0-dev+exp,true,email,email.local_id,keyword,extended,,c26dbea0-80d5-463b-b93c-4e8b708219ce,Unique identifier given by the source. +9.0.0-dev+exp,true,email,email.mailfrom.address,keyword,extended,,userx@y.foo.org,Email address from SMTP `MAIL` command. 9.0.0-dev+exp,true,email,email.message_id,wildcard,extended,,81ce15$8r2j59@mail01.example.com,Value from the Message-ID header. 9.0.0-dev+exp,true,email,email.origination_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time the email was composed. +9.0.0-dev+exp,true,email,email.recipient.addresses,keyword,extended,array,"[""Jones@foo.com"", ""Green@foo.com"", ""Brown@foo.com""]",Recipient addresses from the SMTP `RCPT` commands. 9.0.0-dev+exp,true,email,email.reply_to.address,keyword,extended,array,reply.here@example.com,Address replies should be delivered to. 9.0.0-dev+exp,true,email,email.sender.address,keyword,extended,,,Address of the message sender. 9.0.0-dev+exp,true,email,email.subject,keyword,extended,,Please see this important message.,The subject of the email message. diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 52e90b2670..19a63f3d67 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -2819,6 +2819,24 @@ email.local_id: normalize: [] short: Unique identifier given by the source. type: keyword +email.mailfrom.address: + beta: This field is beta and subject to change. + dashed_name: email-mailfrom-address + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field has + a value of `Smith@bar.com`' + example: userx@y.foo.org + flat_name: email.mailfrom.address + ignore_above: 1024 + level: extended + name: mailfrom.address + normalize: [] + short: Email address from SMTP `MAIL` command. + type: keyword email.message_id: dashed_name: email-message-id description: Identifier from the RFC 5322 `Message-ID:` email header that refers @@ -2841,6 +2859,25 @@ email.origination_timestamp: normalize: [] short: Date and time the email was composed. type: date +email.recipient.addresses: + beta: This field is beta and subject to change. + dashed_name: email-recipient-addresses + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + flat_name: email.recipient.addresses + ignore_above: 1024 + level: extended + name: recipient.addresses + normalize: + - array + short: Recipient addresses from the SMTP `RCPT` commands. + type: keyword email.reply_to.address: dashed_name: email-reply-to-address description: The address that replies should be delivered to based on the value diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 6e1b49a8f9..668aa8b58d 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -3785,6 +3785,24 @@ email: normalize: [] short: Unique identifier given by the source. type: keyword + email.mailfrom.address: + beta: This field is beta and subject to change. + dashed_name: email-mailfrom-address + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field + has a value of `Smith@bar.com`' + example: userx@y.foo.org + flat_name: email.mailfrom.address + ignore_above: 1024 + level: extended + name: mailfrom.address + normalize: [] + short: Email address from SMTP `MAIL` command. + type: keyword email.message_id: dashed_name: email-message-id description: Identifier from the RFC 5322 `Message-ID:` email header that refers @@ -3807,6 +3825,25 @@ email: normalize: [] short: Date and time the email was composed. type: date + email.recipient.addresses: + beta: This field is beta and subject to change. + dashed_name: email-recipient-addresses + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + flat_name: email.recipient.addresses + ignore_above: 1024 + level: extended + name: recipient.addresses + normalize: + - array + short: Recipient addresses from the SMTP `RCPT` commands. + type: keyword email.reply_to.address: dashed_name: email-reply-to-address description: The address that replies should be delivered to based on the value diff --git a/experimental/generated/elasticsearch/composable/component/email.json b/experimental/generated/elasticsearch/composable/component/email.json index e79c0bc54b..ee2c4f5488 100644 --- a/experimental/generated/elasticsearch/composable/component/email.json +++ b/experimental/generated/elasticsearch/composable/component/email.json @@ -107,12 +107,28 @@ "ignore_above": 1024, "type": "keyword" }, + "mailfrom": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "message_id": { "type": "wildcard" }, "origination_timestamp": { "type": "date" }, + "recipient": { + "properties": { + "addresses": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "reply_to": { "properties": { "address": { diff --git a/experimental/generated/elasticsearch/legacy/template.json b/experimental/generated/elasticsearch/legacy/template.json index 13003e6b81..72bc2481e1 100644 --- a/experimental/generated/elasticsearch/legacy/template.json +++ b/experimental/generated/elasticsearch/legacy/template.json @@ -1151,12 +1151,28 @@ "ignore_above": 1024, "type": "keyword" }, + "mailfrom": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "message_id": { "type": "wildcard" }, "origination_timestamp": { "type": "date" }, + "recipient": { + "properties": { + "addresses": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "reply_to": { "properties": { "address": { diff --git a/generated/beats/fields.ecs.yml b/generated/beats/fields.ecs.yml index 93b81e44bb..e9c0839ad7 100644 --- a/generated/beats/fields.ecs.yml +++ b/generated/beats/fields.ecs.yml @@ -1864,6 +1864,19 @@ Identifier is not persistent across hops.' example: c26dbea0-80d5-463b-b93c-4e8b708219ce default_field: false + - name: mailfrom.address + level: extended + type: keyword + ignore_above: 1024 + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field + has a value of `Smith@bar.com`' + example: userx@y.foo.org + default_field: false - name: message_id level: extended type: wildcard @@ -1878,6 +1891,19 @@ will fill in this value automatically when the message is sent by a user. example: '2020-11-10T22:12:34.8196921Z' default_field: false + - name: recipient.addresses + level: extended + type: keyword + ignore_above: 1024 + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + default_field: false - name: reply_to.address level: extended type: keyword diff --git a/generated/csv/fields.csv b/generated/csv/fields.csv index 177908f017..57d273dd20 100644 --- a/generated/csv/fields.csv +++ b/generated/csv/fields.csv @@ -222,8 +222,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 9.0.0-dev,true,email,email.direction,keyword,extended,,inbound,Direction of the message. 9.0.0-dev,true,email,email.from.address,keyword,extended,array,sender@example.com,The sender's email address. 9.0.0-dev,true,email,email.local_id,keyword,extended,,c26dbea0-80d5-463b-b93c-4e8b708219ce,Unique identifier given by the source. +9.0.0-dev,true,email,email.mailfrom.address,keyword,extended,,userx@y.foo.org,Email address from SMTP `MAIL` command. 9.0.0-dev,true,email,email.message_id,wildcard,extended,,81ce15$8r2j59@mail01.example.com,Value from the Message-ID header. 9.0.0-dev,true,email,email.origination_timestamp,date,extended,,2020-11-10T22:12:34.8196921Z,Date and time the email was composed. +9.0.0-dev,true,email,email.recipient.addresses,keyword,extended,array,"[""Jones@foo.com"", ""Green@foo.com"", ""Brown@foo.com""]",Recipient addresses from the SMTP `RCPT` commands. 9.0.0-dev,true,email,email.reply_to.address,keyword,extended,array,reply.here@example.com,Address replies should be delivered to. 9.0.0-dev,true,email,email.sender.address,keyword,extended,,,Address of the message sender. 9.0.0-dev,true,email,email.subject,keyword,extended,,Please see this important message.,The subject of the email message. diff --git a/generated/ecs/ecs_flat.yml b/generated/ecs/ecs_flat.yml index 3ab5bc75d5..f343a4f844 100644 --- a/generated/ecs/ecs_flat.yml +++ b/generated/ecs/ecs_flat.yml @@ -2750,6 +2750,24 @@ email.local_id: normalize: [] short: Unique identifier given by the source. type: keyword +email.mailfrom.address: + beta: This field is beta and subject to change. + dashed_name: email-mailfrom-address + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field has + a value of `Smith@bar.com`' + example: userx@y.foo.org + flat_name: email.mailfrom.address + ignore_above: 1024 + level: extended + name: mailfrom.address + normalize: [] + short: Email address from SMTP `MAIL` command. + type: keyword email.message_id: dashed_name: email-message-id description: Identifier from the RFC 5322 `Message-ID:` email header that refers @@ -2772,6 +2790,25 @@ email.origination_timestamp: normalize: [] short: Date and time the email was composed. type: date +email.recipient.addresses: + beta: This field is beta and subject to change. + dashed_name: email-recipient-addresses + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + flat_name: email.recipient.addresses + ignore_above: 1024 + level: extended + name: recipient.addresses + normalize: + - array + short: Recipient addresses from the SMTP `RCPT` commands. + type: keyword email.reply_to.address: dashed_name: email-reply-to-address description: The address that replies should be delivered to based on the value diff --git a/generated/ecs/ecs_nested.yml b/generated/ecs/ecs_nested.yml index c068d535a1..fcce8c44d5 100644 --- a/generated/ecs/ecs_nested.yml +++ b/generated/ecs/ecs_nested.yml @@ -3705,6 +3705,24 @@ email: normalize: [] short: Unique identifier given by the source. type: keyword + email.mailfrom.address: + beta: This field is beta and subject to change. + dashed_name: email-mailfrom-address + description: 'The address from the SMTP `MAIL` command associated with the email, + at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field + has a value of `Smith@bar.com`' + example: userx@y.foo.org + flat_name: email.mailfrom.address + ignore_above: 1024 + level: extended + name: mailfrom.address + normalize: [] + short: Email address from SMTP `MAIL` command. + type: keyword email.message_id: dashed_name: email-message-id description: Identifier from the RFC 5322 `Message-ID:` email header that refers @@ -3727,6 +3745,25 @@ email: normalize: [] short: Date and time the email was composed. type: date + email.recipient.addresses: + beta: This field is beta and subject to change. + dashed_name: email-recipient-addresses + description: 'The email address(es) from SMTP `RCPT` command(s) associated with + the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, + this may contain all email address from the entire transaction.' + example: '["Jones@foo.com", "Green@foo.com", "Brown@foo.com"]' + flat_name: email.recipient.addresses + ignore_above: 1024 + level: extended + name: recipient.addresses + normalize: + - array + short: Recipient addresses from the SMTP `RCPT` commands. + type: keyword email.reply_to.address: dashed_name: email-reply-to-address description: The address that replies should be delivered to based on the value diff --git a/generated/elasticsearch/composable/component/email.json b/generated/elasticsearch/composable/component/email.json index f33c03d837..5b4b8df39d 100644 --- a/generated/elasticsearch/composable/component/email.json +++ b/generated/elasticsearch/composable/component/email.json @@ -107,12 +107,28 @@ "ignore_above": 1024, "type": "keyword" }, + "mailfrom": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "message_id": { "type": "wildcard" }, "origination_timestamp": { "type": "date" }, + "recipient": { + "properties": { + "addresses": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "reply_to": { "properties": { "address": { diff --git a/generated/elasticsearch/legacy/template.json b/generated/elasticsearch/legacy/template.json index 1a8d14b60e..afcc327879 100644 --- a/generated/elasticsearch/legacy/template.json +++ b/generated/elasticsearch/legacy/template.json @@ -1109,12 +1109,28 @@ "ignore_above": 1024, "type": "keyword" }, + "mailfrom": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "message_id": { "type": "wildcard" }, "origination_timestamp": { "type": "date" }, + "recipient": { + "properties": { + "addresses": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "reply_to": { "properties": { "address": { diff --git a/schemas/email.yml b/schemas/email.yml index 9c8b6ff390..4aa5ac5901 100644 --- a/schemas/email.yml +++ b/schemas/email.yml @@ -200,3 +200,30 @@ description: > The name of the application that was used to draft and send the original email message. example: "Spambot v2.5" + + - name: mailfrom.address + level: extended + type: keyword + beta: This field is beta and subject to change. + short: Email address from SMTP `MAIL` command. + description: > + The address from the SMTP `MAIL` command associated with the email, at the observed email relay host. + + This is commonly called `MAIL FROM` value of the SMTP envelope. + + For example, if the SMTP command is `MAIL FROM:`, this field has a value of `Smith@bar.com` + example: "userx@y.foo.org" + + - name: recipient.addresses + level: extended + type: keyword + normalize: [array] + beta: This field is beta and subject to change. + short: Recipient addresses from the SMTP `RCPT` commands. + description: > + The email address(es) from SMTP `RCPT` command(s) associated with the email message. + + This is commonly called `RCPT TO` value of the SMTP envelope. + + As there may be multiple `RCPT` commands during a single message transaction, this may contain all email address from the entire transaction. + example: "[\"Jones@foo.com\", \"Green@foo.com\", \"Brown@foo.com\"]"