From 77a58a890111138c4e03147f047da92f71f4101e Mon Sep 17 00:00:00 2001 From: Andres Date: Mon, 21 Dec 2020 23:13:41 -0500 Subject: [PATCH 1/9] Support emails from aws SES. --- API/Functions/Emails/index.js | 80 +++++++++++++ .../creditCardWithdrawal.parser.js | 16 +++ .../bancolombia/debitWithdrawal.parser.js | 16 +++ .../parsers/bancolombia/payments.parser.js | 14 +++ .../bancolombia/productPayments.parser.js | 14 +++ .../parsers/bancolombia/shopping.parser.js | 15 +++ .../bancolombia/transferReception.parser.js | 16 +++ .../parsers/bancolombia/transfers.parser.js | 14 +++ .../Emails/parsers/pse/payments.parser.js | 13 +++ API/Functions/Emails/parsers/utils.js | 25 ++++ API/Functions/Emails/utils/index.js | 108 ++++++++++++++++++ .../AutoMailChecker/src/mongoose/mongoose.js | 40 ------- README.md | 14 ++- config/example.json | 3 +- package-lock.json | 34 ++---- package.json | 1 + serverless.yml | 105 ++++++++++++++--- shared/database/repos/payment.repo.js | 13 +++ shared/database/repos/user.repo.js | 1 - shared/models/bank.model.js | 3 +- 20 files changed, 459 insertions(+), 86 deletions(-) create mode 100644 API/Functions/Emails/index.js create mode 100644 API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/payments.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/productPayments.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/shopping.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/transferReception.parser.js create mode 100644 API/Functions/Emails/parsers/bancolombia/transfers.parser.js create mode 100644 API/Functions/Emails/parsers/pse/payments.parser.js create mode 100644 API/Functions/Emails/parsers/utils.js create mode 100644 API/Functions/Emails/utils/index.js delete mode 100644 AutomationServices/AutoMailChecker/src/mongoose/mongoose.js diff --git a/API/Functions/Emails/index.js b/API/Functions/Emails/index.js new file mode 100644 index 0000000..64e8ba3 --- /dev/null +++ b/API/Functions/Emails/index.js @@ -0,0 +1,80 @@ +const AWS = require('aws-sdk') +const S3 = new AWS.S3(); +const moment = require('moment') +const { getBanks } = require('../../../shared/database/repos/bank.repo'); +const { getUser } = require('../../../shared/database/repos/user.repo'); +const { create: createPayment } = require('../../../shared/database/repos/payment.repo') +const utils = require('./utils') + +module.exports.process = async (event, context, callback) => { + try { + const mailEvent = event.Records[0].ses + const { source, messageId, timestamp, commonHeaders } = mailEvent.mail + let { subject } = commonHeaders + + // Removing forward subject label + if (subject.includes('Fwd: ')) { + subject = subject.replace('Fwd: ', '') + } + + // Search for bank by subject + const bank = await getBanks({ subject: RegExp(subject) }) + + if (Array.isArray(bank) && bank.length == 1) { + + // Get bank information + const { filters, ignore_phrase, name: bankName } = bank[0] + + // Retrieve email information + const data = await S3.getObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }).promise(); + + if (!([undefined, null].includes(data.Body))) { + const emailData = data.Body.toString('utf-8') + const result = await utils.readRawEmail(emailData) + + for (let index = 0; index < filters.length; index++) { + const filter = filters[index]; + + const res = utils.search(result.html, filter.phrase, filter.parser, ignore_phrase, bankName) + + if (!res) continue + + const user = await getUser({ emails: source }) + + const prePaymentObj = { + bank: bankName, + source: res.TRANSACTION_SOURCE, + destination: res.TRANSACTION_DESTINATION, + amount: res.TRANSACTION_VALUE, + cardType: res.TRANSACTION_CARD_TYPE, + account: res.TRANSACTION_ACCOUNT, + category: res.TRANSACTION_TYPE, + text: res.description, + type: filter.type, + createdBy: 'AUTO_EMAIL_SERVICE', + createdAt: moment(timestamp).format(), + user: user._id, + description: res.DESCRIPTION, + isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false + } + const payment = await createPayment(prePaymentObj) + break; + } + + // Deleting processed Email. + await S3.deleteObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }) + + } else { + console.log(`No Body`) + } + } + } catch (error) { + console.log(error) + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js b/API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js new file mode 100644 index 0000000..680c7f6 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js @@ -0,0 +1,16 @@ +const { amountParser } = require('../utils') +module.exports.creditCardWithdrawalParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim() + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'), text.indexOf(' *')) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT, + TRANSACTION_TYPE: 'withdrawal', + DESCRIPTION: `Withdrawal in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}` + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js b/API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js new file mode 100644 index 0000000..ce99e52 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js @@ -0,0 +1,16 @@ +const { amountParser } = require('../utils') +module.exports.debitWithdrawalParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim() + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'), text.indexOf(' *')) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT, + TRANSACTION_TYPE: 'withdrawal', + DESCRIPTION: `Withdrawal in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}` + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/payments.parser.js b/API/Functions/Emails/parsers/bancolombia/payments.parser.js new file mode 100644 index 0000000..73c9312 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/payments.parser.js @@ -0,0 +1,14 @@ +const { amountParser } = require('../utils') + +module.exports.paymentsParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' a'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf(' a ') + 3), text.indexOf('desde')) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE: null, + TRANSACTION_ACCOUNT + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/productPayments.parser.js b/API/Functions/Emails/parsers/bancolombia/productPayments.parser.js new file mode 100644 index 0000000..5a90923 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/productPayments.parser.js @@ -0,0 +1,14 @@ +const { amountParser } = require('../utils') + +module.exports.productParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf('desde'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('pago de') + 7), text.indexOf(' por')) + text.substring(text.lastIndexOf(' *'), (text.lastIndexOf(' *')+6)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *')+6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE: null, + TRANSACTION_ACCOUNT + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/shopping.parser.js b/API/Functions/Emails/parsers/bancolombia/shopping.parser.js new file mode 100644 index 0000000..ef3e599 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/shopping.parser.js @@ -0,0 +1,15 @@ +const { amountParser } = require('../utils') + +module.exports.shoppingParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1), text.indexOf(' en'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3 ), (text.indexOf(':') - 3)) + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOf(' *')) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *')+6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/transferReception.parser.js b/API/Functions/Emails/parsers/bancolombia/transferReception.parser.js new file mode 100644 index 0000000..e29807d --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/transferReception.parser.js @@ -0,0 +1,16 @@ +const { amountParser } = require('../utils') +module.exports.transferReceptionParser = (text) => { + const TRANSACTION_SOURCE = text.substring((text.indexOf('transferencia de ') + 17), (text.indexOf(' por'))) + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.includes('en la') ? text.indexOf('en la') : text.indexOf('enla'))) + const TRANSACTION_DESTINATION = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + const TRANSACTION_CARD_TYPE = null + const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + + return { + TRANSACTION_SOURCE, + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/bancolombia/transfers.parser.js b/API/Functions/Emails/parsers/bancolombia/transfers.parser.js new file mode 100644 index 0000000..6d4b990 --- /dev/null +++ b/API/Functions/Emails/parsers/bancolombia/transfers.parser.js @@ -0,0 +1,14 @@ +const { amountParser } = require('../utils') +module.exports.transfersParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' desde'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('a cta') + 5), text.indexOf('/') - 5).trim() + const TRANSACTION_CARD_TYPE = null + const TRANSACTION_ACCOUNT = text.substring(text.indexOf('*'), (text.indexOf('*')+6)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/pse/payments.parser.js b/API/Functions/Emails/parsers/pse/payments.parser.js new file mode 100644 index 0000000..cc4ccae --- /dev/null +++ b/API/Functions/Emails/parsers/pse/payments.parser.js @@ -0,0 +1,13 @@ +const { amountParser } = require('../utils') + +module.exports.paymentsParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf(' fecha de transacción:'))) + const TRANSACTION_DESTINATION = text.substring((text.indexOf('empresa:') + 8 ),(text.indexOf('descripción:') - 1)) + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE: null, + TRANSACTION_ACCOUNT: null + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/utils.js b/API/Functions/Emails/parsers/utils.js new file mode 100644 index 0000000..93480a8 --- /dev/null +++ b/API/Functions/Emails/parsers/utils.js @@ -0,0 +1,25 @@ +String.prototype.splice = function (idx, rem, str) { + return this.slice(0, idx) + str + this.slice(idx + Math.abs(rem)); + +}; + + + +module.exports.amountParser = (value) => { + value = value.replace('=','').trim() + // Check for decimal numbers + if (value.includes(',') && value.substring(value.lastIndexOf(',') + 1).length === 2) { + const decimal = value.substring(value.lastIndexOf(',') + 1) + value = value.replace(/\D/g, "") + const endLength = value.length + value = value.splice(endLength - 2, endLength - 1, '.' + decimal) + } else if (value.includes('.') && value.substring(value.lastIndexOf('.') + 1).length === 2) { + const decimal = value.substring(value.lastIndexOf('.') + 1) + value = value.replace(/\D/g, "") + const endLength = value.length + value = value.splice(endLength - 2, endLength - 1, '.' + decimal) + } else { + value = value.replace(/\D/g, "") + } + return parseFloat(value) +} \ No newline at end of file diff --git a/API/Functions/Emails/utils/index.js b/API/Functions/Emails/utils/index.js new file mode 100644 index 0000000..4dfc7f4 --- /dev/null +++ b/API/Functions/Emails/utils/index.js @@ -0,0 +1,108 @@ +const mailparser = require('mailparser').simpleParser +const _ = require('lodash') +const cheerio = require('cheerio'); +const { htmlToText } = require('html-to-text'); +const { paymentsParser: paymentsPSEParser } = require('../parsers/pse/payments.parser') +const { paymentsParser: paymentsBancolombiaParser } = require('../parsers/bancolombia/payments.parser') +const { shoppingParser } = require('../parsers/bancolombia/shopping.parser') +const { productParser } = require('../parsers/bancolombia/productPayments.parser') +const { transfersParser } = require('../parsers/bancolombia/transfers.parser') +const { transferReceptionParser } = require('../parsers/bancolombia/transferReception.parser') +const { debitWithdrawalParser } = require('../parsers/bancolombia/debitWithdrawal.parser') +const { creditCardWithdrawalParser } = require('../parsers/bancolombia/creditCardWithdrawal.parser') + + +function isBase64(str) { + return Buffer.from(str, 'base64').toString('base64') === str +} + +module.exports.readRawEmail = async (body) => { + let textAsHtml = null; + + if (body !== null && isBase64(body.replace(/\r?\n|\r/g, ""))) { + body = Buffer.from(body.replace(/\r?\n|\r/g, ""), 'base64').toString() + textAsHtml = '

' + htmlToText(body).replace(/\r?\n|\r|\t/g, " ") + '

' + } else { + const result = await mailparser(body); + textAsHtml = result.textAsHtml + } + + + return { + html: textAsHtml + } +} + +module.exports.search = (html, filter, parser, skipped_phrase = 'Bancolombia le informa que su factura inscrita', bank_name = "BANCOLOMBIA") => { + if ([null, undefined].includes(parser)) return undefined; //If we don't have a parser, just return null to continue with the next message + const $ = cheerio.load(html) + const res = $('p') + const value = res.text().trim().toLowerCase().replace(/=/g, '') + filter = filter.toLocaleLowerCase() + if (value.includes(filter)) { + if (skipped_phrase === null || skipped_phrase === undefined || !value.includes(skipped_phrase.toLocaleLowerCase())) { // If the phrase do not includes the skipped phrase + let description, parserResult; + + // The fisrt coincidence for the filter phrase + const first = value.indexOf(filter) + + + switch (bank_name) { + case "BANCOLOMBIA": { + const end = value.indexOf('018000931987', first + 1) + 12 + description = value.substring(first, end) + + switch (parser) { + case 'payments': { + parserResult = paymentsBancolombiaParser(description) + break; + } + case 'shopping': { + parserResult = shoppingParser(description) + break; + } + case 'product': { + parserResult = productParser(description) + break; + } + case 'transfer': { + parserResult = transfersParser(description) + break; + } + case 'transferReception': { + parserResult = transferReceptionParser(description) + break; + } + case 'debitWithdrawal': { + parserResult = debitWithdrawalParser(description) + break; + } + case 'creditCardWithdrawal': { + parserResult = creditCardWithdrawalParser(description) + break; + } + } + break; + } + case "PSE": { + /** + * I want to keep on the description the company name where the payment was made + * This value is before the value of the transaction, but some transaction at the first match value + * has a CUS (like a uuid) of the transaction, that's why I need to look for the value 'Valor de la Transacción' + */ + const end = value.indexOf('[http://www.jlnsoftware.com.br', first + 1) + description = value.substring(first, end) + + parserResult = paymentsPSEParser(description) + break; + } + } + + return { + description, + ...parserResult + } + } + } + return undefined +} \ No newline at end of file diff --git a/AutomationServices/AutoMailChecker/src/mongoose/mongoose.js b/AutomationServices/AutoMailChecker/src/mongoose/mongoose.js deleted file mode 100644 index ff02e7b..0000000 --- a/AutomationServices/AutoMailChecker/src/mongoose/mongoose.js +++ /dev/null @@ -1,40 +0,0 @@ -const mongoose = require('mongoose') -const { - MONGO_HOST, - MONGO_PORT, - MONGO_SSL, - MONGO_USER, - MONGO_SECRET, - MONGO_SET, -} = process.env -let connection = null - -const connect = () => { - if (!connection) { - connection = mongoose.connect(MONGO_HOST, { - port: MONGO_PORT, - ssl: !!MONGO_SSL, - user: MONGO_USER, - replicaSet: MONGO_SET, - pass: MONGO_SECRET, - dbName:'test' - }) - return connection - } - return connection -} - -const destroy = () => { - return mongoose.disconnect() -} - -const isConnected = () => { - return mongoose.isConnected() -} - -module.exports = { - db: mongoose.connection.db, - connect, - destroy, - isConnected -} \ No newline at end of file diff --git a/README.md b/README.md index 691e231..a50b803 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +## Deploy SES email parser + +``` + 1. Verify a domain at https://console.aws.amazon.com/ses/home?region=us-east-1#verified-senders-domain: + 2. sls deploy --stage prod # Make sure about the config parameters + 3. Forward your messages to the EMAIL_RECIPIENTS +``` + ## Deploy Client ### Deploy Cognito Pool @@ -158,9 +166,6 @@ sls deploy --stage [DEV/TEST/PROD] "ignore_phrase": "Bancolombia le informa que su factura inscrita", "index_value": { "$numberLong": "0" - }, - "user": { - "$oid": "5fc43e5e607f370ee4f66308" } } // PSE @@ -178,9 +183,6 @@ sls deploy --stage [DEV/TEST/PROD] "ignore_phrase": "Estado de la transacción: Rechazada", "index_value": { "$numberLong": "0" - }, - "user": { - "$oid": "5fc43e5e607f370ee4f66308" } } ] diff --git a/config/example.json b/config/example.json index 043e085..8ba19a2 100644 --- a/config/example.json +++ b/config/example.json @@ -10,5 +10,6 @@ "EMAIL_PASSWORD": "SECRET", "PHONE_NUMBER": "+573000000000", "SECRET_KEY": "vOVH6sdmpNWjRRIqCc7rdxs01lwHzfr3", - "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:XXXXXXXX:userpool/us-east-1_ZZZZZZZ" + "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:XXXXXXXX:userpool/us-east-1_ZZZZZZZ", + "EMAIL_RECIPIENTS": ["finance@domain.tdl"] } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index af9c2a0..b93fea6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2015,10 +2015,9 @@ "dev": true }, "aws-sdk": { - "version": "2.734.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.734.0.tgz", - "integrity": "sha512-F5SrcOm6WyaN5j+pybx97vfSCa0SIz/kukJNSeDzMdk5q0xsKUjtUzL/kc0QwtCspCni0WTNmBKiD+firP5JKQ==", - "dev": true, + "version": "2.815.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.815.0.tgz", + "integrity": "sha512-BXL3Og97rOY9jE7OeYQdKftMAZ3SneFg/rBslyog+W0dTDKq3NBuM3fBWhc3POf26kHcFjsnLIWScM8bWhD4AA==", "requires": { "buffer": "4.9.2", "events": "1.1.1", @@ -2035,7 +2034,6 @@ "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4", @@ -2045,14 +2043,12 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "dev": true + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" } } }, @@ -3879,8 +3875,7 @@ "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", - "dev": true + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, "execa": { "version": "4.0.3", @@ -5708,8 +5703,7 @@ "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", - "dev": true + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, "js-string-escape": { "version": "1.0.1", @@ -7537,8 +7531,7 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", - "dev": true + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, "quoted-printable": { "version": "1.0.1", @@ -7828,8 +7821,7 @@ "sax": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=", - "dev": true + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, "scmp": { "version": "2.1.0", @@ -9674,7 +9666,6 @@ "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", - "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -9683,8 +9674,7 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", - "dev": true + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" } } }, @@ -9903,7 +9893,6 @@ "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", - "dev": true, "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" @@ -9912,8 +9901,7 @@ "xmlbuilder": { "version": "9.0.7", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "dev": true + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" } } }, diff --git a/package.json b/package.json index 2f5603f..6bf95c8 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "author": "", "license": "ISC", "dependencies": { + "aws-sdk": "^2.815.0", "axios": "^0.21.0", "cheerio": "^1.0.0-rc.3", "chrome-aws-lambda": "^5.5.0", diff --git a/serverless.yml b/serverless.yml index f19967d..2594d1e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -8,6 +8,7 @@ package: - node_modules/puppeteer/.local-chromium/** - node_modules/chrome-aws-lambda/** # The function is provided by the layer - devutils/** + - AutomationServices/AutoMailChecker/** provider: name: aws @@ -25,6 +26,21 @@ provider: MONGO_SET: ${file(./config/${opt:stage}.json):MONGO_SET} MONGO_USER: ${file(./config/${opt:stage}.json):MONGO_USER} MONGO_SSL: true + iamRoleStatements: + - Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + Resource: + - Fn::Join: + - '' + - + - Fn::GetAtt: [EmailsBucket, Arn] + - '/*' + - Effect: Allow + Action: ses:SendRawEmail + Resource: + - "*" functions: TwilioIncommingMessage: @@ -195,20 +211,28 @@ functions: handler: API/Functions/Users.postConfirmation name: PostConfirmationUser-${opt:stage} + ProcessEmails: + handler: API/Functions/Emails.process + name: ProcessEmail-${opt:stage} + environment: + BUCKETNAME: finance-emails-${opt:stage} + + # + # DEPRECATED + # + # EmailChecker: + # handler: AutomationServices/AutoMailChecker/checker.start + # name: AutomationServices-EmailChecker-${opt:stage} + # events: + # - schedule: + # name: "cw-start-email-checker" + # description: "Function to connecto to Gmail :D" + # rate: rate(20 minutes) + # enabled: true + # environment: + # EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} + # EMAIL_PASSWORD: ${file(./config/${opt:stage}.json):EMAIL_PASSWORD} - EmailChecker: - handler: AutomationServices/AutoMailChecker/checker.start - name: AutomationServices-EmailChecker-${opt:stage} - events: - - schedule: - name: "cw-start-email-checker" - description: "Function to connecto to Gmail :D" - rate: rate(20 minutes) - enabled: true - environment: - EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} - EMAIL_PASSWORD: ${file(./config/${opt:stage}.json):EMAIL_PASSWORD} - DataCreditoScraper: handler: AutomationServices/DataCreditoScraper/scraper.start name: AutomationServices-DataCreditoScraper-${opt:stage} @@ -225,3 +249,58 @@ functions: environment: EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} SECRET_KEY: ${file(./config/${opt:stage}.json):SECRET_KEY} + +resources: + Resources: + GiveSESPermissionToInvokeProcessEmailLambdaFunction: + Type: AWS::Lambda::Permission + Properties: + FunctionName: { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } + Principal: ses.amazonaws.com + Action: "lambda:InvokeFunction" + SourceAccount: { Ref: AWS::AccountId } + + EmailsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: finance-emails-${opt:stage} + EmailsBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: { Ref: EmailsBucket } + PolicyDocument: + Statement: + - + Action: + - s3:PutObject + Effect: Allow + Resource: + Fn::Join: + - "" + - + - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } + - "/*" + Principal: + Service: ses.amazonaws.com + Condition: + StringEquals: + aws:Referer: { Ref: AWS::AccountId } + + SesLambdaRule: + Type: AWS::SES::ReceiptRule + DependsOn: + - GiveSESPermissionToInvokeProcessEmailLambdaFunction + - EmailsBucketPolicy + Properties: + Rule: + Enabled: true + ScanEnabled: true + Name: finance-email-rule-${opt:stage} + Recipients: ${file(./config/${opt:stage}.json):EMAIL_RECIPIENTS} + Actions: + - LambdaAction: + FunctionArn: + { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } + - S3Action: + BucketName: finance-emails-${opt:stage} + RuleSetName: default-rule-set diff --git a/shared/database/repos/payment.repo.js b/shared/database/repos/payment.repo.js index 31db043..bc83ee4 100644 --- a/shared/database/repos/payment.repo.js +++ b/shared/database/repos/payment.repo.js @@ -14,6 +14,19 @@ module.exports.createMultiple = async (PaymentBodies = []) => { } }; + + +module.exports.create = async (PaymentBody) => { + try { + await connect(); + const payment = await Payments.create(PaymentBody); + await destroy(); + return payment + } catch (error) { + console.log(error); + } +}; + module.exports.getAllByDate = async ({ userId, date }) => { if (!userId) return []; diff --git a/shared/database/repos/user.repo.js b/shared/database/repos/user.repo.js index 06e8fe3..57f5363 100644 --- a/shared/database/repos/user.repo.js +++ b/shared/database/repos/user.repo.js @@ -1,4 +1,3 @@ -const { search } = require("../../../AutomationServices/AutoMailChecker/utils"); const userModel = require("../../models/user.model"); const Users = require("../../models/user.model"); const { connect, destroy, isConnected } = require("../mongo"); diff --git a/shared/models/bank.model.js b/shared/models/bank.model.js index 9292b9c..3d25636 100644 --- a/shared/models/bank.model.js +++ b/shared/models/bank.model.js @@ -21,8 +21,7 @@ const BankSchema = mongoose.Schema( type: String }, ignore_phrase: { type: String, default: null }, - index_value: { type: Number }, - user: { type: mongoose.Types.ObjectId, required: true } + index_value: { type: Number } }, { timestamps: true, From 81238b85dbb2687d27b9b62d5a930a5250b3378f Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 22 Dec 2020 23:03:36 -0500 Subject: [PATCH 2/9] Getting source from commonHeaders. --- API/Functions/Emails/index.js | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/API/Functions/Emails/index.js b/API/Functions/Emails/index.js index 64e8ba3..7bb9ba2 100644 --- a/API/Functions/Emails/index.js +++ b/API/Functions/Emails/index.js @@ -9,8 +9,12 @@ const utils = require('./utils') module.exports.process = async (event, context, callback) => { try { const mailEvent = event.Records[0].ses - const { source, messageId, timestamp, commonHeaders } = mailEvent.mail - let { subject } = commonHeaders + //Informative log until all is working as expected. + console.log(JSON.stringify(mailEvent)) + const { messageId, timestamp, commonHeaders } = mailEvent.mail + let { subject, from } = commonHeaders + + const source = getEmail(from) // Removing forward subject label if (subject.includes('Fwd: ')) { @@ -39,11 +43,11 @@ module.exports.process = async (event, context, callback) => { const filter = filters[index]; const res = utils.search(result.html, filter.phrase, filter.parser, ignore_phrase, bankName) - + if (!res) continue const user = await getUser({ emails: source }) - + const prePaymentObj = { bank: bankName, source: res.TRANSACTION_SOURCE, @@ -77,4 +81,16 @@ module.exports.process = async (event, context, callback) => { } catch (error) { console.log(error) } -} \ No newline at end of file +} + + +const getEmail = (from) => { + let source = undefined; + if (Array.isArray(from) && from.length > 0) { + source = from[0].match(/\<(.*?)\>/g) + if (Array.isArray(source) && source.length > 0) { + source = source[0].replace('<', '').replace('>', '') + } + } + return source +} From 2f819c0a4af17c0d812ffa189357b2640d74ebb8 Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 27 Dec 2020 12:20:00 -0500 Subject: [PATCH 3/9] Getting user email from commonHeaders. --- API/Functions/Emails/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/API/Functions/Emails/index.js b/API/Functions/Emails/index.js index 7bb9ba2..8668a09 100644 --- a/API/Functions/Emails/index.js +++ b/API/Functions/Emails/index.js @@ -12,9 +12,9 @@ module.exports.process = async (event, context, callback) => { //Informative log until all is working as expected. console.log(JSON.stringify(mailEvent)) const { messageId, timestamp, commonHeaders } = mailEvent.mail - let { subject, from } = commonHeaders + let { subject, to } = commonHeaders - const source = getEmail(from) + const source = to[0] // Removing forward subject label if (subject.includes('Fwd: ')) { From 1ba9299b013dab995cffc37e3905c1c4e9b724c8 Mon Sep 17 00:00:00 2001 From: Andres Date: Wed, 3 Feb 2021 19:25:05 -0500 Subject: [PATCH 4/9] test with emails. --- API/Functions/Emails/index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/API/Functions/Emails/index.js b/API/Functions/Emails/index.js index 8668a09..c06fb2e 100644 --- a/API/Functions/Emails/index.js +++ b/API/Functions/Emails/index.js @@ -12,9 +12,10 @@ module.exports.process = async (event, context, callback) => { //Informative log until all is working as expected. console.log(JSON.stringify(mailEvent)) const { messageId, timestamp, commonHeaders } = mailEvent.mail - let { subject, to } = commonHeaders + let { subject, to, from } = commonHeaders - const source = to[0] + + const source = getEmail(to); // Removing forward subject label if (subject.includes('Fwd: ')) { @@ -35,10 +36,14 @@ module.exports.process = async (event, context, callback) => { Key: messageId }).promise(); + console.log(data); + if (!([undefined, null].includes(data.Body))) { const emailData = data.Body.toString('utf-8') const result = await utils.readRawEmail(emailData) + console.log(result); + for (let index = 0; index < filters.length; index++) { const filter = filters[index]; @@ -90,6 +95,8 @@ const getEmail = (from) => { source = from[0].match(/\<(.*?)\>/g) if (Array.isArray(source) && source.length > 0) { source = source[0].replace('<', '').replace('>', '') + }else{ + source = from[0]; } } return source From 3d3c1be1c43eebf9b3cf5761e06593a904d5d43e Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 29 Jun 2021 14:54:10 -0500 Subject: [PATCH 5/9] Implementing SES to read forwarding emails to add payments. --- .../EmailsForwardingReader}/index.js | 22 ++- .../creditCardWithdrawal.parser.js | 4 +- .../bancolombia/debitWithdrawal.parser.js | 4 +- .../parsers/bancolombia/payments.parser.js | 2 +- .../bancolombia/productPayments.parser.js | 2 +- .../parsers/bancolombia/shopping.parser.js | 4 +- .../bancolombia/transferReception.parser.js | 2 +- .../parsers/bancolombia/transfers.parser.js | 2 +- .../parsers/davivienda/payments.parser.js | 18 +++ .../parsers/pse/payments.parser.js | 4 +- .../EmailsForwardingReader}/parsers/utils.js | 6 +- .../EmailsForwardingReader}/utils/index.js | 0 serverless.yml | 152 ++++++++++++------ 13 files changed, 150 insertions(+), 72 deletions(-) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/index.js (85%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/creditCardWithdrawal.parser.js (83%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/debitWithdrawal.parser.js (83%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/payments.parser.js (80%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/productPayments.parser.js (82%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/shopping.parser.js (78%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/transferReception.parser.js (86%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/bancolombia/transfers.parser.js (82%) create mode 100644 AutomationServices/EmailsForwardingReader/parsers/davivienda/payments.parser.js rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/pse/payments.parser.js (79%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/parsers/utils.js (83%) rename {API/Functions/Emails => AutomationServices/EmailsForwardingReader}/utils/index.js (100%) diff --git a/API/Functions/Emails/index.js b/AutomationServices/EmailsForwardingReader/index.js similarity index 85% rename from API/Functions/Emails/index.js rename to AutomationServices/EmailsForwardingReader/index.js index c06fb2e..7f000cc 100644 --- a/API/Functions/Emails/index.js +++ b/AutomationServices/EmailsForwardingReader/index.js @@ -1,21 +1,18 @@ const AWS = require('aws-sdk') const S3 = new AWS.S3(); const moment = require('moment') -const { getBanks } = require('../../../shared/database/repos/bank.repo'); -const { getUser } = require('../../../shared/database/repos/user.repo'); -const { create: createPayment } = require('../../../shared/database/repos/payment.repo') +const { getBanks } = require('../../shared/database/repos/bank.repo'); +const { getUser } = require('../../shared/database/repos/user.repo'); +const { create: createPayment } = require('../../shared/database/repos/payment.repo') const utils = require('./utils') module.exports.process = async (event, context, callback) => { try { const mailEvent = event.Records[0].ses - //Informative log until all is working as expected. - console.log(JSON.stringify(mailEvent)) const { messageId, timestamp, commonHeaders } = mailEvent.mail let { subject, to, from } = commonHeaders - - const source = getEmail(to); + const source = getEmail(to); // Removing forward subject label if (subject.includes('Fwd: ')) { @@ -23,10 +20,11 @@ module.exports.process = async (event, context, callback) => { } // Search for bank by subject - const bank = await getBanks({ subject: RegExp(subject) }) + const banks = await getBanks({}) - if (Array.isArray(bank) && bank.length == 1) { + const bank = banks.filter(bank => subject.includes(bank.subject)); + if (Array.isArray(bank) && bank.length == 1) { // Get bank information const { filters, ignore_phrase, name: bankName } = bank[0] @@ -36,13 +34,11 @@ module.exports.process = async (event, context, callback) => { Key: messageId }).promise(); - console.log(data); - if (!([undefined, null].includes(data.Body))) { const emailData = data.Body.toString('utf-8') const result = await utils.readRawEmail(emailData) - console.log(result); + console.log('result', result); for (let index = 0; index < filters.length; index++) { const filter = filters[index]; @@ -95,7 +91,7 @@ const getEmail = (from) => { source = from[0].match(/\<(.*?)\>/g) if (Array.isArray(source) && source.length > 0) { source = source[0].replace('<', '').replace('>', '') - }else{ + } else { source = from[0]; } } diff --git a/API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/creditCardWithdrawal.parser.js similarity index 83% rename from API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/creditCardWithdrawal.parser.js index 680c7f6..e5b799d 100644 --- a/API/Functions/Emails/parsers/bancolombia/creditCardWithdrawal.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/creditCardWithdrawal.parser.js @@ -2,8 +2,8 @@ const { amountParser } = require('../utils') module.exports.creditCardWithdrawalParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim() - const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'), text.indexOf(' *')) - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/debitWithdrawal.parser.js similarity index 83% rename from API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/debitWithdrawal.parser.js index ce99e52..7c24ca7 100644 --- a/API/Functions/Emails/parsers/bancolombia/debitWithdrawal.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/debitWithdrawal.parser.js @@ -2,8 +2,8 @@ const { amountParser } = require('../utils') module.exports.debitWithdrawalParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim() - const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'), text.indexOf(' *')) - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/API/Functions/Emails/parsers/bancolombia/payments.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/payments.parser.js similarity index 80% rename from API/Functions/Emails/parsers/bancolombia/payments.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/payments.parser.js index 73c9312..4401224 100644 --- a/API/Functions/Emails/parsers/bancolombia/payments.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/payments.parser.js @@ -3,7 +3,7 @@ const { amountParser } = require('../utils') module.exports.paymentsParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' a'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf(' a ') + 3), text.indexOf('desde')) - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/API/Functions/Emails/parsers/bancolombia/productPayments.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/productPayments.parser.js similarity index 82% rename from API/Functions/Emails/parsers/bancolombia/productPayments.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/productPayments.parser.js index 5a90923..8836820 100644 --- a/API/Functions/Emails/parsers/bancolombia/productPayments.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/productPayments.parser.js @@ -3,7 +3,7 @@ const { amountParser } = require('../utils') module.exports.productParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf('desde'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('pago de') + 7), text.indexOf(' por')) + text.substring(text.lastIndexOf(' *'), (text.lastIndexOf(' *')+6)) - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *')+6)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/API/Functions/Emails/parsers/bancolombia/shopping.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/shopping.parser.js similarity index 78% rename from API/Functions/Emails/parsers/bancolombia/shopping.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/shopping.parser.js index ef3e599..4de4f95 100644 --- a/API/Functions/Emails/parsers/bancolombia/shopping.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/shopping.parser.js @@ -3,8 +3,8 @@ const { amountParser } = require('../utils') module.exports.shoppingParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1), text.indexOf(' en'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3 ), (text.indexOf(':') - 3)) - const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOf(' *')) - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *')+6)) + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/API/Functions/Emails/parsers/bancolombia/transferReception.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/transferReception.parser.js similarity index 86% rename from API/Functions/Emails/parsers/bancolombia/transferReception.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/transferReception.parser.js index e29807d..c975f58 100644 --- a/API/Functions/Emails/parsers/bancolombia/transferReception.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/transferReception.parser.js @@ -4,7 +4,7 @@ module.exports.transferReceptionParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.includes('en la') ? text.indexOf('en la') : text.indexOf('enla'))) const TRANSACTION_DESTINATION = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) const TRANSACTION_CARD_TYPE = null - const TRANSACTION_ACCOUNT = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_SOURCE, diff --git a/API/Functions/Emails/parsers/bancolombia/transfers.parser.js b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/transfers.parser.js similarity index 82% rename from API/Functions/Emails/parsers/bancolombia/transfers.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/bancolombia/transfers.parser.js index 6d4b990..df29c61 100644 --- a/API/Functions/Emails/parsers/bancolombia/transfers.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/bancolombia/transfers.parser.js @@ -3,7 +3,7 @@ module.exports.transfersParser = (text) => { const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' desde'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('a cta') + 5), text.indexOf('/') - 5).trim() const TRANSACTION_CARD_TYPE = null - const TRANSACTION_ACCOUNT = text.substring(text.indexOf('*'), (text.indexOf('*')+6)) + const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6)) return { TRANSACTION_VALUE, diff --git a/AutomationServices/EmailsForwardingReader/parsers/davivienda/payments.parser.js b/AutomationServices/EmailsForwardingReader/parsers/davivienda/payments.parser.js new file mode 100644 index 0000000..6a2063b --- /dev/null +++ b/AutomationServices/EmailsForwardingReader/parsers/davivienda/payments.parser.js @@ -0,0 +1,18 @@ +const { amountParser } = require('../utils') + +module.exports.paymentsParser = (text) => { + const TRANSACTION_VALUE = amountParser(text.substring((text.indexOf('valor transacción: ')), text.indexOf('clase de movimiento:'))) + const TRANSACTION_DESTINATION = text.substring(text.indexOf('lugar de transacción: ') + 22, text.indexOf('banco davivienda')) + const TRANSACTION_ACCOUNT = text.substring(text.indexOf('***') + 3, (text.indexOf('***') + 8)) + const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('movimiento de su') + 17, (text.indexOf(' terminada'))) == 'tarjetacrédito' ? 't.cred' : 't.debt' + const TRANSACTION_TYPE = text.substring(text.indexOf('clase de movimiento: ') + 21, (text.indexOf(' .'))) == 'compra' ? 'EXPENSE' : 'INCOME' + + return { + TRANSACTION_VALUE, + TRANSACTION_DESTINATION, + TRANSACTION_CARD_TYPE, + TRANSACTION_ACCOUNT, + TRANSACTION_TYPE, + DESCRIPTION: `Purchase in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}` + } +} \ No newline at end of file diff --git a/API/Functions/Emails/parsers/pse/payments.parser.js b/AutomationServices/EmailsForwardingReader/parsers/pse/payments.parser.js similarity index 79% rename from API/Functions/Emails/parsers/pse/payments.parser.js rename to AutomationServices/EmailsForwardingReader/parsers/pse/payments.parser.js index cc4ccae..61b2be4 100644 --- a/API/Functions/Emails/parsers/pse/payments.parser.js +++ b/AutomationServices/EmailsForwardingReader/parsers/pse/payments.parser.js @@ -1,13 +1,13 @@ const { amountParser } = require('../utils') module.exports.paymentsParser = (text) => { - const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf(' fecha de transacción:'))) + const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf('fecha de transacción:'))) const TRANSACTION_DESTINATION = text.substring((text.indexOf('empresa:') + 8 ),(text.indexOf('descripción:') - 1)) return { TRANSACTION_VALUE, TRANSACTION_DESTINATION, - TRANSACTION_CARD_TYPE: null, + TRANSACTION_CARD_TYPE: 't.debt', TRANSACTION_ACCOUNT: null } } \ No newline at end of file diff --git a/API/Functions/Emails/parsers/utils.js b/AutomationServices/EmailsForwardingReader/parsers/utils.js similarity index 83% rename from API/Functions/Emails/parsers/utils.js rename to AutomationServices/EmailsForwardingReader/parsers/utils.js index 93480a8..b11d132 100644 --- a/API/Functions/Emails/parsers/utils.js +++ b/AutomationServices/EmailsForwardingReader/parsers/utils.js @@ -3,10 +3,14 @@ String.prototype.splice = function (idx, rem, str) { }; +String.prototype.indexOfRegex = function(regex){ + var match = this.match(regex); + return match ? this.indexOf(match[0]) : -1; + } module.exports.amountParser = (value) => { - value = value.replace('=','').trim() + value = value.replace('=', '').trim() // Check for decimal numbers if (value.includes(',') && value.substring(value.lastIndexOf(',') + 1).length === 2) { const decimal = value.substring(value.lastIndexOf(',') + 1) diff --git a/API/Functions/Emails/utils/index.js b/AutomationServices/EmailsForwardingReader/utils/index.js similarity index 100% rename from API/Functions/Emails/utils/index.js rename to AutomationServices/EmailsForwardingReader/utils/index.js diff --git a/serverless.yml b/serverless.yml index 2f26186..51476fb 100644 --- a/serverless.yml +++ b/serverless.yml @@ -11,13 +11,13 @@ package: - devutils/** - AutomationServices/AutoMailChecker/** -custom: - ArnSQSEmails: - dev: ${file(./config/dev.json):USER_SCHEDULE_SQS_URL} - test: - Ref: EmailUsersQueue - prod: - Ref: EmailUsersQueue +# custom: +# ArnSQSEmails: +# dev: ${file(./config/dev.json):USER_SCHEDULE_SQS_URL} +# test: +# Ref: EmailUsersQueue +# prod: +# Ref: EmailUsersQueue provider: name: aws @@ -149,6 +149,7 @@ functions: arn: ${file(./config/${opt:stage}.json):USER_POOL_ARN} claims: - sub + UpdateEmailCredentials: handler: API/Functions/Users.updateEmailSourceCredentials name: UpdateEmailCredentials-put-${opt:stage} @@ -368,20 +369,27 @@ functions: #AUTOMATIONS - EmailChecker: - handler: AutomationServices/AutoMailChecker/checker.start - name: AutomationServices-EmailChecker-${opt:stage} - timeout: 120 - events: - - sqs: - arn: - Fn::GetAtt: - - EmailUsersQueue - - Arn - batchSize: 1 - environment: - EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} - EMAIL_PASSWORD: ${file(./config/${opt:stage}.json):EMAIL_PASSWORD} + + ProcessEmails: + handler: AutomationServices/EmailsForwardingReader.process + name: ProcessEmail-${opt:stage} + environment: + BUCKETNAME: finance-emails-andresmorelos-dev${opt:stage} + + # EmailChecker: + # handler: AutomationServices/AutoMailChecker/checker.start + # name: AutomationServices-EmailChecker-${opt:stage} + # timeout: 120 + # events: + # - sqs: + # arn: + # Fn::GetAtt: + # - EmailUsersQueue + # - Arn + # batchSize: 1 + # environment: + # EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} + # EMAIL_PASSWORD: ${file(./config/${opt:stage}.json):EMAIL_PASSWORD} PaymentsAnalizer: handler: AutomationServices/AnalizerTask/analizer.start @@ -400,8 +408,6 @@ functions: input: ExecutionType: "WEEKLY" environment: - EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} - EMAIL_PASSWORD: ${file(./config/${opt:stage}.json):EMAIL_PASSWORD} TWILIO_ACCESS_TOKEN: ${file(./config/${opt:stage}.json):TWILIO_ACCESS_TOKEN} TWILIO_SECTRET_KEY: ${file(./config/${opt:stage}.json):TWILIO_SECTRET_KEY} @@ -431,30 +437,84 @@ functions: rate: rate(1 day) enabled: true - ScheduleUsersToCheckEmails: - handler: AutomationServices/AutoMailChecker/schedule.start - name: AutomationServices-ScheduleUsers-${opt:stage} - iamRoleStatementsName: write-sqs-finances-${opt:stage}-role - iamRoleStatements: - - Effect: 'Allow' - Action: - - 'sqs:SendMessage' - - 'sqs:SendMessageBatch' - Resource: - Fn::GetAtt: - - EmailUsersQueue - - Arn - events: - - schedule: - enabled: true - rate: rate(20 minutes) - environment: - USER_SCHEDULE_SQS_URL: ${self:custom.ArnSQSEmails.${opt:stage, self:provider.stage}} + # ScheduleUsersToCheckEmails: + # handler: AutomationServices/AutoMailChecker/schedule.start + # name: AutomationServices-ScheduleUsers-${opt:stage} + # iamRoleStatementsName: write-sqs-finances-${opt:stage}-role + # iamRoleStatements: + # - Effect: 'Allow' + # Action: + # - 'sqs:SendMessage' + # - 'sqs:SendMessageBatch' + # Resource: + # Fn::GetAtt: + # - EmailUsersQueue + # - Arn + # events: + # - schedule: + # enabled: true + # rate: rate(20 minutes) + # environment: + # USER_SCHEDULE_SQS_URL: ${self:custom.ArnSQSEmails.${opt:stage, self:provider.stage}} resources: Resources: - EmailUsersQueue: - Type: "AWS::SQS::Queue" + # EmailUsersQueue: + # Type: "AWS::SQS::Queue" + # Properties: + # QueueName: "email-users-checker" + # VisibilityTimeout: 300 + + GiveSESPermissionToInvokeProcessEmailLambdaFunction: + Type: AWS::Lambda::Permission + Properties: + FunctionName: { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } + Principal: ses.amazonaws.com + Action: "lambda:InvokeFunction" + SourceAccount: { Ref: AWS::AccountId } + + EmailsBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: finance-emails-andresmorelos-dev${opt:stage} + + EmailsBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: { Ref: EmailsBucket } + PolicyDocument: + Statement: + - + Action: + - s3:PutObject + Effect: Allow + Resource: + Fn::Join: + - "" + - + - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } + - "/*" + Principal: + Service: ses.amazonaws.com + Condition: + StringEquals: + aws:Referer: { Ref: AWS::AccountId } + + SesLambdaRule: + Type: AWS::SES::ReceiptRule + DependsOn: + - GiveSESPermissionToInvokeProcessEmailLambdaFunction + - EmailsBucketPolicy Properties: - QueueName: "email-users-checker" - VisibilityTimeout: 300 + Rule: + Enabled: true + ScanEnabled: true + Name: finance-email-rule-${opt:stage} + Recipients: ${file(./config/${opt:stage}.json):EMAIL_RECIPIENTS} + Actions: + - LambdaAction: + FunctionArn: + { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } + - S3Action: + BucketName: finance-emails-andresmorelos-dev${opt:stage} + RuleSetName: default-rule-set \ No newline at end of file From ec0d0f5c652a692b8a3c0cb19b9692c96014861f Mon Sep 17 00:00:00 2001 From: Andres Date: Wed, 30 Jun 2021 09:40:08 -0500 Subject: [PATCH 6/9] Accepting emails only with HTML type. --- .../EmailsForwardingReader/index.js | 8 +- .../EmailsForwardingReader/utils/index.js | 4 +- package-lock.json | 77 +++++++++++++------ 3 files changed, 63 insertions(+), 26 deletions(-) diff --git a/AutomationServices/EmailsForwardingReader/index.js b/AutomationServices/EmailsForwardingReader/index.js index 7f000cc..a6f6917 100644 --- a/AutomationServices/EmailsForwardingReader/index.js +++ b/AutomationServices/EmailsForwardingReader/index.js @@ -11,9 +11,11 @@ module.exports.process = async (event, context, callback) => { const mailEvent = event.Records[0].ses const { messageId, timestamp, commonHeaders } = mailEvent.mail let { subject, to, from } = commonHeaders - + + console.log('commonHeaders', commonHeaders) const source = getEmail(to); + console.log('source', source) // Removing forward subject label if (subject.includes('Fwd: ')) { subject = subject.replace('Fwd: ', '') @@ -24,6 +26,7 @@ module.exports.process = async (event, context, callback) => { const bank = banks.filter(bank => subject.includes(bank.subject)); + console.log('bank', bank) if (Array.isArray(bank) && bank.length == 1) { // Get bank information const { filters, ignore_phrase, name: bankName } = bank[0] @@ -36,6 +39,7 @@ module.exports.process = async (event, context, callback) => { if (!([undefined, null].includes(data.Body))) { const emailData = data.Body.toString('utf-8') + console.log(emailData); const result = await utils.readRawEmail(emailData) console.log('result', result); @@ -54,7 +58,7 @@ module.exports.process = async (event, context, callback) => { source: res.TRANSACTION_SOURCE, destination: res.TRANSACTION_DESTINATION, amount: res.TRANSACTION_VALUE, - cardType: res.TRANSACTION_CARD_TYPE, + cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', account: res.TRANSACTION_ACCOUNT, category: res.TRANSACTION_TYPE, text: res.description, diff --git a/AutomationServices/EmailsForwardingReader/utils/index.js b/AutomationServices/EmailsForwardingReader/utils/index.js index 4dfc7f4..e4687f6 100644 --- a/AutomationServices/EmailsForwardingReader/utils/index.js +++ b/AutomationServices/EmailsForwardingReader/utils/index.js @@ -20,11 +20,13 @@ module.exports.readRawEmail = async (body) => { let textAsHtml = null; if (body !== null && isBase64(body.replace(/\r?\n|\r/g, ""))) { + console.log('Base64') body = Buffer.from(body.replace(/\r?\n|\r/g, ""), 'base64').toString() textAsHtml = '

' + htmlToText(body).replace(/\r?\n|\r|\t/g, " ") + '

' } else { + console.log('Parser') const result = await mailparser(body); - textAsHtml = result.textAsHtml + textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' } diff --git a/package-lock.json b/package-lock.json index ee5db23..dedf900 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2052,6 +2052,7 @@ "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4", @@ -2061,12 +2062,14 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true } } }, @@ -3884,7 +3887,8 @@ "events": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "dev": true }, "execa": { "version": "4.0.3", @@ -4939,19 +4943,29 @@ }, "dependencies": { "dom-serializer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.1.0.tgz", - "integrity": "sha512-ox7bvGXt2n+uLWtCRLybYx60IrOlWL/aCebWJk1T0d4m3y2tzf4U3ij9wBMUb6YJZpz06HCCYuyCDveE2xXmzQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.3.2.tgz", + "integrity": "sha512-5c54Bk5Dw4qAxNOI1pFEizPSjVsx5+bpJKmL2kPn8JhBUq2q09tTCa3mjijun2NfK78NMouDYNMBkOrPZiS+ig==", "requires": { "domelementtype": "^2.0.1", - "domhandler": "^3.0.0", + "domhandler": "^4.2.0", "entities": "^2.0.0" + }, + "dependencies": { + "domhandler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.0.tgz", + "integrity": "sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA==", + "requires": { + "domelementtype": "^2.2.0" + } + } } }, "domelementtype": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.0.2.tgz", - "integrity": "sha512-wFwTwCVebUrMgGeAwRL/NhZtHAUyT9n9yg4IMDwf10+6iCMxSkVq9MGCVEH+QZWo1nNidy8kNvwmv4zWHDTqvA==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.2.0.tgz", + "integrity": "sha512-DtBMo82pv1dFtUmHyr48beiuq792Sxohr+8Hm9zoxklYPfa6n0Z3Byjj2IV7bmr2IyqClnqEQhfgHJJ5QF0R5A==" }, "domhandler": { "version": "3.3.0", @@ -4962,19 +4976,29 @@ } }, "domutils": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.4.2.tgz", - "integrity": "sha512-NKbgaM8ZJOecTZsIzW5gSuplsX2IWW2mIK7xVr8hTQF2v1CJWTmLZ1HOCh5sH+IzVPAGE5IucooOkvwBRAdowA==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.7.0.tgz", + "integrity": "sha512-8eaHa17IwJUPAiB+SoTYBo5mCdeMgdcAoXJ59m6DT1vw+5iLS3gNoqYaRowaBKtGVrOF1Jz4yDTgYKLK2kvfJg==", "requires": { "dom-serializer": "^1.0.1", - "domelementtype": "^2.0.1", - "domhandler": "^3.3.0" + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "dependencies": { + "domhandler": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.2.0.tgz", + "integrity": "sha512-zk7sgt970kzPks2Bf+dwT/PLzghLnsivb9CcxkvR8Mzr66Olr0Ofd8neSbglHJHaHa2MadfoSdNlKYAaafmWfA==", + "requires": { + "domelementtype": "^2.2.0" + } + } } }, "entities": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.1.0.tgz", - "integrity": "sha512-hCx1oky9PFrJ611mf0ifBLBRW8lUUVRlFolb5gWRfIELabBlbp9xZvrqZLZAs+NxFnbfQoeGd8wDkygjg7U85w==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" }, "htmlparser2": { "version": "4.1.0", @@ -5712,7 +5736,8 @@ "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "dev": true }, "js-string-escape": { "version": "1.0.1", @@ -7551,7 +7576,8 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true }, "quoted-printable": { "version": "1.0.1", @@ -7827,7 +7853,8 @@ "sax": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=", + "dev": true }, "scmp": { "version": "2.1.0", @@ -9681,6 +9708,7 @@ "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -9689,7 +9717,8 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "dev": true } } }, @@ -9908,6 +9937,7 @@ "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dev": true, "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" @@ -9916,7 +9946,8 @@ "xmlbuilder": { "version": "9.0.7", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "dev": true } } }, From 490ad1c72203a94b8c498b8e272903cfdc94a246 Mon Sep 17 00:00:00 2001 From: Andres Date: Thu, 1 Jul 2021 08:31:54 -0500 Subject: [PATCH 7/9] Removing Console Logs. --- .../EmailsForwardingReader/index.js | 22 +++++++------------ .../EmailsForwardingReader/utils/index.js | 2 -- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/AutomationServices/EmailsForwardingReader/index.js b/AutomationServices/EmailsForwardingReader/index.js index a6f6917..e3516fb 100644 --- a/AutomationServices/EmailsForwardingReader/index.js +++ b/AutomationServices/EmailsForwardingReader/index.js @@ -11,11 +11,9 @@ module.exports.process = async (event, context, callback) => { const mailEvent = event.Records[0].ses const { messageId, timestamp, commonHeaders } = mailEvent.mail let { subject, to, from } = commonHeaders - - console.log('commonHeaders', commonHeaders) + const source = getEmail(to); - console.log('source', source) // Removing forward subject label if (subject.includes('Fwd: ')) { subject = subject.replace('Fwd: ', '') @@ -26,7 +24,6 @@ module.exports.process = async (event, context, callback) => { const bank = banks.filter(bank => subject.includes(bank.subject)); - console.log('bank', bank) if (Array.isArray(bank) && bank.length == 1) { // Get bank information const { filters, ignore_phrase, name: bankName } = bank[0] @@ -39,11 +36,8 @@ module.exports.process = async (event, context, callback) => { if (!([undefined, null].includes(data.Body))) { const emailData = data.Body.toString('utf-8') - console.log(emailData); const result = await utils.readRawEmail(emailData) - console.log('result', result); - for (let index = 0; index < filters.length; index++) { const filter = filters[index]; @@ -58,7 +52,7 @@ module.exports.process = async (event, context, callback) => { source: res.TRANSACTION_SOURCE, destination: res.TRANSACTION_DESTINATION, amount: res.TRANSACTION_VALUE, - cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', + cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', account: res.TRANSACTION_ACCOUNT, category: res.TRANSACTION_TYPE, text: res.description, @@ -73,15 +67,15 @@ module.exports.process = async (event, context, callback) => { break; } - // Deleting processed Email. - await S3.deleteObject({ - Bucket: process.env.BUCKETNAME, - Key: messageId - }) - } else { console.log(`No Body`) } + + // Deleting processed Email. + await S3.deleteObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }) } } catch (error) { console.log(error) diff --git a/AutomationServices/EmailsForwardingReader/utils/index.js b/AutomationServices/EmailsForwardingReader/utils/index.js index e4687f6..7c00147 100644 --- a/AutomationServices/EmailsForwardingReader/utils/index.js +++ b/AutomationServices/EmailsForwardingReader/utils/index.js @@ -20,11 +20,9 @@ module.exports.readRawEmail = async (body) => { let textAsHtml = null; if (body !== null && isBase64(body.replace(/\r?\n|\r/g, ""))) { - console.log('Base64') body = Buffer.from(body.replace(/\r?\n|\r/g, ""), 'base64').toString() textAsHtml = '

' + htmlToText(body).replace(/\r?\n|\r|\t/g, " ") + '

' } else { - console.log('Parser') const result = await mailparser(body); textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' } From 20c7f0ab5e0e97039c034f496cf87b8486094799 Mon Sep 17 00:00:00 2001 From: Andres Date: Fri, 2 Jul 2021 07:42:30 -0500 Subject: [PATCH 8/9] Fixing budget implementation. --- client/src/components/HomeContainer.js | 2 +- serverless.yml | 9 +++++---- shared/database/repos/payment.repo.js | 8 ++++---- shared/database/repos/user.repo.js | 3 +-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/src/components/HomeContainer.js b/client/src/components/HomeContainer.js index 8a798f8..d0ff657 100644 --- a/client/src/components/HomeContainer.js +++ b/client/src/components/HomeContainer.js @@ -189,7 +189,7 @@ class HomeComponent extends React.Component { />
{ - categories.map((item, index) => { + categories && categories.map((item, index) => { const [category] = user && user.categories && user.categories.filter(category => category.label === item.name) const progressValue = category && category.budget && category.budget.progress ? category.budget.progress : 0 diff --git a/serverless.yml b/serverless.yml index e0c6eec..3be69a7 100644 --- a/serverless.yml +++ b/serverless.yml @@ -274,9 +274,10 @@ functions: arn: ${file(./config/${opt:stage}.json):USER_POOL_ARN} claims: - sub + AddBudgetCategory: handler: API/Functions/Users.addBudgetCategory - name: UserRepo-Categories-POST-${opt:stage} + name: UserRepo-Categories-budget-POST-${opt:stage} events: - http: path: /user/categories/budget @@ -394,7 +395,7 @@ functions: handler: AutomationServices/EmailsForwardingReader.process name: ProcessEmail-${opt:stage} environment: - BUCKETNAME: finance-emails-andresmorelos-dev${opt:stage} + BUCKETNAME: finance-emails-${opt:stage} # EmailChecker: # handler: AutomationServices/AutoMailChecker/checker.start @@ -496,7 +497,7 @@ resources: EmailsBucket: Type: AWS::S3::Bucket Properties: - BucketName: finance-emails-andresmorelos-dev${opt:stage} + BucketName: finance-emails-${opt:stage} EmailsBucketPolicy: Type: AWS::S3::BucketPolicy @@ -536,5 +537,5 @@ resources: FunctionArn: { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } - S3Action: - BucketName: finance-emails-andresmorelos-dev${opt:stage} + BucketName: finance-emails-${opt:stage} RuleSetName: default-rule-set \ No newline at end of file diff --git a/shared/database/repos/payment.repo.js b/shared/database/repos/payment.repo.js index 63c05d1..41aad50 100644 --- a/shared/database/repos/payment.repo.js +++ b/shared/database/repos/payment.repo.js @@ -109,12 +109,12 @@ module.exports.updatePayment = async (Payment) => { const { categories, sub } = await getUser({ _id: Payment.user }) - const results = categories.filter(category => category.label = Payment.category) + const results = categories.filter(category => category.label === Payment.category) let category = results[0] const current = category.budget.current + amount - - const result = await updateBudget( + + await updateBudget( { sub, 'categories.value': category.value @@ -135,7 +135,7 @@ module.exports.updatePayment = async (Payment) => { } ); await destroy() - return 0; + return result; }; module.exports.getByCategories = async (userId, date) => { diff --git a/shared/database/repos/user.repo.js b/shared/database/repos/user.repo.js index 430936a..cf9a4ae 100644 --- a/shared/database/repos/user.repo.js +++ b/shared/database/repos/user.repo.js @@ -84,11 +84,10 @@ module.exports.updateBudget = async (searchCriteria, update) => { if (category.budget.current !== update['categories.$.budget'].current) { update['categories.$.budget']['progress'] = (update['categories.$.budget'].current / category.budget.value) * 100 - update['categories.$.budget']['current'] = update['categories.$.budget'].current update['categories.$.budget']['value'] = category.budget.value } const result = await userModel.updateOne({ ...searchCriteria }, update) - await destroy() + return result.nModified > 0 } \ No newline at end of file From d35abd23a768fb57a5d7be075fb3e8dc5a89e514 Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 6 Jul 2021 21:01:20 -0500 Subject: [PATCH 9/9] Accept Gmail Fowarding emails. --- .../EmailFowardingAccept/DISCLAIMER | 3 + .../EmailFowardingAccept/scraper.js | 35 +++++ .../utils/acceptInvitation.js | 40 ++++++ .../EmailsForwardingReader/index.js | 134 ++++++++++-------- .../parsers/gmail/FowardingConfirmation.js | 10 ++ .../EmailsForwardingReader/utils/index.js | 28 +++- README.md | 32 ++++- serverless.yml | 98 +++++++++---- 8 files changed, 291 insertions(+), 89 deletions(-) create mode 100644 AutomationServices/EmailFowardingAccept/DISCLAIMER create mode 100644 AutomationServices/EmailFowardingAccept/scraper.js create mode 100644 AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js create mode 100644 AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js diff --git a/AutomationServices/EmailFowardingAccept/DISCLAIMER b/AutomationServices/EmailFowardingAccept/DISCLAIMER new file mode 100644 index 0000000..a43d4fd --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/DISCLAIMER @@ -0,0 +1,3 @@ +THIS SCRAPER IS A RESEARCH BASED PROJECT, WE DON'T ENCOURAGE THE MISUSE OF THIS TOOL FOR BAD INTENTIONS. + +THE DEVELOPERS ARE NOT RESPONSIBLE FOR ANY MISUSE OF THIS TOOL. \ No newline at end of file diff --git a/AutomationServices/EmailFowardingAccept/scraper.js b/AutomationServices/EmailFowardingAccept/scraper.js new file mode 100644 index 0000000..44a58a2 --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/scraper.js @@ -0,0 +1,35 @@ +const { acceptInvitation } = require('./utils/acceptInvitation') + + + +const start = async (event, context) => { + try { + const [{ destination, url }] = event.Records.map(sqsMessage => { + try { + return JSON.parse(sqsMessage.body); + } catch (e) { + console.error(e); + } + }) + + console.info('Starting Function') + + console.info('Accepting Invitation', url) + + const result = await acceptInvitation(url) + + if (result === 'accepted') { + console.info(`Invitation Acepted for ${destination}`) + } else { + console.error(result) + } + + } catch (error) { + console.error(error) + } + +} + +module.exports = { + start +} \ No newline at end of file diff --git a/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js b/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js new file mode 100644 index 0000000..ad1f5c0 --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js @@ -0,0 +1,40 @@ +const chromium = require('chrome-aws-lambda'); +const puppeteer = require("puppeteer-core") + +const acceptInvitation = async (url) => { + + return new Promise(async (resolve, reject) => { + try { + const browser = await chromium.puppeteer.launch({ + executablePath: await chromium.executablePath, + args: [...chromium.args, '--enable-features=NetworkService'], + defaultViewport: chromium.defaultViewport, + headless: chromium.headless, + }); + + const page = await browser.newPage(); + + await page.goto(url, { + waitUntil: ["networkidle0", "load", "domcontentloaded"] + }); + + await page.waitForTimeout(3000); + + await page.click('input[type=submit]') + + await page.waitForTimeout(1000); + + await browser.close() + + resolve('accepted') + } catch (error) { + reject(error) + } + }); + + +} + +module.exports = { + acceptInvitation +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/index.js b/AutomationServices/EmailsForwardingReader/index.js index e3516fb..1c1a466 100644 --- a/AutomationServices/EmailsForwardingReader/index.js +++ b/AutomationServices/EmailsForwardingReader/index.js @@ -10,73 +10,27 @@ module.exports.process = async (event, context, callback) => { try { const mailEvent = event.Records[0].ses const { messageId, timestamp, commonHeaders } = mailEvent.mail - let { subject, to, from } = commonHeaders + let { subject, to } = commonHeaders - const source = getEmail(to); // Removing forward subject label if (subject.includes('Fwd: ')) { subject = subject.replace('Fwd: ', '') } - // Search for bank by subject - const banks = await getBanks({}) - - const bank = banks.filter(bank => subject.includes(bank.subject)); - - if (Array.isArray(bank) && bank.length == 1) { - // Get bank information - const { filters, ignore_phrase, name: bankName } = bank[0] - - // Retrieve email information - const data = await S3.getObject({ - Bucket: process.env.BUCKETNAME, - Key: messageId - }).promise(); - - if (!([undefined, null].includes(data.Body))) { - const emailData = data.Body.toString('utf-8') - const result = await utils.readRawEmail(emailData) - - for (let index = 0; index < filters.length; index++) { - const filter = filters[index]; - - const res = utils.search(result.html, filter.phrase, filter.parser, ignore_phrase, bankName) - - if (!res) continue - - const user = await getUser({ emails: source }) - - const prePaymentObj = { - bank: bankName, - source: res.TRANSACTION_SOURCE, - destination: res.TRANSACTION_DESTINATION, - amount: res.TRANSACTION_VALUE, - cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', - account: res.TRANSACTION_ACCOUNT, - category: res.TRANSACTION_TYPE, - text: res.description, - type: filter.type, - createdBy: 'AUTO_EMAIL_SERVICE', - createdAt: moment(timestamp).format(), - user: user._id, - description: res.DESCRIPTION, - isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false - } - const payment = await createPayment(prePaymentObj) - break; - } + const emailData = await getEmailData(messageId) + if (emailData !== 'No Body') { + if (subject.includes('Gmail Forwarding Confirmation') > 0) { + await utils.processForwardingConfirmationGmail(emailData.html); } else { - console.log(`No Body`) + const source = getEmail(to); + await processBankEmails(subject, source, emailData, timestamp); } - - // Deleting processed Email. - await S3.deleteObject({ - Bucket: process.env.BUCKETNAME, - Key: messageId - }) } + + await deleteEmailData(messageId); + } catch (error) { console.log(error) } @@ -95,3 +49,71 @@ const getEmail = (from) => { } return source } + +const processBankEmails = async (subject, source, emailData, timestamp) => { + // Search for bank by subject + const banks = await getBanks({}) + + const bank = banks.filter(_bank => subject.includes(_bank.subject)); + + if (Array.isArray(bank) && bank.length == 1) { + // Get bank information + const { filters, ignore_phrase, name: bankName } = bank[0] + + + for (const filter of filters) { + + const res = utils.search(emailData.html, filter.phrase, filter.parser, ignore_phrase, bankName) + + if (!res) continue + + const user = await getUser({ emails: source }) + + const prePaymentObj = { + bank: bankName, + source: res.TRANSACTION_SOURCE, + destination: res.TRANSACTION_DESTINATION, + amount: res.TRANSACTION_VALUE, + cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', + account: res.TRANSACTION_ACCOUNT, + category: res.TRANSACTION_TYPE, + text: res.description, + type: filter.type, + createdBy: 'AUTO_EMAIL_SERVICE', + createdAt: moment(timestamp).format(), + user: user._id, + description: res.DESCRIPTION, + isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false + } + await createPayment(prePaymentObj) + break; + } + + + + } +} + +const getEmailData = async (messageId) => { + // Retrieve email information + const data = await S3.getObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }).promise(); + + if (!([undefined, null].includes(data.Body))) { + + const emailData = data.Body.toString('utf-8') + return await utils.readRawEmail(emailData) + } + + return 'No Body'; +} + +const deleteEmailData = async (messageId) => { + // Deleting processed Email. + await S3.deleteObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }).promise() +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js b/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js new file mode 100644 index 0000000..9838b68 --- /dev/null +++ b/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js @@ -0,0 +1,10 @@ + +module.exports.forwardingConfirmation = (text) => { + const EMAIL_DESTINATION = text.substring(0, text.indexOf(' has requested to')).trim() + const URL_CONFIRMATION = text.substring((text.indexOf('confirm the request:') + 20), text.indexOf('If you')).trim() + + return { + EMAIL_DESTINATION, + URL_CONFIRMATION, + } +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/utils/index.js b/AutomationServices/EmailsForwardingReader/utils/index.js index 7c00147..f1048dc 100644 --- a/AutomationServices/EmailsForwardingReader/utils/index.js +++ b/AutomationServices/EmailsForwardingReader/utils/index.js @@ -1,3 +1,4 @@ +const AWS = require('aws-sdk') const mailparser = require('mailparser').simpleParser const _ = require('lodash') const cheerio = require('cheerio'); @@ -10,6 +11,7 @@ const { transfersParser } = require('../parsers/bancolombia/transfers.parser') const { transferReceptionParser } = require('../parsers/bancolombia/transferReception.parser') const { debitWithdrawalParser } = require('../parsers/bancolombia/debitWithdrawal.parser') const { creditCardWithdrawalParser } = require('../parsers/bancolombia/creditCardWithdrawal.parser') +const { forwardingConfirmation } = require('../parsers/gmail/FowardingConfirmation') function isBase64(str) { @@ -24,7 +26,7 @@ module.exports.readRawEmail = async (body) => { textAsHtml = '

' + htmlToText(body).replace(/\r?\n|\r|\t/g, " ") + '

' } else { const result = await mailparser(body); - textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' + textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' } @@ -105,4 +107,28 @@ module.exports.search = (html, filter, parser, skipped_phrase = 'Bancolombia le } } return undefined +} +module.exports.processForwardingConfirmationGmail = async (html) => { + const $ = cheerio.load(html) + const res = $('p') + const value = res.text().trim().replace(/=/g, '') + const result = await forwardingConfirmation(value); + + + const SQS = new AWS.SQS({ + region: 'us-east-1', + }) + + try { + await SQS.sendMessage({ + QueueUrl: process.env.EMAIL_FORWARDING_CONFIRMATION_SQS, + MessageBody: JSON.stringify({ + destination: result.EMAIL_DESTINATION, + url: result.URL_CONFIRMATION + }) + }).promise() + } catch (error) { + console.log('access denied error', JSON.stringify(error)) + } + } \ No newline at end of file diff --git a/README.md b/README.md index deb3097..a696345 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,37 @@ export default { "MONGO_SECRET": "password", "MONGO_SET": "replicas_only", "MONGO_USER": "FlavioAandres", - "SRV_CONFIG": false // Note the SRV CONFIG flag off + "SRV_CONFIG": false, // Note the SRV CONFIG flag off + "EMAIL_USERNAME": "me@andresmorelos.dev", //Only needed if you will not use the emails process with SES + "EMAIL_PASSWORD": "PASSWORD", //Only needed if you will not use the emails process with SES + "SECRET_KEY": "GENERATE_ME!", + "BODY_REQUEST": "", + "EMAIL_RECIPIENTS": ["finance_email@andresmorelos.dev"], + "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:ID:userpool/us-east-1_ID", + "TWILIO_ACCESS_TOKEN": "TOKEN", + "TWILIO_SECTRET_KEY" : "SECRET_TOKEN", + "TELEGRAM_BOT_KEY": "TELEGRAM_KEY" + } +``` + +```js + { + "MONGO_HOST": "mongodb+srv://USER:PASSWORD@clustername.ixvju.mongodb.net/DATABASE?authSource=admin&replicaSet=REPLICASET&w=majority&readPreference=primary&appname=Personal%20Finances&retryWrites=true&ssl=true", + "MONGO_PORT": 27017, + "MONGO_SECRET": "", + "MONGO_SET": "", + "MONGO_USER": "", + "MONGO_DATABASE": "", + "SRV_CONFIG": true, //Note SRV CONFIG flag on + "EMAIL_USERNAME": "me@andresmorelos.dev", //Only needed if you will not use the emails process with SES + "EMAIL_PASSWORD": "PASSWORD", //Only needed if you will not use the emails process with SES + "SECRET_KEY": "GENERATE_ME!", + "BODY_REQUEST": "", + "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:ID:userpool/us-east-1_ID", + "EMAIL_RECIPIENTS": ["finance_email@andresmorelos.dev"], + "TWILIO_ACCESS_TOKEN": "TOKEN", + "TWILIO_SECTRET_KEY" : "SECRET_TOKEN", + "TELEGRAM_BOT_KEY": "TELEGRAM_KEY" } ``` diff --git a/serverless.yml b/serverless.yml index 033d008..5be00e3 100644 --- a/serverless.yml +++ b/serverless.yml @@ -11,10 +11,17 @@ package: - devutils/** - AutomationServices/AutoMailChecker/** -# custom: -# ArnSQSEmails: +custom: + ArnSQSForwardingConfirmation: + dev: ${file(./config/dev.json):EMAIL_FORWARDING_CONFIRMATION_SQS} + test: + Ref: EmailForwardingAceptationQueue + prod: + Ref: EmailForwardingAceptationQueue + +# ArnSQSEmails: # dev: ${file(./config/dev.json):USER_SCHEDULE_SQS_URL} -# test: +# test: # Ref: EmailUsersQueue # prod: # Ref: EmailUsersQueue @@ -36,16 +43,6 @@ provider: MONGO_USER: ${file(./config/${opt:stage}.json):MONGO_USER} MONGO_SSL: true iamRoleStatements: - - Effect: Allow - Action: - - s3:PutObject - - s3:GetObject - Resource: - - Fn::Join: - - '' - - - - Fn::GetAtt: [EmailsBucket, Arn] - - '/*' - Effect: Allow Action: ses:SendRawEmail Resource: @@ -274,7 +271,7 @@ functions: arn: ${file(./config/${opt:stage}.json):USER_POOL_ARN} claims: - sub - + AddBudgetCategory: handler: API/Functions/Users.addBudgetCategory name: UserRepo-Categories-budget-POST-${opt:stage} @@ -323,7 +320,7 @@ functions: name: PostConfirmationUser-${opt:stage} #INCOMES - + GetIncomes: handler: API/Functions/Incomes.get name: Incomes-GET-${opt:stage} @@ -387,15 +384,48 @@ functions: claims: - sub - - #AUTOMATIONS + EmailForwardingAcceptation: + handler: AutomationServices/EmailFowardingAccept/scraper.start + name: EmailForwardingAcceptation-${opt:stage} + timeout: 120 + memorySize: 2048 + layers: + - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:22 + events: + - sqs: + arn: + Fn::GetAtt: + - EmailForwardingAceptationQueue + - Arn + batchSize: 1 ProcessEmails: handler: AutomationServices/EmailsForwardingReader.process name: ProcessEmail-${opt:stage} - environment: + iamRoleStatementsName: write-sqs-finances-email-forwarding-confirmation-${opt:stage}-role + iamRoleStatements: + - Effect: "Allow" + Action: + - "sqs:SendMessage" + - "sqs:SendMessageBatch" + Resource: + Fn::GetAtt: + - EmailForwardingAceptationQueue + - Arn + - Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:DeleteObject + Resource: + - Fn::Join: + - "" + - - Fn::GetAtt: [EmailsBucket, Arn] + - "/*" + environment: BUCKETNAME: finance-emails-${opt:stage} + EMAIL_FORWARDING_CONFIRMATION_SQS: ${self:custom.ArnSQSForwardingConfirmation.${opt:stage, self:provider.stage}} # EmailChecker: # handler: AutomationServices/AutoMailChecker/checker.start @@ -432,7 +462,7 @@ functions: TWILIO_ACCESS_TOKEN: ${file(./config/${opt:stage}.json):TWILIO_ACCESS_TOKEN} TWILIO_SECTRET_KEY: ${file(./config/${opt:stage}.json):TWILIO_SECTRET_KEY} - ##Deprecated function since DataCredito updates the backend + # Deprecated function since DataCredito updates the backend # DataCreditoScraper: # handler: AutomationServices/DataCreditoScraper/scraper.start # name: AutomationServices-DataCreditoScraper-${opt:stage} @@ -444,6 +474,8 @@ functions: # description: "Function to scraper datacredito data" # rate: cron(0 0 15,30 * ? *) # Every Fifteen days, some entities take a time to report updates # enabled: false + # layers: + # - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:22 # environment: # EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} # SECRET_KEY: ${file(./config/${opt:stage}.json):SECRET_KEY} @@ -462,17 +494,17 @@ functions: # handler: AutomationServices/AutoMailChecker/schedule.start # name: AutomationServices-ScheduleUsers-${opt:stage} # iamRoleStatementsName: write-sqs-finances-${opt:stage}-role - # iamRoleStatements: + # iamRoleStatements: # - Effect: 'Allow' - # Action: + # Action: # - 'sqs:SendMessage' # - 'sqs:SendMessageBatch' # Resource: # Fn::GetAtt: # - EmailUsersQueue # - Arn - # events: - # - schedule: + # events: + # - schedule: # enabled: true # rate: rate(20 minutes) # environment: @@ -486,6 +518,12 @@ resources: # QueueName: "email-users-checker" # VisibilityTimeout: 300 + EmailForwardingAceptationQueue: + Type: "AWS::SQS::Queue" + Properties: + QueueName: "email-forwarding-aceptation" + VisibilityTimeout: 300 + GiveSESPermissionToInvokeProcessEmailLambdaFunction: Type: AWS::Lambda::Permission Properties: @@ -493,10 +531,10 @@ resources: Principal: ses.amazonaws.com Action: "lambda:InvokeFunction" SourceAccount: { Ref: AWS::AccountId } - + EmailsBucket: - Type: AWS::S3::Bucket - Properties: + Type: AWS::S3::Bucket + Properties: BucketName: finance-emails-${opt:stage} EmailsBucketPolicy: @@ -505,15 +543,13 @@ resources: Bucket: { Ref: EmailsBucket } PolicyDocument: Statement: - - - Action: + - Action: - s3:PutObject Effect: Allow Resource: Fn::Join: - "" - - - - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } + - - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } - "/*" Principal: Service: ses.amazonaws.com @@ -538,4 +574,4 @@ resources: { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } - S3Action: BucketName: finance-emails-${opt:stage} - RuleSetName: default-rule-set \ No newline at end of file + RuleSetName: default-rule-set