Merge pull request #28 from FlavioAandres/process_emails_ses

Process emails with AWS ses

AutomationServices/EmailFowardingAccept/DISCLAIMER

@@ -0,0 +1,3 @@
+THIS SCRAPER IS A RESEARCH BASED PROJECT, WE DON'T ENCOURAGE THE MISUSE OF THIS TOOL FOR BAD INTENTIONS.
+
+THE DEVELOPERS ARE NOT RESPONSIBLE FOR ANY MISUSE OF THIS TOOL.

AutomationServices/EmailFowardingAccept/scraper.js

@@ -0,0 +1,35 @@
+const { acceptInvitation } = require('./utils/acceptInvitation')
+
+
+
+const start = async (event, context) => {
+    try {
+        const [{ destination, url }] = event.Records.map(sqsMessage => {
+            try {
+                return JSON.parse(sqsMessage.body);
+            } catch (e) {
+                console.error(e);
+            }
+        })
+
+        console.info('Starting Function')
+
+        console.info('Accepting Invitation', url)
+
+        const result = await acceptInvitation(url)
+
+        if (result === 'accepted') {
+            console.info(`Invitation Acepted for ${destination}`)
+        } else {
+            console.error(result)
+        }
+
+    } catch (error) {
+        console.error(error)
+    }
+
+}
+
+module.exports = {
+    start
+}

AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js

@@ -0,0 +1,40 @@
+const chromium = require('chrome-aws-lambda');
+const puppeteer = require("puppeteer-core")
+
+const acceptInvitation = async (url) => {
+
+    return new Promise(async (resolve, reject) => {
+        try {
+            const browser = await chromium.puppeteer.launch({
+                executablePath: await chromium.executablePath,
+                args: [...chromium.args, '--enable-features=NetworkService'],
+                defaultViewport: chromium.defaultViewport,
+                headless: chromium.headless,
+            });
+
+            const page = await browser.newPage();
+
+            await page.goto(url, {
+                waitUntil: ["networkidle0", "load", "domcontentloaded"]
+            });
+
+            await page.waitForTimeout(3000);
+
+            await page.click('input[type=submit]')
+
+            await page.waitForTimeout(1000);
+
+            await browser.close()
+
+            resolve('accepted')
+        } catch (error) {
+            reject(error)
+        }
+    });
+
+
+}
+
+module.exports = {
+    acceptInvitation
+}

AutomationServices/EmailsForwardingReader/index.js

@@ -0,0 +1,119 @@
+const AWS = require('aws-sdk')
+const S3 = new AWS.S3();
+const moment = require('moment')
+const { getBanks } = require('../../shared/database/repos/bank.repo');
+const { getUser } = require('../../shared/database/repos/user.repo');
+const { create: createPayment } = require('../../shared/database/repos/payment.repo')
+const utils = require('./utils')
+
+module.exports.process = async (event, context, callback) => {
+    try {
+        const mailEvent = event.Records[0].ses
+        const { messageId, timestamp, commonHeaders } = mailEvent.mail
+        let { subject, to } = commonHeaders
+
+
+        // Removing forward subject label
+        if (subject.includes('Fwd: ')) {
+            subject = subject.replace('Fwd: ', '')
+        }
+
+        const emailData = await getEmailData(messageId)
+
+        if (emailData !== 'No Body') {
+            if (subject.includes('Gmail Forwarding Confirmation') > 0) {
+                await utils.processForwardingConfirmationGmail(emailData.html);
+            } else {
+                const source = getEmail(to);
+                await processBankEmails(subject, source, emailData, timestamp);
+            }
+        }
+
+        await deleteEmailData(messageId);
+
+    } catch (error) {
+        console.log(error)
+    }
+}
+
+
+const getEmail = (from) => {
+    let source = undefined;
+    if (Array.isArray(from) && from.length > 0) {
+        source = from[0].match(/\<(.*?)\>/g)
+        if (Array.isArray(source) && source.length > 0) {
+            source = source[0].replace('<', '').replace('>', '')
+        } else {
+            source = from[0];
+        }
+    }
+    return source
+}
+
+const processBankEmails = async (subject, source, emailData, timestamp) => {
+    // Search for bank by subject
+    const banks = await getBanks({})
+
+    const bank = banks.filter(_bank => subject.includes(_bank.subject));
+
+    if (Array.isArray(bank) && bank.length == 1) {
+        // Get bank information
+        const { filters, ignore_phrase, name: bankName } = bank[0]
+
+
+        for (const filter of filters) {
+
+            const res = utils.search(emailData.html, filter.phrase, filter.parser, ignore_phrase, bankName)
+
+            if (!res) continue
+
+            const user = await getUser({ emails: source })
+
+            const prePaymentObj = {
+                bank: bankName,
+                source: res.TRANSACTION_SOURCE,
+                destination: res.TRANSACTION_DESTINATION,
+                amount: res.TRANSACTION_VALUE,
+                cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual',
+                account: res.TRANSACTION_ACCOUNT,
+                category: res.TRANSACTION_TYPE,
+                text: res.description,
+                type: filter.type,
+                createdBy: 'AUTO_EMAIL_SERVICE',
+                createdAt: moment(timestamp).format(),
+                user: user._id,
+                description: res.DESCRIPTION,
+                isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false
+            }
+            await createPayment(prePaymentObj)
+            break;
+        }
+
+
+
+    }
+}
+
+const getEmailData = async (messageId) => {
+    // Retrieve email information
+    const data = await S3.getObject({
+        Bucket: process.env.BUCKETNAME,
+        Key: messageId
+    }).promise();
+
+    if (!([undefined, null].includes(data.Body))) {
+
+        const emailData = data.Body.toString('utf-8')
+        return await utils.readRawEmail(emailData)
+    }
+
+    return 'No Body';
+}
+
+const deleteEmailData = async (messageId) => {
+    // Deleting processed Email.
+    await S3.deleteObject({
+        Bucket: process.env.BUCKETNAME,
+        Key: messageId
+    }).promise()
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/creditCardWithdrawal.parser.js

@@ -0,0 +1,16 @@
+const { amountParser } = require('../utils')
+module.exports.creditCardWithdrawalParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim()
+    const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT,
+        TRANSACTION_TYPE: 'withdrawal',
+        DESCRIPTION: `Withdrawal in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}`
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/debitWithdrawal.parser.js

@@ -0,0 +1,16 @@
+const { amountParser } = require('../utils')
+module.exports.debitWithdrawalParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf('en')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3), text.indexOf('hora')).trim()
+    const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT,
+        TRANSACTION_TYPE: 'withdrawal',
+        DESCRIPTION: `Withdrawal in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}`
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/payments.parser.js

@@ -0,0 +1,14 @@
+const { amountParser } = require('../utils')
+
+module.exports.paymentsParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' a')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf(' a ') + 3), text.indexOf('desde'))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE: null,
+        TRANSACTION_ACCOUNT
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/productPayments.parser.js

@@ -0,0 +1,14 @@
+const { amountParser } = require('../utils')
+
+module.exports.productParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1),text.indexOf('desde')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf('pago de') + 7), text.indexOf(' por')) + text.substring(text.lastIndexOf(' *'), (text.lastIndexOf(' *')+6))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE: null,
+        TRANSACTION_ACCOUNT
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/shopping.parser.js

@@ -0,0 +1,15 @@
+const { amountParser } = require('../utils')
+
+module.exports.shoppingParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$')+1), text.indexOf(' en')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf('en ') + 3 ), (text.indexOf(':') - 3))
+    const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('t.'),text.indexOfRegex(/ \*\d/))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/transferReception.parser.js

@@ -0,0 +1,16 @@
+const { amountParser } = require('../utils')
+module.exports.transferReceptionParser = (text) => {
+    const TRANSACTION_SOURCE = text.substring((text.indexOf('transferencia de ') + 17), (text.indexOf(' por')))
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.includes('en la') ? text.indexOf('en la') : text.indexOf('enla')))
+    const TRANSACTION_DESTINATION = text.substring(text.indexOf(' *'), (text.indexOf(' *') + 6))
+    const TRANSACTION_CARD_TYPE = null
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_SOURCE,
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/bancolombia/transfers.parser.js

@@ -0,0 +1,14 @@
+const { amountParser } = require('../utils')
+module.exports.transfersParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.lastIndexOf('$') + 1), text.indexOf(' desde')))
+    const TRANSACTION_DESTINATION = text.substring((text.indexOf('a cta') + 5), text.indexOf('/') - 5).trim()
+    const TRANSACTION_CARD_TYPE = null
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOfRegex(/ \*\d/), (text.indexOfRegex(/ \*\d/)+6))
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/davivienda/payments.parser.js

@@ -0,0 +1,18 @@
+const { amountParser } = require('../utils')
+
+module.exports.paymentsParser = (text) => {
+    const TRANSACTION_VALUE = amountParser(text.substring((text.indexOf('valor transacción: ')), text.indexOf('clase de movimiento:')))
+    const TRANSACTION_DESTINATION = text.substring(text.indexOf('lugar de transacción: ') + 22, text.indexOf('banco davivienda'))
+    const TRANSACTION_ACCOUNT = text.substring(text.indexOf('***') + 3, (text.indexOf('***') + 8))
+    const TRANSACTION_CARD_TYPE = text.substring(text.indexOf('movimiento de su') + 17, (text.indexOf('  terminada'))) == 'tarjetacrédito' ? 't.cred' : 't.debt'
+    const TRANSACTION_TYPE = text.substring(text.indexOf('clase de movimiento: ') + 21, (text.indexOf(' .'))) == 'compra' ? 'EXPENSE' : 'INCOME'
+
+    return {
+        TRANSACTION_VALUE,
+        TRANSACTION_DESTINATION,
+        TRANSACTION_CARD_TYPE,
+        TRANSACTION_ACCOUNT,
+        TRANSACTION_TYPE,
+        DESCRIPTION: `Purchase in ${TRANSACTION_DESTINATION}, amount ${Intl.NumberFormat('es-co', { style: 'currency', currency: 'COP' }).format(TRANSACTION_VALUE)}`
+    }
+}

AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js

@@ -0,0 +1,10 @@
+
+module.exports.forwardingConfirmation = (text) => {
+    const EMAIL_DESTINATION = text.substring(0, text.indexOf(' has requested to')).trim()
+    const URL_CONFIRMATION = text.substring((text.indexOf('confirm the request:') + 20), text.indexOf('If you')).trim()
+
+    return {
+        EMAIL_DESTINATION,
+        URL_CONFIRMATION,
+    }
+}