From d35abd23a768fb57a5d7be075fb3e8dc5a89e514 Mon Sep 17 00:00:00 2001 From: Andres Date: Tue, 6 Jul 2021 21:01:20 -0500 Subject: [PATCH] Accept Gmail Fowarding emails. --- .../EmailFowardingAccept/DISCLAIMER | 3 + .../EmailFowardingAccept/scraper.js | 35 +++++ .../utils/acceptInvitation.js | 40 ++++++ .../EmailsForwardingReader/index.js | 134 ++++++++++-------- .../parsers/gmail/FowardingConfirmation.js | 10 ++ .../EmailsForwardingReader/utils/index.js | 28 +++- README.md | 32 ++++- serverless.yml | 98 +++++++++---- 8 files changed, 291 insertions(+), 89 deletions(-) create mode 100644 AutomationServices/EmailFowardingAccept/DISCLAIMER create mode 100644 AutomationServices/EmailFowardingAccept/scraper.js create mode 100644 AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js create mode 100644 AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js diff --git a/AutomationServices/EmailFowardingAccept/DISCLAIMER b/AutomationServices/EmailFowardingAccept/DISCLAIMER new file mode 100644 index 0000000..a43d4fd --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/DISCLAIMER @@ -0,0 +1,3 @@ +THIS SCRAPER IS A RESEARCH BASED PROJECT, WE DON'T ENCOURAGE THE MISUSE OF THIS TOOL FOR BAD INTENTIONS. + +THE DEVELOPERS ARE NOT RESPONSIBLE FOR ANY MISUSE OF THIS TOOL. \ No newline at end of file diff --git a/AutomationServices/EmailFowardingAccept/scraper.js b/AutomationServices/EmailFowardingAccept/scraper.js new file mode 100644 index 0000000..44a58a2 --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/scraper.js @@ -0,0 +1,35 @@ +const { acceptInvitation } = require('./utils/acceptInvitation') + + + +const start = async (event, context) => { + try { + const [{ destination, url }] = event.Records.map(sqsMessage => { + try { + return JSON.parse(sqsMessage.body); + } catch (e) { + console.error(e); + } + }) + + console.info('Starting Function') + + console.info('Accepting Invitation', url) + + const result = await acceptInvitation(url) + + if (result === 'accepted') { + console.info(`Invitation Acepted for ${destination}`) + } else { + console.error(result) + } + + } catch (error) { + console.error(error) + } + +} + +module.exports = { + start +} \ No newline at end of file diff --git a/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js b/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js new file mode 100644 index 0000000..ad1f5c0 --- /dev/null +++ b/AutomationServices/EmailFowardingAccept/utils/acceptInvitation.js @@ -0,0 +1,40 @@ +const chromium = require('chrome-aws-lambda'); +const puppeteer = require("puppeteer-core") + +const acceptInvitation = async (url) => { + + return new Promise(async (resolve, reject) => { + try { + const browser = await chromium.puppeteer.launch({ + executablePath: await chromium.executablePath, + args: [...chromium.args, '--enable-features=NetworkService'], + defaultViewport: chromium.defaultViewport, + headless: chromium.headless, + }); + + const page = await browser.newPage(); + + await page.goto(url, { + waitUntil: ["networkidle0", "load", "domcontentloaded"] + }); + + await page.waitForTimeout(3000); + + await page.click('input[type=submit]') + + await page.waitForTimeout(1000); + + await browser.close() + + resolve('accepted') + } catch (error) { + reject(error) + } + }); + + +} + +module.exports = { + acceptInvitation +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/index.js b/AutomationServices/EmailsForwardingReader/index.js index e3516fb..1c1a466 100644 --- a/AutomationServices/EmailsForwardingReader/index.js +++ b/AutomationServices/EmailsForwardingReader/index.js @@ -10,73 +10,27 @@ module.exports.process = async (event, context, callback) => { try { const mailEvent = event.Records[0].ses const { messageId, timestamp, commonHeaders } = mailEvent.mail - let { subject, to, from } = commonHeaders + let { subject, to } = commonHeaders - const source = getEmail(to); // Removing forward subject label if (subject.includes('Fwd: ')) { subject = subject.replace('Fwd: ', '') } - // Search for bank by subject - const banks = await getBanks({}) - - const bank = banks.filter(bank => subject.includes(bank.subject)); - - if (Array.isArray(bank) && bank.length == 1) { - // Get bank information - const { filters, ignore_phrase, name: bankName } = bank[0] - - // Retrieve email information - const data = await S3.getObject({ - Bucket: process.env.BUCKETNAME, - Key: messageId - }).promise(); - - if (!([undefined, null].includes(data.Body))) { - const emailData = data.Body.toString('utf-8') - const result = await utils.readRawEmail(emailData) - - for (let index = 0; index < filters.length; index++) { - const filter = filters[index]; - - const res = utils.search(result.html, filter.phrase, filter.parser, ignore_phrase, bankName) - - if (!res) continue - - const user = await getUser({ emails: source }) - - const prePaymentObj = { - bank: bankName, - source: res.TRANSACTION_SOURCE, - destination: res.TRANSACTION_DESTINATION, - amount: res.TRANSACTION_VALUE, - cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', - account: res.TRANSACTION_ACCOUNT, - category: res.TRANSACTION_TYPE, - text: res.description, - type: filter.type, - createdBy: 'AUTO_EMAIL_SERVICE', - createdAt: moment(timestamp).format(), - user: user._id, - description: res.DESCRIPTION, - isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false - } - const payment = await createPayment(prePaymentObj) - break; - } + const emailData = await getEmailData(messageId) + if (emailData !== 'No Body') { + if (subject.includes('Gmail Forwarding Confirmation') > 0) { + await utils.processForwardingConfirmationGmail(emailData.html); } else { - console.log(`No Body`) + const source = getEmail(to); + await processBankEmails(subject, source, emailData, timestamp); } - - // Deleting processed Email. - await S3.deleteObject({ - Bucket: process.env.BUCKETNAME, - Key: messageId - }) } + + await deleteEmailData(messageId); + } catch (error) { console.log(error) } @@ -95,3 +49,71 @@ const getEmail = (from) => { } return source } + +const processBankEmails = async (subject, source, emailData, timestamp) => { + // Search for bank by subject + const banks = await getBanks({}) + + const bank = banks.filter(_bank => subject.includes(_bank.subject)); + + if (Array.isArray(bank) && bank.length == 1) { + // Get bank information + const { filters, ignore_phrase, name: bankName } = bank[0] + + + for (const filter of filters) { + + const res = utils.search(emailData.html, filter.phrase, filter.parser, ignore_phrase, bankName) + + if (!res) continue + + const user = await getUser({ emails: source }) + + const prePaymentObj = { + bank: bankName, + source: res.TRANSACTION_SOURCE, + destination: res.TRANSACTION_DESTINATION, + amount: res.TRANSACTION_VALUE, + cardType: res.TRANSACTION_CARD_TYPE ? res.TRANSACTION_CARD_TYPE : 'Manual', + account: res.TRANSACTION_ACCOUNT, + category: res.TRANSACTION_TYPE, + text: res.description, + type: filter.type, + createdBy: 'AUTO_EMAIL_SERVICE', + createdAt: moment(timestamp).format(), + user: user._id, + description: res.DESCRIPTION, + isAccepted: res.TRANSACTION_TYPE === 'withdrawal' ? true : false + } + await createPayment(prePaymentObj) + break; + } + + + + } +} + +const getEmailData = async (messageId) => { + // Retrieve email information + const data = await S3.getObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }).promise(); + + if (!([undefined, null].includes(data.Body))) { + + const emailData = data.Body.toString('utf-8') + return await utils.readRawEmail(emailData) + } + + return 'No Body'; +} + +const deleteEmailData = async (messageId) => { + // Deleting processed Email. + await S3.deleteObject({ + Bucket: process.env.BUCKETNAME, + Key: messageId + }).promise() +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js b/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js new file mode 100644 index 0000000..9838b68 --- /dev/null +++ b/AutomationServices/EmailsForwardingReader/parsers/gmail/FowardingConfirmation.js @@ -0,0 +1,10 @@ + +module.exports.forwardingConfirmation = (text) => { + const EMAIL_DESTINATION = text.substring(0, text.indexOf(' has requested to')).trim() + const URL_CONFIRMATION = text.substring((text.indexOf('confirm the request:') + 20), text.indexOf('If you')).trim() + + return { + EMAIL_DESTINATION, + URL_CONFIRMATION, + } +} \ No newline at end of file diff --git a/AutomationServices/EmailsForwardingReader/utils/index.js b/AutomationServices/EmailsForwardingReader/utils/index.js index 7c00147..f1048dc 100644 --- a/AutomationServices/EmailsForwardingReader/utils/index.js +++ b/AutomationServices/EmailsForwardingReader/utils/index.js @@ -1,3 +1,4 @@ +const AWS = require('aws-sdk') const mailparser = require('mailparser').simpleParser const _ = require('lodash') const cheerio = require('cheerio'); @@ -10,6 +11,7 @@ const { transfersParser } = require('../parsers/bancolombia/transfers.parser') const { transferReceptionParser } = require('../parsers/bancolombia/transferReception.parser') const { debitWithdrawalParser } = require('../parsers/bancolombia/debitWithdrawal.parser') const { creditCardWithdrawalParser } = require('../parsers/bancolombia/creditCardWithdrawal.parser') +const { forwardingConfirmation } = require('../parsers/gmail/FowardingConfirmation') function isBase64(str) { @@ -24,7 +26,7 @@ module.exports.readRawEmail = async (body) => { textAsHtml = '

' + htmlToText(body).replace(/\r?\n|\r|\t/g, " ") + '

' } else { const result = await mailparser(body); - textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' + textAsHtml = result.textAsHtml ? result.textAsHtml : '

' + htmlToText(result.html).replace(/\r?\n|\r|\t/g, " ") + '

' } @@ -105,4 +107,28 @@ module.exports.search = (html, filter, parser, skipped_phrase = 'Bancolombia le } } return undefined +} +module.exports.processForwardingConfirmationGmail = async (html) => { + const $ = cheerio.load(html) + const res = $('p') + const value = res.text().trim().replace(/=/g, '') + const result = await forwardingConfirmation(value); + + + const SQS = new AWS.SQS({ + region: 'us-east-1', + }) + + try { + await SQS.sendMessage({ + QueueUrl: process.env.EMAIL_FORWARDING_CONFIRMATION_SQS, + MessageBody: JSON.stringify({ + destination: result.EMAIL_DESTINATION, + url: result.URL_CONFIRMATION + }) + }).promise() + } catch (error) { + console.log('access denied error', JSON.stringify(error)) + } + } \ No newline at end of file diff --git a/README.md b/README.md index deb3097..a696345 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,37 @@ export default { "MONGO_SECRET": "password", "MONGO_SET": "replicas_only", "MONGO_USER": "FlavioAandres", - "SRV_CONFIG": false // Note the SRV CONFIG flag off + "SRV_CONFIG": false, // Note the SRV CONFIG flag off + "EMAIL_USERNAME": "me@andresmorelos.dev", //Only needed if you will not use the emails process with SES + "EMAIL_PASSWORD": "PASSWORD", //Only needed if you will not use the emails process with SES + "SECRET_KEY": "GENERATE_ME!", + "BODY_REQUEST": "", + "EMAIL_RECIPIENTS": ["finance_email@andresmorelos.dev"], + "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:ID:userpool/us-east-1_ID", + "TWILIO_ACCESS_TOKEN": "TOKEN", + "TWILIO_SECTRET_KEY" : "SECRET_TOKEN", + "TELEGRAM_BOT_KEY": "TELEGRAM_KEY" + } +``` + +```js + { + "MONGO_HOST": "mongodb+srv://USER:PASSWORD@clustername.ixvju.mongodb.net/DATABASE?authSource=admin&replicaSet=REPLICASET&w=majority&readPreference=primary&appname=Personal%20Finances&retryWrites=true&ssl=true", + "MONGO_PORT": 27017, + "MONGO_SECRET": "", + "MONGO_SET": "", + "MONGO_USER": "", + "MONGO_DATABASE": "", + "SRV_CONFIG": true, //Note SRV CONFIG flag on + "EMAIL_USERNAME": "me@andresmorelos.dev", //Only needed if you will not use the emails process with SES + "EMAIL_PASSWORD": "PASSWORD", //Only needed if you will not use the emails process with SES + "SECRET_KEY": "GENERATE_ME!", + "BODY_REQUEST": "", + "USER_POOL_ARN": "arn:aws:cognito-idp:us-east-1:ID:userpool/us-east-1_ID", + "EMAIL_RECIPIENTS": ["finance_email@andresmorelos.dev"], + "TWILIO_ACCESS_TOKEN": "TOKEN", + "TWILIO_SECTRET_KEY" : "SECRET_TOKEN", + "TELEGRAM_BOT_KEY": "TELEGRAM_KEY" } ``` diff --git a/serverless.yml b/serverless.yml index 033d008..5be00e3 100644 --- a/serverless.yml +++ b/serverless.yml @@ -11,10 +11,17 @@ package: - devutils/** - AutomationServices/AutoMailChecker/** -# custom: -# ArnSQSEmails: +custom: + ArnSQSForwardingConfirmation: + dev: ${file(./config/dev.json):EMAIL_FORWARDING_CONFIRMATION_SQS} + test: + Ref: EmailForwardingAceptationQueue + prod: + Ref: EmailForwardingAceptationQueue + +# ArnSQSEmails: # dev: ${file(./config/dev.json):USER_SCHEDULE_SQS_URL} -# test: +# test: # Ref: EmailUsersQueue # prod: # Ref: EmailUsersQueue @@ -36,16 +43,6 @@ provider: MONGO_USER: ${file(./config/${opt:stage}.json):MONGO_USER} MONGO_SSL: true iamRoleStatements: - - Effect: Allow - Action: - - s3:PutObject - - s3:GetObject - Resource: - - Fn::Join: - - '' - - - - Fn::GetAtt: [EmailsBucket, Arn] - - '/*' - Effect: Allow Action: ses:SendRawEmail Resource: @@ -274,7 +271,7 @@ functions: arn: ${file(./config/${opt:stage}.json):USER_POOL_ARN} claims: - sub - + AddBudgetCategory: handler: API/Functions/Users.addBudgetCategory name: UserRepo-Categories-budget-POST-${opt:stage} @@ -323,7 +320,7 @@ functions: name: PostConfirmationUser-${opt:stage} #INCOMES - + GetIncomes: handler: API/Functions/Incomes.get name: Incomes-GET-${opt:stage} @@ -387,15 +384,48 @@ functions: claims: - sub - - #AUTOMATIONS + EmailForwardingAcceptation: + handler: AutomationServices/EmailFowardingAccept/scraper.start + name: EmailForwardingAcceptation-${opt:stage} + timeout: 120 + memorySize: 2048 + layers: + - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:22 + events: + - sqs: + arn: + Fn::GetAtt: + - EmailForwardingAceptationQueue + - Arn + batchSize: 1 ProcessEmails: handler: AutomationServices/EmailsForwardingReader.process name: ProcessEmail-${opt:stage} - environment: + iamRoleStatementsName: write-sqs-finances-email-forwarding-confirmation-${opt:stage}-role + iamRoleStatements: + - Effect: "Allow" + Action: + - "sqs:SendMessage" + - "sqs:SendMessageBatch" + Resource: + Fn::GetAtt: + - EmailForwardingAceptationQueue + - Arn + - Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:DeleteObject + Resource: + - Fn::Join: + - "" + - - Fn::GetAtt: [EmailsBucket, Arn] + - "/*" + environment: BUCKETNAME: finance-emails-${opt:stage} + EMAIL_FORWARDING_CONFIRMATION_SQS: ${self:custom.ArnSQSForwardingConfirmation.${opt:stage, self:provider.stage}} # EmailChecker: # handler: AutomationServices/AutoMailChecker/checker.start @@ -432,7 +462,7 @@ functions: TWILIO_ACCESS_TOKEN: ${file(./config/${opt:stage}.json):TWILIO_ACCESS_TOKEN} TWILIO_SECTRET_KEY: ${file(./config/${opt:stage}.json):TWILIO_SECTRET_KEY} - ##Deprecated function since DataCredito updates the backend + # Deprecated function since DataCredito updates the backend # DataCreditoScraper: # handler: AutomationServices/DataCreditoScraper/scraper.start # name: AutomationServices-DataCreditoScraper-${opt:stage} @@ -444,6 +474,8 @@ functions: # description: "Function to scraper datacredito data" # rate: cron(0 0 15,30 * ? *) # Every Fifteen days, some entities take a time to report updates # enabled: false + # layers: + # - arn:aws:lambda:${self:provider.region}:764866452798:layer:chrome-aws-lambda:22 # environment: # EMAIL_USERNAME: ${file(./config/${opt:stage}.json):EMAIL_USERNAME} # SECRET_KEY: ${file(./config/${opt:stage}.json):SECRET_KEY} @@ -462,17 +494,17 @@ functions: # handler: AutomationServices/AutoMailChecker/schedule.start # name: AutomationServices-ScheduleUsers-${opt:stage} # iamRoleStatementsName: write-sqs-finances-${opt:stage}-role - # iamRoleStatements: + # iamRoleStatements: # - Effect: 'Allow' - # Action: + # Action: # - 'sqs:SendMessage' # - 'sqs:SendMessageBatch' # Resource: # Fn::GetAtt: # - EmailUsersQueue # - Arn - # events: - # - schedule: + # events: + # - schedule: # enabled: true # rate: rate(20 minutes) # environment: @@ -486,6 +518,12 @@ resources: # QueueName: "email-users-checker" # VisibilityTimeout: 300 + EmailForwardingAceptationQueue: + Type: "AWS::SQS::Queue" + Properties: + QueueName: "email-forwarding-aceptation" + VisibilityTimeout: 300 + GiveSESPermissionToInvokeProcessEmailLambdaFunction: Type: AWS::Lambda::Permission Properties: @@ -493,10 +531,10 @@ resources: Principal: ses.amazonaws.com Action: "lambda:InvokeFunction" SourceAccount: { Ref: AWS::AccountId } - + EmailsBucket: - Type: AWS::S3::Bucket - Properties: + Type: AWS::S3::Bucket + Properties: BucketName: finance-emails-${opt:stage} EmailsBucketPolicy: @@ -505,15 +543,13 @@ resources: Bucket: { Ref: EmailsBucket } PolicyDocument: Statement: - - - Action: + - Action: - s3:PutObject Effect: Allow Resource: Fn::Join: - "" - - - - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } + - - { "Fn::GetAtt": ["EmailsBucket", "Arn"] } - "/*" Principal: Service: ses.amazonaws.com @@ -538,4 +574,4 @@ resources: { "Fn::GetAtt": ["ProcessEmailsLambdaFunction", "Arn"] } - S3Action: BucketName: finance-emails-${opt:stage} - RuleSetName: default-rule-set \ No newline at end of file + RuleSetName: default-rule-set