Split functions

serverless.yml

@@ -47,6 +47,10 @@ functions:
           name: orestarScraper
           description: 'Pull Ted Wheeler campaign contribution data every 12 hours'
           rate: rate(12 hours)
-      # - http: 
-      #     method: get
-      #     path: orestarScraper
+  updateDB:
+    handler: src/handler.updateDB
+    events:
+      - http: 
+          method: post
+          path: updateDB
+          cors: true

services/getOrestarFinanceData.ts

@@ -1,11 +1,13 @@
 import chromium from 'chrome-aws-lambda';
+import XLSX from 'xlsx';
 import { existsSync as fileExists, unlink } from 'fs';
+import { OrestarEntry } from './types';
 
 interface OrestarFinanceQueryCriteria {
   candidateName: string;
 }
 
-export default async ({ candidateName }: OrestarFinanceQueryCriteria): Promise<string> => {
+export default async ({ candidateName }: OrestarFinanceQueryCriteria): Promise<OrestarEntry[]> => {
   const browser = await chromium.puppeteer.launch({
     // headless: true,
     // timeout: 0,
@@ -96,5 +98,12 @@ export default async ({ candidateName }: OrestarFinanceQueryCriteria): Promise<s
 
   browser.close();
 
-  return xlsFilename;
+  const workbook = XLSX.readFile(xlsFilename, {
+    bookVBA: true,
+    WTF: true,
+    type: 'file',
+  });
+  const sheetName = workbook.SheetNames[0];
+  const orestarData: OrestarEntry[] = XLSX.utils.sheet_to_json(workbook.Sheets[sheetName]);
+  return orestarData;
 };

services/parseAndSaveContributionData.ts

@@ -1,4 +1,3 @@
-import XLSX from 'xlsx';
 import {
   ContributionType,
   ContributionSubType,
@@ -8,58 +7,7 @@ import {
 import db from '@models/db';
 import addContribution from '@services/addContribution';
 import { reportError } from './bugSnag';
-
-// We are not using the following from Orestar in the OAE database:
-// 'Tran Status'
-// 'Filer'
-// 'Aggregate Amount': number;
-// 'Filer Id'
-// 'Attest By Name'
-// 'Attest Date'
-// 'Due Date'
-// 'Tran Stsfd Ind'
-// 'Filed By Name'
-// 'Filed Date'
-// 'Self Employ Ind'
-// 'Review By Name'
-// 'Review Date'
-// 'Contributor/Payee Committee ID'
-// Also refer to the Orestar User Manual for more info: https://sos.oregon.gov/elections/Documents/orestarTransFiling.pdf
-type OrestarEntry = {
-  'Tran Id': string; // 7 digit number, same as Original Id, unless entry was updated.
-  'Original Id': string; // 7 digit number
-  'Tran Date': string; // MM/DD/YYYY
-  'Tran Status': 'Original' | 'Amended' | 'Deleted';
-  'Filer': string;
-  'Contributor/Payee': string;
-  'Sub Type': string;
-  'Amount': number;
-  'Aggregate Amount': number;
-  'Filer Id': string;
-  'Attest By Name': string;
-  'Attest Date': string;
-  'Due Date': string;
-  'Tran Stsfd Ind': string;
-  'Filed By Name': string;
-  'Filed Date': string;
-  'Book Type': string;
-  'Occptn Txt': string;
-  'Emp Name': string;
-  'Emp City': string;
-  'Emp State': string;
-  'Employ Ind': string;
-  'Self Employ Ind'?: string;
-  'Addr Line1': string;
-  'Addr Line2'?: string;
-  'City': string;
-  'State': string;
-  'Zip': string;
-  'Country': string;
-  'Review By Name'?: string;
-  'Review Date'?: string;
-  'Contributor/Payee Committee ID'?: string;
-  'Purp Desc'?: string;
-}
+import type { OrestarEntry } from './types';
 
 function getContributionSubType(orestarSubType: string): ContributionSubType {
   const subTypeMap = {
@@ -97,17 +45,10 @@ function getContributorType(orestarBookType: string): ContributorType {
   return oaeContributorType;
 }
 
-export async function parseAndSaveContributionData(xlsFilename: string): Promise<void> {
+export async function parseAndSaveContributionData(orestarData: OrestarEntry[]): Promise<void> {
   const connection = await db();
   const contributionRepo = connection.getRepository('external_contributions');
 
-  const workbook = XLSX.readFile(xlsFilename, {
-    bookVBA: true,
-    WTF: true,
-    type: 'file',
-  });
-  const sheetName = workbook.SheetNames[0];
-  const orestarData: OrestarEntry[] = XLSX.utils.sheet_to_json(workbook.Sheets[sheetName]);
   console.log(`total contributions: ${orestarData.length}.`);
 
   Promise.all(orestarData.map(async (orestarEntry: OrestarEntry) => {

services/types.ts

@@ -0,0 +1,51 @@
+// We are not using the following from Orestar in the OAE database:
+// 'Tran Status'
+// 'Filer'
+// 'Aggregate Amount': number;
+// 'Filer Id'
+// 'Attest By Name'
+// 'Attest Date'
+// 'Due Date'
+// 'Tran Stsfd Ind'
+// 'Filed By Name'
+// 'Filed Date'
+// 'Self Employ Ind'
+// 'Review By Name'
+// 'Review Date'
+// 'Contributor/Payee Committee ID'
+// Also refer to the Orestar User Manual for more info: https://sos.oregon.gov/elections/Documents/orestarTransFiling.pdf
+export type OrestarEntry = {
+  'Tran Id': string; // 7 digit number, same as Original Id, unless entry was updated.
+  'Original Id': string; // 7 digit number
+  'Tran Date': string; // MM/DD/YYYY
+  'Tran Status': 'Original' | 'Amended' | 'Deleted';
+  'Filer': string;
+  'Contributor/Payee': string;
+  'Sub Type': string;
+  'Amount': number;
+  'Aggregate Amount': number;
+  'Filer Id': string;
+  'Attest By Name': string;
+  'Attest Date': string;
+  'Due Date': string;
+  'Tran Stsfd Ind': string;
+  'Filed By Name': string;
+  'Filed Date': string;
+  'Book Type': string;
+  'Occptn Txt': string;
+  'Emp Name': string;
+  'Emp City': string;
+  'Emp State': string;
+  'Employ Ind': string;
+  'Self Employ Ind'?: string;
+  'Addr Line1': string;
+  'Addr Line2'?: string;
+  'City': string;
+  'State': string;
+  'Zip': string;
+  'Country': string;
+  'Review By Name'?: string;
+  'Review Date'?: string;
+  'Contributor/Payee Committee ID'?: string;
+  'Purp Desc'?: string;
+}

src/handler.ts

@@ -1,15 +1,30 @@
-import { APIGatewayProxyHandler } from 'aws-lambda';
+import type { APIGatewayProxyHandler } from 'aws-lambda';
 import { parseAndSaveContributionData } from '@services/parseAndSaveContributionData';
 import getOrestarFinanceData from '@services/getOrestarFinanceData';
+import type { OrestarEntry } from '@services/types';
 
 export const orestarScraper: APIGatewayProxyHandler = async () => {
-  const xlsFilename = await getOrestarFinanceData({ candidateName: 'Ted Wheeler' });
-  await parseAndSaveContributionData(xlsFilename);
-
+  const orestarData: OrestarEntry[] = await getOrestarFinanceData({ candidateName: 'Ted Wheeler' });
+  await fetch(process.env.UPDATE_LAMBDA, {
+    method: 'post',
+    body:    JSON.stringify(orestarData),
+    headers: { 'Content-Type': 'application/json' },
+  })
   return {
     statusCode: 200,
     body: JSON.stringify({
       content: 'done!',
     }, null, 2),
   };
 };
+
+export const updateDB: APIGatewayProxyHandler = async (event) => {
+  const jsonData: OrestarEntry[] = JSON.parse(event.body);
+  await parseAndSaveContributionData(jsonData);
+  return {
+    statusCode: 200,
+    body: JSON.stringify({
+      content: 'done!',
+    }, null, 2),
+  };
+}