From 7a776de15228a4ee9c2c5fc767c0c718ededcca0 Mon Sep 17 00:00:00 2001 From: SantriptaSharma Date: Fri, 8 Sep 2023 14:25:13 +0530 Subject: [PATCH] rev: data fetch pipeline reverted to use repository_dispatch --- .github/workflows/update-data.yml | 10 +++-- .gitignore | 1 + src/intents/index.js | 3 +- src/intents/meals.js | 68 ++++++++++++++++--------------- src/utils/fetch-dining.ts | 61 ++++++++++++--------------- src/utils/parse-dining.ts | 52 ++++++++++++++++------- 6 files changed, 108 insertions(+), 87 deletions(-) diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 668d7f4..7ae77b4 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -1,8 +1,7 @@ name: Update Data on: - workflow_dispatch: - schedule: - - cron: "0 */6 * * *" + repository_dispatch: + types: [dining_data] jobs: fetch: @@ -29,7 +28,10 @@ jobs: run: npm i - name: Copy production env - run: echo -e "${{secrets.ENV}}" > .env + run: | + echo -e "${{secrets.ENV}}" > .env + echo -e "MENU_ID=${{github.event.client_payload.id}}" >> .env + echo -e "MENU_YEAR=${{github.event.client_payload.year}}" >> .env - name: Database migrations and generate db types run: npx prisma migrate deploy && npx prisma generate diff --git a/.gitignore b/.gitignore index f89267b..7cace7b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ node_modules respond.js login.js creds.json +out.json local-data *.log *.xlsx diff --git a/src/intents/index.js b/src/intents/index.js index b30cbcd..d01dc41 100644 --- a/src/intents/index.js +++ b/src/intents/index.js @@ -2,9 +2,10 @@ const { createLanguageProcessor } = require('@adiwajshing/whatsapp-info-bot') const { default: helpIntent } = require('@adiwajshing/whatsapp-info-bot/dist/example/intents/help') const create = async() => { + // TODO: update and reintroduce shuttles intent const intents = [ await require('./meals')(), - await require("./shuttle")(), + // await require("./shuttle")(), require('./f_all.json'), require('./gratefulness.json'), diff --git a/src/intents/meals.js b/src/intents/meals.js index 1d104ff..70cb392 100644 --- a/src/intents/meals.js +++ b/src/intents/meals.js @@ -14,27 +14,27 @@ String.prototype.toTitleCase = function () { const mealOptions = [ "breakfast", "lunch", "snacks", "dinner" ] const OUTLET_MENUS = { - 'dhaba': { - document: { - url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/i6V%2FQ%2BJTK4%2FmeKV9puT1UczWmCVVQaCKpwR73bJXspk%3D', - mimetype: 'application/pdf', - name: 'dhaba-menu.pdf' - } - }, - 'asg': { - document: { - url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/xCqevivWF7VTp0qTy%2FuPJjfbpsmAg6A5dSgHjjIZx0w%3D', - mimetype: 'application/pdf', - name: 'asg-menu.pdf' - } - }, - 'fuel zone': { - document: { - url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/Iq9rMPglXt6VlkBbV3dtXfLftJQBsWw5IqKXgqXeWdg%3D', - mimetype: 'application/pdf', - name: 'fuel-zone-menu.pdf' - } - } + // 'dhaba': { + // document: { + // url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/i6V%2FQ%2BJTK4%2FmeKV9puT1UczWmCVVQaCKpwR73bJXspk%3D', + // mimetype: 'application/pdf', + // name: 'dhaba-menu.pdf' + // } + // }, + // 'asg': { + // document: { + // url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/xCqevivWF7VTp0qTy%2FuPJjfbpsmAg6A5dSgHjjIZx0w%3D', + // mimetype: 'application/pdf', + // name: 'asg-menu.pdf' + // } + // }, + // 'fuel zone': { + // document: { + // url: 'https://chatdaddy-media-store.s3.ap-east-1.amazonaws.com/Iq9rMPglXt6VlkBbV3dtXfLftJQBsWw5IqKXgqXeWdg%3D', + // mimetype: 'application/pdf', + // name: 'fuel-zone-menu.pdf' + // } + // } } const locale = "en-IN"; @@ -80,22 +80,24 @@ module.exports = async() => { else { const option = options.meal.toLowerCase() - if (![...mealOptions, "combo"].includes(option)) { + if (![...mealOptions/*, "combo"*/].includes(option)) { throw new Error("Unknown Option: " + option + "; You can ask for " + mealOptions.join(", ")) } - if (option === "combo") { - let week = "wk_" + DateUtils.weekOfYear( date, 1 ).toString() + //TODO: reintroduce static documents, for outlets & VOW Spice + + // if (option === "combo") { + // let week = "wk_" + DateUtils.weekOfYear( date, 1 ).toString() - const arr = mealsData["combo"][week] - let str - if (arr) { - str = Object.keys(arr).map ( key => ("*" + key.toTitleCase() + ":*\n " + arr[key].join("\n ").toTitleCase()) ).join("\n") - } else { - str = "Data not available 😅" - } - return str - } + // const arr = mealsData["combo"][week] + // let str + // if (arr) { + // str = Object.keys(arr).map ( key => ("*" + key.toTitleCase() + ":*\n " + arr[key].join("\n ").toTitleCase()) ).join("\n") + // } else { + // str = "Data not available 😅" + // } + // return str + // } const menu = await prisma.dailyMenu.findFirstOrThrow({ where: { diff --git a/src/utils/fetch-dining.ts b/src/utils/fetch-dining.ts index b1b5179..5b613e8 100644 --- a/src/utils/fetch-dining.ts +++ b/src/utils/fetch-dining.ts @@ -1,54 +1,47 @@ import fs from "fs/promises"; import { google } from "googleapis"; import { PrismaClient } from "@prisma/client"; -import ParseDiningMenu, { FindUniqueItems } from "./parse-dining"; +import ParseDiningMenu, { FindUniqueItemsWithCounts } from "./parse-dining"; -const auth = new google.auth.OAuth2({ - clientId: process.env.OAUTH_CLIENT_ID, - clientSecret: process.env.OAUTH_SECRET, +const drive = google.drive({ + version: "v3", + auth: process.env.MESSCAT_GDRIVE }); -auth.setCredentials({ refresh_token: process.env.GOOGLE_REFRESH }); - -const gmail = google.gmail({ - version: "v1", - auth -}); - -const MENU_MIMETYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'; - -async function GetMenu() +async function GetMenu(): Promise<[string, number] | undefined> { - const {data: {messages: [message]}} = await gmail.users.messages.list({ - userId: "me", - q: `has:attachment to:technology.ministry@ashoka.edu.in`, - maxResults: 1 - }); + const id = process.env.MENU_ID; + const year = parseInt(process.env.MENU_YEAR); + if (id === undefined || year === undefined || Number.isNaN(year)) throw new Error(`Missing parameters: id: ${id}, year: ${year}`); - if (message == undefined) throw new Error("No menu found"); + const file = await drive.files.get({ + fileId: id, + alt: "media", + }, { responseType: "blob"}); - const {data: {payload: {parts}}} = await gmail.users.messages.get({ userId: "me", id: message.id }); - const menu = parts.find(part => part.mimeType == MENU_MIMETYPE); - if (menu == undefined) throw new Error("No matching attachment found"); + const data = (file.data as unknown) as Blob; - const {data: {data}} = await gmail.users.messages.attachments.get({ userId: "me", messageId: message.id, id: menu.body.attachmentId }); - const buf = Buffer.from(data, 'base64'); + if (file.status !== 200) return; + await fs.writeFile("menu.xlsx", Buffer.from(await data.arrayBuffer())); - const menuPath = "menu.xlsx" - await fs.writeFile(menuPath, buf); - - return menuPath; + return ["menu.xlsx", year]; } (async () => { - - const menu = await GetMenu(); const prisma = new PrismaClient(); - const parsed = await ParseDiningMenu(menu); - await fs.rm(menu); + + if (process.argv.length >= 3) + { + process.env.MENU_ID = process.argv[2]; + process.env.MENU_YEAR = new Date().getFullYear().toString(); + } + + const menu = await GetMenu(); + if (menu == undefined) throw new Error("Menu file not found"); + const parsed = await ParseDiningMenu(...menu); parsed.forEach(async weekMenu => { - const itemCounts = FindUniqueItems(weekMenu); + const itemCounts = FindUniqueItemsWithCounts(weekMenu); const itemNames = Array.from(itemCounts.keys()); console.log(itemCounts); diff --git a/src/utils/parse-dining.ts b/src/utils/parse-dining.ts index 4ad8acc..f90024c 100644 --- a/src/utils/parse-dining.ts +++ b/src/utils/parse-dining.ts @@ -1,4 +1,5 @@ import xl from "exceljs"; +import { argv } from "process"; type Meal = "Breakfast" | "Lunch" | "Snacks" | "Dinner"; @@ -14,24 +15,39 @@ type MessMenu = { days: {[k in Meal]: MessItem[]}[] } -async function ParseXlsx(path: string): Promise +// Check (using format specific indicators) whether the sheet is a menu sheet. This should be the case unless they decide to randomly change the format again, in which case we refuse to parse potentially incorrect data. +function isMenuSheet(menu: xl.Worksheet) { + const first = menu.getCell(1, 1); + const [day, date] = [menu.getCell(2, 1), menu.getCell(3, 1)] + + if (!first.isMerged) return false; + if (day.value.toString().trim() != "DAY" || date.value.toString().trim() != "DATE") return false; + + return true; +} + +async function ParseXlsx(path: string, year: number): Promise { const res: MessMenu[] = []; const wb = new xl.Workbook(); await wb.xlsx.readFile(path); wb.worksheets.forEach(menu => { - // TODO: Somehow check whether the sheet is a menu sheet, though this should be the case unless they decide to randomly change the format again + if (!isMenuSheet(menu)) + { + console.error("Sheet is not a menu sheet"); + return; + } const meals: {[k in Meal]: xl.CellValue[][]} = { - Breakfast: menu.getRows(4, 8).map(row => row.values) as xl.CellValue[][], - Lunch: menu.getRows(13, 8).map(row => row.values) as xl.CellValue[][], - Snacks: menu.getRows(22, 5).map(row => row.values) as xl.CellValue[][], - Dinner: menu.getRows(28, 8).map(row => row.values) as xl.CellValue[][] + Breakfast: menu.getRows(5, 12).map(row => row.values) as xl.CellValue[][], + Lunch: menu.getRows(18, 12).map(row => row.values) as xl.CellValue[][], + Snacks: menu.getRows(31, 4).map(row => row.values) as xl.CellValue[][], + Dinner: menu.getRows(36, 10).map(row => row.values) as xl.CellValue[][] } - const title = menu.getCell(1, 1).value as string; - const startString = title.trim().split('(')[1].split('-')[0].trim().replace(/(rd)|(st)|(th)|(nd)/g, "") + ` ${ new Date().getFullYear()}`; + const firstDate = menu.getCell(3, 2).value as string; + const startString = firstDate.trim().replace(/(\d)((rd)|(st)|(th)|(nd))/g, "$1") + ` ${year}`; const startDate = new Date(Date.parse(startString)); startDate.setFullYear(new Date().getFullYear()); const endDate = new Date(startDate); @@ -48,19 +64,19 @@ async function ParseXlsx(path: string): Promise })); const exclude = ["-"]; - + Object.keys(meals).forEach(meal => { const key = meal as Meal; meals[key].forEach((row) => { for (let i = 0; i < 7; i++) { - const name = (row[i + 3] as string ?? "").trim(); + const name = (row[i + 2] as string ?? "").trim(); if (name.length > 0 && !exclude.includes(name)) { - const split = name.split(",").map(s => s.trim()).filter(s => s.length > 0 && !exclude.includes(name)).map(s => s.split("(")[0].trim()).map(item => ({ - category: row[2] as string, + const split = name.split(",").flatMap(s => s.split("/")).map(s => s.trim()).filter(s => s.length > 0 && !exclude.includes(name)).map(s => s.split("(")[0].trim()).map(item => ({ + category: row[1] as string, name: item, meal: key })); @@ -81,7 +97,7 @@ async function ParseXlsx(path: string): Promise return res; } -export function FindUniqueItems(menu: MessMenu): Map +export function FindUniqueItemsWithCounts(menu: MessMenu): Map { const items = new Array(); const counts = new Map(); @@ -101,10 +117,16 @@ export function FindUniqueItems(menu: MessMenu): Map return new Map(items.map(item => [item.name, counts.get(item.name) ?? 0])); } -export default async function ParseDiningMenu(path: string) +export default async function ParseDiningMenu(path: string, year: number) { if (path.endsWith(".xlsx")) { - return await ParseXlsx(path); + return await ParseXlsx(path, year); } +} + +if (argv.length >= 3) { + ParseDiningMenu(argv[2], new Date().getFullYear()).then((thing) => { + console.log(JSON.stringify(thing)); + }); } \ No newline at end of file