diff --git a/.github/workflows/update_google_sheet_members.yml b/.github/workflows/update_google_sheet_members.yml new file mode 100644 index 000000000..60dfb4912 --- /dev/null +++ b/.github/workflows/update_google_sheet_members.yml @@ -0,0 +1,31 @@ +name: Update the google sheet with members + +on: + workflow_dispatch: + schedule: + # Updates every 00:00 every day + - cron: "0 0 * * *" + +env: + NEXT_PUBLIC_BACKEND_BASE_URL: ${{secrets.VITE_BACKEND_BASE_URL}} + MEMBERS_GOOGLE_SPREADSHEET_ID: ${{secrets.MEMBERS_GOOGLE_SPREADSHEET_ID}} + MEMBERS_GOOGLE_SHEET_ID: ${{secrets.MEMBERS_GOOGLE_SHEET_ID}} + GOOGLE_SERVICE_ACCOUNT_JSON: ${{secrets.GOOGLE_SERVICE_ACCOUNT_JSON}} + NEXT_PUBLIC_FIREBASE_API_KEY: ${{secrets.VITE_FIREBASE_API_KEY}} + +jobs: + update_google_sheet_members: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + with: + ref: stable + + - name: Install volta + uses: volta-cli/action@v4 + + - name: Update google sheet members with script + run: | + yarn workspace server update-google-sheet-members diff --git a/server/package.json b/server/package.json index b34525e59..7113cb646 100644 --- a/server/package.json +++ b/server/package.json @@ -32,6 +32,7 @@ "concurrently": "^8.2.2", "dotenv": "^16.4.5", "firebase": "^10.10.0", + "googleapis": "^144.0.0", "nodemon": "^3.1.0", "ts-node": "^10.9.2", "tsc-alias": "^1.8.8", @@ -46,6 +47,7 @@ "serve": "node --es-module-specifier-resolution=node dist/index.js", "token": "ts-node ./tooling/login-prod.ts", "timestamp": "ts-node ./tooling/timestamp-conversion.ts", - "normalise-newlines": "ts-node ./tooling/remove-newlines.ts ./src/middleware/__generated__/swagger.json" + "normalise-newlines": "ts-node ./tooling/remove-newlines.ts ./src/middleware/__generated__/swagger.json", + "update-google-sheet-members": "ts-node ./tooling/update-google-sheet-members.ts" } } diff --git a/server/tooling/update-google-sheet-members.ts b/server/tooling/update-google-sheet-members.ts new file mode 100644 index 000000000..08d985be1 --- /dev/null +++ b/server/tooling/update-google-sheet-members.ts @@ -0,0 +1,239 @@ +import { CombinedUserData } from "service-layer/response-models/UserResponse" +import { firestoreTimestampToDate } from "../src/data-layer/adapters/DateUtils" + +import { google } from "googleapis" +import admin from "firebase-admin" +import dotenv from "dotenv" + +dotenv.config() + +// Environment variables +const BASE_URL = process.env.NEXT_PUBLIC_BACKEND_BASE_URL +const MEMBERS_GOOGLE_SPREADSHEET_ID = process.env.MEMBERS_GOOGLE_SPREADSHEET_ID +const MEMBERS_GOOGLE_SHEET_ID = process.env.MEMBERS_GOOGLE_SHEET_ID +const API_KEY = process.env.NEXT_PUBLIC_FIREBASE_API_KEY +const GOOGLE_SERVICE_ACCOUNT_JSON = process.env.GOOGLE_SERVICE_ACCOUNT_JSON +const USER_ID = "google-sheets-bot" + +admin.initializeApp({ + credential: admin.credential.cert(JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON)) +}) + +const categories = [ + "Email", + "First name", + "Last name", + "Phone number", + "Date of birth", + "Membership", + "Date joined", + "University year", + "Gender", + "Does racing", + "Does ski", + "Does snowboarding", + "Emergency contact", + "Dietary requirements", + "UID" +] + +/** + * Fetches users from the backend + * @param token - The token to authenticate the request + * @param cursor - The cursor to fetch the next page of users + * @returns The data json object from the response + */ +async function fetchUsers(token: string, cursor?: string): Promise { + const res = await fetch( + // Note that VITE_BACKEND_BASE_URL does have a slash at the end + `${BASE_URL}admin/users${cursor ? `?cursor=${cursor}` : ""}`, + { + method: "GET", + headers: { + accept: "application/json", + authorization: `Bearer ${token}` + } + } + ) + if (res.status === 200) { + const data = await res.json() + return data + } else { + throw new Error(`Failed to fetch users, status: ${res.status}`) + } +} + +/** + * Fetches all users from the backend + * @param token - The token to authenticate the request + * @returns An array of CombinedUserData + */ +async function getAllUsers(token: string): Promise { + const allUsers: CombinedUserData[] = [] + let fetchedUsers = await fetchUsers(token) + allUsers.push(...fetchedUsers.data) + while (fetchedUsers.cursor) { + fetchedUsers = await fetchUsers(token, fetchedUsers.cursor) + allUsers.push(...fetchedUsers.data) + } + return allUsers +} + +/** + * Creates google authentication client + * @returns The google auth client + */ +async function authenticateGoogle(): Promise { + const { client_email, private_key } = JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON) + const auth = new google.auth.GoogleAuth({ + credentials: { + client_email, + private_key + }, + scopes: ["https://www.googleapis.com/auth/spreadsheets"] + }) + + const authClient = await auth.getClient() + return authClient +} + +/** + * Append data to Google Sheets using API Key + * @param auth - The google auth client + * @param rows - The rows to append to the Google Sheet + * @returns The response from the Google Sheets API + */ +async function updateGoogleSheet(auth: any, rows: any[]) { + const sheets = google.sheets({ + version: "v4", + auth + }) + + const request = { + spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID, + // Sheet id is something like "Sheet1" + range: MEMBERS_GOOGLE_SHEET_ID + "!A1", + valueInputOption: "RAW", + insertDataOption: "INSERT_ROWS", + resource: { + values: rows + } + } + + try { + const response = (await sheets.spreadsheets.values.append(request)).data + console.log(`${rows.length} rows added to the google sheet.`) + return response + } catch (err) { + return console.error("Failed to add rows to the google sheet.", err) + } +} + +/** + * Clears google sheet data. + * @param auth - The google auth client + */ +async function clearSheet(auth: any) { + const sheets = google.sheets({ + version: "v4", + auth + }) + + const request = { + spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID, + range: MEMBERS_GOOGLE_SHEET_ID + } + try { + await sheets.spreadsheets.values.clear(request) + console.log(`Cleared google sheet.`) + } catch (err) { + console.error("Failed to clear the Google Sheet", err) + } +} + +/** + * Converts user information to an array that google sheets can parse. + * @param users An array of all CombinedUserData + * @returns The mapper user arrays + */ +function mapUsers(users: CombinedUserData[]) { + return users.map((user: CombinedUserData) => [ + user.email, + user.first_name, + user.last_name, + user.phone_number, + firestoreTimestampToDate(user.date_of_birth).toLocaleString([], { + timeZone: "Pacific/Auckland", + hour12: true + }), + user.membership, + user.dateJoined, + user.university_year, + user.gender, + user.does_racing, + user.does_ski, + user.does_snowboarding, + user.emergency_contact, + // Remove stripe id from fields + user.dietary_requirements, + user.uid + ]) +} + +/** + * Code from login-prod.ts to create admin jwt token + * @param uid - The user id to create the token for + * @returns The jwt token + */ +const createIdToken = async () => { + try { + // Ensure that the user exists + try { + await admin.auth().getUser(USER_ID) + } catch (e) { + await admin.auth().createUser({ uid: USER_ID }) + } + await admin + .auth() + .setCustomUserClaims(USER_ID, { member: true, admin: true }) + + const customToken = await admin.auth().createCustomToken(USER_ID) + const res = await fetch( + `https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyCustomToken?key=${API_KEY}`, + { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({ + token: customToken, + returnSecureToken: true + }) + } + ) + + const data = (await res.json()) as any + + return data.idToken + } catch (e) { + console.error(e) + } +} + +/** + * Updates the google sheet with members fetched from backend + * @param token - The token to authenticate the request + */ +async function updateGoogleSheetMembers() { + const token = await createIdToken() + const allUsers: CombinedUserData[] = (await getAllUsers(token)).filter( + (user) => user.membership === "member" + ) + const rows = mapUsers(allUsers) + const auth = await authenticateGoogle() + await clearSheet(auth) // Clear sheet first + await updateGoogleSheet(auth, [categories, ...rows]) +} + +updateGoogleSheetMembers() diff --git a/yarn.lock b/yarn.lock index 7f10456c8..106aedb9f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14521,6 +14521,19 @@ __metadata: languageName: node linkType: hard +"gaxios@npm:^6.0.3": + version: 6.7.1 + resolution: "gaxios@npm:6.7.1" + dependencies: + extend: "npm:^3.0.2" + https-proxy-agent: "npm:^7.0.1" + is-stream: "npm:^2.0.0" + node-fetch: "npm:^2.6.9" + uuid: "npm:^9.0.1" + checksum: 10c0/53e92088470661c5bc493a1de29d05aff58b1f0009ec5e7903f730f892c3642a93e264e61904383741ccbab1ce6e519f12a985bba91e13527678b32ee6d7d3fd + languageName: node + linkType: hard + "gcp-metadata@npm:^6.1.0": version: 6.1.0 resolution: "gcp-metadata@npm:6.1.0" @@ -14912,6 +14925,20 @@ __metadata: languageName: node linkType: hard +"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.7.0": + version: 9.14.1 + resolution: "google-auth-library@npm:9.14.1" + dependencies: + base64-js: "npm:^1.3.0" + ecdsa-sig-formatter: "npm:^1.0.11" + gaxios: "npm:^6.1.1" + gcp-metadata: "npm:^6.1.0" + gtoken: "npm:^7.0.0" + jws: "npm:^4.0.0" + checksum: 10c0/050e16343d93768300a800bc69773d8c451c4e778b0e503fc9dcf72e40e9563c0877f7a79ed06dffad664b49fdd1183080c41f081034b86d54a6795475fb73d2 + languageName: node + linkType: hard + "google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3": version: 9.7.0 resolution: "google-auth-library@npm:9.7.0" @@ -14946,6 +14973,30 @@ __metadata: languageName: node linkType: hard +"googleapis-common@npm:^7.0.0": + version: 7.2.0 + resolution: "googleapis-common@npm:7.2.0" + dependencies: + extend: "npm:^3.0.2" + gaxios: "npm:^6.0.3" + google-auth-library: "npm:^9.7.0" + qs: "npm:^6.7.0" + url-template: "npm:^2.0.8" + uuid: "npm:^9.0.0" + checksum: 10c0/cbbce900582a66c28bb8ccde631bc08202c6fb2e591932b981a23b437b074150051b966d3ad67bcb4b06b4ff5bbbfd8524ac5ca6f7b77b8790f417924bec1f3c + languageName: node + linkType: hard + +"googleapis@npm:^144.0.0": + version: 144.0.0 + resolution: "googleapis@npm:144.0.0" + dependencies: + google-auth-library: "npm:^9.0.0" + googleapis-common: "npm:^7.0.0" + checksum: 10c0/a5ad4c5be32817f7960fac2aa52b1bbc658242ed1874fb08ed84dbdf36b9fa401077e6e425928e453629d14b1f0d54ee31a1dc0959705465b489f31bd3f36f4a + languageName: node + linkType: hard + "gopd@npm:^1.0.1": version: 1.0.1 resolution: "gopd@npm:1.0.1" @@ -20202,6 +20253,15 @@ __metadata: languageName: node linkType: hard +"qs@npm:^6.7.0": + version: 6.13.0 + resolution: "qs@npm:6.13.0" + dependencies: + side-channel: "npm:^1.0.6" + checksum: 10c0/62372cdeec24dc83a9fb240b7533c0fdcf0c5f7e0b83343edd7310f0ab4c8205a5e7c56406531f2e47e1b4878a3821d652be4192c841de5b032ca83619d8f860 + languageName: node + linkType: hard + "querystring-es3@npm:^0.2.1": version: 0.2.1 resolution: "querystring-es3@npm:0.2.1" @@ -21607,6 +21667,7 @@ __metadata: express: "npm:^4.18.2" firebase: "npm:^10.10.0" firebase-admin: "npm:^12.0.0" + googleapis: "npm:^144.0.0" helmet: "npm:^7.1.0" nodemailer: "npm:^6.9.14" nodemon: "npm:^3.1.0" @@ -23939,6 +24000,13 @@ __metadata: languageName: node linkType: hard +"url-template@npm:^2.0.8": + version: 2.0.8 + resolution: "url-template@npm:2.0.8" + checksum: 10c0/56a15057eacbcf05d52b0caed8279c8451b3dd9d32856a1fdd91c6dc84dcb1646f12bafc756b7ade62ca5b1564da8efd7baac5add35868bafb43eb024c62805b + languageName: node + linkType: hard + "url@npm:^0.11.0": version: 0.11.3 resolution: "url@npm:0.11.3"