783 backend write a cron job to automatically update a google sheet with the members

.github/workflows/update_google_sheet_members.yml

@@ -0,0 +1,31 @@
+name: Update the google sheet with members
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Updates every 00:00 every day
+    - cron: "0 0 * * *"
+
+env:
+  NEXT_PUBLIC_BACKEND_BASE_URL: ${{secrets.VITE_BACKEND_BASE_URL}}
+  MEMBERS_GOOGLE_SPREADSHEET_ID: ${{secrets.MEMBERS_GOOGLE_SPREADSHEET_ID}}
+  MEMBERS_GOOGLE_SHEET_ID: ${{secrets.MEMBERS_GOOGLE_SHEET_ID}}
+  GOOGLE_SERVICE_ACCOUNT_JSON: ${{secrets.GOOGLE_SERVICE_ACCOUNT_JSON}}
+  NEXT_PUBLIC_FIREBASE_API_KEY: ${{secrets.VITE_FIREBASE_API_KEY}}
+
+jobs:
+  update_google_sheet_members:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: stable
+
+      - name: Install volta
+        uses: volta-cli/action@v4
+
+      - name: Update google sheet members with script
+        run: |
+          yarn workspace server update-google-sheet-members

server/package.json

@@ -32,6 +32,7 @@
     "concurrently": "^8.2.2",
     "dotenv": "^16.4.5",
     "firebase": "^10.10.0",
+    "googleapis": "^144.0.0",
     "nodemon": "^3.1.0",
     "ts-node": "^10.9.2",
     "tsc-alias": "^1.8.8",
@@ -46,6 +47,7 @@
     "serve": "node --es-module-specifier-resolution=node dist/index.js",
     "token": "ts-node ./tooling/login-prod.ts",
     "timestamp": "ts-node ./tooling/timestamp-conversion.ts",
-    "normalise-newlines": "ts-node ./tooling/remove-newlines.ts ./src/middleware/__generated__/swagger.json"
+    "normalise-newlines": "ts-node ./tooling/remove-newlines.ts ./src/middleware/__generated__/swagger.json",
+    "update-google-sheet-members": "ts-node ./tooling/update-google-sheet-members.ts"
   }
 }

server/tooling/update-google-sheet-members.ts

@@ -0,0 +1,239 @@
+import { CombinedUserData } from "service-layer/response-models/UserResponse"
+import { firestoreTimestampToDate } from "../src/data-layer/adapters/DateUtils"
+
+import { google } from "googleapis"
+import admin from "firebase-admin"
+import dotenv from "dotenv"
+
+dotenv.config()
+
+// Environment variables
+const BASE_URL = process.env.NEXT_PUBLIC_BACKEND_BASE_URL
+const MEMBERS_GOOGLE_SPREADSHEET_ID = process.env.MEMBERS_GOOGLE_SPREADSHEET_ID
+const MEMBERS_GOOGLE_SHEET_ID = process.env.MEMBERS_GOOGLE_SHEET_ID
+const API_KEY = process.env.NEXT_PUBLIC_FIREBASE_API_KEY
+const GOOGLE_SERVICE_ACCOUNT_JSON = process.env.GOOGLE_SERVICE_ACCOUNT_JSON
+const USER_ID = "google-sheets-bot"
+
+admin.initializeApp({
+  credential: admin.credential.cert(JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON))
+})
+
+const categories = [
+  "Email",
+  "First name",
+  "Last name",
+  "Phone number",
+  "Date of birth",
+  "Membership",
+  "Date joined",
+  "University year",
+  "Gender",
+  "Does racing",
+  "Does ski",
+  "Does snowboarding",
+  "Emergency contact",
+  "Dietary requirements",
+  "UID"
+]
+
+/**
+ * Fetches users from the backend
+ * @param token - The token to authenticate the request
+ * @param cursor - The cursor to fetch the next page of users
+ * @returns The data json object from the response
+ */
+async function fetchUsers(token: string, cursor?: string): Promise<any> {
+  const res = await fetch(
+    // Note that VITE_BACKEND_BASE_URL does have a slash at the end
+    `${BASE_URL}admin/users${cursor ? `?cursor=${cursor}` : ""}`,
+    {
+      method: "GET",
+      headers: {
+        accept: "application/json",
+        authorization: `Bearer ${token}`
+      }
+    }
+  )
+  if (res.status === 200) {
+    const data = await res.json()
+    return data
+  } else {
+    throw new Error(`Failed to fetch users, status: ${res.status}`)
+  }
+}
+
+/**
+ * Fetches all users from the backend
+ * @param token - The token to authenticate the request
+ * @returns An array of CombinedUserData
+ */
+async function getAllUsers(token: string): Promise<CombinedUserData[]> {
+  const allUsers: CombinedUserData[] = []
+  let fetchedUsers = await fetchUsers(token)
+  allUsers.push(...fetchedUsers.data)
+  while (fetchedUsers.cursor) {
+    fetchedUsers = await fetchUsers(token, fetchedUsers.cursor)
+    allUsers.push(...fetchedUsers.data)
+  }
+  return allUsers
+}
+
+/**
+ * Creates google authentication client
+ * @returns The google auth client
+ */
+async function authenticateGoogle(): Promise<any> {
+  const { client_email, private_key } = JSON.parse(GOOGLE_SERVICE_ACCOUNT_JSON)
+  const auth = new google.auth.GoogleAuth({
+    credentials: {
+      client_email,
+      private_key
+    },
+    scopes: ["https://www.googleapis.com/auth/spreadsheets"]
+  })
+
+  const authClient = await auth.getClient()
+  return authClient
+}
+
+/**
+ * Append data to Google Sheets using API Key
+ * @param auth - The google auth client
+ * @param rows - The rows to append to the Google Sheet
+ * @returns The response from the Google Sheets API
+ */
+async function updateGoogleSheet(auth: any, rows: any[]) {
+  const sheets = google.sheets({
+    version: "v4",
+    auth
+  })
+
+  const request = {
+    spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID,
+    // Sheet id is something like "Sheet1"
+    range: MEMBERS_GOOGLE_SHEET_ID + "!A1",
+    valueInputOption: "RAW",
+    insertDataOption: "INSERT_ROWS",
+    resource: {
+      values: rows
+    }
+  }
+
+  try {
+    const response = (await sheets.spreadsheets.values.append(request)).data
+    console.log(`${rows.length} rows added to the google sheet.`)
+    return response
+  } catch (err) {
+    return console.error("Failed to add rows to the google sheet.", err)
+  }
+}
+
+/**
+ * Clears google sheet data.
+ * @param auth - The google auth client
+ */
+async function clearSheet(auth: any) {
+  const sheets = google.sheets({
+    version: "v4",
+    auth
+  })
+
+  const request = {
+    spreadsheetId: MEMBERS_GOOGLE_SPREADSHEET_ID,
+    range: MEMBERS_GOOGLE_SHEET_ID
+  }
+  try {
+    await sheets.spreadsheets.values.clear(request)
+    console.log(`Cleared google sheet.`)
+  } catch (err) {
+    console.error("Failed to clear the Google Sheet", err)
+  }
+}
+
+/**
+ * Converts user information to an array that google sheets can parse.
+ * @param users An array of all CombinedUserData
+ * @returns The mapper user arrays
+ */
+function mapUsers(users: CombinedUserData[]) {
+  return users.map((user: CombinedUserData) => [
+    user.email,
+    user.first_name,
+    user.last_name,
+    user.phone_number,
+    firestoreTimestampToDate(user.date_of_birth).toLocaleString([], {
+      timeZone: "Pacific/Auckland",
+      hour12: true
+    }),
+    user.membership,
+    user.dateJoined,
+    user.university_year,
+    user.gender,
+    user.does_racing,
+    user.does_ski,
+    user.does_snowboarding,
+    user.emergency_contact,
+    // Remove stripe id from fields
+    user.dietary_requirements,
+    user.uid
+  ])
+}
+
+/**
+ * Code from login-prod.ts to create admin jwt token
+ * @param uid - The user id to create the token for
+ * @returns The jwt token
+ */
+const createIdToken = async () => {
+  try {
+    // Ensure that the user exists
+    try {
+      await admin.auth().getUser(USER_ID)
+    } catch (e) {
+      await admin.auth().createUser({ uid: USER_ID })
+    }
+    await admin
+      .auth()
+      .setCustomUserClaims(USER_ID, { member: true, admin: true })
+
+    const customToken = await admin.auth().createCustomToken(USER_ID)
+    const res = await fetch(
+      `https://www.googleapis.com/identitytoolkit/v3/relyingparty/verifyCustomToken?key=${API_KEY}`,
+      {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify({
+          token: customToken,
+          returnSecureToken: true
+        })
+      }
+    )
+
+    const data = (await res.json()) as any
+
+    return data.idToken
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+/**
+ * Updates the google sheet with members fetched from backend
+ * @param token - The token to authenticate the request
+ */
+async function updateGoogleSheetMembers() {
+  const token = await createIdToken()
+  const allUsers: CombinedUserData[] = (await getAllUsers(token)).filter(
+    (user) => user.membership === "member"
+  )
+  const rows = mapUsers(allUsers)
+  const auth = await authenticateGoogle()
+  await clearSheet(auth) // Clear sheet first
+  await updateGoogleSheet(auth, [categories, ...rows])
+}
+
+updateGoogleSheetMembers()

yarn.lock

@@ -14521,6 +14521,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"gaxios@npm:^6.0.3":
+  version: 6.7.1
+  resolution: "gaxios@npm:6.7.1"
+  dependencies:
+    extend: "npm:^3.0.2"
+    https-proxy-agent: "npm:^7.0.1"
+    is-stream: "npm:^2.0.0"
+    node-fetch: "npm:^2.6.9"
+    uuid: "npm:^9.0.1"
+  checksum: 10c0/53e92088470661c5bc493a1de29d05aff58b1f0009ec5e7903f730f892c3642a93e264e61904383741ccbab1ce6e519f12a985bba91e13527678b32ee6d7d3fd
+  languageName: node
+  linkType: hard
+
 "gcp-metadata@npm:^6.1.0":
   version: 6.1.0
   resolution: "gcp-metadata@npm:6.1.0"
@@ -14912,6 +14925,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"google-auth-library@npm:^9.0.0, google-auth-library@npm:^9.7.0":
+  version: 9.14.1
+  resolution: "google-auth-library@npm:9.14.1"
+  dependencies:
+    base64-js: "npm:^1.3.0"
+    ecdsa-sig-formatter: "npm:^1.0.11"
+    gaxios: "npm:^6.1.1"
+    gcp-metadata: "npm:^6.1.0"
+    gtoken: "npm:^7.0.0"
+    jws: "npm:^4.0.0"
+  checksum: 10c0/050e16343d93768300a800bc69773d8c451c4e778b0e503fc9dcf72e40e9563c0877f7a79ed06dffad664b49fdd1183080c41f081034b86d54a6795475fb73d2
+  languageName: node
+  linkType: hard
+
 "google-auth-library@npm:^9.3.0, google-auth-library@npm:^9.6.3":
   version: 9.7.0
   resolution: "google-auth-library@npm:9.7.0"
@@ -14946,6 +14973,30 @@ __metadata:
   languageName: node
   linkType: hard
 
+"googleapis-common@npm:^7.0.0":
+  version: 7.2.0
+  resolution: "googleapis-common@npm:7.2.0"
+  dependencies:
+    extend: "npm:^3.0.2"
+    gaxios: "npm:^6.0.3"
+    google-auth-library: "npm:^9.7.0"
+    qs: "npm:^6.7.0"
+    url-template: "npm:^2.0.8"
+    uuid: "npm:^9.0.0"
+  checksum: 10c0/cbbce900582a66c28bb8ccde631bc08202c6fb2e591932b981a23b437b074150051b966d3ad67bcb4b06b4ff5bbbfd8524ac5ca6f7b77b8790f417924bec1f3c
+  languageName: node
+  linkType: hard
+
+"googleapis@npm:^144.0.0":
+  version: 144.0.0
+  resolution: "googleapis@npm:144.0.0"
+  dependencies:
+    google-auth-library: "npm:^9.0.0"
+    googleapis-common: "npm:^7.0.0"
+  checksum: 10c0/a5ad4c5be32817f7960fac2aa52b1bbc658242ed1874fb08ed84dbdf36b9fa401077e6e425928e453629d14b1f0d54ee31a1dc0959705465b489f31bd3f36f4a
+  languageName: node
+  linkType: hard
+
 "gopd@npm:^1.0.1":
   version: 1.0.1
   resolution: "gopd@npm:1.0.1"
@@ -20202,6 +20253,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"qs@npm:^6.7.0":
+  version: 6.13.0
+  resolution: "qs@npm:6.13.0"
+  dependencies:
+    side-channel: "npm:^1.0.6"
+  checksum: 10c0/62372cdeec24dc83a9fb240b7533c0fdcf0c5f7e0b83343edd7310f0ab4c8205a5e7c56406531f2e47e1b4878a3821d652be4192c841de5b032ca83619d8f860
+  languageName: node
+  linkType: hard
+
 "querystring-es3@npm:^0.2.1":
   version: 0.2.1
   resolution: "querystring-es3@npm:0.2.1"
@@ -21607,6 +21667,7 @@ __metadata:
     express: "npm:^4.18.2"
     firebase: "npm:^10.10.0"
     firebase-admin: "npm:^12.0.0"
+    googleapis: "npm:^144.0.0"
     helmet: "npm:^7.1.0"
     nodemailer: "npm:^6.9.14"
     nodemon: "npm:^3.1.0"
@@ -23939,6 +24000,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"url-template@npm:^2.0.8":
+  version: 2.0.8
+  resolution: "url-template@npm:2.0.8"
+  checksum: 10c0/56a15057eacbcf05d52b0caed8279c8451b3dd9d32856a1fdd91c6dc84dcb1646f12bafc756b7ade62ca5b1564da8efd7baac5add35868bafb43eb024c62805b
+  languageName: node
+  linkType: hard
+
 "url@npm:^0.11.0":
   version: 0.11.3
   resolution: "url@npm:0.11.3"