Update Google auth libraries and code (#71)

Since these scripts were last updated, Google has deprecated some of the tools we used and made major version upgrades to others. This updates things to use currently supported approaches.

This also updates the `.youtube-upload-credentials.json.enc` authorization file.

.circleci/config.yml

@@ -1,19 +1,24 @@
-version: 2
+version: 2.1
 
 jobs:
-  build:
+  upload-zoom-recordings:
+    parameters:
+      delete_after_upload:
+        description: 'Delete Zoom recordings after uploading to YouTube'
+        type: boolean
+        default: true
+      dry_run:
+        description: 'Dry run: report planned work, but do not actually upload or change anything'
+        type: boolean
+        default: false
     docker:
-      - image: cimg/python:3.10
+      - image: cimg/python:3.12
     environment:
-      - EDGI_ZOOM_DELETE_AFTER_UPLOAD: '1'
+      - EDGI_ZOOM_DELETE_AFTER_UPLOAD: << parameters.delete_after_upload >>
+      - EDGI_DRY_RUN: << parameters.dry_run >>
     steps:
       - checkout
 
-      - run:
-          name: skip destructive action on non-main
-          command: |
-            echo '[[ "main" != "$CIRCLE_BRANCH" ]] && unset EDGI_ZOOM_DELETE_AFTER_UPLOAD' >> $BASH_ENV
-
       - run:
           name: decrypt files
           command: |
@@ -38,21 +43,23 @@ jobs:
           paths:
             - ~/.cache/pip
           key: v1-pip-{{ checksum "requirements.txt" }}
-      
+
       - run:
           name: run script
           command: |
             . venv/bin/activate
             python3 scripts/upload_zoom_recordings.py
 
 workflows:
-  version: 2
   commit:
     jobs:
-      - build
+      - upload-zoom-recordings:
+          delete_after_upload: false
+          dry_run: true
   youtube-upload:
     jobs:
-      - build
+      - upload-zoom-recordings:
+          delete_after_upload: true
     triggers:
       - schedule:
           # Every hour in the afternoon at :10 on weekdays

.editorconfig

@@ -0,0 +1,14 @@
+[*]
+indent_style = space
+indent_size = 4
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.py]
+max_line_length = 140
+
+[*.rst]
+indent_size = 2
+
+[*.{yaml,yml}]
+indent_size = 2

.github/workflows/zoom-upload.yml

@@ -13,6 +13,10 @@ on:
         description: 'Delete recordings from Zoom after uploading to YouTube'
         type: boolean
         default: true
+      dry_run:
+        description: 'Dry run: report planned work, but do not actually upload or change anything'
+        type: boolean
+        default: false
 
   pull_request: {}
 
@@ -32,7 +36,7 @@ jobs:
 
       - name: Install Dependencies
         run: pip install -r requirements.txt
-      
+
       - name: Decrypt Credential Files
         env:
           EDGI_ZOOM_API_SECRET: ${{ secrets.EDGI_ZOOM_API_SECRET }}
@@ -43,6 +47,7 @@ jobs:
       - name: Upload
         env:
           EDGI_ZOOM_DELETE_AFTER_UPLOAD: ${{ github.event_name == 'schedule' || inputs.delete_after_upload }}
+          EDGI_DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run }}
           DEFAULT_YOUTUBE_PLAYLIST: ${{ secrets.DEFAULT_YOUTUBE_PLAYLIST }}
           EDGI_ZOOM_ACCOUNT_ID: ${{ secrets.EDGI_ZOOM_ACCOUNT_ID }}
           EDGI_ZOOM_CLIENT_ID: ${{ secrets.EDGI_ZOOM_CLIENT_ID }}

requirements.txt

@@ -1,9 +1,7 @@
 requests
 zoomus
 click
-oauth2client
-google-api-python-client ~=1.7.11
-google-auth ~=1.11.2
-google-auth-oauthlib ~=0.4.1
-google-auth-httplib2 ~=0.0.3
-httplib2 ~=0.18.1
+google-api-python-client ~=2.141.0
+google-auth ~=2.34.0
+google-auth-oauthlib ~=1.2.1
+httplib2 ~=0.22.0

scripts/auth.py

@@ -1,12 +1,4 @@
-from __future__ import print_function
-import httplib2
-import os
-import sys
-
-from apiclient import discovery
-from oauth2client import client
-from oauth2client import tools
-from oauth2client.file import Storage
+from google_auth_oauthlib.flow import InstalledAppFlow
 
 # If modifying these scopes, delete your previously saved credentials
 SCOPES = ['https://www.googleapis.com/auth/youtube.upload', 'https://www.googleapis.com/auth/youtube.force-ssl']
@@ -34,20 +26,18 @@ def get_credentials():
     Returns:
         Credentials, the obtained credential.
     """
-    home_dir = os.path.expanduser('~')
-    credential_path = CREDS_FILENAME
-
-    store = Storage(credential_path)
-    credentials = store.get()
-    if not credentials or credentials.invalid:
-        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
-        flow.user_agent = APPLICATION_NAME
-        credentials = tools.run_flow(flow, store)
-    return credentials
+    flow = InstalledAppFlow.from_client_secrets_file(
+        CLIENT_SECRET_FILE,
+        scopes=SCOPES
+    )
+    flow.run_local_server()
+    return flow.credentials
 
 def main():
     credentials = get_credentials()
-    http = credentials.authorize(httplib2.Http())
+    with open(CREDS_FILENAME, 'w+') as file:
+        file.write(credentials.to_json())
+        print(f'Wrote new credentials to {CREDS_FILENAME}.')
 
 if __name__ == '__main__':
     main()

scripts/lib/youtube.py

@@ -11,10 +11,10 @@
 import sys
 
 import google.oauth2.credentials
-import google_auth_oauthlib.flow
 from googleapiclient.discovery import build, build_from_document
 from googleapiclient.errors import HttpError, UnknownApiNameOrVersion
 from googleapiclient.http import MediaFileUpload
+from google.auth.exceptions import GoogleAuthError
 
 
 # Explicitly tell the underlying HTTP transport library not to retry, since
@@ -57,8 +57,6 @@ def parse_youtube_http_error(error):
     
     Oddly, it doesn't match up with the format of YouTube errors here:
     https://developers.google.com/youtube/v3/docs/core_errors
-
-    ¯\_(ツ)_/¯
     """
     try:
         data = json.loads(error.content.decode("utf-8"))["error"]
@@ -90,6 +88,19 @@ def get_youtube_client(credentials_path):
         raise
 
 
+def validate_youtube_credentials(youtube) -> bool:
+    """
+    Make a basic API request to validate the given credentials work. Returns a
+    boolean indicating whether credentials are valid.
+    """
+    try:
+        request = youtube.playlists().list(part='id,contentDetails', mine=True)
+        request.execute()
+        return True
+    except GoogleAuthError:
+        return False
+
+
 def upload_video(youtube, file, title='Test Title', description=None,
                  category=None, tags=None, privacy_status='private',
                  recording_date=None, license=None):
@@ -116,7 +127,7 @@ def upload_video(youtube, file, title='Test Title', description=None,
     if license:
         body['status']['license'] = license
 
-  # Call the API's videos.insert method to create and upload the video.
+    # Call the API's videos.insert method to create and upload the video.
     insert_request = youtube.videos().insert(
         part=','.join(body.keys()),
         body=body,

scripts/upload_zoom_recordings.py

@@ -26,19 +26,15 @@
 #     See README for how to generate this files.
 
 from datetime import datetime
-import functools
-import json
 import os
 import re
 import requests
-from subprocess import check_output, CalledProcessError, PIPE
 import sys
 import tempfile
 from urllib.parse import urlparse
 from zoomus import ZoomClient
 from lib.constants import USER_TYPES, VIDEO_CATEGORY_IDS
-from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist
-from types import SimpleNamespace
+from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist, validate_youtube_credentials
 
 YOUTUBE_CREDENTIALS_PATH = '.youtube-upload-credentials.json'
 ZOOM_CLIENT_ID = os.environ['EDGI_ZOOM_CLIENT_ID']
@@ -47,6 +43,7 @@
 
 def is_truthy(x): return x.lower() in ['true', '1', 'y', 'yes']
 ZOOM_DELETE_AFTER_UPLOAD = is_truthy(os.environ.get('EDGI_ZOOM_DELETE_AFTER_UPLOAD', ''))
+DRY_RUN = is_truthy(os.environ.get('EDGI_DRY_RUN', ''))
 
 MEETINGS_TO_RECORD = ['EDGI Community Standup']
 DEFAULT_YOUTUBE_PLAYLIST = 'Uploads from Zoom'
@@ -87,7 +84,15 @@ def download_file(url, download_path, query=None):
     return filepath
 
 def main():
+    if DRY_RUN:
+        print('⚠️ This is a dry run! Videos will not actually be uploaded.\n')
+
     youtube = get_youtube_client(YOUTUBE_CREDENTIALS_PATH)
+    if not validate_youtube_credentials(youtube):
+        print(f'The credentials in {YOUTUBE_CREDENTIALS_PATH} were not valid!')
+        print('Please use `python scripts/auth.py` to re-authorize.')
+        return sys.exit(1)
+
     with tempfile.TemporaryDirectory() as tmpdirname:
         print('Creating tmp dir: ' + tmpdirname)
         meetings = client.recording.list(user_id=user_id).json()['meetings']
@@ -100,10 +105,10 @@ def main():
             if meeting['topic'] not in MEETINGS_TO_RECORD and DO_FILTER:
                 print('  Skipping...')
                 continue
-            
+
             videos = [file for file in meeting['recording_files']
-                    if file['file_type'].lower() == 'mp4']
-            
+                      if file['file_type'].lower() == 'mp4']
+
             if len(videos) == 0:
                 print(f'  No videos to upload: {meeting["topic"]}')
                 continue
@@ -120,26 +125,27 @@ def main():
                 # `config.get()` so we get an exception if things have changed and
                 # this data is no longer available.
                 filepath = download_file(url,
-                                        tmpdirname,
-                                        query={"access_token": client.config["token"]})
+                                         tmpdirname,
+                                         query={"access_token": client.config["token"]})
+
+                recording_date = fix_date(meeting['start_time'])
                 title = f'{meeting["topic"]} - {pretty_date(meeting["start_time"])}'
 
-                # These characters don't work within Python subprocess commands
-                chars_to_strip = '<>'
-                title = re.sub('['+chars_to_strip+']', '', title)
-
-                video_id = upload_video(youtube,
-                                filepath,
-                                title=title,
-                                category=VIDEO_CATEGORY_IDS["Science & Technology"],
-                                license=DEFAULT_VIDEO_LICENSE,
-                                recording_date=fix_date(meeting['start_time']),
-                                privacy_status='unlisted')
-
+                print(f'  Uploading {filepath}\n    {title=}\n    {recording_date=}')
+                if not DRY_RUN:
+                    video_id = upload_video(youtube,
+                                            filepath,
+                                            title=title,
+                                            category=VIDEO_CATEGORY_IDS["Science & Technology"],
+                                            license=DEFAULT_VIDEO_LICENSE,
+                                            recording_date=recording_date,
+                                            privacy_status='unlisted')
+
                 # Add all videos to default playlist
                 print('  Adding to main playlist: Uploads from Zoom')
-                add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted')
-
+                if not DRY_RUN:
+                    add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted')
+
                 # Add to additional playlists
                 playlist_name = ''
                 if any(x in meeting['topic'].lower() for x in ['web mon', 'website monitoring', 'wm']):
@@ -150,7 +156,7 @@ def main():
 
                 if 'community call' in meeting['topic'].lower():
                     playlist_name = 'Community Calls'
-                
+
                 if 'edgi introductions' in meeting['topic'].lower():
                     playlist_name = 'EDGI Introductions'
 
@@ -159,14 +165,15 @@ def main():
 
                 if playlist_name:
                     print(f'  Adding to call playlist: {playlist_name}')
-                    add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted')
+                    if not DRY_RUN:
+                        add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted')
 
-                if ZOOM_DELETE_AFTER_UPLOAD:
+                if ZOOM_DELETE_AFTER_UPLOAD and not DRY_RUN:
                     # Just delete the video for now, since that takes the most storage space.
                     # We should save the chat log transcript in a comment on the video.
-                    
+
                     # We're using the zoom api directly instead of zoomus, because zoomus only implements
-                    # deleting all recorded files related to the meeting using the v2 API, 
+                    # deleting all recorded files related to the meeting using the v2 API,
                     # while we still want to retain the audio and chat files for backup.
                     url = f'https://api.zoom.us/v2/meetings/{file["meeting_id"]}/recordings/{file["id"]}'
                     querystring = {"action":"trash"}
@@ -176,7 +183,7 @@ def main():
                         print(f'  Deleted {file["file_type"]} file from Zoom for recording: {meeting["topic"]}')
                     else:
                         print(f'  The file could not be deleted. We received this response: {response.status_code}. Please check https://marketplace.zoom.us/docs/api-reference/zoom-api/cloud-recording/recordingdeleteone for what that could mean.')
-                    
+
 
 if __name__ == '__main__':
     main()