From 54e8461289353bfdd79cd6acd63428dbbf739407 Mon Sep 17 00:00:00 2001 From: Rob Brackett Date: Fri, 23 Aug 2024 13:33:51 -0700 Subject: [PATCH] Update Google auth libraries and code (#71) Since these scripts were last updated, Google has deprecated some of the tools we used and made major version upgrades to others. This updates things to use currently supported approaches. This also updates the `.youtube-upload-credentials.json.enc` authorization file. --- .circleci/config.yml | 33 +++++++------ .editorconfig | 14 ++++++ .github/workflows/zoom-upload.yml | 7 ++- .youtube-upload-credentials.json.enc | Bin 1360 -> 784 bytes requirements.txt | 10 ++-- scripts/auth.py | 30 ++++-------- scripts/lib/youtube.py | 19 ++++++-- scripts/upload_zoom_recordings.py | 67 +++++++++++++++------------ 8 files changed, 106 insertions(+), 74 deletions(-) create mode 100644 .editorconfig diff --git a/.circleci/config.yml b/.circleci/config.yml index 073255b..111900d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,19 +1,24 @@ -version: 2 +version: 2.1 jobs: - build: + upload-zoom-recordings: + parameters: + delete_after_upload: + description: 'Delete Zoom recordings after uploading to YouTube' + type: boolean + default: true + dry_run: + description: 'Dry run: report planned work, but do not actually upload or change anything' + type: boolean + default: false docker: - - image: cimg/python:3.10 + - image: cimg/python:3.12 environment: - - EDGI_ZOOM_DELETE_AFTER_UPLOAD: '1' + - EDGI_ZOOM_DELETE_AFTER_UPLOAD: << parameters.delete_after_upload >> + - EDGI_DRY_RUN: << parameters.dry_run >> steps: - checkout - - run: - name: skip destructive action on non-main - command: | - echo '[[ "main" != "$CIRCLE_BRANCH" ]] && unset EDGI_ZOOM_DELETE_AFTER_UPLOAD' >> $BASH_ENV - - run: name: decrypt files command: | @@ -38,7 +43,7 @@ jobs: paths: - ~/.cache/pip key: v1-pip-{{ checksum "requirements.txt" }} - + - run: name: run script command: | @@ -46,13 +51,15 @@ jobs: python3 scripts/upload_zoom_recordings.py workflows: - version: 2 commit: jobs: - - build + - upload-zoom-recordings: + delete_after_upload: false + dry_run: true youtube-upload: jobs: - - build + - upload-zoom-recordings: + delete_after_upload: true triggers: - schedule: # Every hour in the afternoon at :10 on weekdays diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..918045a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +max_line_length = 140 + +[*.rst] +indent_size = 2 + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.github/workflows/zoom-upload.yml b/.github/workflows/zoom-upload.yml index 3857c85..64637b7 100644 --- a/.github/workflows/zoom-upload.yml +++ b/.github/workflows/zoom-upload.yml @@ -13,6 +13,10 @@ on: description: 'Delete recordings from Zoom after uploading to YouTube' type: boolean default: true + dry_run: + description: 'Dry run: report planned work, but do not actually upload or change anything' + type: boolean + default: false pull_request: {} @@ -32,7 +36,7 @@ jobs: - name: Install Dependencies run: pip install -r requirements.txt - + - name: Decrypt Credential Files env: EDGI_ZOOM_API_SECRET: ${{ secrets.EDGI_ZOOM_API_SECRET }} @@ -43,6 +47,7 @@ jobs: - name: Upload env: EDGI_ZOOM_DELETE_AFTER_UPLOAD: ${{ github.event_name == 'schedule' || inputs.delete_after_upload }} + EDGI_DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run }} DEFAULT_YOUTUBE_PLAYLIST: ${{ secrets.DEFAULT_YOUTUBE_PLAYLIST }} EDGI_ZOOM_ACCOUNT_ID: ${{ secrets.EDGI_ZOOM_ACCOUNT_ID }} EDGI_ZOOM_CLIENT_ID: ${{ secrets.EDGI_ZOOM_CLIENT_ID }} diff --git a/.youtube-upload-credentials.json.enc b/.youtube-upload-credentials.json.enc index 55612ff5d7fdfba573a158517bbef4f7a0be3c2e..9cd495a8689dc543dd7bd53863d1e459dbd5a1d6 100644 GIT binary patch literal 784 zcmV+r1MmD(VQh3|WM5y8>}kmSHZEfn?-o^ewx3VM{1gUxD&gfY;K!X(i?8y_nR^^+ z*ZdCP$}A1Camvb~s_rurb$&@iM+2 zoC_PQ=x1`qm8aAMtEVQMjY_tp>P0{u-|VjA{%W6L^; zQ2{I72R3Oey!s?G#yk|xtTbZ3G(sYEmKm6vxvFlO$P1$2<~#zP>jH|-RX=;n$-H}N zT;>QVr?Q=-Qo&LrW7#L@X4%D5%SVr1vSDXVh>5bk#4FB`zCkDF(%@Rdy;xmnkYxPf z2PG$dRU>9nabaSXs5X)sBl=45 ztAM|;;=*Tz1gSv9Kksv}UsLX@nEEbw=uBhvKHN8@k(fA9p`<2UtK7tQbeDTYjJ{vUzIKwrQSJSiJevf%s_}G7-S~ymQAPJsu^oxgXlvxstzx1}KyjQ?G zSsVdS+FNdH%?_CjjlA>npa470qWaT;{hH8U2oF$SfrN_TrCe&u)!=xeCS)<>hX**x zlPf!JRCLfvwFd_tG+9O7XXBbx0OhEhX}#;eDobwq8+$@jKtG}1@r+O7a)d!c?wq69 zKw1YVegpcb<~S6ZW0{?RXOD&7Fv_TK&IOOq&^NkKjWmF4t68tv*v%T0Sa}caDTs^I zNHZ6KOhNN5#`PW!wqWahxrFGPRTEJ7`OC#f_tPFoC54Q9Kz_7~VZ3GNwvbO{$`z~E zHAfs}ZLUM1L)G_zZ@wH8aJ(p)4J!Kx2Oxkt3XsSy1%}NC)(9XjV=NR^xH^)DQ}w2K zY#zXX1fXyI;y%nX3|?@RvZ@^ceuPzkLW)$85s OAQbSA61*}ahW$|7ErcZi literal 1360 zcmV-W1+V&3VQh3|WM5z6YD+9IrqAD#;4$#5*S|uYBxA%KYprstRx#yN3L{*i3IL6K zmbTtnOG`C)R$!=*${gi?y+EYC9}JsP--@4V!w<}%n2iWt0}xNR_TxL0e|4P}It4~* zjHwUobGa6tX=NL_yqap88crL5DEd1eH_*qT463bnYqTJc?&c~_I=LICpW9%8APHIR zVk_C~dU8Z;138J7T=PKonv$)1Ms3L?JY;QGagwuA?;ykD;(M^rl;Cqy1VY`rswH|n%Czi~*zuvZ$ac{HQ>-!pW9KiF01h0yzlq%7IYq$W7WH2e~x zskypA`?2`4;k0l-TB!ywBFN%Mxy@LG+|oj_A{xP?6F?E$V#viQ?}CqKPvlw4Q= z7E+}pi&Gs6=nzodL{GU4-`ttI?FwSCa3%)*Z(o@qlBEM33@SFYFwa7^vUP{lx@pj| zRguz|v2a4VB~j%>oTE8K>|4+ZK66$`oK1{QxtiPGNFfi4HQx1fa~oC?87BX9i2t!l zZY3R|viCZoXS1kjgFpa{jOSWI6)0_L(GA1=(IUQ+^8L_Q^Q&IVQQ2sxn{LG?cx}qv^6APjDMBzGIGTO0Cr5P{SzNqo6;tea?*$ukq6twNC06~*ZK&#Ld zo|F=DkSWr?+k{1yIl>19haen}%T?}n{u6fgu3D7fetET}BNEkm?U{zs`fnZ;V0bM; zm}Y>Euu^Q7HoPzwa^3|IiedFsi4X(>W^$<~g6m6DfXo_%|HWA@%xucr?uUzsxtdc$ z_}-naiUp?Z)F-wdFj=fKA=)y<-pKOr+YGt|tF|(*qbqya^S^#Zu%^s&@*kV_xWfrfg`?e@X z&4$XR!`kNY&2aIu1QYHtAQ@T*9oaEHsW~-%|VMK|Ru_%7xK$kvAE5d~IJ}bIVXMZr8J=8(?TO4Q+6>y+nbBTRH SBzkqPtC5?H#KY1a)tDZ7#H1 bool: + """ + Make a basic API request to validate the given credentials work. Returns a + boolean indicating whether credentials are valid. + """ + try: + request = youtube.playlists().list(part='id,contentDetails', mine=True) + request.execute() + return True + except GoogleAuthError: + return False + + def upload_video(youtube, file, title='Test Title', description=None, category=None, tags=None, privacy_status='private', recording_date=None, license=None): @@ -116,7 +127,7 @@ def upload_video(youtube, file, title='Test Title', description=None, if license: body['status']['license'] = license - # Call the API's videos.insert method to create and upload the video. + # Call the API's videos.insert method to create and upload the video. insert_request = youtube.videos().insert( part=','.join(body.keys()), body=body, diff --git a/scripts/upload_zoom_recordings.py b/scripts/upload_zoom_recordings.py index a5fd46a..565c8e5 100644 --- a/scripts/upload_zoom_recordings.py +++ b/scripts/upload_zoom_recordings.py @@ -26,19 +26,15 @@ # See README for how to generate this files. from datetime import datetime -import functools -import json import os import re import requests -from subprocess import check_output, CalledProcessError, PIPE import sys import tempfile from urllib.parse import urlparse from zoomus import ZoomClient from lib.constants import USER_TYPES, VIDEO_CATEGORY_IDS -from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist -from types import SimpleNamespace +from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist, validate_youtube_credentials YOUTUBE_CREDENTIALS_PATH = '.youtube-upload-credentials.json' ZOOM_CLIENT_ID = os.environ['EDGI_ZOOM_CLIENT_ID'] @@ -47,6 +43,7 @@ def is_truthy(x): return x.lower() in ['true', '1', 'y', 'yes'] ZOOM_DELETE_AFTER_UPLOAD = is_truthy(os.environ.get('EDGI_ZOOM_DELETE_AFTER_UPLOAD', '')) +DRY_RUN = is_truthy(os.environ.get('EDGI_DRY_RUN', '')) MEETINGS_TO_RECORD = ['EDGI Community Standup'] DEFAULT_YOUTUBE_PLAYLIST = 'Uploads from Zoom' @@ -87,7 +84,15 @@ def download_file(url, download_path, query=None): return filepath def main(): + if DRY_RUN: + print('⚠️ This is a dry run! Videos will not actually be uploaded.\n') + youtube = get_youtube_client(YOUTUBE_CREDENTIALS_PATH) + if not validate_youtube_credentials(youtube): + print(f'The credentials in {YOUTUBE_CREDENTIALS_PATH} were not valid!') + print('Please use `python scripts/auth.py` to re-authorize.') + return sys.exit(1) + with tempfile.TemporaryDirectory() as tmpdirname: print('Creating tmp dir: ' + tmpdirname) meetings = client.recording.list(user_id=user_id).json()['meetings'] @@ -100,10 +105,10 @@ def main(): if meeting['topic'] not in MEETINGS_TO_RECORD and DO_FILTER: print(' Skipping...') continue - + videos = [file for file in meeting['recording_files'] - if file['file_type'].lower() == 'mp4'] - + if file['file_type'].lower() == 'mp4'] + if len(videos) == 0: print(f' No videos to upload: {meeting["topic"]}') continue @@ -120,26 +125,27 @@ def main(): # `config.get()` so we get an exception if things have changed and # this data is no longer available. filepath = download_file(url, - tmpdirname, - query={"access_token": client.config["token"]}) + tmpdirname, + query={"access_token": client.config["token"]}) + + recording_date = fix_date(meeting['start_time']) title = f'{meeting["topic"]} - {pretty_date(meeting["start_time"])}' - # These characters don't work within Python subprocess commands - chars_to_strip = '<>' - title = re.sub('['+chars_to_strip+']', '', title) - - video_id = upload_video(youtube, - filepath, - title=title, - category=VIDEO_CATEGORY_IDS["Science & Technology"], - license=DEFAULT_VIDEO_LICENSE, - recording_date=fix_date(meeting['start_time']), - privacy_status='unlisted') - + print(f' Uploading {filepath}\n {title=}\n {recording_date=}') + if not DRY_RUN: + video_id = upload_video(youtube, + filepath, + title=title, + category=VIDEO_CATEGORY_IDS["Science & Technology"], + license=DEFAULT_VIDEO_LICENSE, + recording_date=recording_date, + privacy_status='unlisted') + # Add all videos to default playlist print(' Adding to main playlist: Uploads from Zoom') - add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted') - + if not DRY_RUN: + add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted') + # Add to additional playlists playlist_name = '' if any(x in meeting['topic'].lower() for x in ['web mon', 'website monitoring', 'wm']): @@ -150,7 +156,7 @@ def main(): if 'community call' in meeting['topic'].lower(): playlist_name = 'Community Calls' - + if 'edgi introductions' in meeting['topic'].lower(): playlist_name = 'EDGI Introductions' @@ -159,14 +165,15 @@ def main(): if playlist_name: print(f' Adding to call playlist: {playlist_name}') - add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted') + if not DRY_RUN: + add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted') - if ZOOM_DELETE_AFTER_UPLOAD: + if ZOOM_DELETE_AFTER_UPLOAD and not DRY_RUN: # Just delete the video for now, since that takes the most storage space. # We should save the chat log transcript in a comment on the video. - + # We're using the zoom api directly instead of zoomus, because zoomus only implements - # deleting all recorded files related to the meeting using the v2 API, + # deleting all recorded files related to the meeting using the v2 API, # while we still want to retain the audio and chat files for backup. url = f'https://api.zoom.us/v2/meetings/{file["meeting_id"]}/recordings/{file["id"]}' querystring = {"action":"trash"} @@ -176,7 +183,7 @@ def main(): print(f' Deleted {file["file_type"]} file from Zoom for recording: {meeting["topic"]}') else: print(f' The file could not be deleted. We received this response: {response.status_code}. Please check https://marketplace.zoom.us/docs/api-reference/zoom-api/cloud-recording/recordingdeleteone for what that could mean.') - + if __name__ == '__main__': main()