diff --git a/.circleci/config.yml b/.circleci/config.yml index 073255b..111900d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,19 +1,24 @@ -version: 2 +version: 2.1 jobs: - build: + upload-zoom-recordings: + parameters: + delete_after_upload: + description: 'Delete Zoom recordings after uploading to YouTube' + type: boolean + default: true + dry_run: + description: 'Dry run: report planned work, but do not actually upload or change anything' + type: boolean + default: false docker: - - image: cimg/python:3.10 + - image: cimg/python:3.12 environment: - - EDGI_ZOOM_DELETE_AFTER_UPLOAD: '1' + - EDGI_ZOOM_DELETE_AFTER_UPLOAD: << parameters.delete_after_upload >> + - EDGI_DRY_RUN: << parameters.dry_run >> steps: - checkout - - run: - name: skip destructive action on non-main - command: | - echo '[[ "main" != "$CIRCLE_BRANCH" ]] && unset EDGI_ZOOM_DELETE_AFTER_UPLOAD' >> $BASH_ENV - - run: name: decrypt files command: | @@ -38,7 +43,7 @@ jobs: paths: - ~/.cache/pip key: v1-pip-{{ checksum "requirements.txt" }} - + - run: name: run script command: | @@ -46,13 +51,15 @@ jobs: python3 scripts/upload_zoom_recordings.py workflows: - version: 2 commit: jobs: - - build + - upload-zoom-recordings: + delete_after_upload: false + dry_run: true youtube-upload: jobs: - - build + - upload-zoom-recordings: + delete_after_upload: true triggers: - schedule: # Every hour in the afternoon at :10 on weekdays diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..918045a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.py] +max_line_length = 140 + +[*.rst] +indent_size = 2 + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.github/workflows/zoom-upload.yml b/.github/workflows/zoom-upload.yml index 3857c85..64637b7 100644 --- a/.github/workflows/zoom-upload.yml +++ b/.github/workflows/zoom-upload.yml @@ -13,6 +13,10 @@ on: description: 'Delete recordings from Zoom after uploading to YouTube' type: boolean default: true + dry_run: + description: 'Dry run: report planned work, but do not actually upload or change anything' + type: boolean + default: false pull_request: {} @@ -32,7 +36,7 @@ jobs: - name: Install Dependencies run: pip install -r requirements.txt - + - name: Decrypt Credential Files env: EDGI_ZOOM_API_SECRET: ${{ secrets.EDGI_ZOOM_API_SECRET }} @@ -43,6 +47,7 @@ jobs: - name: Upload env: EDGI_ZOOM_DELETE_AFTER_UPLOAD: ${{ github.event_name == 'schedule' || inputs.delete_after_upload }} + EDGI_DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run }} DEFAULT_YOUTUBE_PLAYLIST: ${{ secrets.DEFAULT_YOUTUBE_PLAYLIST }} EDGI_ZOOM_ACCOUNT_ID: ${{ secrets.EDGI_ZOOM_ACCOUNT_ID }} EDGI_ZOOM_CLIENT_ID: ${{ secrets.EDGI_ZOOM_CLIENT_ID }} diff --git a/.youtube-upload-credentials.json.enc b/.youtube-upload-credentials.json.enc index 55612ff..9cd495a 100644 Binary files a/.youtube-upload-credentials.json.enc and b/.youtube-upload-credentials.json.enc differ diff --git a/requirements.txt b/requirements.txt index 68653d2..36032ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,7 @@ requests zoomus click -oauth2client -google-api-python-client ~=1.7.11 -google-auth ~=1.11.2 -google-auth-oauthlib ~=0.4.1 -google-auth-httplib2 ~=0.0.3 -httplib2 ~=0.18.1 +google-api-python-client ~=2.141.0 +google-auth ~=2.34.0 +google-auth-oauthlib ~=1.2.1 +httplib2 ~=0.22.0 diff --git a/scripts/auth.py b/scripts/auth.py index 5e3c0e3..ba503e2 100644 --- a/scripts/auth.py +++ b/scripts/auth.py @@ -1,12 +1,4 @@ -from __future__ import print_function -import httplib2 -import os -import sys - -from apiclient import discovery -from oauth2client import client -from oauth2client import tools -from oauth2client.file import Storage +from google_auth_oauthlib.flow import InstalledAppFlow # If modifying these scopes, delete your previously saved credentials SCOPES = ['https://www.googleapis.com/auth/youtube.upload', 'https://www.googleapis.com/auth/youtube.force-ssl'] @@ -34,20 +26,18 @@ def get_credentials(): Returns: Credentials, the obtained credential. """ - home_dir = os.path.expanduser('~') - credential_path = CREDS_FILENAME - - store = Storage(credential_path) - credentials = store.get() - if not credentials or credentials.invalid: - flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES) - flow.user_agent = APPLICATION_NAME - credentials = tools.run_flow(flow, store) - return credentials + flow = InstalledAppFlow.from_client_secrets_file( + CLIENT_SECRET_FILE, + scopes=SCOPES + ) + flow.run_local_server() + return flow.credentials def main(): credentials = get_credentials() - http = credentials.authorize(httplib2.Http()) + with open(CREDS_FILENAME, 'w+') as file: + file.write(credentials.to_json()) + print(f'Wrote new credentials to {CREDS_FILENAME}.') if __name__ == '__main__': main() diff --git a/scripts/lib/youtube.py b/scripts/lib/youtube.py index e875230..6974fbb 100644 --- a/scripts/lib/youtube.py +++ b/scripts/lib/youtube.py @@ -11,10 +11,10 @@ import sys import google.oauth2.credentials -import google_auth_oauthlib.flow from googleapiclient.discovery import build, build_from_document from googleapiclient.errors import HttpError, UnknownApiNameOrVersion from googleapiclient.http import MediaFileUpload +from google.auth.exceptions import GoogleAuthError # Explicitly tell the underlying HTTP transport library not to retry, since @@ -57,8 +57,6 @@ def parse_youtube_http_error(error): Oddly, it doesn't match up with the format of YouTube errors here: https://developers.google.com/youtube/v3/docs/core_errors - - ¯\_(ツ)_/¯ """ try: data = json.loads(error.content.decode("utf-8"))["error"] @@ -90,6 +88,19 @@ def get_youtube_client(credentials_path): raise +def validate_youtube_credentials(youtube) -> bool: + """ + Make a basic API request to validate the given credentials work. Returns a + boolean indicating whether credentials are valid. + """ + try: + request = youtube.playlists().list(part='id,contentDetails', mine=True) + request.execute() + return True + except GoogleAuthError: + return False + + def upload_video(youtube, file, title='Test Title', description=None, category=None, tags=None, privacy_status='private', recording_date=None, license=None): @@ -116,7 +127,7 @@ def upload_video(youtube, file, title='Test Title', description=None, if license: body['status']['license'] = license - # Call the API's videos.insert method to create and upload the video. + # Call the API's videos.insert method to create and upload the video. insert_request = youtube.videos().insert( part=','.join(body.keys()), body=body, diff --git a/scripts/upload_zoom_recordings.py b/scripts/upload_zoom_recordings.py index a5fd46a..565c8e5 100644 --- a/scripts/upload_zoom_recordings.py +++ b/scripts/upload_zoom_recordings.py @@ -26,19 +26,15 @@ # See README for how to generate this files. from datetime import datetime -import functools -import json import os import re import requests -from subprocess import check_output, CalledProcessError, PIPE import sys import tempfile from urllib.parse import urlparse from zoomus import ZoomClient from lib.constants import USER_TYPES, VIDEO_CATEGORY_IDS -from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist -from types import SimpleNamespace +from lib.youtube import get_youtube_client, upload_video, add_video_to_playlist, validate_youtube_credentials YOUTUBE_CREDENTIALS_PATH = '.youtube-upload-credentials.json' ZOOM_CLIENT_ID = os.environ['EDGI_ZOOM_CLIENT_ID'] @@ -47,6 +43,7 @@ def is_truthy(x): return x.lower() in ['true', '1', 'y', 'yes'] ZOOM_DELETE_AFTER_UPLOAD = is_truthy(os.environ.get('EDGI_ZOOM_DELETE_AFTER_UPLOAD', '')) +DRY_RUN = is_truthy(os.environ.get('EDGI_DRY_RUN', '')) MEETINGS_TO_RECORD = ['EDGI Community Standup'] DEFAULT_YOUTUBE_PLAYLIST = 'Uploads from Zoom' @@ -87,7 +84,15 @@ def download_file(url, download_path, query=None): return filepath def main(): + if DRY_RUN: + print('⚠️ This is a dry run! Videos will not actually be uploaded.\n') + youtube = get_youtube_client(YOUTUBE_CREDENTIALS_PATH) + if not validate_youtube_credentials(youtube): + print(f'The credentials in {YOUTUBE_CREDENTIALS_PATH} were not valid!') + print('Please use `python scripts/auth.py` to re-authorize.') + return sys.exit(1) + with tempfile.TemporaryDirectory() as tmpdirname: print('Creating tmp dir: ' + tmpdirname) meetings = client.recording.list(user_id=user_id).json()['meetings'] @@ -100,10 +105,10 @@ def main(): if meeting['topic'] not in MEETINGS_TO_RECORD and DO_FILTER: print(' Skipping...') continue - + videos = [file for file in meeting['recording_files'] - if file['file_type'].lower() == 'mp4'] - + if file['file_type'].lower() == 'mp4'] + if len(videos) == 0: print(f' No videos to upload: {meeting["topic"]}') continue @@ -120,26 +125,27 @@ def main(): # `config.get()` so we get an exception if things have changed and # this data is no longer available. filepath = download_file(url, - tmpdirname, - query={"access_token": client.config["token"]}) + tmpdirname, + query={"access_token": client.config["token"]}) + + recording_date = fix_date(meeting['start_time']) title = f'{meeting["topic"]} - {pretty_date(meeting["start_time"])}' - # These characters don't work within Python subprocess commands - chars_to_strip = '<>' - title = re.sub('['+chars_to_strip+']', '', title) - - video_id = upload_video(youtube, - filepath, - title=title, - category=VIDEO_CATEGORY_IDS["Science & Technology"], - license=DEFAULT_VIDEO_LICENSE, - recording_date=fix_date(meeting['start_time']), - privacy_status='unlisted') - + print(f' Uploading {filepath}\n {title=}\n {recording_date=}') + if not DRY_RUN: + video_id = upload_video(youtube, + filepath, + title=title, + category=VIDEO_CATEGORY_IDS["Science & Technology"], + license=DEFAULT_VIDEO_LICENSE, + recording_date=recording_date, + privacy_status='unlisted') + # Add all videos to default playlist print(' Adding to main playlist: Uploads from Zoom') - add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted') - + if not DRY_RUN: + add_video_to_playlist(youtube, video_id, title=DEFAULT_YOUTUBE_PLAYLIST, privacy='unlisted') + # Add to additional playlists playlist_name = '' if any(x in meeting['topic'].lower() for x in ['web mon', 'website monitoring', 'wm']): @@ -150,7 +156,7 @@ def main(): if 'community call' in meeting['topic'].lower(): playlist_name = 'Community Calls' - + if 'edgi introductions' in meeting['topic'].lower(): playlist_name = 'EDGI Introductions' @@ -159,14 +165,15 @@ def main(): if playlist_name: print(f' Adding to call playlist: {playlist_name}') - add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted') + if not DRY_RUN: + add_video_to_playlist(youtube, video_id, title=playlist_name, privacy='unlisted') - if ZOOM_DELETE_AFTER_UPLOAD: + if ZOOM_DELETE_AFTER_UPLOAD and not DRY_RUN: # Just delete the video for now, since that takes the most storage space. # We should save the chat log transcript in a comment on the video. - + # We're using the zoom api directly instead of zoomus, because zoomus only implements - # deleting all recorded files related to the meeting using the v2 API, + # deleting all recorded files related to the meeting using the v2 API, # while we still want to retain the audio and chat files for backup. url = f'https://api.zoom.us/v2/meetings/{file["meeting_id"]}/recordings/{file["id"]}' querystring = {"action":"trash"} @@ -176,7 +183,7 @@ def main(): print(f' Deleted {file["file_type"]} file from Zoom for recording: {meeting["topic"]}') else: print(f' The file could not be deleted. We received this response: {response.status_code}. Please check https://marketplace.zoom.us/docs/api-reference/zoom-api/cloud-recording/recordingdeleteone for what that could mean.') - + if __name__ == '__main__': main()