Skip to content

Commit

Permalink
Merge branch 'master' into sictiru
Browse files Browse the repository at this point in the history
* master: (97 commits)
  Cleanup postgres
  Adding support for hyphens in youtube usernames.
  New APNS updating cert instructions.
  Fiddling with metrics server.
  Handling broken youtube channel
  Youtube username/title
  Handling youtube usernames that are actually handles.
  Handling @handle youtube.com feeds when adding a feed.
  Users who are too far into paging now get a 404
  Updating youtube fetcher to use channels/playlists/users for everything, no longer relying on RSS/xml url.
  Updating certs.
  Handling too many feeds tasked.
  Bumping concurrency for feed fetchers.
  Icon importer also uses new consul domain.
  User not user agent
  Adding individual IP addresses from hetzner.
  Moving hetzner migration doc to archive
  Formatting
  Adding dwww
  Moving node consul
  ...
  • Loading branch information
samuelclay committed Apr 29, 2024
2 parents 904a7cb + 0d0231e commit 93d9612
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion utils/youtube_fetcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,8 @@ def fetch(self):
def extract_username(self, url):
if "gdata.youtube.com" in url:
try:
username_groups = re.search(r"gdata.youtube.com/feeds/\w+/users/(\w+)/", url)
# Also handle usernames like `user-name`
username_groups = re.search(r"gdata.youtube.com/feeds/\w+/users/([^/]+)/", url)
if not username_groups:
return
return username_groups.group(1)
Expand Down Expand Up @@ -160,6 +161,7 @@ def fetch_videos(self, video_ids):
return videos

def fetch_channel_videos(self, channel_id):
logging.debug(" ***> ~FBFetching YouTube channel: ~SB%s" % channel_id)
channel_json = requests.get(
"https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&id=%s&key=%s"
% (channel_id, settings.YOUTUBE_API_KEY)
Expand All @@ -176,6 +178,7 @@ def fetch_channel_videos(self, channel_id):
return self.fetch_playlist_videos(uploads_list_id, title, description)

def fetch_playlist_videos(self, list_id, title=None, description=None):
logging.debug(" ***> ~FBFetching YouTube playlist: ~SB%s" % list_id)
if not title and not description:
playlist_json = requests.get(
"https://www.googleapis.com/youtube/v3/playlists?part=snippet&id=%s&key=%s"
Expand All @@ -201,6 +204,7 @@ def fetch_playlist_videos(self, list_id, title=None, description=None):
return video_ids, title, description

def fetch_user_videos(self, username, username_key="forUsername"):
logging.debug(" ***> ~FBFetching YouTube user: ~SB%s" % username)
channel_json = requests.get(
"https://www.googleapis.com/youtube/v3/channels?part=snippet,contentDetails&%s=%s&key=%s"
% (username_key, username, settings.YOUTUBE_API_KEY)
Expand Down

0 comments on commit 93d9612

Please sign in to comment.