diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b18cf91..2f4ed250 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Fixed +### Changed + +- Differentiate user uploaded shorts, lives & long videos (#367) + +### Fixed +- Corrected the short video resolution in the UI (#366) - Check for empty playlists after filtering, and after downloading videos (#375) ## [3.2.1] - 2024-11-01 diff --git a/scraper/src/youtube2zim/playlists/scraper.py b/scraper/src/youtube2zim/playlists/scraper.py index 1be64081..c3301bd6 100644 --- a/scraper/src/youtube2zim/playlists/scraper.py +++ b/scraper/src/youtube2zim/playlists/scraper.py @@ -91,7 +91,9 @@ def run(self): ( playlists, main_channel_id, - uploads_playlist_id, + user_long_uploads_playlist_id, + user_short_uploads_playlist_id, + user_lives_playlist_id, is_playlist, ) = extract_playlists_details_from(self.youtube_id) @@ -106,10 +108,6 @@ def run(self): shutil.rmtree(self.build_dir, ignore_errors=True) for playlist in playlists: - if playlist.playlist_id == uploads_playlist_id: - logger.info(f"Skipping playlist {playlist.playlist_id} (uploads one)") - continue - logger.info(f"Executing youtube2zim for playlist {playlist.playlist_id}") success, process = self.run_playlist_zim(playlist) if success: diff --git a/scraper/src/youtube2zim/schemas.py b/scraper/src/youtube2zim/schemas.py index 4748eb3c..edba89c0 100644 --- a/scraper/src/youtube2zim/schemas.py +++ b/scraper/src/youtube2zim/schemas.py @@ -105,7 +105,10 @@ class Channel(CamelModel): profile_path: str | None = None banner_path: str | None = None joined_date: str - main_playlist: str | None = None + first_playlist: str | None = None + user_long_uploads_playlist: str | None = None + user_short_uploads_playlist: str | None = None + user_lives_playlist: str | None = None playlist_count: int diff --git a/scraper/src/youtube2zim/scraper.py b/scraper/src/youtube2zim/scraper.py index 68678032..95fa75d4 100644 --- a/scraper/src/youtube2zim/scraper.py +++ b/scraper/src/youtube2zim/scraper.py @@ -170,7 +170,9 @@ def __init__( # process-related self.playlists = [] - self.uploads_playlist_id = None + self.user_long_uploads_playlist_id = None + self.user_short_uploads_playlist_id = None + self.user_lives_playlist_id = None self.videos_ids = [] self.video_ids_count = 0 self.videos_processed = 0 @@ -229,30 +231,6 @@ def banner_path(self): def is_single_channel(self): return len({pl.creator_id for pl in self.playlists}) == 1 - @property - def sorted_playlists(self): - """sorted list of playlists (by title) but with Uploads one at first if any""" - if len(self.playlists) <= 1: - return self.playlists - - sorted_playlists = sorted(self.playlists, key=lambda x: x.title) - index = 0 - # make sure our Uploads, special playlist is first - if self.uploads_playlist_id: - try: - index = [ - index - for index, p in enumerate(sorted_playlists) - if p.playlist_id == self.uploads_playlist_id - ][-1] - except Exception: - index = 0 - return ( - [sorted_playlists[index]] - + sorted_playlists[0:index] - + sorted_playlists[index + 1 :] - ) - def run(self): """execute the scraper step by step""" @@ -552,7 +530,9 @@ def extract_playlists(self): ( self.playlists, self.main_channel_id, - self.uploads_playlist_id, + self.user_long_uploads_playlist_id, + self.user_short_uploads_playlist_id, + self.user_lives_playlist_id, self.is_playlist, ) = extract_playlists_details_from(self.youtube_id) @@ -1045,6 +1025,7 @@ def generate_video_object(video) -> Video: author = videos_channels[video_id] subtitles_list = get_subtitles(video_id) channel_data = get_channel_json(author["channelId"]) + return Video( id=video_id, title=video["snippet"]["title"], @@ -1151,10 +1132,13 @@ def get_playlist_slug(playlist) -> str: ) # write playlists JSON files - playlist_list = [] - home_playlist_list = [] + playlist_list: list[PlaylistPreview] = [] + home_playlist_list: list[Playlist] = [] + + user_long_uploads_playlist_slug = None + user_short_uploads_playlist_slug = None + user_lives_playlist_slug = None - main_playlist_slug = None empty_playlists = list( filter(lambda playlist: len(get_videos_list(playlist)) == 0, self.playlists) ) @@ -1167,10 +1151,6 @@ def get_playlist_slug(playlist) -> str: if len(self.playlists) == 0: raise Exception("No playlist succeeded to download") - main_playlist_slug = get_playlist_slug( - self.playlists[0] - ) # set first playlist as main playlist - for playlist in self.playlists: playlist_slug = get_playlist_slug(playlist) playlist_path = f"playlists/{playlist_slug}.json" @@ -1195,16 +1175,15 @@ def get_playlist_slug(playlist) -> str: # modify playlist object for preview on homepage playlist_obj.videos = playlist_obj.videos[:12] - if playlist.playlist_id == self.uploads_playlist_id: - main_playlist_slug = ( - playlist_slug # set uploads playlist as main playlist - ) - # insert uploads playlist at the beginning of the list - playlist_list.insert(0, generate_playlist_preview_object(playlist)) - home_playlist_list.insert(0, playlist_obj) + home_playlist_list.append(playlist_obj) + if playlist.playlist_id == self.user_long_uploads_playlist_id: + user_long_uploads_playlist_slug = playlist_slug + elif playlist.playlist_id == self.user_short_uploads_playlist_id: + user_short_uploads_playlist_slug = playlist_slug + elif playlist.playlist_id == self.user_lives_playlist_id: + user_lives_playlist_slug = playlist_slug else: playlist_list.append(generate_playlist_preview_object(playlist)) - home_playlist_list.append(playlist_obj) # write playlists.json file self.zim_file.add_item_for( @@ -1241,7 +1220,10 @@ def get_playlist_slug(playlist) -> str: channel_description=channel_data["snippet"]["description"], profile_path="profile.jpg", banner_path="banner.jpg", - main_playlist=main_playlist_slug, + first_playlist=home_playlist_list[0].slug, + user_long_uploads_playlist=user_long_uploads_playlist_slug, + user_short_uploads_playlist=user_short_uploads_playlist_slug, + user_lives_playlist=user_lives_playlist_slug, playlist_count=len(self.playlists), joined_date=channel_data["snippet"]["publishedAt"], ).model_dump_json(by_alias=True, indent=2), diff --git a/scraper/src/youtube2zim/youtube.py b/scraper/src/youtube2zim/youtube.py index fd565f5b..9f7ede30 100644 --- a/scraper/src/youtube2zim/youtube.py +++ b/scraper/src/youtube2zim/youtube.py @@ -56,6 +56,10 @@ def __init__( @classmethod def from_id(cls, playlist_id): playlist_json = get_playlist_json(playlist_id) + if playlist_json is None: + raise PlaylistNotFoundError( + f"Invalid playlistId `{playlist_id}`: Not Found" + ) return Playlist( playlist_id=playlist_id, title=playlist_json["snippet"]["title"], @@ -176,10 +180,13 @@ def get_playlist_json(playlist_id): req.raise_for_status() try: playlist_json = req.json()["items"][0] + total_results = req.json().get("pageInfo", {}).get("totalResults", 0) + if total_results == 0: + logger.error(f"Playlist `{playlist_id}`: No Item Available") + return None except IndexError: - raise PlaylistNotFoundError( - f"Invalid playlistId `{playlist_id}`: Not Found" - ) from None + logger.error(f"Invalid playlistId `{playlist_id}`: Not Found") + return None save_json(YOUTUBE.cache_dir, fname, playlist_json) return playlist_json @@ -336,8 +343,9 @@ def skip_outofrange_videos(date_range, item): def extract_playlists_details_from(youtube_id: str): """prepare a list of Playlist from user request""" - uploads_playlist_id = None - main_channel_id = None + main_channel_id = user_long_uploads_playlist_id = user_short_uploads_playlist_id = ( + user_lives_playlist_id + ) = None if "," not in youtube_id: try: # first try to consider passed ID is a channel ID (or username or handle) @@ -347,11 +355,36 @@ def extract_playlists_details_from(youtube_id: str): playlist_ids = [ p["id"] for p in get_channel_playlists_json(main_channel_id) ] - # we always include uploads playlist (contains everything) - playlist_ids += [ - channel_json["contentDetails"]["relatedPlaylists"]["uploads"] - ] - uploads_playlist_id = playlist_ids[-1] + + # Get special playlists JSON objects + user_long_uploads_json = get_playlist_json("UULF" + main_channel_id[2:]) + user_short_uploads_json = get_playlist_json("UUSH" + main_channel_id[2:]) + user_lives_json = get_playlist_json("UULV" + main_channel_id[2:]) + + # Extract special playlists IDs if the JSON objects are not None + user_long_uploads_playlist_id = ( + user_long_uploads_json["id"] if user_long_uploads_json else None + ) + user_short_uploads_playlist_id = ( + user_short_uploads_json["id"] if user_short_uploads_json else None + ) + user_lives_playlist_id = user_lives_json["id"] if user_lives_json else None + + # Add special playlists if they exists, in proper order + playlist_ids = ( + list( + filter( + None, + [ + user_long_uploads_playlist_id, + user_short_uploads_playlist_id, + user_lives_playlist_id, + ], + ) + ) + + playlist_ids + ) + is_playlist = False except ChannelNotFoundError: # channel not found, then ID should be a playlist @@ -370,6 +403,8 @@ def extract_playlists_details_from(youtube_id: str): # dict.fromkeys maintains the order of playlist_ids while removing duplicates [Playlist.from_id(playlist_id) for playlist_id in dict.fromkeys(playlist_ids)], main_channel_id, - uploads_playlist_id, + user_long_uploads_playlist_id, + user_short_uploads_playlist_id, + user_lives_playlist_id, is_playlist, ) diff --git a/scraper/tests-integration/integration.py b/scraper/tests-integration/integration.py index 8241a565..d48be851 100644 --- a/scraper/tests-integration/integration.py +++ b/scraper/tests-integration/integration.py @@ -47,7 +47,7 @@ def test_zim_channel_json(): assert channel_json["id"] == "UC8elThf5TGMpQfQc_VE917Q" assert channel_json["channelName"] == "openZIM_testing" - assert channel_json["mainPlaylist"] == "uploads_from_openzim_testing-917Q" + assert channel_json["firstPlaylist"] == "uploads_from_openzim_testing-917Q" def test_zim_videos(): diff --git a/zimui/cypress/fixtures/channel/channel.json b/zimui/cypress/fixtures/channel/channel.json index 19696e8b..eba49a46 100644 --- a/zimui/cypress/fixtures/channel/channel.json +++ b/zimui/cypress/fixtures/channel/channel.json @@ -7,5 +7,5 @@ "profilePath": "profile.jpg", "bannerPath": "banner.jpg", "joinedDate": "2024-06-04T13:30:16.232286Z", - "mainPlaylist": "uploads_from_openzim_testing-917Q" + "firstPlaylist": "uploads_from_openzim_testing-917Q" } diff --git a/zimui/src/assets/main.css b/zimui/src/assets/main.css index 251eb294..b73d2399 100644 --- a/zimui/src/assets/main.css +++ b/zimui/src/assets/main.css @@ -6,6 +6,7 @@ html { overflow: auto !important; font-family: 'Roboto', sans-serif; + overflow-y: scroll !important; } body { diff --git a/zimui/src/assets/vjs-youtube.css b/zimui/src/assets/vjs-youtube.css index dce27734..722df671 100644 --- a/zimui/src/assets/vjs-youtube.css +++ b/zimui/src/assets/vjs-youtube.css @@ -54,3 +54,38 @@ .vjs-youtube .vjs-tech canvas { border-radius: 8px; } + + +.video-js.vjs-fluid, +.video-js.vjs-16-9, +.video-js.vjs-4-3, +video.video-js, +video.vjs-tech { + max-height: calc(100vh - 64px); + position: relative !important; + width: 100%; + height: auto !important; + max-width: 100% !important; + padding-top: 0 !important; + line-height: 0; +} +.vjs-control-bar { + line-height: 1; +} + +/* Fullscreen styles */ +.video-js.vjs-fullscreen { + display: flex; + align-items: center; + justify-content: center; + background-color: black; + text-align: center; +} + +.video-js.vjs-fullscreen video { + margin: auto; + width: auto !important; + height: 100% !important; + max-height: 100vh; + object-fit: contain; +} \ No newline at end of file diff --git a/zimui/src/components/channel/ChannelHeader.vue b/zimui/src/components/channel/ChannelHeader.vue index 4194b7a4..ba8d03ec 100644 --- a/zimui/src/components/channel/ChannelHeader.vue +++ b/zimui/src/components/channel/ChannelHeader.vue @@ -20,23 +20,34 @@ onMounted(async () => { } }) -const tabs = [ - { - id: 0, - title: 'Videos', - to: { name: 'videos' } - }, - { - id: 1, - title: 'Playlists', - to: { name: 'playlists' } +// Computed tabs array based on store data +const tabs = computed(() => { + const baseTabs = [ + { id: 0, title: 'Home', to: { name: 'home' } } + ]; + + if (main.channel?.userLongUploadsPlaylist) { + baseTabs.push({ id: 1, title: 'Videos', to: { name: 'videos' } }); + } + + if (main.channel?.userShortUploadsPlaylist) { + baseTabs.push({ id: 2, title: 'Shorts', to: { name: 'shorts' } }); + } + + if (main.channel?.userLivesPlaylist) { + baseTabs.push({ id: 3, title: 'Lives', to: { name: 'lives' } }); } -] + + baseTabs.push({ id: 4, title: 'Playlists', to: { name: 'playlists' } }); + + return baseTabs; +}); + // Hide tabs if there is only one playlist const hideTabs = computed(() => main.channel?.playlistCount === 1) -const tab = ref(tabs[0].id) +const tab = ref(tabs.value[0]?.id || 0); diff --git a/zimui/src/components/channel/tabs/VideosListTab.vue b/zimui/src/components/channel/tabs/ChannelHomeListTab.vue similarity index 93% rename from zimui/src/components/channel/tabs/VideosListTab.vue rename to zimui/src/components/channel/tabs/ChannelHomeListTab.vue index b6078cf0..423cd580 100644 --- a/zimui/src/components/channel/tabs/VideosListTab.vue +++ b/zimui/src/components/channel/tabs/ChannelHomeListTab.vue @@ -12,7 +12,7 @@ const isLoading = ref(true) // Watch for changes in the main playlist watch( - () => main.channel?.mainPlaylist, + () => main.channel?.id, () => { fetchData() } @@ -20,7 +20,7 @@ watch( // Fetch the videos for the main playlist const fetchData = async function () { - if (main.channel?.mainPlaylist) { + if (main.channel?.id) { try { const resp = await main.fetchHomePlaylists() if (resp) { diff --git a/zimui/src/components/channel/tabs/ChannelHomeTab.vue b/zimui/src/components/channel/tabs/ChannelHomeTab.vue new file mode 100644 index 00000000..f548874f --- /dev/null +++ b/zimui/src/components/channel/tabs/ChannelHomeTab.vue @@ -0,0 +1,19 @@ + + + diff --git a/zimui/src/components/channel/tabs/GenericTab.vue b/zimui/src/components/channel/tabs/GenericTab.vue new file mode 100755 index 00000000..26ba5c14 --- /dev/null +++ b/zimui/src/components/channel/tabs/GenericTab.vue @@ -0,0 +1,72 @@ + + + diff --git a/zimui/src/components/channel/tabs/LivesTab.vue b/zimui/src/components/channel/tabs/LivesTab.vue new file mode 100644 index 00000000..aab3f8a1 --- /dev/null +++ b/zimui/src/components/channel/tabs/LivesTab.vue @@ -0,0 +1,7 @@ + + + diff --git a/zimui/src/components/channel/tabs/PlaylistsTab.vue b/zimui/src/components/channel/tabs/PlaylistsTab.vue index a06c7039..a97a602a 100644 --- a/zimui/src/components/channel/tabs/PlaylistsTab.vue +++ b/zimui/src/components/channel/tabs/PlaylistsTab.vue @@ -13,7 +13,7 @@ const isLoading = ref(true) // Watch for changes in the main playlist watch( - () => main.channel?.mainPlaylist, + () => main.channel?.id, () => { fetchData() } @@ -21,7 +21,7 @@ watch( // Fetch the playlists for the playlist tab const fetchData = async function () { - if (main.channel?.mainPlaylist) { + if (main.channel?.id) { try { const resp = await main.fetchPlaylists() if (resp) { diff --git a/zimui/src/components/channel/tabs/ShortsTab.vue b/zimui/src/components/channel/tabs/ShortsTab.vue new file mode 100644 index 00000000..d9bfa11d --- /dev/null +++ b/zimui/src/components/channel/tabs/ShortsTab.vue @@ -0,0 +1,7 @@ + + + diff --git a/zimui/src/components/channel/tabs/VideosTab.vue b/zimui/src/components/channel/tabs/VideosTab.vue index e3fc8c2e..82ff56f7 100644 --- a/zimui/src/components/channel/tabs/VideosTab.vue +++ b/zimui/src/components/channel/tabs/VideosTab.vue @@ -1,19 +1,7 @@ diff --git a/zimui/src/router/index.ts b/zimui/src/router/index.ts index 42f93300..4b1634a0 100644 --- a/zimui/src/router/index.ts +++ b/zimui/src/router/index.ts @@ -4,7 +4,10 @@ import HomeView from '../views/HomeView.vue' import VideoPlayerView from '../views/VideoPlayerView.vue' import NotFoundView from '../views/NotFoundView.vue' +import ChannelHomeTab from '@/components/channel/tabs/ChannelHomeTab.vue' import VideosTab from '@/components/channel/tabs/VideosTab.vue' +import ShortsTab from '@/components/channel/tabs/ShortsTab.vue' +import LivesTab from '@/components/channel/tabs/LivesTab.vue' import PlaylistsTab from '@/components/channel/tabs/PlaylistsTab.vue' import PlaylistView from '@/views/PlaylistView.vue' @@ -15,13 +18,28 @@ const router = createRouter({ path: '/', name: 'home', component: HomeView, - redirect: '/videos', + redirect: '/channel-home', children: [ + { + path: 'channel-home', + name: 'channel-home', + component: ChannelHomeTab + }, { path: 'videos', name: 'videos', component: VideosTab }, + { + path: 'shorts', + name: 'shorts', + component: ShortsTab + }, + { + path: 'lives', + name: 'lives', + component: LivesTab + }, { path: 'playlists', name: 'playlists', diff --git a/zimui/src/types/Channel.ts b/zimui/src/types/Channel.ts index 21039f27..f243a077 100644 --- a/zimui/src/types/Channel.ts +++ b/zimui/src/types/Channel.ts @@ -7,7 +7,10 @@ export interface Channel { profilePath?: string bannerPath?: string joinedDate: string - mainPlaylist?: string + firstPlaylist?: string + userLongUploadsPlaylist?: string + userShortUploadsPlaylist?: string + userLivesPlaylist?: string playlistCount: number }