From b1eecd593ccfb98f3d0f6e00be21fca0e05a8449 Mon Sep 17 00:00:00 2001 From: rafaelgomesxyz Date: Mon, 3 May 2021 18:12:37 -0300 Subject: [PATCH] Begin implementing automatic sync --- src/api/TmdbApi.ts | 2 - src/api/WrongItemApi.ts | 2 - src/common/BrowserStorage.tsx | 3 + src/modules/background/background.ts | 22 ++++++- src/modules/options/OptionsApp.tsx | 5 +- .../components/StreamingServiceOption.tsx | 4 +- src/streaming-services/common/Api.ts | 4 +- src/streaming-services/common/AutoSync.ts | 66 +++++++++++++++++++ src/streaming-services/common/SyncPage.tsx | 26 ++++++-- src/streaming-services/hbo-go/HboGoApi.ts | 27 ++++++-- src/streaming-services/netflix/NetflixApi.ts | 24 +++++-- src/streaming-services/nrk/NrkApi.ts | 26 ++++++-- src/streaming-services/streaming-services.ts | 7 ++ src/streaming-services/viaplay/ViaplayApi.ts | 26 ++++++-- 14 files changed, 211 insertions(+), 33 deletions(-) create mode 100644 src/streaming-services/common/AutoSync.ts diff --git a/src/api/TmdbApi.ts b/src/api/TmdbApi.ts index f21099bd..93d3f4fd 100644 --- a/src/api/TmdbApi.ts +++ b/src/api/TmdbApi.ts @@ -5,7 +5,6 @@ import { RequestException, Requests } from '../common/Requests'; import { Item } from '../models/Item'; import { TraktItem } from '../models/TraktItem'; import { secrets } from '../secrets'; -import { getSyncStore } from '../streaming-services/common/common'; import { StreamingServiceId } from '../streaming-services/streaming-services'; export interface TmdbConfigResponse { @@ -205,7 +204,6 @@ class _TmdbApi { for (const item of missingItems) { item.imageUrl = item.imageUrl ?? this.PLACEHOLDER_IMAGE; } - await getSyncStore(serviceId).update(); }; loadItemImage = async (item: Item): Promise => { diff --git a/src/api/WrongItemApi.ts b/src/api/WrongItemApi.ts index 8010a666..f8dacb10 100644 --- a/src/api/WrongItemApi.ts +++ b/src/api/WrongItemApi.ts @@ -3,7 +3,6 @@ import { CacheValues } from '../common/Cache'; import { Messaging } from '../common/Messaging'; import { Requests } from '../common/Requests'; import { CorrectionSuggestion, Item } from '../models/Item'; -import { getSyncStore } from '../streaming-services/common/common'; import { StreamingServiceId } from '../streaming-services/streaming-services'; class _WrongItemApi { @@ -74,7 +73,6 @@ class _WrongItemApi { for (const item of missingItems) { item.correctionSuggestions = item.correctionSuggestions ?? null; } - await getSyncStore(serviceId).update(); }; loadItemSuggestions = async (item: Item): Promise => { diff --git a/src/common/BrowserStorage.tsx b/src/common/BrowserStorage.tsx index 42a0b6d6..f7b415ea 100644 --- a/src/common/BrowserStorage.tsx +++ b/src/common/BrowserStorage.tsx @@ -34,6 +34,7 @@ export type StreamingServiceValue = { autoSync: boolean; autoSyncDays: number; lastSync: number; + lastSyncId: string; }; export type ThemeValue = 'light' | 'dark' | 'system'; @@ -178,6 +179,7 @@ class _BrowserStorage { autoSync: false, autoSyncDays: 7, lastSync: 0, + lastSyncId: '', }, ]) ) as Record, @@ -275,6 +277,7 @@ class _BrowserStorage { autoSync: false, autoSyncDays: 7, lastSync: 0, + lastSyncId: '', }, ]) ) as Record; diff --git a/src/modules/background/background.ts b/src/modules/background/background.ts index 5af3b172..d97986bb 100644 --- a/src/modules/background/background.ts +++ b/src/modules/background/background.ts @@ -13,6 +13,7 @@ import { RequestDetails, Requests } from '../../common/Requests'; import { Shared } from '../../common/Shared'; import { Item } from '../../models/Item'; import { TraktItem } from '../../models/TraktItem'; +import { AutoSync } from '../../streaming-services/common/AutoSync'; import { StreamingServiceId, streamingServices } from '../../streaming-services/streaming-services'; export type MessageRequest = @@ -130,10 +131,27 @@ const init = async () => { browser.tabs.onRemoved.addListener((tabId) => void onTabRemoved(tabId)); browser.storage.onChanged.addListener(onStorageChanged); if (storage.options?.streamingServices) { - const scrobblerEnabled = (Object.entries(storage.options.streamingServices) as [ + const streamingServiceEntries = Object.entries(storage.options.streamingServices) as [ StreamingServiceId, StreamingServiceValue - ][]).some( + ][]; + + const now = Math.trunc(Date.now() / 1e3); + const servicesToSync = streamingServiceEntries.filter( + ([streamingServiceId, value]) => + streamingServices[streamingServiceId].hasSync && + streamingServices[streamingServiceId].hasAutoSync && + value.sync && + value.autoSync && + value.autoSyncDays > 0 && + value.lastSync > 0 && + now - value.lastSync >= value.autoSyncDays * 86400 + ); + if (servicesToSync.length > 0) { + void AutoSync.sync(servicesToSync, now); + } + + const scrobblerEnabled = streamingServiceEntries.some( ([streamingServiceId, value]) => streamingServices[streamingServiceId].hasScrobbler && value.scrobble ); diff --git a/src/modules/options/OptionsApp.tsx b/src/modules/options/OptionsApp.tsx index 5a1c2026..6a0fb579 100644 --- a/src/modules/options/OptionsApp.tsx +++ b/src/modules/options/OptionsApp.tsx @@ -135,7 +135,10 @@ export const OptionsApp: React.FC = () => { if (streamingServiceValue.sync && !service.hasSync) { streamingServiceValue.sync = false; } - if (streamingServiceValue.autoSync && (!service.hasSync || !streamingServiceValue.sync)) { + if ( + streamingServiceValue.autoSync && + (!service.hasSync || !service.hasAutoSync || !streamingServiceValue.sync) + ) { streamingServiceValue.autoSync = false; } if (streamingServiceValue.scrobble || streamingServiceValue.sync) { diff --git a/src/modules/options/components/StreamingServiceOption.tsx b/src/modules/options/components/StreamingServiceOption.tsx index f45ea09d..32799ebd 100644 --- a/src/modules/options/components/StreamingServiceOption.tsx +++ b/src/modules/options/components/StreamingServiceOption.tsx @@ -73,13 +73,13 @@ export const StreamingServiceOption: React.FC = ( ; + abstract loadHistory(itemsToLoad: number, lastSync?: number, lastSyncId?: string): Promise; loadTraktHistory = async (items: Item[]) => { const missingItems = items.filter((item) => typeof item.trakt === 'undefined'); @@ -38,7 +37,6 @@ export abstract class Api { } await Promise.all(promises); await BrowserStorage.set({ traktCache }, false); - await getSyncStore(this.id).update(); } catch (err) { if (!(err as RequestException).canceled) { Errors.error('Failed to load Trakt history.', err); diff --git a/src/streaming-services/common/AutoSync.ts b/src/streaming-services/common/AutoSync.ts new file mode 100644 index 00000000..e365e4a1 --- /dev/null +++ b/src/streaming-services/common/AutoSync.ts @@ -0,0 +1,66 @@ +import { TraktSync } from '../../api/TraktSync'; +import { + BrowserStorage, + Option, + StorageValuesOptions, + StreamingServiceValue, +} from '../../common/BrowserStorage'; +import '../pages'; +import { StreamingServiceId } from '../streaming-services'; +import { getApi, getSyncStore } from './common'; + +const addOptionToSave = ( + optionsToSave: StorageValuesOptions, + option: Option +) => { + optionsToSave[option.id] = option.value; +}; + +class _AutoSync { + sync = async (serviceEntries: [StreamingServiceId, StreamingServiceValue][], now: number) => { + const optionsToSave = {} as StorageValuesOptions; + const options = await BrowserStorage.getOptions(); + const syncOptions = await BrowserStorage.getSyncOptions(); + + for (const [serviceId, serviceValue] of serviceEntries) { + try { + const api = getApi(serviceId); + const store = getSyncStore(serviceId); + + await api.loadHistory(0, serviceValue.lastSync, serviceValue.lastSyncId); + options.streamingServices.value[serviceId].lastSync = now; + + let items = store.data.items.filter( + (item) => !item.percentageWatched || item.percentageWatched >= 10.0 + ); + + if (items.length === 0) { + continue; + } + + await api.loadTraktHistory(items); + + items = items + .filter((item) => item.trakt && !item.trakt.watchedAt) + .map((item) => ({ ...item, isSelected: true })); + + if (items.length === 0) { + continue; + } + + await TraktSync.sync(items, syncOptions.addWithReleaseDate.value); + + options.streamingServices.value[serviceId].lastSyncId = items[0].id; + } catch (err) { + // Do nothing + } + } + + for (const option of Object.values(options) as Option[]) { + addOptionToSave(optionsToSave, option); + } + await BrowserStorage.set({ options: optionsToSave }, true); + }; +} + +export const AutoSync = new _AutoSync(); diff --git a/src/streaming-services/common/SyncPage.tsx b/src/streaming-services/common/SyncPage.tsx index c3073d10..0d290673 100644 --- a/src/streaming-services/common/SyncPage.tsx +++ b/src/streaming-services/common/SyncPage.tsx @@ -9,7 +9,9 @@ import { TraktSync } from '../../api/TraktSync'; import { WrongItemApi } from '../../api/WrongItemApi'; import { BrowserStorage, + Option, Options, + StorageValuesOptions, StorageValuesSyncOptions, SyncOptions, } from '../../common/BrowserStorage'; @@ -31,7 +33,6 @@ import { HistoryList } from '../../modules/history/components/HistoryList'; import { HistoryOptionsList } from '../../modules/history/components/HistoryOptionsList'; import { StreamingServiceId } from '../streaming-services'; import { Api } from './Api'; -import { getApi, getSyncStore } from './common'; import { SyncStore } from './SyncStore'; interface PageProps { @@ -60,6 +61,13 @@ interface Content { hasReachedEnd: boolean; } +const addOptionToSave = ( + optionsToSave: StorageValuesOptions, + option: Option +) => { + optionsToSave[option.id] = option.value; +}; + export const SyncPage: React.FC = (props: PageProps) => { const { serviceId, serviceName, store, api } = props; @@ -125,6 +133,13 @@ export const SyncPage: React.FC = (props: PageProps) => { isLoading: true, })); await TraktSync.sync(content.visibleItems, syncOptionsContent.options.addWithReleaseDate.value); + const optionsToSave = {} as StorageValuesOptions; + const options = await BrowserStorage.getOptions(); + options.streamingServices.value[serviceId].lastSync = Math.trunc(Date.now() / 1e3); + for (const option of Object.values(options) as Option[]) { + addOptionToSave(optionsToSave, option); + } + await BrowserStorage.set({ options: optionsToSave }, true); setContent((prevContent) => ({ ...prevContent, isLoading: false, @@ -188,7 +203,7 @@ export const SyncPage: React.FC = (props: PageProps) => { if (!traktCache) { traktCache = {}; } - await getApi(serviceId).loadTraktItemHistory(data.item, traktCache, { + await api.loadTraktItemHistory(data.item, traktCache, { type: data.type, traktId: data.traktId, url: data.url, @@ -205,7 +220,7 @@ export const SyncPage: React.FC = (props: PageProps) => { } } await BrowserStorage.set({ traktCache }, false); - await getSyncStore(serviceId).update(); + await store.update(); }; const onMissingWatchedDateAdded = async (data: MissingWatchedDateAddedData): Promise => { @@ -227,7 +242,7 @@ export const SyncPage: React.FC = (props: PageProps) => { break; // no-default } - await getSyncStore(serviceId).update(); + await store.update(); }; const onHistorySyncSuccess = async (data: HistorySyncSuccessData) => { @@ -361,11 +376,12 @@ export const SyncPage: React.FC = (props: PageProps) => { useEffect(() => { const loadData = async () => { try { - await getApi(serviceId).loadTraktHistory(content.visibleItems); + await api.loadTraktHistory(content.visibleItems); await Promise.all([ WrongItemApi.loadSuggestions(serviceId, content.visibleItems), TmdbApi.loadImages(serviceId, content.visibleItems), ]); + await store.update(); } catch (err) { // Do nothing } diff --git a/src/streaming-services/hbo-go/HboGoApi.ts b/src/streaming-services/hbo-go/HboGoApi.ts index 4cf9e335..e16751ee 100644 --- a/src/streaming-services/hbo-go/HboGoApi.ts +++ b/src/streaming-services/hbo-go/HboGoApi.ts @@ -197,7 +197,11 @@ class _HboGoApi extends Api { .replace(/{ageRating}/i, '0'); }; - loadHistory = async (itemsToLoad: number): Promise => { + loadHistory = async ( + itemsToLoad: number, + lastSync?: number, + lastSyncId?: string + ): Promise => { try { if (!this.isActivated) { await this.activate(); @@ -221,8 +225,23 @@ class _HboGoApi extends Api { if (responseJson) { const responseItems = responseJson.Container[0]?.Contents.Items; if (responseItems && responseItems.length > 0) { - itemsToLoad -= responseItems.length; - historyItems.push(...responseItems); + let filteredItems = []; + if (lastSyncId) { + for (const responseItem of responseItems) { + if (responseItem.Id && responseItem.Id !== lastSyncId) { + filteredItems.push(responseItem); + } else { + break; + } + } + if (filteredItems.length !== responseItems.length) { + hasReachedEnd = true; + } + } else { + filteredItems = responseItems; + } + itemsToLoad -= filteredItems.length; + historyItems.push(...filteredItems); } else { hasReachedEnd = true; } @@ -230,7 +249,7 @@ class _HboGoApi extends Api { hasReachedEnd = true; } nextPage += 1; - } while (!hasReachedEnd && itemsToLoad > 0); + } while (!hasReachedEnd && (itemsToLoad > 0 || lastSyncId)); if (historyItems.length > 0) { items = historyItems.map(this.parseHistoryItem); } diff --git a/src/streaming-services/netflix/NetflixApi.ts b/src/streaming-services/netflix/NetflixApi.ts index cc78f9ef..d83a6518 100644 --- a/src/streaming-services/netflix/NetflixApi.ts +++ b/src/streaming-services/netflix/NetflixApi.ts @@ -201,7 +201,7 @@ class _NetflixApi extends Api { ); }; - loadHistory = async (itemsToLoad: number) => { + loadHistory = async (itemsToLoad: number, lastSync?: number) => { try { if (!this.isActivated) { await this.activate(); @@ -220,13 +220,28 @@ class _NetflixApi extends Api { }); const responseJson = JSON.parse(responseText) as NetflixHistoryResponse; if (responseJson && responseJson.viewedItems.length > 0) { - itemsToLoad -= responseJson.viewedItems.length; - historyItems.push(...responseJson.viewedItems); + let filteredItems = []; + if (lastSync) { + for (const viewedItem of responseJson.viewedItems) { + if (viewedItem.date && Math.trunc(viewedItem.date / 1e3) > lastSync) { + filteredItems.push(viewedItem); + } else { + break; + } + } + if (filteredItems.length !== responseJson.viewedItems.length) { + hasReachedEnd = true; + } + } else { + filteredItems = responseJson.viewedItems; + } + itemsToLoad -= filteredItems.length; + historyItems.push(...filteredItems); } else { hasReachedEnd = true; } nextPage += 1; - } while (!hasReachedEnd && itemsToLoad > 0); + } while (!hasReachedEnd && (itemsToLoad > 0 || lastSync)); if (historyItems.length > 0) { const historyItemsWithMetadata = await this.getHistoryMetadata(historyItems); items = historyItemsWithMetadata.map(this.parseHistoryItem); @@ -292,6 +307,7 @@ class _NetflixApi extends Api { let episode; const isCollection = !historyItem.seasonDescriptor.includes('Season'); if (!isCollection) { + // TODO: Some items don't have a summary response (see Friends pilot). season = historyItem.summary.season; episode = historyItem.summary.episode; } diff --git a/src/streaming-services/nrk/NrkApi.ts b/src/streaming-services/nrk/NrkApi.ts index 28340e44..282fe98b 100644 --- a/src/streaming-services/nrk/NrkApi.ts +++ b/src/streaming-services/nrk/NrkApi.ts @@ -159,7 +159,7 @@ class _NrkApi extends Api { this.isActivated = true; }; - loadHistory = async (itemsToLoad: number) => { + loadHistory = async (itemsToLoad: number, lastSync?: number) => { try { if (!this.isActivated) { await this.activate(); @@ -184,10 +184,28 @@ class _NrkApi extends Api { hasReachedEnd = true; } if (progresses.length > 0) { - itemsToLoad -= progresses.length; - historyItems.push(...progresses); + let filteredItems = []; + if (lastSync) { + for (const progress of progresses) { + if ( + progress.registeredAt && + Math.trunc(new Date(progress.registeredAt).getTime() / 1e3) > lastSync + ) { + filteredItems.push(progress); + } else { + break; + } + } + if (filteredItems.length !== progresses.length) { + hasReachedEnd = true; + } + } else { + filteredItems = progresses; + } + itemsToLoad -= filteredItems.length; + historyItems.push(...filteredItems); } - } while (!hasReachedEnd && itemsToLoad > 0); + } while (!hasReachedEnd && (itemsToLoad > 0 || lastSync)); if (historyItems.length > 0) { const promises = historyItems.map(this.parseHistoryItem); items = await Promise.all(promises); diff --git a/src/streaming-services/streaming-services.ts b/src/streaming-services/streaming-services.ts index 562b899b..81048ec5 100644 --- a/src/streaming-services/streaming-services.ts +++ b/src/streaming-services/streaming-services.ts @@ -13,6 +13,7 @@ export interface StreamingService { hostPatterns: string[]; hasScrobbler: boolean; hasSync: boolean; + hasAutoSync: boolean; } export const streamingServices: Record = { @@ -23,6 +24,7 @@ export const streamingServices: Record = { hostPatterns: ['*://*.primevideo.com/*'], hasScrobbler: true, hasSync: false, + hasAutoSync: false, }, 'hbo-go': { id: 'hbo-go', @@ -31,6 +33,7 @@ export const streamingServices: Record = { hostPatterns: ['*://*.hbogola.com/*', '*://*.hbogo.com.br/*'], hasScrobbler: true, hasSync: true, + hasAutoSync: true, }, netflix: { id: 'netflix', @@ -39,6 +42,7 @@ export const streamingServices: Record = { hostPatterns: ['*://*.netflix.com/*'], hasScrobbler: true, hasSync: true, + hasAutoSync: true, }, nrk: { id: 'nrk', @@ -47,6 +51,7 @@ export const streamingServices: Record = { hostPatterns: ['*://*.nrk.no/*'], hasScrobbler: true, hasSync: true, + hasAutoSync: true, }, viaplay: { id: 'viaplay', @@ -61,6 +66,7 @@ export const streamingServices: Record = { ], hasScrobbler: false, hasSync: true, + hasAutoSync: true, }, 'telia-play': { id: 'telia-play', @@ -69,5 +75,6 @@ export const streamingServices: Record = { hostPatterns: ['*://*.teliaplay.se/*', '*://*.telia.net/*', '*://*.telia.se/*'], hasScrobbler: false, hasSync: true, + hasAutoSync: false, }, }; diff --git a/src/streaming-services/viaplay/ViaplayApi.ts b/src/streaming-services/viaplay/ViaplayApi.ts index d17b1c74..ea4dfd5d 100644 --- a/src/streaming-services/viaplay/ViaplayApi.ts +++ b/src/streaming-services/viaplay/ViaplayApi.ts @@ -107,7 +107,7 @@ class _ViaplayApi extends Api { this.isActivated = true; }; - loadHistory = async (itemsToLoad: number) => { + loadHistory = async (itemsToLoad: number, lastSync?: number) => { try { if (!this.isActivated) { await this.activate(); @@ -133,8 +133,26 @@ class _ViaplayApi extends Api { const viaplayProducts: ViaplayProduct[] = historyPage._embedded['viaplay:products']; if (viaplayProducts && viaplayProducts.length > 0) { - itemsToLoad -= viaplayProducts.length; - historyItems.push(...viaplayProducts); + let filteredItems = []; + if (lastSync) { + for (const viaplayProduct of viaplayProducts) { + if ( + viaplayProduct.user.progress?.updated && + Math.trunc(viaplayProduct.user.progress?.updated / 1e3) > lastSync + ) { + filteredItems.push(viaplayProduct); + } else { + break; + } + } + if (filteredItems.length !== viaplayProducts.length) { + hasReachedEnd = true; + } + } else { + filteredItems = viaplayProducts; + } + itemsToLoad -= filteredItems.length; + historyItems.push(...filteredItems); } else { hasReachedEnd = true; } @@ -143,7 +161,7 @@ class _ViaplayApi extends Api { if (!url) { hasReachedEnd = true; } - } while (!hasReachedEnd && itemsToLoad > 0); + } while (!hasReachedEnd && (itemsToLoad > 0 || lastSync)); if (historyItems.length > 0) { items = historyItems.map(this.parseHistoryItem); }