diff --git a/apps/expo/src/components/player/VideoPlayer.tsx b/apps/expo/src/components/player/VideoPlayer.tsx index 5b21fcd..7f167dc 100644 --- a/apps/expo/src/components/player/VideoPlayer.tsx +++ b/apps/expo/src/components/player/VideoPlayer.tsx @@ -280,6 +280,8 @@ export const VideoPlayer = () => { }; }, [player, meta, removeFromWatchHistory, autoPlay, setMeta, router]); + console.log("loading player", player); + return ( { - throw new Error(`Failed to get streams: ${ err}`); + throw new Error('Failed to get streams: ' + err); }); } } diff --git a/packages/providers/src/__test__/providers/providerUtils.ts b/packages/providers/src/__test__/providers/providerUtils.ts index bb80330..d47b924 100644 --- a/packages/providers/src/__test__/providers/providerUtils.ts +++ b/packages/providers/src/__test__/providers/providerUtils.ts @@ -1,10 +1,9 @@ -import type { ScrapeMedia } from '@/entrypoint/utils/media'; -import type { Sourcerer} from '@/providers/base'; -import { Embed, SourcererEmbed } from '@/providers/base'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { Embed, Sourcerer, SourcererEmbed } from '@/providers/base'; import { buildProviders } from '@/entrypoint/builder'; import { describe, expect, it } from 'vitest'; import { makeStandardFetcher } from '@/fetchers/standardFetch'; -import type { ProviderControls } from '@/entrypoint/controls'; +import { ProviderControls } from '@/entrypoint/controls'; import { NotFoundError } from '@/utils/errors'; import { targets } from '@/entrypoint/utils/targets'; import { getBuiltinEmbeds } from '@/entrypoint/providers'; @@ -42,7 +41,7 @@ export function testSource(ops: TestSourceOptions) { let hasError = false; let streamCount = 0; let embedCount = 0; - const embeds = []; + let embeds = []; try { const result = await providers.runSourceScraper({ id: ops.source.id, diff --git a/packages/providers/src/__test__/providers/testMedia.ts b/packages/providers/src/__test__/providers/testMedia.ts index 3218f10..1c2c3bd 100644 --- a/packages/providers/src/__test__/providers/testMedia.ts +++ b/packages/providers/src/__test__/providers/testMedia.ts @@ -1,4 +1,4 @@ -import type { ScrapeMedia } from '@/entrypoint/utils/media'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; function makeMedia(media: ScrapeMedia): ScrapeMedia { return media; diff --git a/packages/providers/src/__test__/standard/fetchers/simpleProxy.test.ts b/packages/providers/src/__test__/standard/fetchers/simpleProxy.test.ts index 01fb4c5..cef585e 100644 --- a/packages/providers/src/__test__/standard/fetchers/simpleProxy.test.ts +++ b/packages/providers/src/__test__/standard/fetchers/simpleProxy.test.ts @@ -1,6 +1,5 @@ import { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy'; -import type { DefaultedFetcherOptions} from '@/fetchers/types'; -import { FetcherOptions } from '@/fetchers/types'; +import { DefaultedFetcherOptions, FetcherOptions } from '@/fetchers/types'; import { Headers } from 'node-fetch'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/packages/providers/src/__test__/standard/fetchers/standard.test.ts b/packages/providers/src/__test__/standard/fetchers/standard.test.ts index 8e59c62..8699136 100644 --- a/packages/providers/src/__test__/standard/fetchers/standard.test.ts +++ b/packages/providers/src/__test__/standard/fetchers/standard.test.ts @@ -1,5 +1,5 @@ import { makeStandardFetcher } from '@/fetchers/standardFetch'; -import type { DefaultedFetcherOptions } from '@/fetchers/types'; +import { DefaultedFetcherOptions } from '@/fetchers/types'; import { Headers } from 'node-fetch'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/packages/providers/src/__test__/standard/providerTests.ts b/packages/providers/src/__test__/standard/providerTests.ts index 46744ef..f5c87b7 100644 --- a/packages/providers/src/__test__/standard/providerTests.ts +++ b/packages/providers/src/__test__/standard/providerTests.ts @@ -1,7 +1,7 @@ - +// eslint-disable-next-line import/no-extraneous-dependencies import { vi } from 'vitest'; -import type { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; import { makeEmbed, makeSourcerer } from '@/providers/base'; export function makeProviderMocks() { diff --git a/packages/providers/src/__test__/standard/providers/checks.test.ts b/packages/providers/src/__test__/standard/providers/checks.test.ts index a1d8f87..56e1c39 100644 --- a/packages/providers/src/__test__/standard/providers/checks.test.ts +++ b/packages/providers/src/__test__/standard/providers/checks.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '../providerTests'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; -import type { FeatureMap } from '@/entrypoint/utils/targets'; +import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; diff --git a/packages/providers/src/__test__/standard/utils/features.test.ts b/packages/providers/src/__test__/standard/utils/features.test.ts index a7a9349..75a855c 100644 --- a/packages/providers/src/__test__/standard/utils/features.test.ts +++ b/packages/providers/src/__test__/standard/utils/features.test.ts @@ -1,5 +1,4 @@ -import type { FeatureMap, Flags} from '@/entrypoint/utils/targets'; -import { flags, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; +import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { describe, it, expect } from 'vitest'; describe('flagsAllowedInFeatures()', () => { diff --git a/packages/providers/src/dev-cli/browser/.gitignore b/packages/providers/src/dev-cli/browser/.gitignore deleted file mode 100644 index 1521c8b..0000000 --- a/packages/providers/src/dev-cli/browser/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist diff --git a/packages/providers/src/dev-cli/browser/index.html b/packages/providers/src/dev-cli/browser/index.html deleted file mode 100644 index 7709f4b..0000000 --- a/packages/providers/src/dev-cli/browser/index.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - Scraper CLI - - - - - diff --git a/packages/providers/src/dev-cli/browser/index.ts b/packages/providers/src/dev-cli/browser/index.ts deleted file mode 100644 index c753201..0000000 --- a/packages/providers/src/dev-cli/browser/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { makeProviders } from "@/entrypoint/declare"; -import { targets } from "@/entrypoint/utils/targets"; -import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; -import { makeStandardFetcher } from "@/fetchers/standardFetch"; - -(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => { - const providers = makeProviders({ - fetcher: makeStandardFetcher(fetch), - target: targets.BROWSER, - proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch), - }); - if (type === 'source') { - return providers.runSourceScraper(input); - } - if (type === 'embed') { - return providers.runEmbedScraper(input); - } - - throw new Error('Input input type'); -}; diff --git a/packages/providers/src/dev-cli/index.ts b/packages/providers/src/dev-cli/index.ts index 0e6edc2..6edf96a 100644 --- a/packages/providers/src/dev-cli/index.ts +++ b/packages/providers/src/dev-cli/index.ts @@ -6,181 +6,182 @@ import { prompt } from 'enquirer'; import { runScraper } from '@/dev-cli/scraper'; import { processOptions } from '@/dev-cli/validate'; -import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; + +import { getBuiltinEmbeds, getBuiltinSources } from '..'; dotenv.config(); -interface ProviderSourceAnswers { - id: string; - type: string; -} +type ProviderSourceAnswers = { + id: string; + type: string; +}; -interface EmbedSourceAnswers { - url: string; -} +type EmbedSourceAnswers = { + url: string; +}; -interface CommonAnswers { - fetcher: string; - source: string; -} +type CommonAnswers = { + fetcher: string; + source: string; +}; -interface ShowAnswers { - season: string; - episode: string; -} +type ShowAnswers = { + season: string; + episode: string; +}; const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank); const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank); const sources = [...sourceScrapers, ...embedScrapers]; function joinMediaTypes(mediaTypes: string[] | undefined) { - if (mediaTypes) { - const formatted = mediaTypes - .map((type: string) => `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`) - .join(' / '); - - return `(${formatted})`; - } - return ''; // * Embed sources pass through here too + if (mediaTypes) { + const formatted = mediaTypes + .map((type: string) => `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`) + .join(' / '); + + return `(${formatted})`; + } + return ''; // * Embed sources pass through here too } async function runQuestions() { - const options = { - fetcher: 'node-fetch', - sourceId: '', - tmdbId: '', - type: 'movie', - season: '0', - episode: '0', - url: '', - }; - - const answers = await prompt([ - { - type: 'select', - name: 'fetcher', - message: 'Select a fetcher mode', - choices: [ - { - message: 'Native', - name: 'native', - }, - { - message: 'Node fetch', - name: 'node-fetch', - }, - { - message: 'Browser', - name: 'browser', - }, - ], - }, - { - type: 'select', - name: 'source', - message: 'Select a source', - choices: sources.map((source) => ({ - message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), - name: source.id, - })), - }, - ]); - - options.fetcher = answers.fetcher; - options.sourceId = answers.source; - - const source = sources.find(({ id }) => id === answers.source); - - if (!source) { - throw new Error(`No source with ID ${answers.source} found`); - } - - if (source.type === 'embed') { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'url', - message: 'Embed URL', - }, - ]); - - options.url = sourceAnswers.url; - } else { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'id', - message: 'TMDB ID', - }, - { - type: 'select', - name: 'type', - message: 'Media type', - choices: [ - { - message: 'Movie', - name: 'movie', - }, - { - message: 'TV Show', - name: 'show', - }, - ], - }, - ]); - - options.tmdbId = sourceAnswers.id; - options.type = sourceAnswers.type; - - if (sourceAnswers.type === 'show') { - const seriesAnswers = await prompt([ - { - type: 'input', - name: 'season', - message: 'Season', - }, - { - type: 'input', - name: 'episode', - message: 'Episode', - }, - ]); - - options.season = seriesAnswers.season; - options.episode = seriesAnswers.episode; - } - } - - const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options); - await runScraper(providerOptions, validatedSource, validatedOps); + const options = { + fetcher: 'node-fetch', + sourceId: '', + tmdbId: '', + type: 'movie', + season: '0', + episode: '0', + url: '', + }; + + const answers = await prompt([ + { + type: 'select', + name: 'fetcher', + message: 'Select a fetcher mode', + choices: [ + { + message: 'Native', + name: 'native', + }, + { + message: 'Node fetch', + name: 'node-fetch', + }, + { + message: 'Browser', + name: 'browser', + }, + ], + }, + { + type: 'select', + name: 'source', + message: 'Select a source', + choices: sources.map((source) => ({ + message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), + name: source.id, + })), + }, + ]); + + options.fetcher = answers.fetcher; + options.sourceId = answers.source; + + const source = sources.find(({ id }) => id === answers.source); + + if (!source) { + throw new Error(`No source with ID ${answers.source} found`); + } + + if (source.type === 'embed') { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'url', + message: 'Embed URL', + }, + ]); + + options.url = sourceAnswers.url; + } else { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'id', + message: 'TMDB ID', + }, + { + type: 'select', + name: 'type', + message: 'Media type', + choices: [ + { + message: 'Movie', + name: 'movie', + }, + { + message: 'TV Show', + name: 'show', + }, + ], + }, + ]); + + options.tmdbId = sourceAnswers.id; + options.type = sourceAnswers.type; + + if (sourceAnswers.type === 'show') { + const seriesAnswers = await prompt([ + { + type: 'input', + name: 'season', + message: 'Season', + }, + { + type: 'input', + name: 'episode', + message: 'Episode', + }, + ]); + + options.season = seriesAnswers.season; + options.episode = seriesAnswers.episode; + } + } + + const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options); + await runScraper(providerOptions, validatedSource, validatedOps); } async function runCommandLine() { - program - .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') - .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') - .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') - .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') - .option('-s, --season ', "Season number. Only used if type is 'show'", '0') - .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); - - program.parse(); - - const { - providerOptions, - source: validatedSource, - options: validatedOps, - } = await processOptions(sources, program.opts()); - await runScraper(providerOptions, validatedSource, validatedOps); + program + .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') + .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') + .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') + .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') + .option('-s, --season ', "Season number. Only used if type is 'show'", '0') + .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') + .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); + + program.parse(); + + const { + providerOptions, + source: validatedSource, + options: validatedOps, + } = await processOptions(sources, program.opts()); + await runScraper(providerOptions, validatedSource, validatedOps); } if (process.argv.length === 2) { - runQuestions() - .catch(() => console.error('Exited.')) - .finally(() => process.exit(0)); + runQuestions() + .catch(() => console.error('Exited.')) + .finally(() => process.exit(0)); } else { - runCommandLine() - .catch(() => console.error('Exited.')) - .finally(() => process.exit(0)); + runCommandLine() + .catch(() => console.error('Exited.')) + .finally(() => process.exit(0)); } diff --git a/packages/providers/src/dev-cli/logging.ts b/packages/providers/src/dev-cli/logging.ts index 2e41a5f..5495f1d 100644 --- a/packages/providers/src/dev-cli/logging.ts +++ b/packages/providers/src/dev-cli/logging.ts @@ -1,8 +1,7 @@ -import { inspect } from "node:util"; +import { inspect } from 'node:util'; export function logDeepObject(object: Record) { + // This is the dev cli, so we can use console.log // eslint-disable-next-line no-console - console.log( - inspect(object, { showHidden: false, depth: null, colors: true }), - ); + console.log(inspect(object, { showHidden: false, depth: null, colors: true })); } diff --git a/packages/providers/src/dev-cli/scraper.ts b/packages/providers/src/dev-cli/scraper.ts index c03ea00..8c66b89 100644 --- a/packages/providers/src/dev-cli/scraper.ts +++ b/packages/providers/src/dev-cli/scraper.ts @@ -1,34 +1,31 @@ /* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ -import { existsSync } from "fs"; -import { join } from "path"; -import type { CommandLineArguments } from "@/dev-cli/validate"; -import type { ProviderMakerOptions } from "@/entrypoint/declare"; -import type { MetaOutput } from "@/entrypoint/utils/meta"; -import type { Browser } from "puppeteer"; -import type { PreviewServer } from "vite"; -import { getConfig } from "@/dev-cli/config"; -import { logDeepObject } from "@/dev-cli/logging"; -import { getMovieMediaDetails, getShowMediaDetails } from "@/dev-cli/tmdb"; -import { makeProviders } from "@/entrypoint/declare"; -import puppeteer from "puppeteer"; -import Spinnies from "spinnies"; -import { build, preview } from "vite"; +import { existsSync } from 'fs'; +import { join } from 'path'; + +import puppeteer, { Browser } from 'puppeteer'; +import Spinnies from 'spinnies'; +import { PreviewServer, build, preview } from 'vite'; + +import { getConfig } from '@/dev-cli/config'; +import { logDeepObject } from '@/dev-cli/logging'; +import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; +import { CommandLineArguments } from '@/dev-cli/validate'; + +import { MetaOutput, ProviderMakerOptions, makeProviders } from '..'; async function runBrowserScraping( providerOptions: ProviderMakerOptions, source: MetaOutput, options: CommandLineArguments, ) { - if (!existsSync(join(__dirname, "../../lib/index.js"))) - throw new Error("Please compile before running cli in browser mode"); + if (!existsSync(join(__dirname, '../../lib/index.js'))) + throw new Error('Please compile before running cli in browser mode'); const config = getConfig(); if (!config.proxyUrl) - throw new Error( - "Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work", - ); + throw new Error('Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work'); - const root = join(__dirname, "browser"); + const root = join(__dirname, 'browser'); let server: PreviewServer | undefined; let browser: Browser | undefined; try { @@ -41,47 +38,37 @@ async function runBrowserScraping( }); browser = await puppeteer.launch({ headless: true, - args: ["--no-sandbox", "--disable-setuid-sandbox"], + args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); // This is the dev cli, so we can use console.log + // eslint-disable-next-line no-console + page.on('console', (message) => console.log(`${message.type().slice(0, 3).toUpperCase()} ${message.text()}`)); - page.on("console", (message) => - // eslint-disable-next-line no-console - console.log( - `${message.type().slice(0, 3).toUpperCase()} ${message.text()}`, - ), - ); - - if (!server.resolvedUrls?.local.length) - throw new Error("Server did not start"); + if (!server.resolvedUrls?.local.length) throw new Error('Server did not start'); await page.goto(server.resolvedUrls.local[0]); - await page.waitForFunction("!!window.scrape", { timeout: 5000 }); + await page.waitForFunction('!!window.scrape', { timeout: 5000 }); // get input media let input: any; - if (source.type === "embed") { + if (source.type === 'embed') { input = { url: options.url, id: source.id, }; - } else if (source.type === "source") { + } else if (source.type === 'source') { let media; - if (options.type === "movie") { + if (options.type === 'movie') { media = await getMovieMediaDetails(options.tmdbId); } else { - media = await getShowMediaDetails( - options.tmdbId, - options.season, - options.episode, - ); + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); } input = { media, id: source.id, }; } else { - throw new Error("Wrong source input type"); + throw new Error('Wrong source input type'); } return await page.evaluate( @@ -101,28 +88,23 @@ async function runActualScraping( source: MetaOutput, options: CommandLineArguments, ): Promise { - if (options.fetcher === "browser") - return runBrowserScraping(providerOptions, source, options); + if (options.fetcher === 'browser') return runBrowserScraping(providerOptions, source, options); const providers = makeProviders(providerOptions); - if (source.type === "embed") { + if (source.type === 'embed') { return providers.runEmbedScraper({ url: options.url, id: source.id, }); } - if (source.type === "source") { + if (source.type === 'source') { let media; - if (options.type === "movie") { + if (options.type === 'movie') { media = await getMovieMediaDetails(options.tmdbId); } else { - media = await getShowMediaDetails( - options.tmdbId, - options.season, - options.episode, - ); + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); } return providers.runSourceScraper({ @@ -131,7 +113,7 @@ async function runActualScraping( }); } - throw new Error("Invalid source type"); + throw new Error('Invalid source type'); } export async function runScraper( @@ -141,17 +123,17 @@ export async function runScraper( ) { const spinnies = new Spinnies(); - spinnies.add("scrape", { text: `Running ${source.name} scraper` }); + spinnies.add('scrape', { text: `Running ${source.name} scraper` }); try { const result = await runActualScraping(providerOptions, source, options); - spinnies.succeed("scrape", { text: "Done!" }); + spinnies.succeed('scrape', { text: 'Done!' }); logDeepObject(result); } catch (error) { - let message = "Unknown error"; + let message = 'Unknown error'; if (error instanceof Error) { message = error.message; } - spinnies.fail("scrape", { text: `ERROR: ${message}` }); + spinnies.fail('scrape', { text: `ERROR: ${message}` }); console.error(error); } } diff --git a/packages/providers/src/dev-cli/tmdb.ts b/packages/providers/src/dev-cli/tmdb.ts index da66bf5..c03307d 100644 --- a/packages/providers/src/dev-cli/tmdb.ts +++ b/packages/providers/src/dev-cli/tmdb.ts @@ -1,100 +1,101 @@ import { getConfig } from '@/dev-cli/config'; -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; + +import { MovieMedia, ShowMedia } from '..'; export async function makeTMDBRequest(url: string, appendToResponse?: string): Promise { - const headers: { - accept: 'application/json'; - authorization?: string; - } = { - accept: 'application/json', - }; - - const requestURL = new URL(url); - const key = getConfig().tmdbApiKey; - - // * JWT keys always start with ey and are ONLY valid as a header. - // * All other keys are ONLY valid as a query param. - // * Thanks TMDB. - if (key.startsWith('ey')) { - headers.authorization = `Bearer ${key}`; - } else { - requestURL.searchParams.append('api_key', key); - } - - if (appendToResponse) { - requestURL.searchParams.append('append_to_response', appendToResponse); - } - - return fetch(requestURL, { - method: 'GET', - headers, - }); + const headers: { + accept: 'application/json'; + authorization?: string; + } = { + accept: 'application/json', + }; + + const requestURL = new URL(url); + const key = getConfig().tmdbApiKey; + + // * JWT keys always start with ey and are ONLY valid as a header. + // * All other keys are ONLY valid as a query param. + // * Thanks TMDB. + if (key.startsWith('ey')) { + headers.authorization = `Bearer ${key}`; + } else { + requestURL.searchParams.append('api_key', key); + } + + if (appendToResponse) { + requestURL.searchParams.append('append_to_response', appendToResponse); + } + + return fetch(requestURL, { + method: 'GET', + headers, + }); } export async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`, 'external_ids'); - const movie = await response.json(); - - if (movie.success === false) { - throw new Error(movie.status_message); - } - - if (!movie.release_date) { - throw new Error(`${movie.title} has no release_date. Assuming unreleased`); - } - - return { - type: 'movie', - title: movie.title, - releaseYear: Number(movie.release_date.split('-')[0]), - tmdbId: id, - imdbId: movie.imdb_id, - }; + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`, 'external_ids'); + const movie = await response.json(); + + if (movie.success === false) { + throw new Error(movie.status_message); + } + + if (!movie.release_date) { + throw new Error(`${movie.title} has no release_date. Assuming unreleased`); + } + + return { + type: 'movie', + title: movie.title, + releaseYear: Number(movie.release_date.split('-')[0]), + tmdbId: id, + imdbId: movie.imdb_id, + }; } export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { - // * TV shows require the TMDB ID for the series, season, and episode - // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`, 'external_ids'); - const series = await response.json(); - - if (series.success === false) { - throw new Error(series.status_message); - } - - if (!series.first_air_date) { - throw new Error(`${series.name} has no first_air_date. Assuming unaired`); - } - - response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); - const season = await response.json(); - - if (season.success === false) { - throw new Error(season.status_message); - } - - response = await makeTMDBRequest( - `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, - ); - const episode = await response.json(); - - if (episode.success === false) { - throw new Error(episode.status_message); - } - - return { - type: 'show', - title: series.name, - releaseYear: Number(series.first_air_date.split('-')[0]), - tmdbId: id, - episode: { - number: episode.episode_number, - tmdbId: episode.id, - }, - season: { - number: season.season_number, - tmdbId: season.id, - }, - imdbId: series.external_ids.imdb_id, - }; + // * TV shows require the TMDB ID for the series, season, and episode + // * and the name of the series. Needs multiple requests + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`, 'external_ids'); + const series = await response.json(); + + if (series.success === false) { + throw new Error(series.status_message); + } + + if (!series.first_air_date) { + throw new Error(`${series.name} has no first_air_date. Assuming unaired`); + } + + response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); + const season = await response.json(); + + if (season.success === false) { + throw new Error(season.status_message); + } + + response = await makeTMDBRequest( + `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, + ); + const episode = await response.json(); + + if (episode.success === false) { + throw new Error(episode.status_message); + } + + return { + type: 'show', + title: series.name, + releaseYear: Number(series.first_air_date.split('-')[0]), + tmdbId: id, + episode: { + number: episode.episode_number, + tmdbId: episode.id, + }, + season: { + number: season.season_number, + tmdbId: season.id, + }, + imdbId: series.external_ids.imdb_id, + }; } diff --git a/packages/providers/src/dev-cli/validate.ts b/packages/providers/src/dev-cli/validate.ts index aaee51c..dd1f638 100644 --- a/packages/providers/src/dev-cli/validate.ts +++ b/packages/providers/src/dev-cli/validate.ts @@ -1,11 +1,10 @@ import nodeFetch from 'node-fetch'; -import type { Embed, Sourcerer } from '@/providers/base'; -import type { ProviderMakerOptions } from '@/entrypoint/declare'; -import { targets } from '@/entrypoint/utils/targets'; -import { makeStandardFetcher } from '@/fetchers/standardFetch'; +import { Embed, Sourcerer } from '@/providers/base'; -export interface CommandLineArguments { +import { ProviderMakerOptions, makeStandardFetcher, targets } from '..'; + +export type CommandLineArguments = { fetcher: string; sourceId: string; tmdbId: string; @@ -13,9 +12,9 @@ export interface CommandLineArguments { season: string; episode: string; url: string; -} +}; -export async function processOptions(sources: (Embed | Sourcerer)[], options: CommandLineArguments) { +export async function processOptions(sources: Array, options: CommandLineArguments) { const fetcherOptions = ['node-fetch', 'native', 'browser']; if (!fetcherOptions.includes(options.fetcher)) { throw new Error(`Fetcher must be any of: ${fetcherOptions.join()}`); diff --git a/packages/providers/src/entrypoint/builder.ts b/packages/providers/src/entrypoint/builder.ts index d001265..abf8288 100644 --- a/packages/providers/src/entrypoint/builder.ts +++ b/packages/providers/src/entrypoint/builder.ts @@ -1,13 +1,11 @@ -import type { ProviderControls} from '@/entrypoint/controls'; -import { makeControls } from '@/entrypoint/controls'; +import { ProviderControls, makeControls } from '@/entrypoint/controls'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; -import type { Targets} from '@/entrypoint/utils/targets'; -import { getTargetFeatures } from '@/entrypoint/utils/targets'; -import type { Fetcher } from '@/fetchers/types'; -import type { Embed, Sourcerer } from '@/providers/base'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; +import { Embed, Sourcerer } from '@/providers/base'; import { getProviders } from '@/providers/get'; -export interface ProviderBuilder { +export type ProviderBuilder = { setTarget(target: Targets): ProviderBuilder; setFetcher(fetcher: Fetcher): ProviderBuilder; setProxiedFetcher(fetcher: Fetcher): ProviderBuilder; @@ -18,7 +16,7 @@ export interface ProviderBuilder { addBuiltinProviders(): ProviderBuilder; enableConsistentIpForRequests(): ProviderBuilder; build(): ProviderControls; -} +}; export function buildProviders(): ProviderBuilder { let consistentIpForRequests = false; diff --git a/packages/providers/src/entrypoint/controls.ts b/packages/providers/src/entrypoint/controls.ts index 120f921..6599f28 100644 --- a/packages/providers/src/entrypoint/controls.ts +++ b/packages/providers/src/entrypoint/controls.ts @@ -1,14 +1,12 @@ -import type { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events'; -import type { ScrapeMedia } from '@/entrypoint/utils/media'; -import type { MetaOutput} from '@/entrypoint/utils/meta'; -import { getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta'; -import type { FeatureMap } from '@/entrypoint/utils/targets'; +import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta'; +import { FeatureMap } from '@/entrypoint/utils/targets'; import { makeFetcher } from '@/fetchers/common'; -import type { Fetcher } from '@/fetchers/types'; -import type { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base'; +import { Fetcher } from '@/fetchers/types'; +import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base'; import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner'; -import type { RunOutput} from '@/runners/runner'; -import { runAllProviders } from '@/runners/runner'; +import { RunOutput, runAllProviders } from '@/runners/runner'; export interface ProviderControlsInput { fetcher: Fetcher; @@ -16,6 +14,7 @@ export interface ProviderControlsInput { features: FeatureMap; sources: Sourcerer[]; embeds: Embed[]; + proxyStreams?: boolean; // temporary } export interface RunnerOptions { @@ -32,6 +31,10 @@ export interface RunnerOptions { // the media you want to see sources from media: ScrapeMedia; + + // it makes sense to have this in the builder + // but I belive it's more useful in runner ops + disableOpensubtitles?: boolean; } export interface SourceRunnerOptions { @@ -43,6 +46,10 @@ export interface SourceRunnerOptions { // id of the source scraper you want to scrape from id: string; + + // it makes sense to have this in the builder + // but I belive it's more useful in runner ops + disableOpensubtitles?: boolean; } export interface EmbedRunnerOptions { @@ -54,6 +61,10 @@ export interface EmbedRunnerOptions { // id of the embed scraper you want to scrape from id: string; + + // it makes sense to have this in the builder + // but I belive it's more useful in runner ops + disableOpensubtitles?: boolean; } export interface ProviderControls { @@ -87,6 +98,7 @@ export function makeControls(ops: ProviderControlsInput): ProviderControls { features: ops.features, fetcher: makeFetcher(ops.fetcher), proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher), + proxyStreams: ops.proxyStreams, }; return { diff --git a/packages/providers/src/entrypoint/declare.ts b/packages/providers/src/entrypoint/declare.ts index 7aeb3f1..55f08e3 100644 --- a/packages/providers/src/entrypoint/declare.ts +++ b/packages/providers/src/entrypoint/declare.ts @@ -1,8 +1,7 @@ import { makeControls } from '@/entrypoint/controls'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; -import type { Targets} from '@/entrypoint/utils/targets'; -import { getTargetFeatures } from '@/entrypoint/utils/targets'; -import type { Fetcher } from '@/fetchers/types'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; import { getProviders } from '@/providers/get'; export interface ProviderMakerOptions { @@ -19,10 +18,17 @@ export interface ProviderMakerOptions { // Set this to true, if the requests will have the same IP as // the device that the stream will be played on consistentIpForRequests?: boolean; + + // This is temporary + proxyStreams?: boolean; } export function makeProviders(ops: ProviderMakerOptions) { - const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false); + const features = getTargetFeatures( + ops.proxyStreams ? 'any' : ops.target, + ops.consistentIpForRequests ?? false, + ops.proxyStreams, + ); const list = getProviders(features, { embeds: getBuiltinEmbeds(), sources: getBuiltinSources(), @@ -34,5 +40,6 @@ export function makeProviders(ops: ProviderMakerOptions) { features, fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher, + proxyStreams: ops.proxyStreams, }); } diff --git a/packages/providers/src/entrypoint/providers.ts b/packages/providers/src/entrypoint/providers.ts index 623dcbc..e456eb0 100644 --- a/packages/providers/src/entrypoint/providers.ts +++ b/packages/providers/src/entrypoint/providers.ts @@ -1,5 +1,5 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; -import type { Embed, Sourcerer } from '@/providers/base'; +import { Embed, Sourcerer } from '@/providers/base'; export function getBuiltinSources(): Sourcerer[] { return gatherAllSources().filter((v) => !v.disabled); diff --git a/packages/providers/src/entrypoint/utils/events.ts b/packages/providers/src/entrypoint/utils/events.ts index 98ece21..50a8c2a 100644 --- a/packages/providers/src/entrypoint/utils/events.ts +++ b/packages/providers/src/entrypoint/utils/events.ts @@ -1,32 +1,32 @@ export type UpdateEventStatus = 'success' | 'failure' | 'notfound' | 'pending'; -export interface UpdateEvent { +export type UpdateEvent = { id: string; // id presented in start event percentage: number; status: UpdateEventStatus; error?: unknown; // set when status is failure reason?: string; // set when status is not-found -} +}; -export interface InitEvent { +export type InitEvent = { sourceIds: string[]; // list of source ids -} +}; -export interface DiscoverEmbedsEvent { +export type DiscoverEmbedsEvent = { sourceId: string; // list of embeds that will be scraped in order - embeds: { + embeds: Array<{ id: string; embedScraperId: string; - }[]; -} + }>; +}; -export interface SingleScraperEvents { +export type SingleScraperEvents = { update?: (evt: UpdateEvent) => void; -} +}; -export interface FullScraperEvents { +export type FullScraperEvents = { // update progress percentage and status of the currently scraping item update?: (evt: UpdateEvent) => void; @@ -39,9 +39,9 @@ export interface FullScraperEvents { // start scraping an item. start?: (id: string) => void; -} +}; -export interface IndividualScraperEvents { +export type IndividualScraperEvents = { // update progress percentage and status of the currently scraping item update?: (evt: UpdateEvent) => void; -} +}; diff --git a/packages/providers/src/entrypoint/utils/media.ts b/packages/providers/src/entrypoint/utils/media.ts index 6d01752..c137063 100644 --- a/packages/providers/src/entrypoint/utils/media.ts +++ b/packages/providers/src/entrypoint/utils/media.ts @@ -1,9 +1,9 @@ -export interface CommonMedia { +export type CommonMedia = { title: string; releaseYear: number; imdbId?: string; tmdbId: string; -} +}; export type MediaTypes = 'show' | 'movie'; diff --git a/packages/providers/src/entrypoint/utils/meta.ts b/packages/providers/src/entrypoint/utils/meta.ts index 4c2bce2..5e54b2a 100644 --- a/packages/providers/src/entrypoint/utils/meta.ts +++ b/packages/providers/src/entrypoint/utils/meta.ts @@ -1,17 +1,17 @@ -import type { MediaTypes } from '@/entrypoint/utils/media'; -import type { Embed, Sourcerer } from '@/providers/base'; -import type { ProviderList } from '@/providers/get'; +import { MediaTypes } from '@/entrypoint/utils/media'; +import { Embed, Sourcerer } from '@/providers/base'; +import { ProviderList } from '@/providers/get'; -export interface MetaOutput { +export type MetaOutput = { type: 'embed' | 'source'; id: string; rank: number; name: string; - mediaTypes?: MediaTypes[]; -} + mediaTypes?: Array; +}; function formatSourceMeta(v: Sourcerer): MetaOutput { - const types: MediaTypes[] = []; + const types: Array = []; if (v.scrapeMovie) types.push('movie'); if (v.scrapeShow) types.push('show'); return { diff --git a/packages/providers/src/entrypoint/utils/targets.ts b/packages/providers/src/entrypoint/utils/targets.ts index cd0ae8e..b3e4200 100644 --- a/packages/providers/src/entrypoint/utils/targets.ts +++ b/packages/providers/src/entrypoint/utils/targets.ts @@ -9,6 +9,10 @@ export const flags = { // The source/embed is blocking cloudflare ip's // This flag is not compatible with a proxy hosted on cloudflare CF_BLOCKED: 'cf-blocked', + + // Streams and sources with this flag wont be proxied + // And will be exclusive to the extension + PROXY_BLOCKED: 'proxy-blocked', } as const; export type Flags = (typeof flags)[keyof typeof flags]; @@ -29,10 +33,10 @@ export const targets = { export type Targets = (typeof targets)[keyof typeof targets]; -export interface FeatureMap { +export type FeatureMap = { requires: Flags[]; disallowed: Flags[]; -} +}; export const targetToFeatures: Record = { browser: { @@ -53,9 +57,14 @@ export const targetToFeatures: Record = { }, }; -export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap { +export function getTargetFeatures( + target: Targets, + consistentIpForRequests: boolean, + proxyStreams?: boolean, +): FeatureMap { const features = targetToFeatures[target]; if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED); + if (proxyStreams) features.disallowed.push(flags.PROXY_BLOCKED); return features; } diff --git a/packages/providers/src/fetchers/body.ts b/packages/providers/src/fetchers/body.ts index c0c6209..c859a2a 100644 --- a/packages/providers/src/fetchers/body.ts +++ b/packages/providers/src/fetchers/body.ts @@ -1,6 +1,6 @@ import FormData from 'form-data'; -import type { FetcherOptions } from '@/fetchers/types'; +import { FetcherOptions } from '@/fetchers/types'; import { isReactNative } from '@/utils/native'; export interface SeralizedBody { diff --git a/packages/providers/src/fetchers/common.ts b/packages/providers/src/fetchers/common.ts index e427a58..840c7ef 100644 --- a/packages/providers/src/fetchers/common.ts +++ b/packages/providers/src/fetchers/common.ts @@ -1,4 +1,4 @@ -import type { Fetcher, FetcherOptions, UseableFetcher } from '@/fetchers/types'; +import { Fetcher, FetcherOptions, UseableFetcher } from '@/fetchers/types'; export type FullUrlOptions = Pick; @@ -15,7 +15,11 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string { if (rightSide.startsWith('/')) rightSide = rightSide.slice(1); const fullUrl = leftSide + rightSide; - if (!fullUrl.startsWith('http://') && !fullUrl.startsWith('https://')) + + // we need the data scheme for base64 encoded hls playlists + // this is for playlists that themselves have cors but not their parts + // this allows us to proxy them, encode them into base64 and then fetch the parts normally + if (!fullUrl.startsWith('http://') && !fullUrl.startsWith('https://') && !fullUrl.startsWith('data:')) throw new Error(`Invald URL -- URL doesn't start with a http scheme: '${fullUrl}'`); const parsedUrl = new URL(fullUrl); @@ -34,6 +38,7 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher { baseUrl: ops?.baseUrl ?? '', readHeaders: ops?.readHeaders ?? [], body: ops?.body, + credentials: ops?.credentials, }); const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body; output.full = newFetcher; diff --git a/packages/providers/src/fetchers/fetch.ts b/packages/providers/src/fetchers/fetch.ts index c898e8b..741f0de 100644 --- a/packages/providers/src/fetchers/fetch.ts +++ b/packages/providers/src/fetchers/fetch.ts @@ -3,18 +3,19 @@ * Only containing what we need for it to function. */ -export interface FetchOps { +export type FetchOps = { headers: Record; method: string; body: any; -} + credentials?: 'include' | 'same-origin' | 'omit'; +}; -export interface FetchHeaders { +export type FetchHeaders = { get(key: string): string | null; set(key: string, value: string): void; -} +}; -export interface FetchReply { +export type FetchReply = { text(): Promise; json(): Promise; extraHeaders?: FetchHeaders; @@ -22,6 +23,6 @@ export interface FetchReply { headers: FetchHeaders; url: string; status: number; -} +}; export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise; diff --git a/packages/providers/src/fetchers/simpleProxy.ts b/packages/providers/src/fetchers/simpleProxy.ts index 261d0a4..cf8897c 100644 --- a/packages/providers/src/fetchers/simpleProxy.ts +++ b/packages/providers/src/fetchers/simpleProxy.ts @@ -1,7 +1,7 @@ import { makeFullUrl } from '@/fetchers/common'; -import type { FetchLike } from '@/fetchers/fetch'; +import { FetchLike } from '@/fetchers/fetch'; import { makeStandardFetcher } from '@/fetchers/standardFetch'; -import type { Fetcher } from '@/fetchers/types'; +import { Fetcher } from '@/fetchers/types'; const headerMap: Record = { cookie: 'X-Cookie', @@ -25,7 +25,7 @@ export function makeSimpleProxyFetcher(proxyUrl: string, f: FetchLike): Fetcher Object.entries(responseHeaderMap).forEach((entry) => { const value = res.headers.get(entry[0]); if (!value) return; - res.extraHeaders?.set(entry[0].toLowerCase(), value); + res.extraHeaders?.set(entry[1].toLowerCase(), value); }); // set correct final url diff --git a/packages/providers/src/fetchers/standardFetch.ts b/packages/providers/src/fetchers/standardFetch.ts index 682e7cc..41c5500 100644 --- a/packages/providers/src/fetchers/standardFetch.ts +++ b/packages/providers/src/fetchers/standardFetch.ts @@ -1,16 +1,17 @@ import { serializeBody } from '@/fetchers/body'; import { makeFullUrl } from '@/fetchers/common'; -import type { FetchLike, FetchReply } from '@/fetchers/fetch'; -import type { Fetcher } from '@/fetchers/types'; +import { FetchLike, FetchReply } from '@/fetchers/fetch'; +import { Fetcher } from '@/fetchers/types'; function getHeaders(list: string[], res: FetchReply): Headers { const output = new Headers(); list.forEach((header) => { const realHeader = header.toLowerCase(); - const value = res.headers.get(realHeader); + const realValue = res.headers.get(realHeader); const extraValue = res.extraHeaders?.get(realHeader); + const value = extraValue ?? realValue; if (!value) return; - output.set(realHeader, extraValue ?? value); + output.set(realHeader, value); }); return output; } @@ -27,6 +28,7 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { ...ops.headers, }, body: seralizedBody.body, + credentials: ops.credentials, }); let body: any; diff --git a/packages/providers/src/fetchers/types.ts b/packages/providers/src/fetchers/types.ts index f69c2b7..efc21a5 100644 --- a/packages/providers/src/fetchers/types.ts +++ b/packages/providers/src/fetchers/types.ts @@ -1,37 +1,41 @@ -import type * as FormData from 'form-data'; +import * as FormData from 'form-data'; -export interface FetcherOptions { +export type FetcherOptions = { baseUrl?: string; headers?: Record; query?: Record; method?: 'HEAD' | 'GET' | 'POST'; readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; -} + credentials?: 'include' | 'same-origin' | 'omit'; +}; // Version of the options that always has the defaults set // This is to make making fetchers yourself easier -export interface DefaultedFetcherOptions { +export type DefaultedFetcherOptions = { baseUrl?: string; body?: Record | string | FormData; headers: Record; query: Record; readHeaders: string[]; method: 'HEAD' | 'GET' | 'POST'; -} + credentials?: 'include' | 'same-origin' | 'omit'; +}; -export interface FetcherResponse { +export type FetcherResponse = { statusCode: number; headers: Headers; finalUrl: string; body: T; -} +}; // This is the version that will be inputted by library users -export type Fetcher = (url: string, ops: DefaultedFetcherOptions) => Promise>; +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): Promise>; +}; // This is the version that scrapers will be interacting with -export interface UseableFetcher { +export type UseableFetcher = { (url: string, ops?: FetcherOptions): Promise; full: (url: string, ops?: FetcherOptions) => Promise>; -} +}; diff --git a/packages/providers/src/providers/all.ts b/packages/providers/src/providers/all.ts index 2916196..30de4ce 100644 --- a/packages/providers/src/providers/all.ts +++ b/packages/providers/src/providers/all.ts @@ -1,4 +1,4 @@ -import type { Embed, Sourcerer } from '@/providers/base'; +import { Embed, Sourcerer } from '@/providers/base'; import { doodScraper } from '@/providers/embeds/dood'; import { droploadScraper } from '@/providers/embeds/dropload'; import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; @@ -8,27 +8,44 @@ import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; import { streambucketScraper } from '@/providers/embeds/streambucket'; import { streamsbScraper } from '@/providers/embeds/streamsb'; +import { turbovidScraper } from '@/providers/embeds/turbovid'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; import { vTubeScraper } from '@/providers/embeds/vtube'; +import { astraScraper, novaScraper, orionScraper } from '@/providers/embeds/whvx'; +import { autoembedScraper } from '@/providers/sources/autoembed'; +import { catflixScraper } from '@/providers/sources/catflix'; +import { ee3Scraper } from '@/providers/sources/ee3'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; +import { fsharetvScraper } from '@/providers/sources/fsharetv'; import { goMoviesScraper } from '@/providers/sources/gomovies/index'; import { insertunitScraper } from '@/providers/sources/insertunit'; import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { lookmovieScraper } from '@/providers/sources/lookmovie'; import { nsbxScraper } from '@/providers/sources/nsbx'; +import { redStarScraper } from '@/providers/sources/redstar'; import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { tugaflixScraper } from '@/providers/sources/tugaflix'; import { vidsrcScraper } from '@/providers/sources/vidsrc/index'; +import { whvxScraper } from '@/providers/sources/whvx'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { + autoembedBengaliScraper, + autoembedEnglishScraper, + autoembedHindiScraper, + autoembedTamilScraper, + autoembedTeluguScraper, +} from './embeds/autoembed'; import { bflixScraper } from './embeds/bflix'; import { closeLoadScraper } from './embeds/closeload'; import { fileMoonScraper } from './embeds/filemoon'; import { fileMoonMp4Scraper } from './embeds/filemoon/mp4'; -import { deltaScraper } from './embeds/nsbx/delta'; +import { hydraxScraper } from './embeds/hydrax'; +import { alphaScraper, deltaScraper } from './embeds/nsbx'; +import { playm4uNMScraper } from './embeds/playm4u/nm'; import { ridooScraper } from './embeds/ridoo'; import { smashyStreamOScraper } from './embeds/smashystream/opstream'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; @@ -39,9 +56,11 @@ import { vidplayScraper } from './embeds/vidplay'; import { voeScraper } from './embeds/voe'; import { warezcdnembedHlsScraper } from './embeds/warezcdn/hls'; import { warezcdnembedMp4Scraper } from './embeds/warezcdn/mp4'; +import { warezPlayerScraper } from './embeds/warezcdn/warezplayer'; import { wootlyScraper } from './embeds/wootly'; import { goojaraScraper } from './sources/goojara'; import { hdRezkaScraper } from './sources/hdrezka'; +import { m4uScraper } from './sources/m4ufree'; import { nepuScraper } from './sources/nepu'; import { nitesScraper } from './sources/nites'; import { primewireScraper } from './sources/primewire'; @@ -51,9 +70,10 @@ import { soaperTvScraper } from './sources/soapertv'; import { vidSrcToScraper } from './sources/vidsrcto'; import { warezcdnScraper } from './sources/warezcdn'; -export function gatherAllSources(): Sourcerer[] { +export function gatherAllSources(): Array { // all sources are gathered here return [ + catflixScraper, flixhqScraper, remotestreamScraper, kissAsianScraper, @@ -69,16 +89,22 @@ export function gatherAllSources(): Sourcerer[] { nepuScraper, goojaraScraper, hdRezkaScraper, + m4uScraper, primewireScraper, warezcdnScraper, insertunitScraper, nitesScraper, soaperTvScraper, + autoembedScraper, tugaflixScraper, + ee3Scraper, + whvxScraper, + fsharetvScraper, + redStarScraper, ]; } -export function gatherAllEmbeds(): Embed[] { +export function gatherAllEmbeds(): Array { // all embeds are gathered here return [ upcloudScraper, @@ -98,6 +124,7 @@ export function gatherAllEmbeds(): Embed[] { fileMoonScraper, fileMoonMp4Scraper, deltaScraper, + alphaScraper, vidplayScraper, wootlyScraper, doodScraper, @@ -109,6 +136,18 @@ export function gatherAllEmbeds(): Embed[] { vTubeScraper, warezcdnembedHlsScraper, warezcdnembedMp4Scraper, + warezPlayerScraper, bflixScraper, + playm4uNMScraper, + hydraxScraper, + autoembedEnglishScraper, + autoembedHindiScraper, + autoembedBengaliScraper, + autoembedTamilScraper, + autoembedTeluguScraper, + turbovidScraper, + novaScraper, + astraScraper, + orionScraper, ]; } diff --git a/packages/providers/src/providers/base.ts b/packages/providers/src/providers/base.ts index 211daf6..0d43895 100644 --- a/packages/providers/src/providers/base.ts +++ b/packages/providers/src/providers/base.ts @@ -1,20 +1,20 @@ -import type { Flags } from '@/entrypoint/utils/targets'; -import type { Stream } from '@/providers/streams'; -import type { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { Flags } from '@/entrypoint/utils/targets'; +import { Stream } from '@/providers/streams'; +import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; export type MediaScraperTypes = 'show' | 'movie'; -export interface SourcererEmbed { +export type SourcererEmbed = { embedId: string; url: string; -} +}; -export interface SourcererOutput { +export type SourcererOutput = { embeds: SourcererEmbed[]; stream?: Stream[]; -} +}; -export interface SourcererOptions { +export type SourcererOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue @@ -22,7 +22,7 @@ export interface SourcererOptions { flags: Flags[]; scrapeMovie?: (input: MovieScrapeContext) => Promise; scrapeShow?: (input: ShowScrapeContext) => Promise; -} +}; export type Sourcerer = SourcererOptions & { type: 'source'; @@ -42,17 +42,17 @@ export function makeSourcerer(state: SourcererOptions): Sourcerer { }; } -export interface EmbedOutput { +export type EmbedOutput = { stream: Stream[]; -} +}; -export interface EmbedOptions { +export type EmbedOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; scrape: (input: EmbedScrapeContext) => Promise; -} +}; export type Embed = EmbedOptions & { type: 'embed'; diff --git a/packages/providers/src/providers/captions.ts b/packages/providers/src/providers/captions.ts index afbe49e..d64dcc2 100644 --- a/packages/providers/src/providers/captions.ts +++ b/packages/providers/src/providers/captions.ts @@ -6,13 +6,14 @@ export const captionTypes = { }; export type CaptionType = keyof typeof captionTypes; -export interface Caption { +export type Caption = { type: CaptionType; id: string; // only unique per stream + opensubtitles?: boolean; url: string; hasCorsRestrictions: boolean; language: string; -} +}; export function getCaptionTypeFromUrl(url: string): CaptionType | null { const extensions = Object.keys(captionTypes) as CaptionType[]; diff --git a/packages/providers/src/providers/embeds/autoembed.ts b/packages/providers/src/providers/embeds/autoembed.ts new file mode 100644 index 0000000..50d153a --- /dev/null +++ b/packages/providers/src/providers/embeds/autoembed.ts @@ -0,0 +1,54 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +const providers = [ + { + id: 'autoembed-english', + rank: 10, + }, + { + id: 'autoembed-hindi', + rank: 9, + }, + { + id: 'autoembed-tamil', + rank: 8, + }, + { + id: 'autoembed-telugu', + rank: 7, + }, + { + id: 'autoembed-bengali', + rank: 6, + }, +]; + +function embed(provider: { id: string; rank: number }) { + return makeEmbed({ + id: provider.id, + name: provider.id.charAt(0).toUpperCase() + provider.id.slice(1), + rank: provider.rank, + async scrape(ctx) { + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: ctx.url, + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, + }); +} + +export const [ + autoembedEnglishScraper, + autoembedHindiScraper, + autoembedBengaliScraper, + autoembedTamilScraper, + autoembedTeluguScraper, +] = providers.map(embed); diff --git a/packages/providers/src/providers/embeds/closeload.ts b/packages/providers/src/providers/embeds/closeload.ts index 5a620e9..811323a 100644 --- a/packages/providers/src/providers/embeds/closeload.ts +++ b/packages/providers/src/providers/embeds/closeload.ts @@ -5,8 +5,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { NotFoundError } from '@/utils/errors'; import { makeEmbed } from '../base'; -import type { Caption} from '../captions'; -import { getCaptionTypeFromUrl, labelToLanguageCode } from '../captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../captions'; const referer = 'https://ridomovies.tv/'; diff --git a/packages/providers/src/providers/embeds/febbox/common.ts b/packages/providers/src/providers/embeds/febbox/common.ts index b9cdb1d..5d902db 100644 --- a/packages/providers/src/providers/embeds/febbox/common.ts +++ b/packages/providers/src/providers/embeds/febbox/common.ts @@ -1,4 +1,4 @@ -import type { MediaTypes } from '@/entrypoint/utils/media'; +import { MediaTypes } from '@/entrypoint/utils/media'; export const febBoxBase = `https://www.febbox.com`; diff --git a/packages/providers/src/providers/embeds/febbox/fileList.ts b/packages/providers/src/providers/embeds/febbox/fileList.ts index 2651103..593fc77 100644 --- a/packages/providers/src/providers/embeds/febbox/fileList.ts +++ b/packages/providers/src/providers/embeds/febbox/fileList.ts @@ -1,7 +1,6 @@ -import type { MediaTypes } from '@/entrypoint/utils/media'; -import type { FebboxFileList} from '@/providers/embeds/febbox/common'; -import { febBoxBase } from '@/providers/embeds/febbox/common'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { MediaTypes } from '@/entrypoint/utils/media'; +import { FebboxFileList, febBoxBase } from '@/providers/embeds/febbox/common'; +import { EmbedScrapeContext } from '@/utils/context'; export async function getFileList( ctx: EmbedScrapeContext, diff --git a/packages/providers/src/providers/embeds/febbox/hls.ts b/packages/providers/src/providers/embeds/febbox/hls.ts index 443f850..792c112 100644 --- a/packages/providers/src/providers/embeds/febbox/hls.ts +++ b/packages/providers/src/providers/embeds/febbox/hls.ts @@ -1,53 +1,47 @@ -import { makeEmbed } from "@/providers/base"; -import { parseInputUrl } from "@/providers/embeds/febbox/common"; -import { getStreams } from "@/providers/embeds/febbox/fileList"; -import { getSubtitles } from "@/providers/embeds/febbox/subtitles"; -import { showboxBase } from "@/providers/sources/showbox/common"; +import { MediaTypes } from '@/entrypoint/utils/media'; +import { makeEmbed } from '@/providers/base'; +import { parseInputUrl } from '@/providers/embeds/febbox/common'; +import { getStreams } from '@/providers/embeds/febbox/fileList'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; +import { showboxBase } from '@/providers/sources/showbox/common'; // structure: https://www.febbox.com/share/ export function extractShareKey(url: string): string { const parsedUrl = new URL(url); - const shareKey = parsedUrl.pathname.split("/")[2]; + const shareKey = parsedUrl.pathname.split('/')[2]; return shareKey; } export const febboxHlsScraper = makeEmbed({ - id: "febbox-hls", - name: "Febbox (HLS)", + id: 'febbox-hls', + name: 'Febbox (HLS)', rank: 160, disabled: true, async scrape(ctx) { const { type, id, season, episode } = parseInputUrl(ctx.url); const sharelinkResult = await ctx.proxiedFetcher<{ data?: { link?: string }; - }>("/index/share_link", { + }>('/index/share_link', { baseUrl: showboxBase, query: { id, - type: type === "movie" ? "1" : "2", + type: type === 'movie' ? '1' : '2', }, }); - if (!sharelinkResult?.data?.link) throw new Error("No embed url found"); + if (!sharelinkResult?.data?.link) throw new Error('No embed url found'); ctx.progress(30); const shareKey = extractShareKey(sharelinkResult.data.link); const fileList = await getStreams(ctx, shareKey, type, season, episode); const firstStream = fileList[0]; - if (!firstStream) throw new Error("No playable mp4 stream found"); + if (!firstStream) throw new Error('No playable mp4 stream found'); ctx.progress(70); return { stream: [ { - id: "primary", - type: "hls", + id: 'primary', + type: 'hls', flags: [], - captions: await getSubtitles( - ctx, - id, - firstStream.fid, - type, - season, - episode, - ), + captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, }, ], diff --git a/packages/providers/src/providers/embeds/febbox/qualities.ts b/packages/providers/src/providers/embeds/febbox/qualities.ts index 86decdc..b55dd72 100644 --- a/packages/providers/src/providers/embeds/febbox/qualities.ts +++ b/packages/providers/src/providers/embeds/febbox/qualities.ts @@ -1,6 +1,6 @@ import { sendRequest } from '@/providers/sources/showbox/sendRequest'; -import type { StreamFile } from '@/providers/streams'; -import type { ScrapeContext } from '@/utils/context'; +import { StreamFile } from '@/providers/streams'; +import { ScrapeContext } from '@/utils/context'; const allowedQualities = ['360', '480', '720', '1080', '4k']; diff --git a/packages/providers/src/providers/embeds/febbox/subtitles.ts b/packages/providers/src/providers/embeds/febbox/subtitles.ts index 084038d..186c04f 100644 --- a/packages/providers/src/providers/embeds/febbox/subtitles.ts +++ b/packages/providers/src/providers/embeds/febbox/subtitles.ts @@ -1,13 +1,12 @@ -import type { - Caption} from '@/providers/captions'; import { + Caption, getCaptionTypeFromUrl, isValidLanguageCode, removeDuplicatedLanguages as removeDuplicateLanguages, } from '@/providers/captions'; import { captionsDomains } from '@/providers/sources/showbox/common'; import { sendRequest } from '@/providers/sources/showbox/sendRequest'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; interface CaptionApiResponse { data: { diff --git a/packages/providers/src/providers/embeds/filemoon/index.ts b/packages/providers/src/providers/embeds/filemoon/index.ts index 42aebb5..8ed7646 100644 --- a/packages/providers/src/providers/embeds/filemoon/index.ts +++ b/packages/providers/src/providers/embeds/filemoon/index.ts @@ -1,10 +1,11 @@ import { load } from 'cheerio'; import { unpack } from 'unpacker'; -import type { SubtitleResult } from './types'; +import { flags } from '@/entrypoint/utils/targets'; + +import { SubtitleResult } from './types'; import { makeEmbed } from '../../base'; -import type { Caption} from '../../captions'; -import { getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; const evalCodeRegex = /eval\((.*)\)/g; const fileRegex = /file:"(.*?)"/g; @@ -52,7 +53,7 @@ export const fileMoonScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: file[1], - flags: [], + flags: [flags.IP_LOCKED], captions, }, ], diff --git a/packages/providers/src/providers/embeds/filemoon/mp4.ts b/packages/providers/src/providers/embeds/filemoon/mp4.ts index 181453b..35580c6 100644 --- a/packages/providers/src/providers/embeds/filemoon/mp4.ts +++ b/packages/providers/src/providers/embeds/filemoon/mp4.ts @@ -1,3 +1,4 @@ +import { flags } from '@/entrypoint/utils/targets'; import { NotFoundError } from '@/utils/errors'; import { makeEmbed } from '../../base'; @@ -28,7 +29,7 @@ export const fileMoonMp4Scraper = makeEmbed({ url, }, }, - flags: [], + flags: [flags.IP_LOCKED], captions: result.stream[0].captions, }, ], diff --git a/packages/providers/src/providers/embeds/hydrax.ts b/packages/providers/src/providers/embeds/hydrax.ts new file mode 100644 index 0000000..beef294 --- /dev/null +++ b/packages/providers/src/providers/embeds/hydrax.ts @@ -0,0 +1,65 @@ +import { makeEmbed } from '@/providers/base'; + +export const hydraxScraper = makeEmbed({ + id: 'hydrax', + name: 'Hydrax', + rank: 250, + async scrape(ctx) { + // ex-url: https://hihihaha1.xyz/?v=Lgd2uuuTS7 + const embed = await ctx.proxiedFetcher(ctx.url); + + const match = embed.match(/PLAYER\(atob\("(.*?)"/); + if (!match?.[1]) throw new Error('No Data Found'); + + ctx.progress(50); + + const qualityMatch = embed.match(/({"pieceLength.+?})/); + let qualityData: { pieceLength?: string; sd?: string[]; mHd?: string[]; hd?: string[]; fullHd?: string[] } = {}; + if (qualityMatch?.[1]) qualityData = JSON.parse(qualityMatch[1]); + + const data: { id: string; domain: string } = JSON.parse(atob(match[1])); + if (!data.id || !data.domain) throw new Error('Required values missing'); + + const domain = new URL((await ctx.proxiedFetcher.full(`https://${data.domain}`)).finalUrl).hostname; + + ctx.progress(100); + + return { + stream: [ + { + id: 'primary', + type: 'file', + qualities: { + ...(qualityData?.fullHd && { + 1080: { + type: 'mp4', + url: `https://${domain}/whw${data.id}`, + }, + }), + ...(qualityData?.hd && { + 720: { + type: 'mp4', + url: `https://${domain}/www${data.id}`, + }, + }), + ...(qualityData?.mHd && { + 480: { + type: 'mp4', + url: `https://${domain}/${data.id}`, + }, + }), + 360: { + type: 'mp4', + url: `https://${domain}/${data.id}`, + }, + }, + headers: { + Referer: ctx.url.replace(new URL(ctx.url).hostname, 'abysscdn.com'), + }, + captions: [], + flags: [], + }, + ], + }; + }, +}); diff --git a/packages/providers/src/providers/embeds/nsbx.ts b/packages/providers/src/providers/embeds/nsbx.ts new file mode 100644 index 0000000..495486b --- /dev/null +++ b/packages/providers/src/providers/embeds/nsbx.ts @@ -0,0 +1,54 @@ +import { EmbedOutput, makeEmbed } from '@/providers/base'; +import { NotFoundError } from '@/utils/errors'; + +const providers = [ + { + id: 'delta', + rank: 699, + }, + { + id: 'alpha', + rank: 695, + }, +]; + +function embed(provider: { id: string; rank: number }) { + return makeEmbed({ + id: provider.id, + name: provider.id.charAt(0).toUpperCase() + provider.id.slice(1), + rank: provider.rank, + disabled: false, + async scrape(ctx) { + const [query, baseUrl] = ctx.url.split('|'); + + const search = await ctx.fetcher.full('/search', { + query: { + query, + provider: provider.id, + }, + credentials: 'include', + baseUrl, + }); + + if (search.statusCode === 429) throw new Error('Rate limited'); + if (search.statusCode !== 200) throw new NotFoundError('Failed to search'); + + ctx.progress(50); + + const result = await ctx.fetcher('/provider', { + query: { + resourceId: search.body.url, + provider: provider.id, + }, + credentials: 'include', + baseUrl, + }); + + ctx.progress(100); + + return result as EmbedOutput; + }, + }); +} + +export const [deltaScraper, alphaScraper] = providers.map(embed); diff --git a/packages/providers/src/providers/embeds/nsbx/delta.ts b/packages/providers/src/providers/embeds/nsbx/delta.ts deleted file mode 100644 index 283ddde..0000000 --- a/packages/providers/src/providers/embeds/nsbx/delta.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { EmbedOutput} from '@/providers/base'; -import { makeEmbed } from '@/providers/base'; -import { headers } from '@/providers/sources/nsbx'; - -export const deltaScraper = makeEmbed({ - id: 'delta', - name: 'Delta', - rank: 200, - disabled: false, - async scrape(ctx) { - const url = `https://api.nsbx.ru/provider?resourceId=${encodeURIComponent(ctx.url)}&provider=delta`; - const result = await ctx.fetcher(url, { - headers, - }); - - return result as EmbedOutput; - }, -}); diff --git a/packages/providers/src/providers/embeds/playm4u/nm.ts b/packages/providers/src/providers/embeds/playm4u/nm.ts new file mode 100644 index 0000000..32429fb --- /dev/null +++ b/packages/providers/src/providers/embeds/playm4u/nm.ts @@ -0,0 +1,123 @@ +import { load } from 'cheerio'; +import crypto from 'crypto-js'; + +import { makeEmbed } from '@/providers/base'; + +const { AES, MD5 } = crypto; + +// I didn't even care to take a look at the code +// it poabably could be better, +// i don't care +// Thanks Paradox_77 +function mahoaData(input: string, key: string) { + const a = AES.encrypt(input, key).toString(); + + const b = a + .replace('U2FsdGVkX1', '') + .replace(/\//g, '|a') + .replace(/\+/g, '|b') + .replace(/\\=/g, '|c') + .replace(/\|/g, '-z'); + return b; +} + +function caesarShift(str: string, amount: number) { + if (amount < 0) { + return caesarShift(str, amount + 26); + } + let output = ''; + for (let i = 0; i < str.length; i++) { + let c = str[i]; + if (c.match(/[a-z]/i)) { + const code = str.charCodeAt(i); + if (code >= 65 && code <= 90) { + c = String.fromCharCode(((code - 65 + amount) % 26) + 65); + } else if (code >= 97 && code <= 122) { + c = String.fromCharCode(((code - 97 + amount) % 26) + 97); + } + } + output += c; + } + return output; +} + +function stringToHex(tmp: string) { + let str = ''; + for (let i = 0; i < tmp.length; i++) { + str += tmp[i].charCodeAt(0).toString(16); + } + return str; +} + +function generateResourceToken(idUser: string, idFile: string, domainRef: string) { + const dataToken = stringToHex( + caesarShift(mahoaData(`Win32|${idUser}|${idFile}|${domainRef}`, MD5('plhq@@@2022').toString()), 22), + ); + const resourceToken = `${dataToken}|${MD5(`${dataToken}plhq@@@22`).toString()}`; + return resourceToken; +} + +const apiUrl = 'https://api-post-iframe-rd.playm4u.xyz/api/playiframe'; + +type apiRes = { + status: number; + // i only came across url-m3u8 + type: 'url-m3u8'; + data: string; + cache: boolean; + sub?: string | undefined; + subs?: string | undefined; +}; + +export const playm4uNMScraper = makeEmbed({ + id: 'playm4u-nm', + name: 'PlayM4U', + rank: 240, + scrape: async (ctx) => { + // ex: https://play9str.playm4u.xyz/play/648f159ba3115a6f00744a16 + const mainPage$ = load(await ctx.proxiedFetcher(ctx.url)); + + const script = mainPage$(`script:contains("${apiUrl}")`).text(); + if (!script) throw new Error('Failed to get script'); + + ctx.progress(50); + + const domainRef = 'https://ww2.m4ufree.tv'; + const idFile = script.match(/var\s?idfile\s?=\s?"(.*)";/im)?.[1]; + const idUser = script.match(/var\s?iduser\s?=\s?"(.*)";/im)?.[1]; + if (!idFile || !idUser) throw new Error('Failed to get ids'); + + const charecters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789=+'; + + const apiRes: apiRes = await ctx.proxiedFetcher(apiUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + namekey: 'playm4u03', + token: Array.from({ length: 100 }, () => charecters.charAt(Math.floor(Math.random() * charecters.length))).join( + '', + ), + referrer: domainRef, + data: generateResourceToken(idUser, idFile, domainRef), + }), + }); + + if (!apiRes.data || apiRes.type !== 'url-m3u8') throw new Error('Failed to get the stream'); + + ctx.progress(100); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: apiRes.data, + captions: [], + flags: [], + }, + ], + }; + }, +}); diff --git a/packages/providers/src/providers/embeds/smashystream/video1.ts b/packages/providers/src/providers/embeds/smashystream/video1.ts index a0e5772..19af888 100644 --- a/packages/providers/src/providers/embeds/smashystream/video1.ts +++ b/packages/providers/src/providers/embeds/smashystream/video1.ts @@ -1,13 +1,12 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; -import type { Caption} from '@/providers/captions'; -import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; import { NotFoundError } from '@/utils/errors'; -interface FPlayerResponse { +type FPlayerResponse = { sourceUrls: string[]; subtitles: string | null; -} +}; // if you don't understand how this is reversed // check https://discord.com/channels/871713465100816424/1186646348137775164/1225644477188935770 diff --git a/packages/providers/src/providers/embeds/streamsb.ts b/packages/providers/src/providers/embeds/streamsb.ts index 084c59d..7e5c5ad 100644 --- a/packages/providers/src/providers/embeds/streamsb.ts +++ b/packages/providers/src/providers/embeds/streamsb.ts @@ -5,8 +5,8 @@ import FormData from 'form-data'; import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; -import type { StreamFile } from '@/providers/streams'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { StreamFile } from '@/providers/streams'; +import { EmbedScrapeContext } from '@/utils/context'; async function fetchCaptchaToken(ctx: EmbedScrapeContext, domain: string, recaptchaKey: string) { const domainHash = Base64.stringify(Utf8.parse(domain)).replace(/=/g, '.'); @@ -150,7 +150,7 @@ export const streamsbScraper = makeEmbed({ (a, v) => { a[v.quality] = { type: 'mp4', - url: v.url!, + url: v.url as string, }; return a; }, diff --git a/packages/providers/src/providers/embeds/turbovid.ts b/packages/providers/src/providers/embeds/turbovid.ts new file mode 100644 index 0000000..7f7e3fc --- /dev/null +++ b/packages/providers/src/providers/embeds/turbovid.ts @@ -0,0 +1,79 @@ +import { makeEmbed } from '@/providers/base'; + +// Thanks to Paradox_77 for helping with the decryption +function hexToChar(hex: string): string { + return String.fromCharCode(parseInt(hex, 16)); +} + +function decrypt(data: string, key: string): string { + const formatedData = data.match(/../g)?.map(hexToChar).join('') || ''; + return formatedData + .split('') + .map((char, i) => String.fromCharCode(char.charCodeAt(0) ^ key.charCodeAt(i % key.length))) + .join(''); +} + +export const turbovidScraper = makeEmbed({ + id: 'turbovid', + name: 'Turbovid', + rank: 122, + async scrape(ctx) { + const baseUrl = new URL(ctx.url).origin; + const embedPage = await ctx.proxiedFetcher(ctx.url); + + ctx.progress(30); + + // the whitespace is for future-proofing the regex a bit + const apkey = embedPage.match(/const\s+apkey\s*=\s*"(.*?)";/)?.[1]; + const xxid = embedPage.match(/const\s+xxid\s*=\s*"(.*?)";/)?.[1]; + + if (!apkey || !xxid) throw new Error('Failed to get required values'); + + // json isn't parsed by proxiedFetcher due to content-type being text/html + const juiceKey = JSON.parse( + await ctx.proxiedFetcher('/api/cucked/juice_key', { + baseUrl, + headers: { + referer: ctx.url, + }, + }), + ).juice; + + if (!juiceKey) throw new Error('Failed to fetch the key'); + + ctx.progress(60); + + const data = JSON.parse( + await ctx.proxiedFetcher('/api/cucked/the_juice/', { + baseUrl, + query: { + [apkey]: xxid, + }, + headers: { + referer: ctx.url, + }, + }), + ).data; + + if (!data) throw new Error('Failed to fetch required data'); + + ctx.progress(90); + + const playlist = decrypt(data, juiceKey); + + return { + stream: [ + { + type: 'hls', + id: 'primary', + playlist, + headers: { + referer: baseUrl, + }, + flags: [], + captions: [], + }, + ], + }; + }, +}); diff --git a/packages/providers/src/providers/embeds/upcloud.ts b/packages/providers/src/providers/embeds/upcloud.ts index ad30d53..63d8407 100644 --- a/packages/providers/src/providers/embeds/upcloud.ts +++ b/packages/providers/src/providers/embeds/upcloud.ts @@ -2,8 +2,7 @@ import crypto from 'crypto-js'; import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; -import type { Caption} from '@/providers/captions'; -import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; const origin = 'https://rabbitstream.net'; const referer = 'https://rabbitstream.net/'; diff --git a/packages/providers/src/providers/embeds/vidplay/common.ts b/packages/providers/src/providers/embeds/vidplay/common.ts index f4f9a02..f10681a 100644 --- a/packages/providers/src/providers/embeds/vidplay/common.ts +++ b/packages/providers/src/providers/embeds/vidplay/common.ts @@ -1,6 +1,6 @@ import { makeFullUrl } from '@/fetchers/common'; import { decodeData } from '@/providers/sources/vidsrcto/common'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.online'; export const referer = `${vidplayBase}/`; diff --git a/packages/providers/src/providers/embeds/vidplay/index.ts b/packages/providers/src/providers/embeds/vidplay/index.ts index 741bd80..4084e82 100644 --- a/packages/providers/src/providers/embeds/vidplay/index.ts +++ b/packages/providers/src/providers/embeds/vidplay/index.ts @@ -1,9 +1,9 @@ +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; -import type { Caption} from '@/providers/captions'; -import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; import { getFileUrl } from './common'; -import type { SubtitleResult, ThumbnailTrack, VidplaySourceResponse } from './types'; +import { SubtitleResult, ThumbnailTrack, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', @@ -54,7 +54,7 @@ export const vidplayScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: source, - flags: [], + flags: [flags.PROXY_BLOCKED], headers: { Referer: url.origin, Origin: url.origin, diff --git a/packages/providers/src/providers/embeds/vidplay/types.ts b/packages/providers/src/providers/embeds/vidplay/types.ts index 14fc36a..8810647 100644 --- a/packages/providers/src/providers/embeds/vidplay/types.ts +++ b/packages/providers/src/providers/embeds/vidplay/types.ts @@ -1,4 +1,4 @@ -export interface VidplaySourceResponse { +export type VidplaySourceResponse = { result: | { sources: { @@ -10,7 +10,7 @@ export interface VidplaySourceResponse { }[]; } | number; -} +}; export type SubtitleResult = { file: string; @@ -18,7 +18,7 @@ export type SubtitleResult = { kind: string; }[]; -export interface ThumbnailTrack { +export type ThumbnailTrack = { type: 'vtt'; url: string; -} +}; diff --git a/packages/providers/src/providers/embeds/warezcdn/common.ts b/packages/providers/src/providers/embeds/warezcdn/common.ts index 1f35b6f..79501c2 100644 --- a/packages/providers/src/providers/embeds/warezcdn/common.ts +++ b/packages/providers/src/providers/embeds/warezcdn/common.ts @@ -1,5 +1,5 @@ import { warezcdnPlayerBase } from '@/providers/sources/warezcdn/common'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; function decrypt(input: string) { @@ -38,6 +38,7 @@ export async function getDecryptedId(ctx: EmbedScrapeContext) { const allowanceKey = page.match(/let allowanceKey = "(.*?)";/)?.[1]; if (!allowanceKey) throw new NotFoundError('Failed to get allowanceKey'); + // this endpoint is removed hence the method no longer works const streamData = await ctx.proxiedFetcher('/functions.php', { baseUrl: warezcdnPlayerBase, method: 'POST', diff --git a/packages/providers/src/providers/embeds/warezcdn/hls.ts b/packages/providers/src/providers/embeds/warezcdn/hls.ts index 809d2ff..4595c9b 100644 --- a/packages/providers/src/providers/embeds/warezcdn/hls.ts +++ b/packages/providers/src/providers/embeds/warezcdn/hls.ts @@ -1,6 +1,6 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { getDecryptedId } from './common'; @@ -21,6 +21,8 @@ async function getVideowlUrlStream(ctx: EmbedScrapeContext, decryptedId: string) export const warezcdnembedHlsScraper = makeEmbed({ id: 'warezcdnembedhls', // WarezCDN is both a source and an embed host name: 'WarezCDN HLS', + // method no longer works + disabled: true, rank: 83, async scrape(ctx) { const decryptedId = await getDecryptedId(ctx); diff --git a/packages/providers/src/providers/embeds/warezcdn/mp4.ts b/packages/providers/src/providers/embeds/warezcdn/mp4.ts index 8347e96..89d1798 100644 --- a/packages/providers/src/providers/embeds/warezcdn/mp4.ts +++ b/packages/providers/src/providers/embeds/warezcdn/mp4.ts @@ -1,7 +1,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { warezcdnWorkerProxy } from '@/providers/sources/warezcdn/common'; -import type { EmbedScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { getDecryptedId } from './common'; @@ -25,8 +25,9 @@ async function checkUrls(ctx: EmbedScrapeContext, fileId: string) { export const warezcdnembedMp4Scraper = makeEmbed({ id: 'warezcdnembedmp4', // WarezCDN is both a source and an embed host name: 'WarezCDN MP4', + // method no longer works rank: 82, - disabled: false, + disabled: true, async scrape(ctx) { const decryptedId = await getDecryptedId(ctx); diff --git a/packages/providers/src/providers/embeds/warezcdn/warezplayer.ts b/packages/providers/src/providers/embeds/warezcdn/warezplayer.ts new file mode 100644 index 0000000..5eb9b66 --- /dev/null +++ b/packages/providers/src/providers/embeds/warezcdn/warezplayer.ts @@ -0,0 +1,58 @@ +import { makeEmbed } from '@/providers/base'; +import { warezcdnApiBase, warezcdnPlayerBase } from '@/providers/sources/warezcdn/common'; + +export const warezPlayerScraper = makeEmbed({ + id: 'warezplayer', + name: 'warezPLAYER', + rank: 85, + async scrape(ctx) { + const page = await ctx.proxiedFetcher.full(`/player.php`, { + baseUrl: warezcdnPlayerBase, + headers: { + Referer: `${warezcdnApiBase}/getEmbed.php?${new URLSearchParams({ + id: ctx.url, + sv: 'warezcdn', + })}`, + }, + query: { + id: ctx.url, + }, + }); + // ex url: https://basseqwevewcewcewecwcw.xyz/video/0e4a2c65bdaddd66a53422d93daebe68 + const playerPageUrl = new URL(page.finalUrl); + + const hash = playerPageUrl.pathname.split('/')[2]; + const playerApiRes = await ctx.proxiedFetcher('/player/index.php', { + baseUrl: playerPageUrl.origin, + query: { + data: hash, + do: 'getVideo', + }, + method: 'POST', + body: new URLSearchParams({ + hash, + }), + headers: { + 'X-Requested-With': 'XMLHttpRequest', + }, + }); + const sources = JSON.parse(playerApiRes); // json isn't parsed by fetcher due to content-type being text/html. + if (!sources.videoSource) throw new Error('Playlist not found'); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + flags: [], + captions: [], + playlist: sources.videoSource, + headers: { + // without this it returns "security error" + Accept: '*/*', + }, + }, + ], + }; + }, +}); diff --git a/packages/providers/src/providers/embeds/whvx.ts b/packages/providers/src/providers/embeds/whvx.ts new file mode 100644 index 0000000..3017f3d --- /dev/null +++ b/packages/providers/src/providers/embeds/whvx.ts @@ -0,0 +1,71 @@ +import { EmbedOutput, makeEmbed } from '@/providers/base'; +import { baseUrl } from '@/providers/sources/whvx'; +import { NotFoundError } from '@/utils/errors'; + +const providers = [ + { + id: 'nova', + rank: 720, + }, + { + id: 'astra', + rank: 710, + }, + { + id: 'orion', + rank: 700, + disabled: true, + }, +]; + +export const headers = { + Origin: 'https://www.vidbinge.com', + Referer: 'https://www.vidbinge.com', +}; + +function embed(provider: { id: string; rank: number; disabled?: boolean }) { + return makeEmbed({ + id: provider.id, + name: provider.id.charAt(0).toUpperCase() + provider.id.slice(1), + rank: provider.rank, + disabled: provider.disabled, + async scrape(ctx) { + let progress = 50; + const interval = setInterval(() => { + if (progress < 100) { + progress += 1; + ctx.progress(progress); + } + }, 100); + + try { + const search = await ctx.fetcher.full( + `${baseUrl}/search?query=${encodeURIComponent(ctx.url)}&provider=${provider.id}`, + { headers }, + ); + + if (search.statusCode === 429) { + throw new Error('Rate limited'); + } else if (search.statusCode !== 200) { + throw new NotFoundError('Failed to search'); + } + + const result = await ctx.fetcher( + `${baseUrl}/source?resourceId=${encodeURIComponent(search.body.url)}&provider=${provider.id}`, + { headers }, + ); + + clearInterval(interval); + ctx.progress(100); + + return result as EmbedOutput; + } catch (error) { + clearInterval(interval); + ctx.progress(100); + throw new NotFoundError('Failed to search'); + } + }, + }); +} + +export const [novaScraper, astraScraper, orionScraper] = providers.map(embed); diff --git a/packages/providers/src/providers/get.ts b/packages/providers/src/providers/get.ts index e454692..900bbd1 100644 --- a/packages/providers/src/providers/get.ts +++ b/packages/providers/src/providers/get.ts @@ -1,6 +1,5 @@ -import type { FeatureMap} from '@/entrypoint/utils/targets'; -import { flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; -import type { Embed, Sourcerer } from '@/providers/base'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; +import { Embed, Sourcerer } from '@/providers/base'; import { hasDuplicates } from '@/utils/predicates'; export interface ProviderList { diff --git a/packages/providers/src/providers/sources/autoembed.ts b/packages/providers/src/providers/sources/autoembed.ts new file mode 100644 index 0000000..e4d9a2e --- /dev/null +++ b/packages/providers/src/providers/sources/autoembed.ts @@ -0,0 +1,46 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +const baseUrl = 'https://autoembed.cc/'; + +async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const playerPage = await ctx.proxiedFetcher(`/embed/player.php`, { + baseUrl, + query: { + id: ctx.media.tmdbId, + ...(ctx.media.type === 'show' && { + s: ctx.media.season.number.toString(), + e: ctx.media.episode.number.toString(), + }), + }, + }); + + const fileDataMatch = playerPage.match(/"file": (\[.*?\])/s); + if (!fileDataMatch[1]) throw new NotFoundError('No data found'); + + const fileData: { title: string; file: string }[] = JSON.parse(fileDataMatch[1].replace(/,\s*\]$/, ']')); + + const embeds: SourcererEmbed[] = []; + + for (const stream of fileData) { + const url = stream.file; + if (!url) continue; + embeds.push({ embedId: `autoembed-${stream.title.toLowerCase().trim()}`, url }); + } + + return { + embeds, + }; +} + +export const autoembedScraper = makeSourcerer({ + id: 'autoembed', + name: 'Autoembed', + rank: 10, + disabled: true, + flags: [flags.CORS_ALLOWED], + scrapeMovie: comboScraper, + scrapeShow: comboScraper, +}); diff --git a/packages/providers/src/providers/sources/catflix.ts b/packages/providers/src/providers/sources/catflix.ts new file mode 100644 index 0000000..03c9456 --- /dev/null +++ b/packages/providers/src/providers/sources/catflix.ts @@ -0,0 +1,74 @@ +import { load } from 'cheerio'; + +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { compareMedia } from '@/utils/compare'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +const baseUrl = 'https://catflix.su'; + +async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const searchPage = await ctx.proxiedFetcher('/', { + baseUrl, + query: { + s: ctx.media.title, + }, + }); + + ctx.progress(40); + + const $search = load(searchPage); + const searchResults: { title: string; year?: number | undefined; url: string }[] = []; + + $search('li').each((_, element) => { + const title = $search(element).find('h2').first().text().trim(); + // the year is always present, but I sitll decided to make it nullable since the impl isn't as future-proof + const year = Number($search(element).find('.text-xs > span').eq(1).text().trim()) || undefined; + const url = $search(element).find('a').attr('href'); + + if (!title || !url) return; + + searchResults.push({ title, year, url }); + }); + + let watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url; + if (!watchPageUrl) throw new NotFoundError('No watchable item found'); + + ctx.progress(60); + + if (ctx.media.type === 'show') { + const match = watchPageUrl.match(/\/series\/([^/]+)\/?/); + if (!match) throw new Error('Failed to parse watch page url'); + watchPageUrl = watchPageUrl.replace( + `/series/${match[1]}`, + `/episode/${match[1]}-${ctx.media.season.number}x${ctx.media.episode.number}`, + ); + } + + const watchPage = load(await ctx.proxiedFetcher(watchPageUrl)); + + ctx.progress(80); + + const url = watchPage('iframe').first().attr('src'); // I couldn't think of a better way + if (!url) throw new Error('Failed to find embed url'); + + ctx.progress(90); + + return { + embeds: [ + { + embedId: 'turbovid', + url, + }, + ], + }; +} + +export const catflixScraper = makeSourcerer({ + id: 'catflix', + name: 'Catflix', + rank: 122, + flags: [], + scrapeMovie: comboScraper, + scrapeShow: comboScraper, +}); diff --git a/packages/providers/src/providers/sources/ee3/common.ts b/packages/providers/src/providers/sources/ee3/common.ts new file mode 100644 index 0000000..f1201cc --- /dev/null +++ b/packages/providers/src/providers/sources/ee3/common.ts @@ -0,0 +1,7 @@ +export const useAltEndpoint: boolean = false; + +export const baseUrl = useAltEndpoint ? 'https://rips.cc' : 'https://ee3.me'; + +export const username = '_sf_'; + +export const password = 'defonotscraping'; diff --git a/packages/providers/src/providers/sources/ee3/index.ts b/packages/providers/src/providers/sources/ee3/index.ts new file mode 100644 index 0000000..4424386 --- /dev/null +++ b/packages/providers/src/providers/sources/ee3/index.ts @@ -0,0 +1,97 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { Caption } from '@/providers/captions'; +import { compareMedia } from '@/utils/compare'; +import { MovieScrapeContext } from '@/utils/context'; +import { makeCookieHeader } from '@/utils/cookie'; +import { NotFoundError } from '@/utils/errors'; + +import { baseUrl, password, username } from './common'; +import { itemDetails, renewResponse } from './types'; +import { login, parseSearch } from './utils'; + +// this source only has movies +async function comboScraper(ctx: MovieScrapeContext): Promise { + const pass = await login(username, password, ctx); + if (!pass) throw new Error('Login failed'); + + const search = parseSearch( + await ctx.proxiedFetcher('/get', { + baseUrl, + method: 'POST', + body: new URLSearchParams({ query: ctx.media.title, action: 'search' }), + headers: { + cookie: makeCookieHeader({ PHPSESSID: pass }), + }, + }), + ); + + const id = search.find((v) => v && compareMedia(ctx.media, v.title, v.year))?.id; + if (!id) throw new NotFoundError('No watchable item found'); + + const details: itemDetails = JSON.parse( + await ctx.proxiedFetcher('/get', { + baseUrl, + method: 'POST', + body: new URLSearchParams({ id, action: 'get_movie_info' }), + headers: { + cookie: makeCookieHeader({ PHPSESSID: pass }), + }, + }), + ); + if (!details.message.video) throw new Error('Failed to get the stream'); + + const keyParams: renewResponse = JSON.parse( + await ctx.proxiedFetcher('/renew', { + baseUrl, + method: 'POST', + headers: { + cookie: makeCookieHeader({ PHPSESSID: pass }), + }, + }), + ); + if (!keyParams.k) throw new Error('Failed to get the key'); + + const server = details.message.server === '1' ? 'https://vid.ee3.me/vid/' : 'https://vault.rips.cc/video/'; + const k = keyParams.k; + const url = `${server}${details.message.video}?${new URLSearchParams({ k })}`; + const captions: Caption[] = []; + + // this how they actually deal with subtitles + if (details.message.subs?.toLowerCase() === 'yes' && details.message.imdbID) { + captions.push({ + id: `https://rips.cc/subs/${details.message.imdbID}.vtt`, + url: `https://rips.cc/subs/${details.message.imdbID}.vtt`, + type: 'vtt', + hasCorsRestrictions: false, + language: 'en', + }); + } + + return { + embeds: [], + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + captions, + qualities: { + // should be unknown, but all the videos are 720p + 720: { + type: 'mp4', + url, + }, + }, + }, + ], + }; +} + +export const ee3Scraper = makeSourcerer({ + id: 'ee3', + name: 'EE3', + rank: 111, + flags: [flags.CORS_ALLOWED], + scrapeMovie: comboScraper, +}); diff --git a/packages/providers/src/providers/sources/ee3/types.ts b/packages/providers/src/providers/sources/ee3/types.ts new file mode 100644 index 0000000..6728c9c --- /dev/null +++ b/packages/providers/src/providers/sources/ee3/types.ts @@ -0,0 +1,31 @@ +export interface itemDetails { + status: number; + message: { + id: string; + imdbID: string; + title: string; + video: string; + server: string; + year: string; + image: string; + glow: string; + rating: string; + watch_count: string; + datetime?: string | null; + requested_by?: string | null; + subs?: string | null; + time?: string | null; + duration?: string | null; + }; +} + +export interface renewResponse { + k: string; + msg?: string | null; + status: number | string | null; +} + +export interface loginResponse { + status: number; + message: string; +} diff --git a/packages/providers/src/providers/sources/ee3/utils.ts b/packages/providers/src/providers/sources/ee3/utils.ts new file mode 100644 index 0000000..a797977 --- /dev/null +++ b/packages/providers/src/providers/sources/ee3/utils.ts @@ -0,0 +1,46 @@ +import { load } from 'cheerio'; + +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { parseSetCookie } from '@/utils/cookie'; + +import { baseUrl } from './common'; +import { loginResponse } from './types'; + +export async function login( + user: string, + pass: string, + ctx: ShowScrapeContext | MovieScrapeContext, +): Promise { + const req = await ctx.proxiedFetcher.full('/login', { + baseUrl, + method: 'POST', + body: new URLSearchParams({ user, pass, action: 'login' }), + readHeaders: ['Set-Cookie'], + }); + const res: loginResponse = JSON.parse(req.body); + + const cookie = parseSetCookie( + // It retruns a cookie even when the login failed + // I have the backup cookie here just in case + res.status === 1 ? (req.headers.get('Set-Cookie') ?? '') : 'PHPSESSID=mk2p73c77qc28o5i5120843ruu;', + ); + + return cookie.PHPSESSID.value; +} + +export function parseSearch(body: string): { title: string; year: number; id: string }[] { + const result: { title: string; year: number; id: string }[] = []; + + const $ = load(body); + $('div').each((_, element) => { + const title = $(element).find('.title').text().trim(); + const year = parseInt($(element).find('.details span').first().text().trim(), 10); + const id = $(element).find('.control-buttons').attr('data-id'); + + if (title && year && id) { + result.push({ title, year, id }); + } + }); + + return result; +} diff --git a/packages/providers/src/providers/sources/flixhq/index.ts b/packages/providers/src/providers/sources/flixhq/index.ts index c037835..d0a2898 100644 --- a/packages/providers/src/providers/sources/flixhq/index.ts +++ b/packages/providers/src/providers/sources/flixhq/index.ts @@ -1,6 +1,5 @@ import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererEmbed} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { vidCloudScraper } from '@/providers/embeds/vidcloud'; import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape'; diff --git a/packages/providers/src/providers/sources/flixhq/scrape.ts b/packages/providers/src/providers/sources/flixhq/scrape.ts index e43c2c6..5555be5 100644 --- a/packages/providers/src/providers/sources/flixhq/scrape.ts +++ b/packages/providers/src/providers/sources/flixhq/scrape.ts @@ -1,8 +1,8 @@ import { load } from 'cheerio'; -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; export async function getFlixhqSourceDetails(ctx: ScrapeContext, sourceId: string): Promise { diff --git a/packages/providers/src/providers/sources/flixhq/search.ts b/packages/providers/src/providers/sources/flixhq/search.ts index 3d86aca..bcab033 100644 --- a/packages/providers/src/providers/sources/flixhq/search.ts +++ b/packages/providers/src/providers/sources/flixhq/search.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { compareMedia, compareTitle } from '@/utils/compare'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia | ShowMedia): Promise { const searchResults = await ctx.proxiedFetcher(`/search/${media.title.replaceAll(/[^a-z0-9A-Z]/g, '-')}`, { diff --git a/packages/providers/src/providers/sources/fsharetv.ts b/packages/providers/src/providers/sources/fsharetv.ts new file mode 100644 index 0000000..6193607 --- /dev/null +++ b/packages/providers/src/providers/sources/fsharetv.ts @@ -0,0 +1,94 @@ +import { load } from 'cheerio'; + +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { FileBasedStream } from '@/providers/streams'; +import { compareMedia } from '@/utils/compare'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; +import { getValidQualityFromString } from '@/utils/quality'; + +const baseUrl = 'https://fsharetv.co'; + +async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const searchPage = await ctx.proxiedFetcher('/search', { + baseUrl, + query: { + q: ctx.media.title, + }, + }); + + const search$ = load(searchPage); + const searchResults: { title: string; year?: number; url: string }[] = []; + + search$('.movie-item').each((_, element) => { + const [, title, year] = + search$(element) + .find('b') + .text() + ?.match(/^(.*?)\s*(?:\(?\s*(\d{4})(?:\s*-\s*\d{0,4})?\s*\)?)?\s*$/) || []; + const url = search$(element).find('a').attr('href'); + if (!title || !url) return; + + searchResults.push({ title, year: Number(year) ?? undefined, url }); + }); + + const watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url; + if (!watchPageUrl) throw new NotFoundError('No watchable item found'); + + const watchPage = await ctx.proxiedFetcher(watchPageUrl.replace('/movie', '/w'), { baseUrl }); + + const fileId = watchPage.match(/Movie\.setSource\('([^']*)'/)?.[1]; + if (!fileId) throw new Error('File ID not found'); + + const apiRes: { data: { file: { sources: { src: string; quality: string | number }[] } } } = await ctx.proxiedFetcher( + `/api/file/${fileId}/source`, + { + baseUrl, + query: { + type: 'watch', + }, + }, + ); + if (!apiRes.data.file.sources.length) throw new Error('No sources found'); + + // this is to get around a ext bug where it doesn't send the headers to the second req after redir + const mediaBase = new URL((await ctx.proxiedFetcher.full(apiRes.data.file.sources[0].src, { baseUrl })).finalUrl) + .origin; + + const qualities = apiRes.data.file.sources.reduce( + (acc, source) => { + const quality = typeof source.quality === 'number' ? source.quality.toString() : source.quality; + const validQuality = getValidQualityFromString(quality); + acc[validQuality] = { + type: 'mp4', + url: `${mediaBase}${source.src.replace('/api', '')}`, + }; + return acc; + }, + {} as FileBasedStream['qualities'], + ); + + return { + embeds: [], + stream: [ + { + id: 'primary', + type: 'file', + flags: [], + headers: { + referer: 'https://fsharetv.co', + }, + qualities, + captions: [], + }, + ], + }; +} + +export const fsharetvScraper = makeSourcerer({ + id: 'fsharetv', + name: 'FshareTV', + rank: 93, + flags: [], + scrapeMovie: comboScraper, +}); diff --git a/packages/providers/src/providers/sources/gomovies/index.ts b/packages/providers/src/providers/sources/gomovies/index.ts index d2e1463..419efba 100644 --- a/packages/providers/src/providers/sources/gomovies/index.ts +++ b/packages/providers/src/providers/sources/gomovies/index.ts @@ -131,7 +131,7 @@ export const goMoviesScraper = makeSourcerer({ .filter((embed) => embed.url) .map((embed) => ({ embedId: embed.embedId, - url: embed.url!, + url: embed.url as string, })); if (filteredEmbeds.length === 0) throw new Error('No valid embeds found.'); @@ -213,7 +213,7 @@ export const goMoviesScraper = makeSourcerer({ .filter((embed) => embed.url) .map((embed) => ({ embedId: embed.embedId, - url: embed.url!, + url: embed.url as string, })); if (filteredEmbeds.length === 0) throw new Error('No valid embeds found.'); diff --git a/packages/providers/src/providers/sources/gomovies/source.ts b/packages/providers/src/providers/sources/gomovies/source.ts index 3d55a7b..4ef93bc 100644 --- a/packages/providers/src/providers/sources/gomovies/source.ts +++ b/packages/providers/src/providers/sources/gomovies/source.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { gomoviesBase } from '.'; diff --git a/packages/providers/src/providers/sources/goojara/getEmbeds.ts b/packages/providers/src/providers/sources/goojara/getEmbeds.ts index 8adacc0..7b26d6c 100644 --- a/packages/providers/src/providers/sources/goojara/getEmbeds.ts +++ b/packages/providers/src/providers/sources/goojara/getEmbeds.ts @@ -1,10 +1,9 @@ import { load } from 'cheerio'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { makeCookieHeader, parseSetCookie } from '@/utils/cookie'; -import type { EmbedsResult} from './type'; -import { baseUrl, baseUrl2 } from './type'; +import { EmbedsResult, baseUrl, baseUrl2 } from './type'; export async function getEmbeds(ctx: ScrapeContext, id: string): Promise { const data = await ctx.fetcher.full(`/${id}`, { @@ -23,7 +22,7 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise => { const result = await searchAndFindMediaId(ctx); - if (!result?.id) throw new NotFoundError('No result found'); + if (!result || !result.id) throw new NotFoundError('No result found'); const translatorId = await getTranslatorId(result.url, result.id, ctx); if (!translatorId) throw new NotFoundError('No translator id found'); diff --git a/packages/providers/src/providers/sources/hdrezka/types.ts b/packages/providers/src/providers/sources/hdrezka/types.ts index 22fc129..d7ccdc2 100644 --- a/packages/providers/src/providers/sources/hdrezka/types.ts +++ b/packages/providers/src/providers/sources/hdrezka/types.ts @@ -1,6 +1,6 @@ -import type { ScrapeMedia } from "@/entrypoint/utils/media"; +import { ScrapeMedia } from '@/index'; -export interface VideoLinks { +export type VideoLinks = { success: boolean; message: string; premium_content: number; @@ -10,7 +10,7 @@ export interface VideoLinks { subtitle_lns: boolean; subtitle_def: boolean; thumbnails: string; -} +}; export interface MovieData { id: string | null; diff --git a/packages/providers/src/providers/sources/hdrezka/utils.ts b/packages/providers/src/providers/sources/hdrezka/utils.ts index 57c6469..a88c5a3 100644 --- a/packages/providers/src/providers/sources/hdrezka/utils.ts +++ b/packages/providers/src/providers/sources/hdrezka/utils.ts @@ -1,5 +1,5 @@ import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import type { FileBasedStream } from '@/providers/streams'; +import { FileBasedStream } from '@/providers/streams'; import { NotFoundError } from '@/utils/errors'; import { getValidQualityFromString } from '@/utils/quality'; diff --git a/packages/providers/src/providers/sources/insertunit/captions.ts b/packages/providers/src/providers/sources/insertunit/captions.ts index 863f533..881c9c2 100644 --- a/packages/providers/src/providers/sources/insertunit/captions.ts +++ b/packages/providers/src/providers/sources/insertunit/captions.ts @@ -1,7 +1,6 @@ -import type { Caption} from '@/providers/captions'; -import { removeDuplicatedLanguages } from '@/providers/captions'; +import { Caption, removeDuplicatedLanguages } from '@/providers/captions'; -import type { Subtitle } from './types'; +import { Subtitle } from './types'; export async function getCaptions(data: Subtitle[]) { let captions: Caption[] = []; diff --git a/packages/providers/src/providers/sources/insertunit/index.ts b/packages/providers/src/providers/sources/insertunit/index.ts index 898f5fd..9a54866 100644 --- a/packages/providers/src/providers/sources/insertunit/index.ts +++ b/packages/providers/src/providers/sources/insertunit/index.ts @@ -1,10 +1,10 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; -import type { Caption } from '@/providers/captions'; +import { Caption } from '@/providers/captions'; import { NotFoundError } from '@/utils/errors'; import { getCaptions } from './captions'; -import type { Season } from './types'; +import { Season } from './types'; const insertUnitBase = 'https://api.insertunit.ws/'; @@ -80,7 +80,7 @@ export const insertunitScraper = makeSourcerer({ let captions: Caption[] = []; - if (subtitleJSONData?.[1] != null) { + if (subtitleJSONData != null && subtitleJSONData[1] != null) { const subtitleData = JSON.parse(subtitleJSONData[1]); captions = await getCaptions(subtitleData); } diff --git a/packages/providers/src/providers/sources/kissasian/search.ts b/packages/providers/src/providers/sources/kissasian/search.ts index 039f39d..bbf1a01 100644 --- a/packages/providers/src/providers/sources/kissasian/search.ts +++ b/packages/providers/src/providers/sources/kissasian/search.ts @@ -1,7 +1,7 @@ import { load } from 'cheerio'; import FormData from 'form-data'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { kissasianBase } from './common'; diff --git a/packages/providers/src/providers/sources/lookmovie/index.ts b/packages/providers/src/providers/sources/lookmovie/index.ts index b84ee2e..80f816d 100644 --- a/packages/providers/src/providers/sources/lookmovie/index.ts +++ b/packages/providers/src/providers/sources/lookmovie/index.ts @@ -1,7 +1,6 @@ import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { scrape, searchAndFindMedia } from './util'; diff --git a/packages/providers/src/providers/sources/lookmovie/type.ts b/packages/providers/src/providers/sources/lookmovie/type.ts index d8f7478..47ccc55 100644 --- a/packages/providers/src/providers/sources/lookmovie/type.ts +++ b/packages/providers/src/providers/sources/lookmovie/type.ts @@ -1,4 +1,4 @@ -import type { MovieMedia } from '@/entrypoint/utils/media'; +import { MovieMedia } from '@/entrypoint/utils/media'; // ! Types interface BaseConfig { @@ -35,7 +35,9 @@ export interface ShowDataResult { episodes: episodeObj[]; } -type VideoSources = Record; +interface VideoSources { + [key: string]: string; +} interface VideoSubtitles { id?: number; diff --git a/packages/providers/src/providers/sources/lookmovie/util.ts b/packages/providers/src/providers/sources/lookmovie/util.ts index e523251..f453dbc 100644 --- a/packages/providers/src/providers/sources/lookmovie/util.ts +++ b/packages/providers/src/providers/sources/lookmovie/util.ts @@ -1,9 +1,9 @@ -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { compareMedia } from '@/utils/compare'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import type { Result, ResultItem, ShowDataResult, episodeObj } from './type'; +import { Result, ResultItem, ShowDataResult, episodeObj } from './type'; import { getVideo } from './video'; export const baseUrl = 'https://lmscript.xyz'; diff --git a/packages/providers/src/providers/sources/lookmovie/video.ts b/packages/providers/src/providers/sources/lookmovie/video.ts index 7990330..8e8e3c4 100644 --- a/packages/providers/src/providers/sources/lookmovie/video.ts +++ b/packages/providers/src/providers/sources/lookmovie/video.ts @@ -1,9 +1,8 @@ -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; -import type { Caption} from '@/providers/captions'; -import { labelToLanguageCode, removeDuplicatedLanguages } from '@/providers/captions'; -import type { ScrapeContext } from '@/utils/context'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { Caption, labelToLanguageCode, removeDuplicatedLanguages } from '@/providers/captions'; +import { ScrapeContext } from '@/utils/context'; -import type { StreamsDataResult } from './type'; +import { StreamsDataResult } from './type'; import { baseUrl } from './util'; export async function getVideoSources( diff --git a/packages/providers/src/providers/sources/m4ufree.ts b/packages/providers/src/providers/sources/m4ufree.ts new file mode 100644 index 0000000..6cbb2bc --- /dev/null +++ b/packages/providers/src/providers/sources/m4ufree.ts @@ -0,0 +1,156 @@ +// kinda based on m4uscraper by Paradox_77 +// thanks Paradox_77 +import { load } from 'cheerio'; + +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; +import { compareMedia } from '@/utils/compare'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { makeCookieHeader, parseSetCookie } from '@/utils/cookie'; +import { NotFoundError } from '@/utils/errors'; + +let baseUrl = 'https://m4ufree.tv'; + +const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { + // this redirects to ww1.m4ufree.tv or ww2.m4ufree.tv + // if i explicitly keep the base ww1 while the load balancers thinks ww2 is optimal + // it will keep redirecting all the requests + // not only that but the last iframe request will fail + const homePage = await ctx.proxiedFetcher.full(baseUrl); + baseUrl = new URL(homePage.finalUrl).origin; + + const searchSlug = ctx.media.title + .replace(/'/g, '') + .replace(/!|@|%|\^|\*|\(|\)|\+|=|<|>|\?|\/|,|\.|:|;|'| |"|&|#|\[|\]|~|$|_/g, '-') + .replace(/-+-/g, '-') + .replace(/^-+|-+$/g, '') + .replace(/Ă¢â‚¬â€œ/g, ''); + + const searchPage$ = load( + await ctx.proxiedFetcher(`/search/${searchSlug}.html`, { + baseUrl, + query: { + type: ctx.media.type === 'movie' ? 'movie' : 'tvs', + }, + }), + ); + + const searchResults: { title: string; year: number | undefined; url: string }[] = []; + searchPage$('.item').each((_, element) => { + const [, title, year] = + searchPage$(element) + // the title emement on their page is broken + // it just breaks when the titles are too big + .find('.imagecover a') + .attr('title') + // ex-titles: Home Alone 1990, Avengers Endgame (2019), The Curse (2023-) + ?.match(/^(.*?)\s*(?:\(?\s*(\d{4})(?:\s*-\s*\d{0,4})?\s*\)?)?\s*$/) || []; + const url = searchPage$(element).find('a').attr('href'); + + if (!title || !url) return; + + searchResults.push({ title, year: year ? parseInt(year, 10) : undefined, url }); + }); + + const watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url; + if (!watchPageUrl) throw new NotFoundError('No watchable item found'); + + ctx.progress(25); + + const watchPage = await ctx.proxiedFetcher.full(watchPageUrl, { + baseUrl, + readHeaders: ['Set-Cookie'], + }); + + ctx.progress(50); + + let watchPage$ = load(watchPage.body); + + const csrfToken = watchPage$('script:contains("_token:")') + .html() + ?.match(/_token:\s?'(.*)'/m)?.[1]; + if (!csrfToken) throw new Error('Failed to find csrfToken'); + + const laravelSession = parseSetCookie(watchPage.headers.get('Set-Cookie') ?? '').laravel_session; + if (!laravelSession?.value) throw new Error('Failed to find cookie'); + + const cookie = makeCookieHeader({ [laravelSession.name]: laravelSession.value }); + + if (ctx.media.type === 'show') { + const s = ctx.media.season.number < 10 ? `0${ctx.media.season.number}` : ctx.media.season.number.toString(); + const e = ctx.media.episode.number < 10 ? `0${ctx.media.episode.number}` : ctx.media.episode.number.toString(); + + const episodeToken = watchPage$(`button:contains("S${s}-E${e}")`).attr('idepisode'); + if (!episodeToken) throw new Error('Failed to find episodeToken'); + + watchPage$ = load( + await ctx.proxiedFetcher('/ajaxtv', { + baseUrl, + method: 'POST', + body: new URLSearchParams({ + idepisode: episodeToken, + _token: csrfToken, + }), + headers: { + cookie, + }, + }), + ); + } + + ctx.progress(75); + + const embeds: SourcererEmbed[] = []; + + const sources: { name: string; data: string }[] = watchPage$('div.row.justify-content-md-center div.le-server') + .map((_, element) => { + const name = watchPage$(element).find('span').text().toLowerCase().replace('#', ''); + const data = watchPage$(element).find('span').attr('data'); + + if (!data || !name) return null; + return { name, data }; + }) + .get(); + + for (const source of sources) { + let embedId; + if (source.name === 'm') + embedId = 'playm4u-m'; // TODO + else if (source.name === 'nm') embedId = 'playm4u-nm'; + else if (source.name === 'h') embedId = 'hydrax'; + else continue; + + const iframePage$ = load( + await ctx.proxiedFetcher('/ajax', { + baseUrl, + method: 'POST', + body: new URLSearchParams({ + m4u: source.data, + _token: csrfToken, + }), + headers: { + cookie, + }, + }), + ); + + const url = iframePage$('iframe').attr('src'); + if (!url) continue; + + ctx.progress(100); + + embeds.push({ embedId, url }); + } + + return { + embeds, + }; +}; + +export const m4uScraper = makeSourcerer({ + id: 'm4ufree', + name: 'M4UFree', + rank: 125, + flags: [], + scrapeMovie: universalScraper, + scrapeShow: universalScraper, +}); diff --git a/packages/providers/src/providers/sources/nepu/index.ts b/packages/providers/src/providers/sources/nepu/index.ts index cc4aa3e..8f3a094 100644 --- a/packages/providers/src/providers/sources/nepu/index.ts +++ b/packages/providers/src/providers/sources/nepu/index.ts @@ -1,15 +1,14 @@ import { load } from 'cheerio'; -import type { SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { compareTitle } from '@/utils/compare'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import type { SearchResults } from './types'; +import { SearchResults } from './types'; -const nepuBase = 'https://nepu.to'; -const nepuReferer = `${nepuBase}/`; +const nepuBase = 'https://nepu.io'; +const nepuReferer = 'https://nepu.to'; const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { const searchResultRequest = await ctx.proxiedFetcher('/ajax/posts', { @@ -64,11 +63,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => captions: [], playlist: streamUrl[1], type: 'hls', - flags: [], headers: { - Origin: nepuBase, - Referer: nepuReferer, + Origin: nepuReferer, + Referer: `${nepuReferer}/`, }, + flags: [], }, ], } as SourcererOutput; @@ -78,8 +77,8 @@ export const nepuScraper = makeSourcerer({ id: 'nepu', name: 'Nepu', rank: 80, - flags: [], disabled: true, + flags: [], scrapeMovie: universalScraper, scrapeShow: universalScraper, }); diff --git a/packages/providers/src/providers/sources/nepu/types.ts b/packages/providers/src/providers/sources/nepu/types.ts index 1a00719..200995a 100644 --- a/packages/providers/src/providers/sources/nepu/types.ts +++ b/packages/providers/src/providers/sources/nepu/types.ts @@ -1,8 +1,8 @@ -export interface SearchResults { +export type SearchResults = { data: { id: number; name: string; url: string; type: 'Movie' | 'Serie'; }[]; -} +}; diff --git a/packages/providers/src/providers/sources/nites.ts b/packages/providers/src/providers/sources/nites.ts index 822ec1b..4a3570a 100644 --- a/packages/providers/src/providers/sources/nites.ts +++ b/packages/providers/src/providers/sources/nites.ts @@ -1,9 +1,8 @@ import { load } from 'cheerio'; -import type { SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { compareMedia } from '@/utils/compare'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; const baseUrl = 'https://w1.nites.is'; diff --git a/packages/providers/src/providers/sources/nsbx.ts b/packages/providers/src/providers/sources/nsbx.ts index 4eb2d1b..199e627 100644 --- a/packages/providers/src/providers/sources/nsbx.ts +++ b/packages/providers/src/providers/sources/nsbx.ts @@ -1,14 +1,8 @@ import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererEmbed, SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -export const headers = { - Origin: 'https://extension.works.again.with.nsbx', - Referer: 'https://extension.works.again.with.nsbx', -}; - async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { const query = { title: ctx.media.title, @@ -16,30 +10,31 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis tmdbId: ctx.media.tmdbId, imdbId: ctx.media.imdbId, type: ctx.media.type, - season: '', - episode: '', + ...(ctx.media.type === 'show' && { + season: ctx.media.season.number.toString(), + episode: ctx.media.episode.number.toString(), + }), }; - if (ctx.media.type === 'show') { - query.season = ctx.media.season.number.toString(); - query.episode = ctx.media.episode.number.toString(); - } + const res: { providers: string[]; endpoint: string } = await ctx.fetcher('https://api.nsbx.ru/status'); - const result = await ctx.fetcher(`https://api.nsbx.ru/search?query=${encodeURIComponent(JSON.stringify(query))}`, { - headers, - }); + if (res.providers?.length === 0) throw new NotFoundError('No providers available'); + if (!res.endpoint) throw new Error('No endpoint returned'); - if (result.embeds.length === 0) throw new NotFoundError('No watchable item found'); + const embeds = res.providers.map((provider: string) => ({ + embedId: provider, + url: `${JSON.stringify(query)}|${res.endpoint}`, + })); return { - embeds: result.embeds as SourcererEmbed[], + embeds, }; } export const nsbxScraper = makeSourcerer({ id: 'nsbx', name: 'NSBX', - rank: 150, + rank: 129, flags: [flags.CORS_ALLOWED], disabled: false, scrapeMovie: comboScraper, diff --git a/packages/providers/src/providers/sources/primewire/index.ts b/packages/providers/src/providers/sources/primewire/index.ts index 81a0afb..42c78cd 100644 --- a/packages/providers/src/providers/sources/primewire/index.ts +++ b/packages/providers/src/providers/sources/primewire/index.ts @@ -2,7 +2,7 @@ import { load } from 'cheerio'; import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { primewireApiKey, primewireBase } from './common'; @@ -80,7 +80,7 @@ async function getStreams(title: string) { export const primewireScraper = makeSourcerer({ id: 'primewire', name: 'Primewire', - rank: 110, + rank: 1, flags: [flags.CORS_ALLOWED], async scrapeMovie(ctx) { if (!ctx.media.imdbId) throw new Error('No imdbId provided'); diff --git a/packages/providers/src/providers/sources/redstar.ts b/packages/providers/src/providers/sources/redstar.ts new file mode 100644 index 0000000..18190df --- /dev/null +++ b/packages/providers/src/providers/sources/redstar.ts @@ -0,0 +1,43 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + try { + const res = await ctx.fetcher.full(`https://red-star.ningai.workers.dev/scrape/showbox`, { + query: { + type: ctx.media.type, + title: ctx.media.title, + releaseYear: ctx.media.releaseYear.toString(), + tmdbId: ctx.media.tmdbId, + imdbId: ctx.media.imdbId ?? '', + ...(ctx.media.type === 'show' && { + episodeNumber: ctx.media.episode.number.toString(), + episodeTmdbId: ctx.media.episode.tmdbId, + seasonNumber: ctx.media.season.number.toString(), + seasonTmdbId: ctx.media.season.tmdbId, + }), + }, + }); + + if (res.statusCode === 200 && res.body.stream?.length) + return { stream: res.body.stream, embeds: [] } as SourcererOutput; + if (res.statusCode === 404) throw new NotFoundError('No watchable item found'); + + throw new Error(res.body.message ?? 'An error has occurred!'); + } catch (e: any) { + if (e instanceof NotFoundError) throw new NotFoundError(e.message); + throw new Error(e.message ?? 'An error has occurred!'); + } +}; + +export const redStarScraper = makeSourcerer({ + id: 'redstar', + name: 'redStar', + disabled: true, + rank: 131, + flags: [flags.CORS_ALLOWED], + scrapeMovie: universalScraper, + scrapeShow: universalScraper, +}); diff --git a/packages/providers/src/providers/sources/ridomovies/index.ts b/packages/providers/src/providers/sources/ridomovies/index.ts index 1017ee7..638ea13 100644 --- a/packages/providers/src/providers/sources/ridomovies/index.ts +++ b/packages/providers/src/providers/sources/ridomovies/index.ts @@ -1,14 +1,13 @@ import { load } from 'cheerio'; import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererEmbed} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; import { closeLoadScraper } from '@/providers/embeds/closeload'; import { ridooScraper } from '@/providers/embeds/ridoo'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import type { IframeSourceResult, SearchResult } from './types'; +import { IframeSourceResult, SearchResult } from './types'; const ridoMoviesBase = `https://ridomovies.tv`; const ridoMoviesApiBase = `${ridoMoviesBase}/core/api`; diff --git a/packages/providers/src/providers/sources/ridomovies/types.ts b/packages/providers/src/providers/sources/ridomovies/types.ts index d5ba25d..a030738 100644 --- a/packages/providers/src/providers/sources/ridomovies/types.ts +++ b/packages/providers/src/providers/sources/ridomovies/types.ts @@ -65,14 +65,14 @@ export interface SearchResultItem { contentable: Contentable; } -export interface SearchResult { +export type SearchResult = { data: { items: SearchResultItem[]; }; -} +}; -export interface IframeSourceResult { +export type IframeSourceResult = { data: { url: string; }[]; -} +}; diff --git a/packages/providers/src/providers/sources/showbox/index.ts b/packages/providers/src/providers/sources/showbox/index.ts index 32fec19..9901a52 100644 --- a/packages/providers/src/providers/sources/showbox/index.ts +++ b/packages/providers/src/providers/sources/showbox/index.ts @@ -1,9 +1,8 @@ import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { compareTitle } from '@/utils/compare'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { sendRequest } from './sendRequest'; diff --git a/packages/providers/src/providers/sources/smashystream/index.ts b/packages/providers/src/providers/sources/smashystream/index.ts index 8dfea61..cc91cef 100644 --- a/packages/providers/src/providers/sources/smashystream/index.ts +++ b/packages/providers/src/providers/sources/smashystream/index.ts @@ -1,9 +1,8 @@ import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { smashyStreamOScraper } from '@/providers/embeds/smashystream/opstream'; import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { // theres no point in fetching the player page @@ -12,17 +11,17 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr const query = ctx.media.type === 'movie' ? `?tmdb=${ctx.media.tmdbId}` - : `?tmdbId=${ctx.media.tmdbId}&season=${ctx.media.season.number}&episode=${ctx.media.episode.number}`; + : `?tmdb=${ctx.media.tmdbId}&season=${ctx.media.season.number}&episode=${ctx.media.episode.number}`; return { embeds: [ { embedId: smashyStreamFScraper.id, - url: `https://embed.smashystream.com/video1dn.php${query}`, + url: `https://embed.smashystream.com/videofeee.php${query}`, }, { embedId: smashyStreamOScraper.id, - url: `https://embed.smashystream.com/videoop.php${query}`, + url: `https://embed.smashystream.com/shortmoviec.php${query}`, }, ], }; @@ -32,6 +31,7 @@ export const smashyStreamScraper = makeSourcerer({ id: 'smashystream', name: 'SmashyStream', rank: 30, + disabled: true, flags: [flags.CORS_ALLOWED], scrapeMovie: universalScraper, scrapeShow: universalScraper, diff --git a/packages/providers/src/providers/sources/soapertv/index.ts b/packages/providers/src/providers/sources/soapertv/index.ts index 4d8cbba..54e371f 100644 --- a/packages/providers/src/providers/sources/soapertv/index.ts +++ b/packages/providers/src/providers/sources/soapertv/index.ts @@ -1,14 +1,14 @@ import { load } from 'cheerio'; import { flags } from '@/entrypoint/utils/targets'; -import type { Caption} from '@/providers/captions'; -import { labelToLanguageCode } from '@/providers/captions'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { Caption, labelToLanguageCode } from '@/providers/captions'; +import { Stream } from '@/providers/streams'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; +import { convertPlaylistsToDataUrls } from '@/utils/playlist'; -import type { InfoResponse } from './types'; -import type { SourcererOutput} from '../../base'; -import { makeSourcerer } from '../../base'; +import { InfoResponse } from './types'; +import { SourcererOutput, makeSourcerer } from '../../base'; const baseUrl = 'https://soaper.tv'; @@ -43,13 +43,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr const contentPage$ = load(contentPage); const pass = contentPage$('#hId').attr('value'); - const param = contentPage$('#divU').text(); - if (!pass || !param) throw new NotFoundError('Content not found'); + if (!pass) throw new NotFoundError('Content not found'); const formData = new URLSearchParams(); formData.append('pass', pass); - formData.append('param', param); formData.append('e2', '0'); formData.append('server', '0'); @@ -92,20 +90,22 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr stream: [ { id: 'primary', - playlist: streamResJson.val, + playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val}`), type: 'hls', - flags: [flags.IP_LOCKED], + proxyDepth: 2, + flags: [flags.CORS_ALLOWED], captions, }, ...(streamResJson.val_bak ? [ { id: 'backup', - playlist: streamResJson.val_bak, - type: 'hls' as const, - flags: [flags.IP_LOCKED], + playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val_bak}`), + type: 'hls', + flags: [flags.CORS_ALLOWED], + proxyDepth: 2, captions, - }, + } as Stream, ] : []), ], @@ -115,8 +115,8 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr export const soaperTvScraper = makeSourcerer({ id: 'soapertv', name: 'SoaperTV', - rank: 115, - flags: [flags.IP_LOCKED], + rank: 126, + flags: [flags.CORS_ALLOWED], scrapeMovie: universalScraper, scrapeShow: universalScraper, }); diff --git a/packages/providers/src/providers/sources/tugaflix/index.ts b/packages/providers/src/providers/sources/tugaflix/index.ts index 379e805..eda8042 100644 --- a/packages/providers/src/providers/sources/tugaflix/index.ts +++ b/packages/providers/src/providers/sources/tugaflix/index.ts @@ -1,8 +1,7 @@ import { load } from 'cheerio'; import { flags } from '@/entrypoint/utils/targets'; -import type { SourcererEmbed} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; import { compareMedia } from '@/utils/compare'; import { NotFoundError } from '@/utils/errors'; diff --git a/packages/providers/src/providers/sources/vidsrc/scrape-movie.ts b/packages/providers/src/providers/sources/vidsrc/scrape-movie.ts index 8adebab..585eb31 100644 --- a/packages/providers/src/providers/sources/vidsrc/scrape-movie.ts +++ b/packages/providers/src/providers/sources/vidsrc/scrape-movie.ts @@ -1,5 +1,5 @@ import { getVidSrcMovieSources } from '@/providers/sources/vidsrc/scrape'; -import type { MovieScrapeContext } from '@/utils/context'; +import { MovieScrapeContext } from '@/utils/context'; export async function scrapeMovie(ctx: MovieScrapeContext) { return { diff --git a/packages/providers/src/providers/sources/vidsrc/scrape-show.ts b/packages/providers/src/providers/sources/vidsrc/scrape-show.ts index 33bbe12..ff5d2a4 100644 --- a/packages/providers/src/providers/sources/vidsrc/scrape-show.ts +++ b/packages/providers/src/providers/sources/vidsrc/scrape-show.ts @@ -1,5 +1,5 @@ import { getVidSrcShowSources } from '@/providers/sources/vidsrc/scrape'; -import type { ShowScrapeContext } from '@/utils/context'; +import { ShowScrapeContext } from '@/utils/context'; export async function scrapeShow(ctx: ShowScrapeContext) { return { diff --git a/packages/providers/src/providers/sources/vidsrc/scrape.ts b/packages/providers/src/providers/sources/vidsrc/scrape.ts index a05d95b..81dceff 100644 --- a/packages/providers/src/providers/sources/vidsrc/scrape.ts +++ b/packages/providers/src/providers/sources/vidsrc/scrape.ts @@ -1,10 +1,10 @@ import { load } from 'cheerio'; -import type { SourcererEmbed } from '@/providers/base'; +import { SourcererEmbed } from '@/providers/base'; import { streambucketScraper } from '@/providers/embeds/streambucket'; import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; import { vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; function decodeSrc(encoded: string, seed: string) { let decoded = ''; diff --git a/packages/providers/src/providers/sources/vidsrcto/common.ts b/packages/providers/src/providers/sources/vidsrcto/common.ts index 33a3bb5..33c436e 100644 --- a/packages/providers/src/providers/sources/vidsrcto/common.ts +++ b/packages/providers/src/providers/sources/vidsrcto/common.ts @@ -1,10 +1,10 @@ // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! -const DECRYPTION_KEY = "WXrUARXb1aDLaZjI"; +const DECRYPTION_KEY = 'WXrUARXb1aDLaZjI'; export const decodeBase64UrlSafe = (str: string) => { - const standardizedInput = str.replace(/_/g, "/").replace(/-/g, "+"); + const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); const decodedData = atob(standardizedInput); const bytes = new Uint8Array(decodedData.length); @@ -26,21 +26,17 @@ export const decodeData = (key: string, data: any) => { } index1 = 0; let index2 = 0; - let finalKey = ""; + let finalKey = ''; for (let char = 0; char < data.length; char += 1) { index1 = (index1 + 1) % 256; index2 = (index2 + state[index1]) % 256; const temp = state[index1]; state[index1] = state[index2]; state[index2] = temp; - if (typeof data[char] === "string") { - finalKey += String.fromCharCode( - data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256], - ); - } else if (typeof data[char] === "number") { - finalKey += String.fromCharCode( - data[char] ^ state[(state[index1] + state[index2]) % 256], - ); + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); } } return finalKey; diff --git a/packages/providers/src/providers/sources/vidsrcto/index.ts b/packages/providers/src/providers/sources/vidsrcto/index.ts index 08abe6d..d048b69 100644 --- a/packages/providers/src/providers/sources/vidsrcto/index.ts +++ b/packages/providers/src/providers/sources/vidsrcto/index.ts @@ -1,11 +1,11 @@ import { load } from 'cheerio'; -import type { SourcererEmbed, SourcererOutput} from '@/providers/base'; -import { makeSourcerer } from '@/providers/base'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { decryptSourceUrl } from './common'; -import type { SourceResult, SourcesResult } from './types'; +import { SourceResult, SourcesResult } from './types'; const vidSrcToBase = 'https://vidsrc.to'; const referer = `${vidSrcToBase}/`; @@ -82,8 +82,9 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr export const vidSrcToScraper = makeSourcerer({ id: 'vidsrcto', name: 'VidSrcTo', + disabled: true, scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [], + flags: [flags.PROXY_BLOCKED], rank: 130, }); diff --git a/packages/providers/src/providers/sources/vidsrcto/types.ts b/packages/providers/src/providers/sources/vidsrcto/types.ts index 1471ca3..0694b15 100644 --- a/packages/providers/src/providers/sources/vidsrcto/types.ts +++ b/packages/providers/src/providers/sources/vidsrcto/types.ts @@ -1,7 +1,7 @@ -export interface VidSrcToResponse { +export type VidSrcToResponse = { status: number; result: T; -} +}; export type SourcesResult = VidSrcToResponse< { diff --git a/packages/providers/src/providers/sources/warezcdn/common.ts b/packages/providers/src/providers/sources/warezcdn/common.ts index 4d60aee..182b2b6 100644 --- a/packages/providers/src/providers/sources/warezcdn/common.ts +++ b/packages/providers/src/providers/sources/warezcdn/common.ts @@ -1,4 +1,4 @@ -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; export const warezcdnBase = 'https://embed.warezcdn.com'; export const warezcdnApiBase = 'https://warezcdn.com/embed'; diff --git a/packages/providers/src/providers/sources/warezcdn/index.ts b/packages/providers/src/providers/sources/warezcdn/index.ts index 3cc097b..a119f8f 100644 --- a/packages/providers/src/providers/sources/warezcdn/index.ts +++ b/packages/providers/src/providers/sources/warezcdn/index.ts @@ -1,52 +1,45 @@ -import type { SourcererEmbed } from "@/providers/base"; -import { flags } from "@/entrypoint/utils/targets"; -import { makeSourcerer } from "@/providers/base"; -import { mixdropScraper } from "@/providers/embeds/mixdrop"; -import { warezcdnembedHlsScraper } from "@/providers/embeds/warezcdn/hls"; -import { warezcdnembedMp4Scraper } from "@/providers/embeds/warezcdn/mp4"; -import { NotFoundError } from "@/utils/errors"; -import { load } from "cheerio"; - -import { getExternalPlayerUrl, warezcdnBase } from "./common"; -import { SerieAjaxResponse } from "./types"; +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; +import { warezcdnembedHlsScraper } from '@/providers/embeds/warezcdn/hls'; +import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4'; +import { warezPlayerScraper } from '@/providers/embeds/warezcdn/warezplayer'; +import { NotFoundError } from '@/utils/errors'; + +import { getExternalPlayerUrl, warezcdnBase } from './common'; +import { SerieAjaxResponse } from './types'; export const warezcdnScraper = makeSourcerer({ - id: "warezcdn", - name: "WarezCDN", + id: 'warezcdn', + name: 'WarezCDN', rank: 81, flags: [flags.CORS_ALLOWED], scrapeMovie: async (ctx) => { - if (!ctx.media.imdbId) - throw new NotFoundError("This source requires IMDB id."); + if (!ctx.media.imdbId) throw new NotFoundError('This source requires IMDB id.'); - const serversPage = await ctx.proxiedFetcher( - `/filme/${ctx.media.imdbId}`, - { - baseUrl: warezcdnBase, - }, - ); + const serversPage = await ctx.proxiedFetcher(`/filme/${ctx.media.imdbId}`, { + baseUrl: warezcdnBase, + }); const $ = load(serversPage); - const embedsHost = $(".hostList.active [data-load-embed]").get(); + const embedsHost = $('.hostList.active [data-load-embed]').get(); const embeds: SourcererEmbed[] = []; embedsHost.forEach(async (element) => { - const embedHost = $(element).attr("data-load-embed-host")!; - const embedUrl = $(element).attr("data-load-embed")!; - - if (embedHost === "mixdrop") { - const realEmbedUrl = await getExternalPlayerUrl( - ctx, - "mixdrop", - embedUrl, - ); - if (!realEmbedUrl) throw new Error("Could not find embed url"); + const embedHost = $(element).attr('data-load-embed-host')!; + const embedUrl = $(element).attr('data-load-embed')!; + + if (embedHost === 'mixdrop') { + const realEmbedUrl = await getExternalPlayerUrl(ctx, 'mixdrop', embedUrl); + if (!realEmbedUrl) throw new Error('Could not find embed url'); embeds.push({ embedId: mixdropScraper.id, url: realEmbedUrl, }); - } else if (embedHost === "warezcdn") { + } else if (embedHost === 'warezcdn') { embeds.push( { embedId: warezcdnembedHlsScraper.id, @@ -56,6 +49,10 @@ export const warezcdnScraper = makeSourcerer({ embedId: warezcdnembedMp4Scraper.id, url: embedUrl, }, + { + embedId: warezPlayerScraper.id, + url: embedUrl, + }, ); } }); @@ -65,21 +62,18 @@ export const warezcdnScraper = makeSourcerer({ }; }, scrapeShow: async (ctx) => { - if (!ctx.media.imdbId) - throw new NotFoundError("This source requires IMDB id."); + if (!ctx.media.imdbId) throw new NotFoundError('This source requires IMDB id.'); const url = `${warezcdnBase}/serie/${ctx.media.imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; const serversPage = await ctx.proxiedFetcher(url); - const episodeId = serversPage.match( - /\$\('\[data-load-episode-content="(\d+)"\]'\)/, - )?.[1]; + const episodeId = serversPage.match(/\$\('\[data-load-episode-content="(\d+)"\]'\)/)?.[1]; - if (!episodeId) throw new NotFoundError("Failed to find episode id"); + if (!episodeId) throw new NotFoundError('Failed to find episode id'); const streamsData = await ctx.proxiedFetcher(`/serieAjax.php`, { - method: "POST", + method: 'POST', baseUrl: warezcdnBase, body: new URLSearchParams({ getAudios: episodeId, @@ -87,25 +81,25 @@ export const warezcdnScraper = makeSourcerer({ headers: { Origin: warezcdnBase, Referer: url, - "X-Requested-With": "XMLHttpRequest", + 'X-Requested-With': 'XMLHttpRequest', }, }); const streams: SerieAjaxResponse = JSON.parse(streamsData); - const list = streams.list["0"]; + const list = streams.list['0']; const embeds: SourcererEmbed[] = []; // 3 means ok - if (list.mixdropStatus === "3") { - const realEmbedUrl = await getExternalPlayerUrl(ctx, "mixdrop", list.id); - if (!realEmbedUrl) throw new Error("Could not find embed url"); + if (list.mixdropStatus === '3') { + const realEmbedUrl = await getExternalPlayerUrl(ctx, 'mixdrop', list.id); + if (!realEmbedUrl) throw new Error('Could not find embed url'); embeds.push({ embedId: mixdropScraper.id, url: realEmbedUrl, }); } - if (list.warezcdnStatus === "3") { + if (list.warezcdnStatus === '3') { embeds.push( { embedId: warezcdnembedHlsScraper.id, diff --git a/packages/providers/src/providers/sources/warezcdn/types.ts b/packages/providers/src/providers/sources/warezcdn/types.ts index a7c7cec..38711ff 100644 --- a/packages/providers/src/providers/sources/warezcdn/types.ts +++ b/packages/providers/src/providers/sources/warezcdn/types.ts @@ -7,7 +7,9 @@ interface Data { warezcdnStatus: string; } -type List = Record; +type List = { + [key: string]: Data; +}; export interface SerieAjaxResponse { list: List; diff --git a/packages/providers/src/providers/sources/whvx.ts b/packages/providers/src/providers/sources/whvx.ts new file mode 100644 index 0000000..ad9bff4 --- /dev/null +++ b/packages/providers/src/providers/sources/whvx.ts @@ -0,0 +1,42 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +export const baseUrl = 'https://api.whvx.net'; + +async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const query = { + title: ctx.media.title, + releaseYear: ctx.media.releaseYear, + tmdbId: ctx.media.tmdbId, + imdbId: ctx.media.imdbId, + type: ctx.media.type, + ...(ctx.media.type === 'show' && { + season: ctx.media.season.number.toString(), + episode: ctx.media.episode.number.toString(), + }), + }; + + const res: { providers: string[] } = await ctx.fetcher('/status', { baseUrl }); + + if (res.providers?.length === 0) throw new NotFoundError('No providers available'); + + const embeds = res.providers.map((provider: string) => ({ + embedId: provider, + url: JSON.stringify(query), + })); + + return { + embeds, + }; +} + +export const whvxScraper = makeSourcerer({ + id: 'whvx', + name: 'VidBinge', + rank: 128, + flags: [flags.CORS_ALLOWED], + scrapeMovie: comboScraper, + scrapeShow: comboScraper, +}); diff --git a/packages/providers/src/providers/sources/zoechip/common.ts b/packages/providers/src/providers/sources/zoechip/common.ts index 070d55b..55b37ac 100644 --- a/packages/providers/src/providers/sources/zoechip/common.ts +++ b/packages/providers/src/providers/sources/zoechip/common.ts @@ -3,17 +3,17 @@ import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; import { vidCloudScraper } from '@/providers/embeds/vidcloud'; import { getZoeChipSourceURL, getZoeChipSources } from '@/providers/sources/zoechip/scrape'; -import type { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; export const zoeBase = 'https://zoechip.cc'; -export interface ZoeChipSourceDetails { +export type ZoeChipSourceDetails = { type: string; // Only seen "iframe" so far link: string; sources: string[]; // Never seen this populated, assuming it's a string array tracks: string[]; // Never seen this populated, assuming it's a string array title: string; -} +}; export async function formatSource( ctx: MovieScrapeContext | ShowScrapeContext, diff --git a/packages/providers/src/providers/sources/zoechip/scrape-movie.ts b/packages/providers/src/providers/sources/zoechip/scrape-movie.ts index bdabad2..86161fc 100644 --- a/packages/providers/src/providers/sources/zoechip/scrape-movie.ts +++ b/packages/providers/src/providers/sources/zoechip/scrape-movie.ts @@ -1,6 +1,6 @@ import { createZoeChipStreamData } from '@/providers/sources/zoechip/common'; import { getZoeChipMovieID } from '@/providers/sources/zoechip/search'; -import type { MovieScrapeContext } from '@/utils/context'; +import { MovieScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; export async function scrapeMovie(ctx: MovieScrapeContext) { diff --git a/packages/providers/src/providers/sources/zoechip/scrape-show.ts b/packages/providers/src/providers/sources/zoechip/scrape-show.ts index 71f8cc9..fe9f4eb 100644 --- a/packages/providers/src/providers/sources/zoechip/scrape-show.ts +++ b/packages/providers/src/providers/sources/zoechip/scrape-show.ts @@ -1,7 +1,7 @@ import { createZoeChipStreamData } from '@/providers/sources/zoechip/common'; import { getZoeChipEpisodeID, getZoeChipSeasonID } from '@/providers/sources/zoechip/scrape'; import { getZoeChipShowID } from '@/providers/sources/zoechip/search'; -import type { ShowScrapeContext } from '@/utils/context'; +import { ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; export async function scrapeShow(ctx: ShowScrapeContext) { diff --git a/packages/providers/src/providers/sources/zoechip/scrape.ts b/packages/providers/src/providers/sources/zoechip/scrape.ts index 5c42ccd..d3eb183 100644 --- a/packages/providers/src/providers/sources/zoechip/scrape.ts +++ b/packages/providers/src/providers/sources/zoechip/scrape.ts @@ -1,9 +1,8 @@ import { load } from 'cheerio'; -import type { ShowMedia } from '@/entrypoint/utils/media'; -import type { ZoeChipSourceDetails} from '@/providers/sources/zoechip/common'; -import { zoeBase } from '@/providers/sources/zoechip/common'; -import type { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { ShowMedia } from '@/entrypoint/utils/media'; +import { ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common'; +import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; export async function getZoeChipSources(ctx: MovieScrapeContext | ShowScrapeContext, id: string) { // Movies use /ajax/episode/list/ID diff --git a/packages/providers/src/providers/sources/zoechip/search.ts b/packages/providers/src/providers/sources/zoechip/search.ts index 9feb4bc..f3a838d 100644 --- a/packages/providers/src/providers/sources/zoechip/search.ts +++ b/packages/providers/src/providers/sources/zoechip/search.ts @@ -1,9 +1,9 @@ import { load } from 'cheerio'; -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { zoeBase } from '@/providers/sources/zoechip/common'; import { compareMedia } from '@/utils/compare'; -import type { ScrapeContext } from '@/utils/context'; +import { ScrapeContext } from '@/utils/context'; export async function getZoeChipSearchResults(ctx: ScrapeContext, media: MovieMedia | ShowMedia) { const titleCleaned = media.title.toLocaleLowerCase().replace(/ /g, '-'); diff --git a/packages/providers/src/providers/streams.ts b/packages/providers/src/providers/streams.ts index c371857..113b9de 100644 --- a/packages/providers/src/providers/streams.ts +++ b/packages/providers/src/providers/streams.ts @@ -1,26 +1,26 @@ -import type { Flags } from '@/entrypoint/utils/targets'; -import type { Caption } from '@/providers/captions'; +import { Flags } from '@/entrypoint/utils/targets'; +import { Caption } from '@/providers/captions'; -export interface StreamFile { +export type StreamFile = { type: 'mp4'; url: string; -} +}; export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k'; -interface ThumbnailTrack { +type ThumbnailTrack = { type: 'vtt'; url: string; -} +}; -interface StreamCommon { +type StreamCommon = { id: string; // only unique per output flags: Flags[]; captions: Caption[]; thumbnailTrack?: ThumbnailTrack; headers?: Record; // these headers HAVE to be set to watch the stream preferredHeaders?: Record; // these headers are optional, would improve the stream -} +}; export type FileBasedStream = StreamCommon & { type: 'file'; @@ -30,6 +30,7 @@ export type FileBasedStream = StreamCommon & { export type HlsBasedStream = StreamCommon & { type: 'hls'; playlist: string; + proxyDepth?: 0 | 1 | 2; }; export type Stream = FileBasedStream | HlsBasedStream; diff --git a/packages/providers/src/runners/individualRunner.ts b/packages/providers/src/runners/individualRunner.ts index a4c956c..f96ab3c 100644 --- a/packages/providers/src/runners/individualRunner.ts +++ b/packages/providers/src/runners/individualRunner.ts @@ -1,22 +1,25 @@ -import type { IndividualScraperEvents } from '@/entrypoint/utils/events'; -import type { ScrapeMedia } from '@/entrypoint/utils/media'; -import type { FeatureMap} from '@/entrypoint/utils/targets'; -import { flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; -import type { UseableFetcher } from '@/fetchers/types'; -import type { EmbedOutput, SourcererOutput } from '@/providers/base'; -import type { ProviderList } from '@/providers/get'; -import type { ScrapeContext } from '@/utils/context'; +import { IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; +import { UseableFetcher } from '@/fetchers/types'; +import { EmbedOutput, SourcererOutput } from '@/providers/base'; +import { ProviderList } from '@/providers/get'; +import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; +import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles'; +import { requiresProxy, setupProxy } from '@/utils/proxy'; import { isValidStream, validatePlayableStreams } from '@/utils/valid'; -export interface IndividualSourceRunnerOptions { +export type IndividualSourceRunnerOptions = { features: FeatureMap; fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; media: ScrapeMedia; id: string; events?: IndividualScraperEvents; -} + proxyStreams?: boolean; // temporary + disableOpensubtitles?: boolean; +}; export async function scrapeInvidualSource( list: ProviderList, @@ -56,6 +59,10 @@ export async function scrapeInvidualSource( output.stream = output.stream .filter((stream) => isValidStream(stream)) .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + + output.stream = output.stream.map((stream) => + requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream, + ); } if (!output) throw new Error('output is null'); @@ -67,6 +74,15 @@ export async function scrapeInvidualSource( return true; }); + // opensubtitles + if (!ops.disableOpensubtitles) + for (const embed of output.embeds) + embed.url = `${embed.url}${btoa('MEDIA=')}${btoa( + `${ops.media.imdbId}${ + ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' + }`, + )}`; + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) throw new NotFoundError('No streams found'); @@ -74,19 +90,35 @@ export async function scrapeInvidualSource( if (output.stream && output.stream.length > 0 && output.embeds.length === 0) { const playableStreams = await validatePlayableStreams(output.stream, ops, sourceScraper.id); if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); + + // opensubtitles + if (!ops.disableOpensubtitles) + for (const playableStream of playableStreams) { + playableStream.captions = await addOpenSubtitlesCaptions( + playableStream.captions, + ops, + btoa( + `${ops.media.imdbId}${ + ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' + }`, + ), + ); + } output.stream = playableStreams; } return output; } -export interface IndividualEmbedRunnerOptions { +export type IndividualEmbedRunnerOptions = { features: FeatureMap; fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; url: string; id: string; events?: IndividualScraperEvents; -} + proxyStreams?: boolean; // temporary + disableOpensubtitles?: boolean; +}; export async function scrapeIndividualEmbed( list: ProviderList, @@ -95,10 +127,14 @@ export async function scrapeIndividualEmbed( const embedScraper = list.embeds.find((v) => ops.id === v.id); if (!embedScraper) throw new Error('Embed with ID not found'); + let url = ops.url; + let media; + if (ops.url.includes(btoa('MEDIA='))) [url, media] = url.split(btoa('MEDIA=')); + const output = await embedScraper.scrape({ fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher, - url: ops.url, + url, progress(val) { ops.events?.update?.({ id: embedScraper.id, @@ -113,8 +149,17 @@ export async function scrapeIndividualEmbed( .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); if (output.stream.length === 0) throw new NotFoundError('No streams found'); + output.stream = output.stream.map((stream) => + requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream, + ); + const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id); if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); + + if (media && !ops.disableOpensubtitles) + for (const playableStream of playableStreams) + playableStream.captions = await addOpenSubtitlesCaptions(playableStream.captions, ops, media); + output.stream = playableStreams; return output; diff --git a/packages/providers/src/runners/runner.ts b/packages/providers/src/runners/runner.ts index 95d1ae8..6c8ccaa 100644 --- a/packages/providers/src/runners/runner.ts +++ b/packages/providers/src/runners/runner.ts @@ -1,34 +1,35 @@ -import type { FullScraperEvents, UpdateEvent } from "@/entrypoint/utils/events"; -import type { ScrapeMedia } from "@/entrypoint/utils/media"; -import type { FeatureMap } from "@/entrypoint/utils/targets"; -import type { UseableFetcher } from "@/fetchers/types"; -import type { EmbedOutput, SourcererOutput } from "@/providers/base"; -import type { ProviderList } from "@/providers/get"; -import type { Stream } from "@/providers/streams"; -import type { ScrapeContext } from "@/utils/context"; -import { flagsAllowedInFeatures } from "@/entrypoint/utils/targets"; -import { NotFoundError } from "@/utils/errors"; -import { reorderOnIdList } from "@/utils/list"; -import { isValidStream, validatePlayableStream } from "@/utils/valid"; - -export interface RunOutput { +import { FullScraperEvents, UpdateEvent } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; +import { UseableFetcher } from '@/fetchers/types'; +import { EmbedOutput, SourcererOutput } from '@/providers/base'; +import { ProviderList } from '@/providers/get'; +import { Stream } from '@/providers/streams'; +import { ScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; +import { reorderOnIdList } from '@/utils/list'; +import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles'; +import { requiresProxy, setupProxy } from '@/utils/proxy'; +import { isValidStream, validatePlayableStream } from '@/utils/valid'; + +export type RunOutput = { sourceId: string; embedId?: string; stream: Stream; -} +}; -export interface SourceRunOutput { +export type SourceRunOutput = { sourceId: string; stream: Stream[]; embeds: []; -} +}; -export interface EmbedRunOutput { +export type EmbedRunOutput = { embedId: string; stream: Stream[]; -} +}; -export interface ProviderRunnerOptions { +export type ProviderRunnerOptions = { fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; features: FeatureMap; @@ -36,22 +37,19 @@ export interface ProviderRunnerOptions { embedOrder?: string[]; events?: FullScraperEvents; media: ScrapeMedia; -} - -export async function runAllProviders( - list: ProviderList, - ops: ProviderRunnerOptions, -): Promise { - const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter( - (source) => { - if (ops.media.type === "movie") return !!source.scrapeMovie; - if (ops.media.type === "show") return !!source.scrapeShow; - return false; - }, - ); + proxyStreams?: boolean; // temporary + disableOpensubtitles?: boolean; +}; + +export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise { + const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((source) => { + if (ops.media.type === 'movie') return !!source.scrapeMovie; + if (ops.media.type === 'show') return !!source.scrapeShow; + return false; + }); const embeds = reorderOnIdList(ops.embedOrder ?? [], list.embeds); const embedIds = embeds.map((embed) => embed.id); - let lastId = ""; + let lastId = ''; const contextBase: ScrapeContext = { fetcher: ops.fetcher, @@ -60,7 +58,7 @@ export async function runAllProviders( ops.events?.update?.({ id: lastId, percentage: val, - status: "pending", + status: 'pending', }); }, }; @@ -76,12 +74,12 @@ export async function runAllProviders( // run source scrapers let output: SourcererOutput | null = null; try { - if (ops.media.type === "movie" && source.scrapeMovie) + if (ops.media.type === 'movie' && source.scrapeMovie) output = await source.scrapeMovie({ ...contextBase, media: ops.media, }); - else if (ops.media.type === "show" && source.scrapeShow) + else if (ops.media.type === 'show' && source.scrapeShow) output = await source.scrapeShow({ ...contextBase, media: ops.media, @@ -89,18 +87,20 @@ export async function runAllProviders( if (output) { output.stream = (output.stream ?? []) .filter(isValidStream) - .filter((stream) => - flagsAllowedInFeatures(ops.features, stream.flags), - ); + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + + output.stream = output.stream.map((stream) => + requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream, + ); } if (!output || (!output.stream?.length && !output.embeds.length)) { - throw new NotFoundError("No streams found"); + throw new NotFoundError('No streams found'); } } catch (error) { const updateParams: UpdateEvent = { id: source.id, percentage: 100, - status: error instanceof NotFoundError ? "notfound" : "failure", + status: error instanceof NotFoundError ? 'notfound' : 'failure', reason: error instanceof NotFoundError ? error.message : undefined, error: error instanceof NotFoundError ? undefined : error, }; @@ -108,16 +108,25 @@ export async function runAllProviders( ops.events?.update?.(updateParams); continue; } - if (!output) throw new Error("Invalid media type"); + if (!output) throw new Error('Invalid media type'); // return stream is there are any if (output.stream?.[0]) { - const playableStream = await validatePlayableStream( - output.stream[0], - ops, - source.id, - ); - if (!playableStream) throw new NotFoundError("No streams found"); + const playableStream = await validatePlayableStream(output.stream[0], ops, source.id); + if (!playableStream) throw new NotFoundError('No streams found'); + + // opensubtitles + if (!ops.disableOpensubtitles) + playableStream.captions = await addOpenSubtitlesCaptions( + playableStream.captions, + ops, + btoa( + `${ops.media.imdbId}${ + ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' + }`, + ), + ); + return { sourceId: source.id, stream: playableStream, @@ -130,14 +139,12 @@ export async function runAllProviders( const e = list.embeds.find((v) => v.id === embed.embedId); return e && !e.disabled; }) - .sort( - (a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId), - ); + .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); if (sortedEmbeds.length > 0) { ops.events?.discoverEmbeds?.({ embeds: sortedEmbeds.map((embed, i) => ({ - id: [source.id, i].join("-"), + id: [source.id, i].join('-'), embedScraperId: embed.embedId, })), sourceId: source.id, @@ -146,10 +153,10 @@ export async function runAllProviders( for (const [ind, embed] of sortedEmbeds.entries()) { const scraper = embeds.find((v) => v.id === embed.embedId); - if (!scraper) throw new Error("Invalid embed returned"); + if (!scraper) throw new Error('Invalid embed returned'); // run embed scraper - const id = [source.id, ind].join("-"); + const id = [source.id, ind].join('-'); ops.events?.start?.(id); lastId = id; @@ -161,24 +168,33 @@ export async function runAllProviders( }); embedOutput.stream = embedOutput.stream .filter(isValidStream) - .filter((stream) => - flagsAllowedInFeatures(ops.features, stream.flags), - ); + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + embedOutput.stream = embedOutput.stream.map((stream) => + requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream, + ); if (embedOutput.stream.length === 0) { - throw new NotFoundError("No streams found"); + throw new NotFoundError('No streams found'); } - const playableStream = await validatePlayableStream( - embedOutput.stream[0]!, - ops, - embed.embedId, - ); - if (!playableStream) throw new NotFoundError("No streams found"); + const playableStream = await validatePlayableStream(embedOutput.stream[0], ops, embed.embedId); + if (!playableStream) throw new NotFoundError('No streams found'); + + // opensubtitles + if (!ops.disableOpensubtitles) + playableStream.captions = await addOpenSubtitlesCaptions( + playableStream.captions, + ops, + btoa( + `${ops.media.imdbId}${ + ops.media.type === 'show' ? `.${ops.media.season.number}.${ops.media.episode.number}` : '' + }`, + ), + ); embedOutput.stream = [playableStream]; } catch (error) { const updateParams: UpdateEvent = { - id: source.id, + id, percentage: 100, - status: error instanceof NotFoundError ? "notfound" : "failure", + status: error instanceof NotFoundError ? 'notfound' : 'failure', reason: error instanceof NotFoundError ? error.message : undefined, error: error instanceof NotFoundError ? undefined : error, }; @@ -190,7 +206,7 @@ export async function runAllProviders( return { sourceId: source.id, embedId: scraper.id, - stream: embedOutput.stream[0]!, + stream: embedOutput.stream[0], }; } } diff --git a/packages/providers/src/utils/compare.ts b/packages/providers/src/utils/compare.ts index e57ece2..7db784e 100644 --- a/packages/providers/src/utils/compare.ts +++ b/packages/providers/src/utils/compare.ts @@ -1,4 +1,4 @@ -import type { CommonMedia } from '@/entrypoint/utils/media'; +import { CommonMedia } from '@/entrypoint/utils/media'; export function normalizeTitle(title: string): string { let titleTrimmed = title.trim().toLowerCase(); diff --git a/packages/providers/src/utils/context.ts b/packages/providers/src/utils/context.ts index e6baa46..6f16bca 100644 --- a/packages/providers/src/utils/context.ts +++ b/packages/providers/src/utils/context.ts @@ -1,15 +1,15 @@ -import type { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; -import type { UseableFetcher } from '@/fetchers/types'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { UseableFetcher } from '@/fetchers/types'; -export interface ScrapeContext { +export type ScrapeContext = { proxiedFetcher: UseableFetcher; fetcher: UseableFetcher; progress(val: number): void; -} +}; -export interface EmbedInput { +export type EmbedInput = { url: string; -} +}; export type EmbedScrapeContext = EmbedInput & ScrapeContext; diff --git a/packages/providers/src/utils/cookie.ts b/packages/providers/src/utils/cookie.ts index 20fd3e4..9f40f85 100644 --- a/packages/providers/src/utils/cookie.ts +++ b/packages/providers/src/utils/cookie.ts @@ -13,7 +13,8 @@ export function makeCookieHeader(cookies: Record): string { } export function parseSetCookie(headerValue: string): Record { - const parsedCookies = setCookieParser.parse(headerValue, { + const splitHeaderValue = setCookieParser.splitCookiesString(headerValue); + const parsedCookies = setCookieParser.parse(splitHeaderValue, { map: true, }); return parsedCookies; diff --git a/packages/providers/src/utils/native.ts b/packages/providers/src/utils/native.ts index e72e9a2..cc91cdb 100644 --- a/packages/providers/src/utils/native.ts +++ b/packages/providers/src/utils/native.ts @@ -1,7 +1,7 @@ export const isReactNative = () => { try { - // eslint-disable-next-line import/no-extraneous-dependencies, global-require - require("react-native"); + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + require('react-native'); return true; } catch (e) { return false; diff --git a/packages/providers/src/utils/opensubtitles.ts b/packages/providers/src/utils/opensubtitles.ts new file mode 100644 index 0000000..559536c --- /dev/null +++ b/packages/providers/src/utils/opensubtitles.ts @@ -0,0 +1,49 @@ +import { Caption, labelToLanguageCode, removeDuplicatedLanguages } from '@/providers/captions'; +import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner'; +import { ProviderRunnerOptions } from '@/runners/runner'; + +export async function addOpenSubtitlesCaptions( + captions: Caption[], + ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions, + media: string, +): Promise { + try { + const [imdbId, season, episode] = atob(media) + .split('.') + .map((x, i) => (i === 0 ? x : Number(x) || null)); + if (!imdbId) return captions; + const Res: { + LanguageName: string; + SubDownloadLink: string; + SubFormat: 'srt' | 'vtt'; + }[] = await ops.proxiedFetcher( + `https://rest.opensubtitles.org/search/${ + season && episode ? `episode-${episode}/` : '' + }imdbid-${(imdbId as string).slice(2)}${season && episode ? `/season-${season}` : ''}`, + { + headers: { + 'X-User-Agent': 'VLSub 0.10.2', + }, + }, + ); + + const openSubtilesCaptions: Caption[] = []; + for (const caption of Res) { + const url = caption.SubDownloadLink.replace('.gz', '').replace('download/', 'download/subencoding-utf8/'); + const language = labelToLanguageCode(caption.LanguageName); + if (!url || !language) continue; + else + openSubtilesCaptions.push({ + id: url, + opensubtitles: true, + url, + type: caption.SubFormat || 'srt', + hasCorsRestrictions: false, + language, + }); + } + return [...captions, ...removeDuplicatedLanguages(openSubtilesCaptions)]; + } catch { + return captions; + } +} diff --git a/packages/providers/src/utils/playlist.ts b/packages/providers/src/utils/playlist.ts new file mode 100644 index 0000000..2972bb1 --- /dev/null +++ b/packages/providers/src/utils/playlist.ts @@ -0,0 +1,25 @@ +import { parse, stringify } from 'hls-parser'; +import { MasterPlaylist } from 'hls-parser/types'; + +import { UseableFetcher } from '@/fetchers/types'; + +export async function convertPlaylistsToDataUrls( + fetcher: UseableFetcher, + playlistUrl: string, + headers?: Record, +) { + const playlistData = await fetcher(playlistUrl, { headers }); + const playlist = parse(playlistData); + + if (playlist.isMasterPlaylist) { + await Promise.all( + (playlist as MasterPlaylist).variants.map(async (variant) => { + const variantPlaylistData = await fetcher(variant.uri, { headers }); + const variantPlaylist = parse(variantPlaylistData); + variant.uri = `data:application/vnd.apple.mpegurl;base64,${btoa(stringify(variantPlaylist))}`; + }), + ); + } + + return `data:application/vnd.apple.mpegurl;base64,${btoa(stringify(playlist))}`; +} diff --git a/packages/providers/src/utils/predicates.ts b/packages/providers/src/utils/predicates.ts index e6704f7..f581b2f 100644 --- a/packages/providers/src/utils/predicates.ts +++ b/packages/providers/src/utils/predicates.ts @@ -1,3 +1,3 @@ -export function hasDuplicates(values: T[]): boolean { +export function hasDuplicates(values: Array): boolean { return new Set(values).size !== values.length; } diff --git a/packages/providers/src/utils/proxy.ts b/packages/providers/src/utils/proxy.ts new file mode 100644 index 0000000..e1d7976 --- /dev/null +++ b/packages/providers/src/utils/proxy.ts @@ -0,0 +1,44 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { Stream } from '@/providers/streams'; + +export function requiresProxy(stream: Stream): boolean { + if (!stream.flags.includes(flags.CORS_ALLOWED) || !!(stream.headers && Object.keys(stream.headers).length > 0)) + return true; + return false; +} + +export function setupProxy(stream: Stream): Stream { + const headers = stream.headers && Object.keys(stream.headers).length > 0 ? stream.headers : undefined; + + const options = { + ...(stream.type === 'hls' && { depth: stream.proxyDepth ?? 0 }), + }; + + const payload: { + type?: 'hls' | 'mp4'; + url?: string; + headers?: Record; + options?: { depth?: 0 | 1 | 2 }; + } = { + headers, + options, + }; + + if (stream.type === 'hls') { + payload.type = 'hls'; + payload.url = stream.playlist; + stream.playlist = `https://proxy.nsbx.ru/proxy?${new URLSearchParams({ payload: Buffer.from(JSON.stringify(payload)).toString('base64url') })}`; + } + + if (stream.type === 'file') { + payload.type = 'mp4'; + Object.entries(stream.qualities).forEach((entry) => { + payload.url = entry[1].url; + entry[1].url = `https://proxy.nsbx.ru/proxy?${new URLSearchParams({ payload: Buffer.from(JSON.stringify(payload)).toString('base64url') })}`; + }); + } + + stream.headers = {}; + stream.flags = [flags.CORS_ALLOWED]; + return stream; +} diff --git a/packages/providers/src/utils/quality.ts b/packages/providers/src/utils/quality.ts index 6915672..8854ca5 100644 --- a/packages/providers/src/utils/quality.ts +++ b/packages/providers/src/utils/quality.ts @@ -1,4 +1,4 @@ -import type { Qualities } from '@/providers/streams'; +import { Qualities } from '@/providers/streams'; export function getValidQualityFromString(quality: string): Qualities { switch (quality.toLowerCase().replace('p', '')) { diff --git a/packages/providers/src/utils/valid.ts b/packages/providers/src/utils/valid.ts index 22f05df..9d83176 100644 --- a/packages/providers/src/utils/valid.ts +++ b/packages/providers/src/utils/valid.ts @@ -1,20 +1,27 @@ -import type { Stream } from "@/providers/streams"; -import type { IndividualEmbedRunnerOptions } from "@/runners/individualRunner"; -import type { ProviderRunnerOptions } from "@/runners/runner"; -import { warezcdnembedMp4Scraper } from "@/providers/embeds/warezcdn/mp4"; +import { alphaScraper, deltaScraper } from '@/providers/embeds/nsbx'; +import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4'; +import { astraScraper, novaScraper, orionScraper } from '@/providers/embeds/whvx'; +import { Stream } from '@/providers/streams'; +import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner'; +import { ProviderRunnerOptions } from '@/runners/runner'; -const SKIP_VALIDATION_CHECK_IDS = [warezcdnembedMp4Scraper.id]; +const SKIP_VALIDATION_CHECK_IDS = [ + warezcdnembedMp4Scraper.id, + deltaScraper.id, + alphaScraper.id, + novaScraper.id, + astraScraper.id, + orionScraper.id, +]; export function isValidStream(stream: Stream | undefined): boolean { if (!stream) return false; - if (stream.type === "hls") { + if (stream.type === 'hls') { if (!stream.playlist) return false; return true; } - if (stream.type === "file") { - const validQualities = Object.values(stream.qualities).filter( - (v) => v.url.length > 0, - ); + if (stream.type === 'file') { + const validQualities = Object.values(stream.qualities).filter((v) => v.url.length > 0); if (validQualities.length === 0) return false; return true; } @@ -30,9 +37,12 @@ export async function validatePlayableStream( ): Promise { if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return stream; - if (stream.type === "hls") { + if (stream.type === 'hls') { + // dirty temp fix for base64 urls to prep for fmhy poll + if (stream.playlist.startsWith('data:')) return stream; + const result = await ops.proxiedFetcher.full(stream.playlist, { - method: "GET", + method: 'GET', headers: { ...stream.preferredHeaders, ...stream.headers, @@ -41,15 +51,15 @@ export async function validatePlayableStream( if (result.statusCode < 200 || result.statusCode >= 400) return null; return stream; } - if (stream.type === "file") { + if (stream.type === 'file') { const validQualitiesResults = await Promise.all( Object.values(stream.qualities).map((quality) => ops.proxiedFetcher.full(quality.url, { - method: "GET", + method: 'GET', headers: { ...stream.preferredHeaders, ...stream.headers, - Range: "bytes=0-1", + Range: 'bytes=0-1', }, }), ), @@ -57,10 +67,7 @@ export async function validatePlayableStream( // remove invalid qualities from the stream const validQualities = stream.qualities; Object.keys(stream.qualities).forEach((quality, index) => { - if ( - validQualitiesResults[index]!.statusCode < 200 || - validQualitiesResults[index]!.statusCode >= 400 - ) { + if (validQualitiesResults[index].statusCode < 200 || validQualitiesResults[index].statusCode >= 400) { delete validQualities[quality as keyof typeof stream.qualities]; } }); @@ -78,9 +85,7 @@ export async function validatePlayableStreams( ): Promise { if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return streams; - return ( - await Promise.all( - streams.map((stream) => validatePlayableStream(stream, ops, sourcererId)), - ) - ).filter((v) => v !== null) as Stream[]; + return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops, sourcererId)))).filter( + (v) => v !== null, + ) as Stream[]; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0018797..00d7c59 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -295,108 +295,18 @@ importers: '@movie-web/providers': specifier: workspace:* version: link:../providers - '@nabla/vite-plugin-eslint': - specifier: ^2.0.2 - version: 2.0.4(eslint@8.57.0)(vite@5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) - '@types/cookie': - specifier: ^0.6.0 - version: 0.6.0 - '@types/crypto-js': - specifier: ^4.2.2 - version: 4.2.2 - '@types/node-fetch': - specifier: ^2.6.11 - version: 2.6.11 - '@types/randombytes': - specifier: ^2.0.3 - version: 2.0.3 - '@types/set-cookie-parser': - specifier: ^2.4.7 - version: 2.4.10 - '@types/spinnies': - specifier: ^0.5.3 - version: 0.5.3 - '@typescript-eslint/eslint-plugin': - specifier: ^7.4.0 - version: 7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0)(typescript@5.5.4) - '@typescript-eslint/parser': - specifier: ^7.4.0 - version: 7.18.0(eslint@8.57.0)(typescript@5.5.4) - '@vitest/coverage-v8': - specifier: ^1.4.0 - version: 1.6.0(vitest@1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) - commander: - specifier: ^12.0.0 - version: 12.1.0 - cross-env: - specifier: ^7.0.3 - version: 7.0.3 - dotenv: - specifier: ^16.4.5 - version: 16.4.5 - enquirer: - specifier: ^2.4.1 - version: 2.4.1 - eslint: - specifier: ^8.57.0 - version: 8.57.0 - eslint-config-airbnb-base: - specifier: ^15.0.0 - version: 15.0.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0) - eslint-config-prettier: - specifier: ^9.1.0 - version: 9.1.0(eslint@8.57.0) - eslint-import-resolver-typescript: - specifier: ^3.6.1 - version: 3.6.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@8.57.0) - eslint-plugin-import: - specifier: ^2.29.1 - version: 2.29.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) - eslint-plugin-prettier: - specifier: ^5.1.3 - version: 5.2.1(@types/eslint@8.56.11)(eslint-config-prettier@9.1.0(eslint@8.57.0))(eslint@8.57.0)(prettier@3.3.3) - node-fetch: - specifier: ^3.3.2 - version: 3.3.2 parse-hls: specifier: ^1.0.7 version: 1.0.7 prettier: specifier: ^3.2.5 version: 3.3.3 - puppeteer: - specifier: ^22.6.1 - version: 22.15.0(typescript@5.5.4) - spinnies: - specifier: ^0.5.1 - version: 0.5.1 srt-webvtt: specifier: ^2.0.0 version: 2.0.0 tmdb-ts: specifier: ^1.6.1 version: 1.8.0 - tsc-alias: - specifier: ^1.8.8 - version: 1.8.10 - tsconfig-paths: - specifier: ^4.2.0 - version: 4.2.0 - typescript: - specifier: ^5.4.3 - version: 5.5.4 - vite: - specifier: ^5.2.7 - version: 5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) - vite-node: - specifier: ^1.4.0 - version: 1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) - vite-plugin-dts: - specifier: ^3.8.1 - version: 3.9.1(@types/node@22.2.0)(rollup@4.20.0)(typescript@5.5.4)(vite@5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) - vitest: - specifier: ^1.4.0 - version: 1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) devDependencies: '@movie-web/eslint-config': specifier: workspace:^0.2.0 @@ -407,6 +317,12 @@ importers: '@movie-web/tsconfig': specifier: workspace:^0.1.0 version: link:../../tooling/typescript + eslint: + specifier: ^8.57.0 + version: 8.57.0 + typescript: + specifier: ^5.4.3 + version: 5.5.4 packages/providers: dependencies: @@ -422,6 +338,9 @@ importers: form-data: specifier: ^4.0.0 version: 4.0.0 + hls-parser: + specifier: ^0.13.3 + version: 0.13.3 iso-639-1: specifier: ^3.1.2 version: 3.1.2 @@ -431,8 +350,11 @@ importers: node-fetch: specifier: ^3.3.2 version: 3.3.2 + react-native: + specifier: 0.74.5 + version: 0.74.5(@babel/core@7.25.2)(@babel/preset-env@7.25.3(@babel/core@7.25.2))(@types/react@18.3.3)(react@18.3.1) set-cookie-parser: - specifier: ^2.6.0 + specifier: ^2.7.0 version: 2.7.0 unpacker: specifier: ^1.0.1 @@ -447,6 +369,9 @@ importers: '@movie-web/tsconfig': specifier: workspace:^0.1.0 version: link:../../tooling/typescript + '@nabla/vite-plugin-eslint': + specifier: ^2.0.4 + version: 2.0.4(eslint@8.57.0)(vite@5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) '@types/cookie': specifier: ^0.6.0 version: 0.6.0 @@ -460,16 +385,22 @@ importers: specifier: ^2.0.3 version: 2.0.3 '@types/set-cookie-parser': - specifier: ^2.4.7 + specifier: ^2.4.10 version: 2.4.10 '@types/spinnies': specifier: ^0.5.3 version: 0.5.3 + '@typescript-eslint/eslint-plugin': + specifier: ^7.4.0 + version: 7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint@8.57.0)(typescript@5.5.4) + '@typescript-eslint/parser': + specifier: ^7.4.0 + version: 7.18.0(eslint@8.57.0)(typescript@5.5.4) '@vitest/coverage-v8': - specifier: ^1.4.0 + specifier: ^1.6.0 version: 1.6.0(vitest@1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) commander: - specifier: ^12.0.0 + specifier: ^12.1.0 version: 12.1.0 cross-env: specifier: ^7.0.3 @@ -480,17 +411,35 @@ importers: enquirer: specifier: ^2.4.1 version: 2.4.1 + eslint: + specifier: ^8.57.0 + version: 8.57.0 + eslint-config-airbnb-base: + specifier: ^15.0.0 + version: 15.0.0(eslint-plugin-import@2.29.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-config-prettier: + specifier: ^9.1.0 + version: 9.1.0(eslint@8.57.0) + eslint-import-resolver-typescript: + specifier: ^3.6.1 + version: 3.6.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-plugin-import@2.29.1)(eslint@8.57.0) + eslint-plugin-import: + specifier: ^2.29.1 + version: 2.29.1(@typescript-eslint/parser@7.18.0(eslint@8.57.0)(typescript@5.5.4))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) + eslint-plugin-prettier: + specifier: ^5.2.1 + version: 5.2.1(@types/eslint@8.56.11)(eslint-config-prettier@9.1.0(eslint@8.57.0))(eslint@8.57.0)(prettier@3.3.3) prettier: specifier: ^3.2.5 version: 3.3.3 puppeteer: - specifier: ^22.6.1 + specifier: ^22.15.0 version: 22.15.0(typescript@5.5.4) spinnies: specifier: ^0.5.1 version: 0.5.1 tsc-alias: - specifier: ^1.8.8 + specifier: ^1.8.10 version: 1.8.10 tsconfig-paths: specifier: ^4.2.0 @@ -499,16 +448,16 @@ importers: specifier: ^5.4.3 version: 5.5.4 vite: - specifier: ^5.2.7 + specifier: ^5.3.5 version: 5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) vite-node: - specifier: ^1.4.0 + specifier: ^1.6.0 version: 1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) vite-plugin-dts: - specifier: ^3.8.1 + specifier: ^3.9.1 version: 3.9.1(@types/node@22.2.0)(rollup@4.20.0)(typescript@5.5.4)(vite@5.4.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6)) vitest: - specifier: ^1.4.0 + specifier: ^1.6.0 version: 1.6.0(@types/node@22.2.0)(lightningcss@1.22.0)(terser@5.31.6) packages/tmdb: @@ -5045,6 +4994,9 @@ packages: hey-listen@1.0.8: resolution: {integrity: sha512-COpmrF2NOg4TBWUJ5UVyaCU2A88wEMkUPK4hNqyCkqHbxT92BbvfjoSozkAIIm6XhicGlJHhFdullInrdhwU8Q==} + hls-parser@0.13.3: + resolution: {integrity: sha512-DXqW7bwx9j2qFcAXS/LBJTDJWitxknb6oUnsnTvECHrecPvPbhRgIu45OgNDUU6gpwKxMJx40SHRRUUhdIM2gA==} + hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} @@ -14222,6 +14174,8 @@ snapshots: hey-listen@1.0.8: {} + hls-parser@0.13.3: {} + hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1