Skip to content

Commit

Permalink
add: initial support for cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@hexdecimal16
hexdecimal16 committed Aug 20, 2024
1 parent 3a5e5a2 commit 281e7d5
Show file tree
Hide file tree
Showing 5 changed files with 132 additions and 19 deletions.
4 changes: 3 additions & 1 deletion .env
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
PORT=3000
NODE_TLS_REJECT_UNAUTHORIZED=0
PROXY_URL="https://www.proxy-list.download/api/v1/get?type=https"
PROXY_URL="https://www.proxy-list.download/api/v1/get?type=https"
CACHE_ENABLED=false
REDIS_URL=""
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"hpagent": "^1.2.0",
"openai": "^4.0.0",
"puppeteer": "^23.1.0",
"redis": "^4.7.0",
"sharp": "^0.33.5",
"stremio-addon-sdk": "^1.6.10"
},
Expand Down
68 changes: 50 additions & 18 deletions src/addon.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
import { addonBuilder, serveHTTP, Args, MetaDetail } from 'stremio-addon-sdk';
import * as fs from 'fs';
import axios, { AxiosError } from 'axios';
import axios from 'axios';
import sharp from 'sharp';
import * as cheerio from 'cheerio';
import * as dotenv from 'dotenv';
import * as path from 'path';
import manifest from './manifest';
import fakeUa from 'fake-useragent'
import { fetchNetwork } from './network';
import { createCacheClient } from './cache'

// Load environment variables from .env file
dotenv.config();

// Create a new addon builder
const builder = new addonBuilder(manifest);
const cinemeta_catalog = 'https://cinemeta-catalogs.strem.io';


async function addRatingToImage(base64String: string, ratingMap: { [key: string]: string }): Promise<string> {
try {
Expand Down Expand Up @@ -132,7 +135,7 @@ async function getRatingsFromGoogle(query: string): Promise<{ [key: string]: str
'User-Agent': fakeUa()
};
const response = await fetchNetwork(`https://www.google.com/search?q=${encodeURIComponent(query)}`, 'axios', false);

const $ = cheerio.load(response);

let ratingsDiv = $('div.Ap5OSd').first();
Expand Down Expand Up @@ -246,6 +249,7 @@ function firstResolved<T>(promises: Promise<T>[]): Promise<T> {
// async function scrapeRatings(imdbId: string, type: string, browser?: Browser, axiosInstance?: AxiosInstance): Promise<MetaDetail> {
async function scrapeRatings(imdbId: string, type: string): Promise<MetaDetail> {

const cacheClient = await createCacheClient();
const metadata = await getMetadata(imdbId, type);
try {
const cleanTitle = metadata.name;
Expand All @@ -256,24 +260,49 @@ async function scrapeRatings(imdbId: string, type: string): Promise<MetaDetail>
return metadata;
}

const query = `${cleanTitle} - ${type}`;
// try to get ratings from cache
const ratingKeys = ['imdb', 'metacritic', 'rotten_tomatoes'];
let ratingMap: { [key: string]: string } = {};
let yahooFallback = false;
try {
ratingMap = await firstResolved([
getRatingsFromGoogle(query),
getRatingsFromBing(query),
]);
} catch (error) {
yahooFallback = true;
}
let isRatingCached = false;
ratingKeys.forEach(async (key) => {
if (cacheClient === null) {
return;
}
const cacheKey = `${imdbId}:${key}`;
const rating = await cacheClient.get(cacheKey);
if (rating) {
description += `(${key.replace("_", " ")}: ${rating}) `;
ratingMap[key] = rating;
isRatingCached = true;
}
});
if (!isRatingCached) {

const query = `${cleanTitle} - ${type}`;
let yahooFallback = false;
try {
ratingMap = await firstResolved([
getRatingsFromGoogle(query),
getRatingsFromBing(query),
]);
} catch (error) {
yahooFallback = true;
}

if (yahooFallback) {
ratingMap = await getRatingsFromYahoo(query);
}
if (yahooFallback) {
ratingMap = await getRatingsFromYahoo(query);
}

for (const [key, value] of Object.entries(ratingMap)) {
description += `(${key.replace("_", " ")}: ${value}) `;
for (const [key, value] of Object.entries(ratingMap)) {
description += `(${key.replace("_", " ")}: ${value}) `;
// Cache the rating for 1 day
if (cacheClient === null) {
continue;
}
const caccheKey = `${imdbId}:${key}`;
await cacheClient.set(caccheKey, value)
await cacheClient.expire(caccheKey, 86400);
}
}

if (metadata.poster) {
Expand All @@ -290,6 +319,10 @@ async function scrapeRatings(imdbId: string, type: string): Promise<MetaDetail>
} catch (error) {
console.error(`Error fetching ratings: ${(error as Error).message}`);
return metadata;
} finally {
if (cacheClient) {
cacheClient.disconnect();
}
}
}

Expand All @@ -308,7 +341,6 @@ builder.defineMetaHandler(async (args: { id: string, type: string }) => {
});


const cinemeta_catalog = 'https://cinemeta-catalogs.strem.io';

// Fetch trending catalog
async function trendingCatalog(type: string, extra: any): Promise<any> {
Expand Down
17 changes: 17 additions & 0 deletions src/cache.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { createClient } from 'redis';

async function createCacheClient() {
const isCacheEnabled = process.env.CACHE_ENABLED === 'true';
if (!isCacheEnabled) {
return null;
}

const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
const client = await createClient({ url: redisUrl })
.on('error', err => console.log('Redis Client Error', err))
.connect();
return client;
}


export { createCacheClient };
61 changes: 61 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,40 @@
unbzip2-stream "^1.4.3"
yargs "^17.7.2"

"@redis/[email protected]":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==

"@redis/[email protected]":
version "1.6.0"
resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.6.0.tgz#dcf4ae1319763db6fdddd6de7f0af68a352c30ea"
integrity sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg==
dependencies:
cluster-key-slot "1.1.2"
generic-pool "3.9.0"
yallist "4.0.0"

"@redis/[email protected]":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea"
integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==

"@redis/[email protected]":
version "1.0.7"
resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.7.tgz#016257fcd933c4cbcb9c49cde8a0961375c6893b"
integrity sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==

"@redis/[email protected]":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.2.0.tgz#50976fd3f31168f585666f7922dde111c74567b8"
integrity sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==

"@redis/[email protected]":
version "1.1.0"
resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.1.0.tgz#cba454c05ec201bd5547aaf55286d44682ac8eb5"
integrity sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==

"@tootallnate/quickjs-emscripten@^0.23.0":
version "0.23.0"
resolved "https://registry.yarnpkg.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz#db4ecfd499a9765ab24002c3b696d02e6d32a12c"
Expand Down Expand Up @@ -590,6 +624,11 @@ cliui@^8.0.1:
strip-ansi "^6.0.1"
wrap-ansi "^7.0.0"

[email protected]:
version "1.1.2"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==

color-convert@^1.9.0:
version "1.9.3"
resolved "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz"
Expand Down Expand Up @@ -1072,6 +1111,11 @@ function-bind@^1.1.2:
resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"
integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==

[email protected]:
version "3.9.0"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==

get-caller-file@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
Expand Down Expand Up @@ -1747,6 +1791,18 @@ readdirp@~3.6.0:
dependencies:
picomatch "^2.2.1"

redis@^4.7.0:
version "4.7.0"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.0.tgz#b401787514d25dd0cfc22406d767937ba3be55d6"
integrity sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==
dependencies:
"@redis/bloom" "1.2.0"
"@redis/client" "1.6.0"
"@redis/graph" "1.1.1"
"@redis/json" "1.0.7"
"@redis/search" "1.2.0"
"@redis/time-series" "1.1.0"

require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
Expand Down Expand Up @@ -2249,6 +2305,11 @@ y18n@^5.0.5:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==

[email protected]:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

yargs-parser@^21.1.1:
version "21.1.1"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"
Expand Down

0 comments on commit 281e7d5

Please sign in to comment.