add: support for database

.env

@@ -1,4 +1,5 @@
 PORT=3000
 NODE_TLS_REJECT_UNAUTHORIZED=0
 PROXY_URL="https://www.proxy-list.download/api/v1/get?type=https"
-REDIS_URL=""
+REDIS_URL=""
+DATABASE_URL=""

package.json

@@ -17,6 +17,7 @@
     "fake-useragent": "^1.0.1",
     "hpagent": "^1.2.0",
     "openai": "^4.0.0",
+    "pg": "^8.12.0",
     "puppeteer": "^23.1.0",
     "redis": "^4.7.0",
     "sharp": "^0.33.5",
@@ -26,6 +27,7 @@
     "@types/express": "^4.17.21",
     "@types/fake-useragent": "^1.0.0",
     "@types/node": "^22.4.0",
+    "@types/pg": "^8.11.6",
     "@types/stremio-addon-sdk": "^1.6.11",
     "nodemon": "^3.1.4",
     "ts-node": "^10.9.2",

src/handlers/catalogHandler.ts

@@ -1,15 +1,20 @@
 import axios from 'axios';
-import { scrapeRatings } from '../utils/ratingScrapers';
+import { getRatingsfromDB, scrapeRatings } from '../utils/ratingScrapers';
 import { CINEMETA_BASE_URL, CINEMETA_CATALOG_URL } from '../constants/urls';
 import { closeCacheClient, getCacheClient } from '../cache';
 import { DEFAULT_PROVIDERS } from '../constants/costants';
+import { isDatabaseConnected } from '../repository';
 
 async function fetchCatalog(url: string, providers: string[]): Promise<any> {
     const response = await axios.get(url);
-    response.data.metas = await Promise.all(response.data.metas.map(async (meta: any) => {
-        return scrapeRatings(meta.id, meta.type, providers);
-    }));
-    return response.data;
+    if (isDatabaseConnected()) {
+        return getRatingsfromDB(response.data, providers);
+    } else {
+        response.data.metas = await Promise.all(response.data.metas.map(async (meta: any) => {
+            return scrapeRatings(meta.id, meta.type, providers);
+        }));
+        return response.data;
+    }
 }
 
 export async function trendingCatalog(type: string, extra: any, providers: string[]): Promise<any> {

src/index.ts

@@ -3,6 +3,8 @@ import { handleMetaRequest } from "./handlers/metaHandler";
 import { handleCatalogRequest } from "./handlers/catalogHandler";
 import manifest from "./manifest";
 import dotenv from "dotenv";
+import { closeClient, getClient } from "./repository";
+import { get } from "http";
 
 dotenv.config();
 
@@ -11,21 +13,28 @@ const builder = new addonBuilder(manifest);
 // Catalog Handlers
 builder.defineCatalogHandler(async (args: Args) => {
     console.log("CatalogHandler args:", args);
+    await getClient();
     try {
         return await handleCatalogRequest(args);
     } catch (error) {
         console.error("Error in CatalogHandler:", error);
         return { metas: [] };
+    } finally{
+        console.log("CatalogHandler finally");
+        closeClient();
     }
 });
 
 // Meta Handlers
 builder.defineMetaHandler(async (args: { type: ContentType, id: string }) => {
+    await getClient();
     try {
         return { meta: await handleMetaRequest(args) };
     } catch (error) {
         console.error("Error in MetaHandler:", error);
         return { meta: {} as any };
+    } finally {
+        closeClient();
     }
 });
 

src/repository/index.ts

@@ -0,0 +1,43 @@
+import pg from 'pg'
+const { Client } = pg
+
+let client: pg.Client | null = null
+
+export async function getClient(): Promise<pg.Client | null> {
+    const connectionStr = process.env.DATABASE_URL
+    if (!connectionStr) {
+        console.error('DATABASE_URL so not using database')
+        return null;
+    }
+    if (client) {
+        console.error('Already connected to database')
+        return client
+    }
+    client = new Client({connectionString: connectionStr})
+}
+
+export async function closeClient() {
+    if (client) {
+        await client.end()
+        client = null
+    }
+}
+
+export async function getRatingsfromTTIDs(ttids: string[]): Promise<Record<string, Record<string, string>>> {
+    if (!client) {
+        throw new Error('Not connected to database')
+    }
+    const query = `SELECT ttid, provder, rating FROM ratings WHERE ttid = ANY($1)`
+    const { rows } = await client.query(query, [ttids])
+    return rows.reduce((acc: Record<string, Record<string, string>>, row: any) => {
+        if (!acc[row.ttid]) {
+            acc[row.ttid] = {}
+        }
+        acc[row.ttid][row.provider] = row.rating
+        return acc
+    });
+}
+
+export function isDatabaseConnected(): boolean {
+    return client != null
+}

src/utils/ratingScrapers.ts

@@ -4,6 +4,7 @@ import { fetchBingRatings, fetchGoogleRatings, fetchYahooRatings, getMetadata }
 import { RedisClientType } from 'redis';
 import { addRatingToImage } from './image';
 import axios from 'axios';
+import { getRatingsfromTTIDs } from '../repository';
 
 export async function getRatingsFromGoogle(query: string, imdbId: string, cacheClient: RedisClientType | null): Promise<Record<string, string>> {
     try {
@@ -136,3 +137,36 @@ async function getRatingsFromCache(imdbId: string, cacheClient: RedisClientType
 
     return ratingMap;
 }
+
+
+export async function getRatingsfromDB(metas: MetaDetail[], providers: string[]): Promise<MetaDetail[]> {
+    const ttids = metas.map(meta => meta.id);
+    const ratings = await getRatingsfromTTIDs(ttids);
+
+    const modifiedMetaPromises = await metas.map(async meta => {
+        if (!ratings[meta.id]) {
+            return meta;
+        }
+
+        // update description with ratings
+        meta.description = meta.description || '';
+        let filteredRatings = {};
+
+        for (const [key, value] of Object.entries(ratings[meta.id])) {
+            if (providers.includes('all') || providers.includes(key)) {
+                meta.description += `(${key.replace('_', ' ')}: ${value}) `;
+                filteredRatings[key] = value;
+            }
+        }
+
+        if (meta.poster && Object.keys(ratings[meta.id]).length > 0) {
+            const response = await axios.get(meta.poster, { responseType: 'arraybuffer' });
+            const posterBase64 = Buffer.from(response.data).toString('base64');
+            const modifiedPoster = await addRatingToImage(posterBase64, filteredRatings);
+            meta.poster = modifiedPoster;
+        }
+        return meta;
+    });
+
+    return await Promise.all(modifiedMetaPromises);
+}

yarn.lock

@@ -315,6 +315,15 @@
   dependencies:
     undici-types "~5.26.4"
 
+"@types/pg@^8.11.6":
+  version "8.11.6"
+  resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.11.6.tgz#a2d0fb0a14b53951a17df5197401569fb9c0c54b"
+  integrity sha512-/2WmmBXHLsfRqzfHW7BNZ8SbYzE8OSk7i3WjFYvfgRHj7S1xj+16Je5fUKv3lVdVzk/zn9TXOqf+avFCFIE0yQ==
+  dependencies:
+    "@types/node" "*"
+    pg-protocol "*"
+    pg-types "^4.0.1"
+
 "@types/qs@*":
   version "6.9.15"
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.15.tgz#adde8a060ec9c305a82de1babc1056e73bd64dce"
@@ -1628,6 +1637,11 @@ object-inspect@^1.13.1:
   resolved "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz"
   integrity sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==
 
+obuf@~1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e"
+  integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==
+
 [email protected]:
   version "2.4.1"
   resolved "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz"
@@ -1750,6 +1764,80 @@ pend@~1.2.0:
   resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
   integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==
 
+pg-cloudflare@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98"
+  integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==
+
+pg-connection-string@^2.6.4:
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d"
+  integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==
+
+[email protected]:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
+  integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==
+
+[email protected]:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/pg-numeric/-/pg-numeric-1.0.2.tgz#816d9a44026086ae8ae74839acd6a09b0636aa3a"
+  integrity sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==
+
+pg-pool@^3.6.2:
+  version "3.6.2"
+  resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2"
+  integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==
+
+pg-protocol@*, pg-protocol@^1.6.1:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3"
+  integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==
+
+pg-types@^2.1.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3"
+  integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==
+  dependencies:
+    pg-int8 "1.0.1"
+    postgres-array "~2.0.0"
+    postgres-bytea "~1.0.0"
+    postgres-date "~1.0.4"
+    postgres-interval "^1.1.0"
+
+pg-types@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-4.0.2.tgz#399209a57c326f162461faa870145bb0f918b76d"
+  integrity sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==
+  dependencies:
+    pg-int8 "1.0.1"
+    pg-numeric "1.0.2"
+    postgres-array "~3.0.1"
+    postgres-bytea "~3.0.0"
+    postgres-date "~2.1.0"
+    postgres-interval "^3.0.0"
+    postgres-range "^1.1.1"
+
+pg@^8.12.0:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79"
+  integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==
+  dependencies:
+    pg-connection-string "^2.6.4"
+    pg-pool "^3.6.2"
+    pg-protocol "^1.6.1"
+    pg-types "^2.1.0"
+    pgpass "1.x"
+  optionalDependencies:
+    pg-cloudflare "^1.1.1"
+
+[email protected]:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d"
+  integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==
+  dependencies:
+    split2 "^4.1.0"
+
 picocolors@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
@@ -1760,6 +1848,55 @@ picomatch@^2.0.4, picomatch@^2.2.1:
   resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
+postgres-array@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e"
+  integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==
+
+postgres-array@~3.0.1:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-3.0.2.tgz#68d6182cb0f7f152a7e60dc6a6889ed74b0a5f98"
+  integrity sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==
+
+postgres-bytea@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35"
+  integrity sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==
+
+postgres-bytea@~3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-3.0.0.tgz#9048dc461ac7ba70a6a42d109221619ecd1cb089"
+  integrity sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==
+  dependencies:
+    obuf "~1.1.2"
+
+postgres-date@~1.0.4:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8"
+  integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==
+
+postgres-date@~2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-2.1.0.tgz#b85d3c1fb6fb3c6c8db1e9942a13a3bf625189d0"
+  integrity sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==
+
+postgres-interval@^1.1.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695"
+  integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==
+  dependencies:
+    xtend "^4.0.0"
+
+postgres-interval@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-3.0.0.tgz#baf7a8b3ebab19b7f38f07566c7aab0962f0c86a"
+  integrity sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==
+
+postgres-range@^1.1.1:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/postgres-range/-/postgres-range-1.1.4.tgz#a59c5f9520909bcec5e63e8cf913a92e4c952863"
+  integrity sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==
+
 progress@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
@@ -2069,6 +2206,11 @@ source-map@~0.6.1:
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
+split2@^4.1.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4"
+  integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==
+
 sprintf-js@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a"
@@ -2372,6 +2514,11 @@ ws@^8.18.0:
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
   integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
 
+xtend@^4.0.0:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
+  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
+
 y18n@^5.0.5:
   version "5.0.8"
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"