Latest from quaid

.env

@@ -0,0 +1 @@
+DB_PATH=/mount/projs.sqlite

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.idea

index.js

@@ -0,0 +1,89 @@
+import * as items from './lib/items.js';
+import * as rolimons from "./lib/rolimons.js";
+import db from "./lib/db.js";
+import {sleep, repeat, print} from "./lib/globals.js";
+import {Worker} from 'node:worker_threads';
+
+const getAllItems = async () => (await db.get('items')) || {};
+
+const updateMissingItems = async () => {
+    const itemsStored = Object.keys(await getAllItems());
+    const ollieItems = Object.values(items.getOllieData());
+
+    const missingItems = ollieItems
+        .filter(ollieItem => !itemsStored.find(item => Number(item) === ollieItem.id));
+
+    for (const missingItem of missingItems) {
+        print(`Scraping missing data for "${missingItem.name}" (${missingItem.id})...`);
+
+        // fetch all points for the item from rolimons
+        const allPoints = await rolimons.fetchItemSales(missingItem.id);
+        if (!allPoints) {
+            print(`Failed to scrape data for "${missingItem.name}" (${missingItem.id})`);
+            continue;
+        }
+
+        // condense each array into a single array of objects
+        const mappedPoints = allPoints.timestamp_list.map((point, index) => ({
+            saleId: allPoints.sale_id_list[index],
+            timestamp: point,
+            rap: allPoints.sale_rap_list[index],
+            // value: allPoints.sale_value_list[index],
+            price: allPoints.sale_price_list[index],
+        }));
+
+        // store the data for later use
+        await db.set(`items.${missingItem.id}.points`, mappedPoints);
+
+        print(`Saved ${mappedPoints.length} points for "${missingItem.name}" (${missingItem.id})...`);
+
+        // courtesy sleep yw rolimon
+        await sleep(4 * 1000);
+    }
+};
+const handleNewSales = async () => {
+    const newSales = await rolimons.fetchAllRecentSales();
+    if (!newSales) return;
+
+    for (const saleData of newSales) {
+        const [
+            timestamp,
+            _unknown,
+            itemId,
+            oldRap,
+            newRap,
+            saleId,
+        ] = saleData;
+        const itemSales = await db.get(`items.${itemId}.points`);
+        if (!itemSales) {
+            print(`Found sale for ${itemId}, but previous data hasn't been scraped yet`);
+            continue;
+        }
+
+        // check if sale is a duplicate
+        if (itemSales.find(sale => sale.saleId === saleId)) continue;
+
+        const estimatedPrice = oldRap + (newRap - oldRap) * 10;
+        await db.push(`items.${itemId}.points`, {
+            saleId,
+            timestamp,
+            rap: newRap,
+            price: estimatedPrice,
+        });
+
+        print(`Found new sale for ${itemId} (${oldRap.toLocaleString()} -> ${newRap.toLocaleString()} | approx. ${estimatedPrice.toLocaleString()})`);
+    }
+};
+
+const main = async () => {
+    print(`Fetching data from api.danny.ink...`);
+    await items.refreshOllieData();
+
+    repeat(items.refreshOllieData, 60 * 1000);
+    repeat(updateMissingItems, 5 * 60 * 1000);
+    repeat(handleNewSales, 20 * 1000);
+}
+
+main();
+
+new Worker('./lib/server.js')

lib/calculations.js

@@ -0,0 +1,11 @@
+export const filterUnneeded = array => {
+    const sorted = array.sort((a, b) => a - b);
+    const q1 = sorted[Math.floor(sorted.length / 4)];
+    const q3 = sorted[Math.floor(sorted.length * (3 / 4))];
+    const iqr = q3 - q1;
+    const maxValue = q3 + iqr * 1.1;
+
+    return sorted.filter(i => i < maxValue && i > 0);
+}
+
+export const mean = array => array.reduce((a, b) => a + b) / array.length;

lib/db.js

@@ -0,0 +1,7 @@
+import {QuickDB} from 'quick.db';
+
+const db = new QuickDB({
+    filePath: process.env.DB_PATH || 'projs.sqlite',
+});
+
+export default db;

lib/globals.js

@@ -0,0 +1,8 @@
+export const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+export const print = (...args) => console.log(`[${new Date().toLocaleTimeString()}]`, ...args);
+export const repeat = async (fn, delay = 1000) => {
+    for (;;) {
+        await fn();
+        await sleep(delay);
+    }
+}

lib/items.js

@@ -0,0 +1,24 @@
+import superagent from 'superagent';
+import {print, sleep} from "./globals.js";
+
+let ollieData = {};
+export const getOllieData = () => ollieData;
+
+export const refreshOllieData = () => new Promise(resolve => {
+    superagent('GET', `https://api.danny.ink/api/itemdetails`)
+        .then(resp => {
+            ollieData = resp.body;
+            resolve();
+        })
+        .catch(async error => {
+            if (!error.response) {
+                print(`Internal error fetching ollie data:`, error);
+                return resolve();
+            }
+
+            print(`Error fetching ollie data:`, error.text);
+
+            await sleep(25 * 1000);
+            return refreshOllieData().then(resolve);
+        })
+});

lib/rolimons.js

@@ -0,0 +1,53 @@
+import superagent from 'superagent';
+import {print} from "./globals.js";
+
+const agent = superagent.agent()
+    .set('user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36')
+
+export const fetchItemSales = itemId => new Promise(resolve => {
+    agent.get(`https://www.rolimons.com/itemsales/${itemId}`)
+        .then(resp => {
+            if (!resp || !resp.text) return resolve();
+
+            return resolve(
+                JSON.parse(resp.text.split('var item_sales            = ')[1].split(';')[0])
+            )
+        })
+        .catch(error => {
+            if (!error.response) {
+                print(`Internal error fetching item sales for ${itemId}:`, error);
+                return resolve();
+            }
+
+            if (error.response.status === 403) {
+                print(`Request blocked when fetching item sales for ${itemId}:`, error);
+                return resolve();
+            }
+
+            print(`Error fetching item sales for ${itemId}:`, error);
+            resolve();
+        })
+});
+
+export const fetchAllRecentSales = () => new Promise(resolve => {
+    agent.get(`https://www.rolimons.com/api/activity`)
+        .then(resp => {
+            if (!resp.body || !resp.body.activities) return resolve();
+
+            return resolve(resp.body.activities);
+        })
+        .catch(error => {
+            if (!error.response) {
+                print(`Internal error fetching recent sales:`, error);
+                return resolve();
+            }
+
+            if (error.response.status === 403) {
+                print(`Request blocked when fetching recent sales:`, error);
+                return resolve();
+            }
+
+            print(`Error fetching recent sales:`, error);
+            resolve();
+        })
+});

lib/server.js

@@ -0,0 +1,49 @@
+import express from 'express';
+import db from './db.js';
+import * as calculations from './calculations.js';
+import {print} from "./globals.js";
+
+let cachedResponse = {};
+
+const app = express();
+app.use(express.json());
+
+app.get('/', async (req, resp) => {
+    if (cachedResponse.iat && Date.now() - cachedResponse.iat < 1000 * 60 * 5)
+        return resp.send(cachedResponse);
+
+    const items = await db.get('items');
+
+    let response = {
+        iat: Date.now(),
+        items: {}
+    };
+
+    for (const itemId in items) {
+        const item = items[itemId];
+        const points = item.points.reverse().slice(0, 150);
+
+        const filteredPriceOutliars = calculations.filterUnneeded(
+            points.map(i => i.price)
+        );
+        const filteredRapOutliars = calculations.filterUnneeded(
+            points.map(i => i.rap)
+        );
+
+        if (filteredPriceOutliars.length < 3 || filteredRapOutliars.length < 3) {
+            response.items[itemId] = null;
+            continue;
+        }
+
+        response.items[itemId] = Math.floor(calculations.mean([
+            calculations.mean(filteredRapOutliars), calculations.mean(filteredPriceOutliars)
+        ]))
+    }
+
+    cachedResponse = response;
+    return resp.send(response);
+});
+
+app.listen(3000, () => {
+    print('Server started on port 3000');
+});