diff --git a/package-lock.json b/package-lock.json index f6a5bc63..09b3956b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13427,7 +13427,8 @@ "node_modules/idb": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.0.tgz", - "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==" + "integrity": "sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==", + "license": "ISC" }, "node_modules/identity-obj-proxy": { "version": "3.0.0", diff --git a/src/main/Scrapers/X Corp/bookmarks.js b/src/main/Scrapers/X Corp/bookmarks.js index f04f5c10..7650f002 100644 --- a/src/main/Scrapers/X Corp/bookmarks.js +++ b/src/main/Scrapers/X Corp/bookmarks.js @@ -51,43 +51,43 @@ async function checkIfBookmarkExists(id, platformId, company, name, currentBookm return false; } -async function checkBigData(company, name) { +async function checkTwitterCredentials(company, name) { const userData = await ipcRenderer.invoke('get-user-data-path'); - const bigDataPath = path.join( + const twitterCredentialsPath = path.join( userData, 'surfer_data', company, name, - 'bigData.json', + 'twitterCredentials.json', ); - const fileExists = await fs.existsSync(bigDataPath); + const fileExists = await fs.existsSync(twitterCredentialsPath); if (fileExists) { - const fileContent = fs.readFileSync(bigDataPath, 'utf-8'); + const fileContent = fs.readFileSync(twitterCredentialsPath, 'utf-8'); return JSON.parse(fileContent); } return null; }; async function exportBookmarks(id, platformId, filename, company, name) { - let bigData; + let twitterCredentials; if (!window.location.href.includes('x.com')) { bigStepper(id, 'Navigating to Twitter'); customConsoleLog(id, 'Navigating to Twitter'); window.location.assign('https://x.com/i/bookmarks/all'); - ipcRenderer.send('get-big-data', company, name); + ipcRenderer.send('get-twitter-credentials', company, name); } // Wait for bigData to be available - while (!bigData) { + while (!twitterCredentials) { await wait(0.5); - bigData = await checkBigData(company, name); + twitterCredentials = await checkTwitterCredentials(company, name); } - customConsoleLog(id, 'bigData obtained!') + customConsoleLog(id, 'twitterCredentials obtained!') // Run API requests to get bookmarks try { - const bookmarks = await getBookmarks(id, bigData); + const bookmarks = await getBookmarks(id, twitterCredentials); customConsoleLog(id, `Retrieved ${bookmarks.length} bookmarks`); // let bookmarkArray = []; @@ -132,29 +132,28 @@ async function exportBookmarks(id, platformId, filename, company, name) { // } // } - customConsoleLog(id, `Exporting ${bookmarkArray.length} bookmarks`); bigStepper(id, 'Exporting data'); // ipcRenderer.send('handle-update-complete', id, platformId, company, name); return bookmarks; } catch (error) { - console.error(id, `Error fetching bookmarks: ${error.message}`); + customConsoleLog(id, `Error fetching bookmarks: ${error.message}`); return 'ERROR'; } } -async function getBookmarks(id, bigData, cursor = "", totalImported = 0, allBookmarks = []) { +async function getBookmarks(id, twitterCredentials, cursor = "", totalImported = 0, allBookmarks = []) { const headers = new Headers(); - headers.append('Cookie', bigData.cookie); - headers.append('X-Csrf-token', bigData.csrf); - headers.append('Authorization', bigData.auth); + headers.append('Cookie', twitterCredentials.cookie); + headers.append('X-Csrf-token', twitterCredentials.csrf); + headers.append('Authorization', twitterCredentials.auth); const variables = { count: 100, cursor: cursor, includePromotedContent: false, }; const API_URL = `https://x.com/i/api/graphql/${ - bigData.bookmarksApiId + twitterCredentials.bookmarksApiId }/Bookmarks?features=${encodeURIComponent( JSON.stringify(features) )}&variables=${encodeURIComponent(JSON.stringify(variables))}`; @@ -194,7 +193,7 @@ async function getBookmarks(id, bigData, cursor = "", totalImported = 0, allBook if (nextCursor && newBookmarksCount > 0) { console.log('TRYING TO GET MORE BOOKMARKS!!!') - return await getBookmarks(id, bigData, nextCursor, totalImported, allBookmarks); + return await getBookmarks(id, twitterCredentials, nextCursor, totalImported, allBookmarks); } else { customConsoleLog(id, 'No new bookmarks found, returning all bookmarks'); diff --git a/src/main/main.ts b/src/main/main.ts index c6a9c2dd..51523ea9 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -3,7 +3,6 @@ dotenv.config(); import {} from '../../'; import path from 'path'; import MenuBuilder from './utils/menu'; -import yauzl from 'yauzl'; import { getFilesInFolder } from './utils/util'; import { app, @@ -22,13 +21,9 @@ import { import { autoUpdater } from 'electron-updater'; import log from 'electron-log'; import { resolveHtmlPath } from './utils/util'; -import { createClient } from '@supabase/supabase-js'; import fs from 'fs'; -import { PythonUtils } from './utils/python'; -import { mboxParser } from 'mbox-parser'; - - -const pythonUtils = new PythonUtils(); +import { convertMboxToJson, findMboxFile, extractZip, getTotalFolderSize } from './utils/helpers'; +import { getImessageData } from './utils/imessage'; let appIcon: Tray | null = null; @@ -43,6 +38,7 @@ let downloadingItems = new Map(); import express from 'express'; import cors from 'cors'; +import { getTwitterCredentials } from './utils/network'; const expressApp = express(); expressApp.use(cors()); expressApp.use(express.json()); @@ -69,109 +65,56 @@ expressApp.post('/api/get', async (req, res) => { expressApp.post('/api/export', async (req, res) => { console.log('EXPORT REQUEST: ', req.body); const { platformId } = req.body; + try { - mainWindow?.webContents.send('element-found', platformId); - let currentRun = null; - const startTime = Date.now(); - const timeout = 180000; // Increased to 30 seconds to allow for export completion - - // First, wait for the run to start - while (!currentRun && Date.now() - startTime < timeout) { - const currentRuns = JSON.parse( - fs.readFileSync( - path.join(app.getPath('userData'), 'runs.json'), - 'utf8', - ), - ); - currentRun = currentRuns - .filter( - (r: any) => r.platformId === platformId && r.status === 'running', - ) - .sort( - (a: any, b: any) => - new Date(b.startDate).getTime() - new Date(a.startDate).getTime(), - )[0]; - - if (!currentRun) { - await new Promise((resolve) => setTimeout(resolve, 500)); - } - } - if (!currentRun) { - throw new Error('Timed out waiting for run to start'); - } + // Start export + mainWindow?.webContents.send('element-found', platformId); - console.log('Found running run:', currentRun); + // Get initial run with timeout + const currentRun: any = await new Promise((resolve) => { + ipcMain.once('run-started', (event, run) => resolve(run)); + }); - // Now wait for the run to complete - let finalRun = null; - while (!finalRun && Date.now() - startTime < timeout) { - const currentRuns = JSON.parse( - fs.readFileSync( - path.join(app.getPath('userData'), 'runs.json'), - 'utf8', - ), - ); - finalRun = currentRuns.filter( - (r: any) => r.name === currentRun.name && r.company === currentRun.company && r.status === 'success', - ).pop(); + console.log('Found current run:', currentRun.id); - console.log('finalRun: ', finalRun); + // Monitor run status with timeout + const finalRun = await new Promise((resolve) => { + const checkRunStatus = async () => { + mainWindow?.webContents.send('get-runs-request'); + const runsResponse: any = await new Promise((resolve) => { + ipcMain.on('get-runs-response', (event, runs) => resolve(runs)); + }); - if (!finalRun) { - await new Promise((resolve) => setTimeout(resolve, 500)); - } - } + const finalRun = runsResponse.find( + (r: any) => r.id === currentRun.id, + ); + if (finalRun?.status === 'success') { + clearInterval(statusInterval); + resolve(finalRun); + } + }; - if (!finalRun) { - throw new Error('Timed out waiting for run to complete'); + const statusInterval = setInterval(checkRunStatus, 1000); + }); + + console.log('final run status: ', finalRun.status); + // Process results + const latestRunPath = finalRun.exportPath; + if (!fs.existsSync(latestRunPath)) { + throw new Error('Export path not found'); } - console.log('Run completed, waiting for file to be created'); - - await new Promise((resolve) => setTimeout(resolve, 10000)); - - // // Read the exported file - // const folderPath = path.join( - // app.getPath('userData'), - // 'surfer_data', - // finalRun.company, - // finalRun.name, - // ); - // // Get all platform folders (e.g. bookmarks-001-timestamp) - // const platformFolders = fs.readdirSync(folderPath); - - // // Filter to folders matching the platform ID and sort by timestamp - // const runFolders = platformFolders - // .filter(folder => folder.startsWith(finalRun.platformId)) - // .sort((a, b) => { - // const timestampA = parseInt(a.split('-').pop() || '0'); - // const timestampB = parseInt(b.split('-').pop() || '0'); - // return timestampB - timestampA; - // }); - - // if (runFolders.length === 0) { - // throw new Error('No export folders found'); - // } - - // Get the latest run folder - const latestRunPath = finalRun.exportPath; - - // Find the JSON file in that folder const files = fs.readdirSync(latestRunPath); - const jsonFile = files.find(file => file.endsWith('.json')); + const jsonFile = files.find((file) => file.endsWith('.json')); if (!jsonFile) { - throw new Error('No JSON file found in latest export folder'); + throw new Error('No JSON file found in export folder'); } const filePath = path.join(latestRunPath, jsonFile); - - if (!fs.existsSync(filePath)) { - throw new Error('Export file not found'); - } - const fileData = JSON.parse(fs.readFileSync(filePath, 'utf8')); + res.json({ success: true, data: fileData, @@ -180,7 +123,10 @@ expressApp.post('/api/export', async (req, res) => { }); } catch (error) { console.error('Export error:', error); - res.status(500).json({ success: false, error: error.message }); + res.status(500).json({ + success: false, + error: error.message || 'Unknown export error', + }); } }); @@ -190,31 +136,6 @@ expressApp.listen(port, () => { }); -ipcMain.on('run-started', (event, run) => { - const runsPath = path.join(app.getPath('userData'), 'runs.json'); - if (!fs.existsSync(runsPath)) { - fs.writeFileSync(runsPath, JSON.stringify([])); - } - const runs = JSON.parse(fs.readFileSync(runsPath, 'utf8')); - if (!runs.find((r: any) => r.id === run.id)) { - runs.push(run); - fs.writeFileSync(runsPath, JSON.stringify(runs, null, 2)); - } -}); - - -ipcMain.on('run-finished', (event, company, name, runID, folderPath) => { - const runsPath = path.join(app.getPath('userData'), 'runs.json'); - const runs = JSON.parse(fs.readFileSync(runsPath, 'utf8')); - const run = runs.filter((r: any) => r.name === name && r.company === company && r.status === 'running').pop(); - console.log('this run: ', run); - run.status = 'success'; - run.exportPath = folderPath; - console.log('folder/filepath for run: ', folderPath); - fs.writeFileSync(runsPath, JSON.stringify(runs, null, 2)); -}); - - ipcMain.on('connect-platform', (event, platform: any) => { const { company, name, connectURL, connectSelector, id } = platform; //console.log('CONNECTING TO PLATFORM: ', company, name, connectURL, connectSelector); @@ -265,64 +186,8 @@ ipcMain.on('connect-platform', (event, platform: any) => { }); }); -ipcMain.on('get-big-data', async (event, company, name) => { - const userData = app.getPath('userData'); - const bigDataPath = path.join( - userData, - 'surfer_data', - company, - name, - 'bigData.json', - ); - console.log('CALLING GET BIG DATA!!!!!!!!') - return new Promise((resolve) => { - session.defaultSession.webRequest.onBeforeSendHeaders( - { urls: ['*://*.twitter.com/*', '*://*.x.com/*'] }, - (details: any, callback) => { - console.log('getting twitter requests ig') - if (details.url.includes('/Bookmarks?variables')) { - console.log('getting big data!'); - const bookmarksUrlPattern = - /https:\/\/x\.com\/i\/api\/graphql\/([^/]+)\/Bookmarks\?/; - const match = details.url.match(bookmarksUrlPattern); - - let result = { - bookmarksApiId: null as string | null, - auth: null as string | null, - cookie: null as string | null, - csrf: null as string | null, - }; - - if (match) { - result.bookmarksApiId = match[1]; - } - - result.auth = details.requestHeaders['authorization'] || null; - result.cookie = details.requestHeaders['Cookie'] || null; - result.csrf = details.requestHeaders['x-csrf-token'] || null; - - if ( - result.bookmarksApiId && - result.auth && - result.cookie && - result.csrf - ) { - console.log('got twitter credentials!'); - - // Create the directory if it doesn't exist - fs.mkdirSync(path.dirname(bigDataPath), { recursive: true }); - - // Write the bigData to the file - fs.writeFileSync(bigDataPath, JSON.stringify(result, null, 2)); - - event.sender.send('got-big-data', result); - } - } - - callback({ requestHeaders: details.requestHeaders }); - }, - ); - }); +ipcMain.on('get-twitter-credentials', async (event, company, name) => { + return await getTwitterCredentials(company, name); }); ipcMain.handle('check-connected-platforms', async (event, platforms) => { @@ -467,84 +332,9 @@ ipcMain.on('get-version-number', (event) => { ipcMain.handle('get-imessage-data', async (event, company: string, name: string, id: string) => { //if (process.platform === 'win32') { - const username = process.env.USERNAME || process.env.USER; - const defaultPath = - process.platform === 'win32' - ? path.join('C:', 'Users', username, 'Apple', 'MobileSync', 'Backup') - : path.join( - '/Users', - username, - 'Library', - 'Application Support', - 'MobileSync', - 'Backup', - ); - - if (!fs.existsSync(defaultPath)) { - console.log('NEED TO BACKUP YOUR IMESSAGE FOLDER!'); - return null; - } - - const result = await dialog.showOpenDialog({ - properties: ['openDirectory'], - title: 'Select iMessages Folder', - buttonLabel: 'Select', - defaultPath: defaultPath, - }); - - if (result.filePaths.length > 0) { - const selectedFolder = result.filePaths[0]; - mainWindow?.webContents.send('console-log', id, 'Got folder, now exporting iMessages (will take a few minutes)'); - - try { - const scriptOutput = await pythonUtils.iMessageScript( - process.platform, - selectedFolder, - company, - name, - id, - ); - mainWindow?.webContents.send( - 'console-log', - id, - 'iMessage export complete!' - ); - // Assuming the last line of the output is the JSON file path - const folderPath = scriptOutput.split('\n').pop()?.trim(); - console.log('JSON file path:', folderPath); - mainWindow?.webContents.send('export-complete', company, name, id, folderPath, getTotalFolderSize(folderPath)); - return folderPath; - } catch (error) { - console.error('Error running iMessage script:', error); - return null; - } - } - // } else if (process.platform === 'darwin') { - // console.log('Mac is being added soon!'); - // return null; - // } else { - // console.log('Unsupported platform:', process.platform); - // return null; - // } + return getImessageData(event, company, name, id); }); -function getTotalFolderSize(folderPath: string): number { - let totalSize = 0; - const files = fs.readdirSync(folderPath); - - for (const file of files) { - const filePath = path.join(folderPath, file); - const stats = fs.statSync(filePath); - - if (stats.isFile()) { - totalSize += stats.size; - } else if (stats.isDirectory()) { - totalSize += getTotalFolderSize(filePath); - } - } - - return totalSize; -} if (process.env.NODE_ENV === 'production') { const sourceMapSupport = require('source-map-support'); @@ -560,7 +350,7 @@ class AppUpdater { } } -let mainWindow: BrowserWindow | null = null; +export let mainWindow: BrowserWindow | null = null; const isDebug = process.env.NODE_ENV === 'development' || process.env.DEBUG_PROD === 'true'; @@ -729,151 +519,6 @@ export const createWindow = async (visible: boolean = true) => { mainWindow.webContents.send('update-web-artifact', data); }); - function extractZip(source, target) { - return new Promise((resolve, reject) => { - yauzl.open(source, { lazyEntries: true }, (err, zipfile) => { - if (err) return reject(err); - - zipfile.readEntry(); - zipfile.on('entry', (entry) => { - const fullPath = path.join(target, entry.fileName); - const directory = path.dirname(fullPath); - - if (/\/$/.test(entry.fileName)) { - // Directory entry - try { - fs.mkdirSync(fullPath, { recursive: true }); - zipfile.readEntry(); - } catch (err) { - reject(err); - } - } else { - // File entry - try { - fs.mkdirSync(directory, { recursive: true }); - zipfile.openReadStream(entry, (err, readStream) => { - if (err) return reject(err); - const writeStream = fs.createWriteStream(fullPath); - readStream.on('end', () => { - writeStream.end(); - zipfile.readEntry(); - }); - readStream.pipe(writeStream); - }); - } catch (err) { - reject(err); - } - } - }); - - zipfile.on('end', resolve); - zipfile.on('error', reject); - }); - }); - } - -async function parseConversationsJSON(extractPath: string) { - const conversationsFilePath = path.join(extractPath, 'conversations.json'); - const parsedConversationsFilePath = path.join( - extractPath, - 'basic_conversations.json', - ); - console.log('Parsing conversations.json...'); - - let formattedConversations = []; - - if (fs.existsSync(conversationsFilePath)) { - const conversationsData = fs.readFileSync(conversationsFilePath, 'utf8'); - const parsedData = JSON.parse(conversationsData); - - formattedConversations = parsedData.map((conversation: any) => { - const messages = Object.values(conversation.mapping) - .filter( - (node: any) => node.message && node.message.author.role !== 'system', - ) - .sort((a: any, b: any) => { - const timeA = a.message.create_time || 0; - const timeB = b.message.create_time || 0; - return timeA - timeB; - }) - .map((node: any) => ({ - message: node.message.content.parts.join('\n'), - role: node.message.author.role === 'assistant' ? 'AI' : 'human', - })) - .filter(msg => msg.message.trim() !== ''); // Filter out blank or whitespace-only messages - - return { - title: conversation.title, - timestamp: conversation.create_time, - conversation: messages, - }; - }); - - console.log('Conversations parsed successfully'); - } else { - console.warn('Conversations.json file not found in:', extractPath); - } - - // Write the formatted conversations to parsed_conversations.json - try { - fs.writeFileSync( - parsedConversationsFilePath, - JSON.stringify(formattedConversations, null, 2), - ); - console.log('Parsed conversations written to parsed_conversations.json'); - } catch (error) { - console.error('Error writing to parsed_conversations.json:', error); - } - - return formattedConversations; -} - -async function convertMboxToJson( - mboxFilePath: string, - jsonOutputPath: string, - id: string, - company: string, - name: string, - runID: string, -): Promise { - return new Promise((resolve, reject) => { - const readStream = fs.createReadStream(mboxFilePath); - - const data = { - company, - name, - runID, - timestamp: Date.now(), - content: [], - }; - - mboxParser(readStream) - .then((messages) => { - messages.forEach((message) => { - const jsonMessage = { - accountID: id, - from: message.from?.text, - to: message.to?.text || message.to, - subject: message.subject, - timestamp: message.date, - body: message.text, - added_to_db: new Date().toISOString(), - }; - - data.content.push(jsonMessage); - }); - - fs.writeFileSync(jsonOutputPath, JSON.stringify(data, null, 2)); - console.log('MBOX to JSON conversion completed'); - resolve(); - }) - .catch((error) => { - console.error('Error parsing MBOX:', error); - reject(error); - }); - }); -} - let lastDownloadUrl = ''; let lastDownloadTime = 0; @@ -981,12 +626,6 @@ async function convertMboxToJson( await extractZip(filePath, extractPath); console.log('Outer ZIP extracted to:', extractPath); - // // parsing conversations.json - - // if (extractPath.includes('ChatGPT')) { - // await parseConversationsJSON(extractPath) - // } - // Find the inner ZIP file const innerZipFile = fs.readdirSync(extractPath).find(file => file.endsWith('.zip')); if (innerZipFile) { @@ -1005,20 +644,7 @@ async function convertMboxToJson( if (url.includes('takeout-download.usercontent.google.com')) { // Function to recursively find the MBOX file - const findMboxFile = (dir) => { - const files = fs.readdirSync(dir); - for (const file of files) { - const filePath = path.join(dir, file); - const stat = fs.statSync(filePath); - if (stat.isDirectory()) { - const result = findMboxFile(filePath); - if (result) return result; - } else if (file.toLowerCase().endsWith('.mbox')) { - return filePath; - } - } - return null; - }; + const mboxFilePath = findMboxFile(extractPath); if (mboxFilePath) { @@ -1252,22 +878,12 @@ ipcMain.on('handle-update-complete', (event, runID, platformId, company, name, c ipcMain.on('handle-export', (event, runID, platformId, filename, company, name, content, isUpdated) => { - console.log( - 'handling export for: ', - company, - ', specific name: ', - name, - ', runID: ', - runID, - ); - const userData = app.getPath('userData'); const surferDataPath = path.join(userData, 'surfer_data'); const platformPath = path.join(surferDataPath, company); const namePath = path.join(platformPath, name); const idPath = path.join(namePath, runID); - // Create necessary folders [surferDataPath, platformPath, namePath, idPath].forEach((dir) => { if (!fs.existsSync(dir)) { @@ -1277,15 +893,12 @@ ipcMain.on('handle-export', (event, runID, platformId, filename, company, name, const timestamp = Date.now(); let fileName; - - fileName = `${platformId}.json`; - - +// if (isUpdated) { +// fileName = `${platformId}.json`; +// } +// else { fileName = `${platformId}_${timestamp}.json`; - const filePath = path.join(idPath, fileName); - - console.log('exporting this for twitter: ', content); // Prepare the data object const exportData = { company, @@ -1388,10 +1001,6 @@ ipcMain.handle('restart-app', () => { }); app.on('window-all-closed', () => { - // Terminate the llamafile server if it's running - - // killNitroServer(); - try { downloadingItems.forEach((item, key) => { if (item) { @@ -1403,8 +1012,6 @@ app.on('window-all-closed', () => { console.log('DOWNLOAD CANCEL ERROR: ', error); } - // Respect the OSX convention of having the application in memory even - // after all windows have been closed if (process.platform !== 'darwin') { app.quit(); } diff --git a/src/main/utils/helpers.ts b/src/main/utils/helpers.ts new file mode 100644 index 00000000..81695d82 --- /dev/null +++ b/src/main/utils/helpers.ts @@ -0,0 +1,127 @@ +import fs from 'fs'; +import path from 'path'; +import { mboxParser } from 'mbox-parser'; +import yauzl from 'yauzl'; + +export async function convertMboxToJson( + mboxFilePath: string, + jsonOutputPath: string, + id: string, + company: string, + name: string, + runID: string, +): Promise { + return new Promise((resolve, reject) => { + const readStream = fs.createReadStream(mboxFilePath); + + const data = { + company, + name, + runID, + timestamp: Date.now(), + content: [], + }; + + mboxParser(readStream) + .then((messages) => { + messages.forEach((message) => { + const jsonMessage = { + accountID: id, + from: message.from?.text, + to: message.to?.text || message.to, + subject: message.subject, + timestamp: message.date, + body: message.text, + added_to_db: new Date().toISOString(), + }; + + data.content.push(jsonMessage); + }); + + fs.writeFileSync(jsonOutputPath, JSON.stringify(data, null, 2)); + console.log('MBOX to JSON conversion completed'); + resolve(); + }) + .catch((error) => { + console.error('Error parsing MBOX:', error); + reject(error); + }); + }); +} + +export const findMboxFile = (dir: string): string | null => { + const files = fs.readdirSync(dir); + for (const file of files) { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + if (stat.isDirectory()) { + const result = findMboxFile(filePath); + if (result) return result; + } else if (file.toLowerCase().endsWith('.mbox')) { + return filePath; + } + } + return null; +}; + +export function extractZip(source: string, target: string) { + return new Promise((resolve, reject) => { + yauzl.open(source, { lazyEntries: true }, (err, zipfile) => { + if (err) return reject(err); + + zipfile.readEntry(); + zipfile.on('entry', (entry) => { + const fullPath = path.join(target, entry.fileName); + const directory = path.dirname(fullPath); + + if (/\/$/.test(entry.fileName)) { + // Directory entry + try { + fs.mkdirSync(fullPath, { recursive: true }); + zipfile.readEntry(); + } catch (err) { + reject(err); + } + } else { + // File entry + try { + fs.mkdirSync(directory, { recursive: true }); + zipfile.openReadStream(entry, (err, readStream) => { + if (err) return reject(err); + const writeStream = fs.createWriteStream(fullPath); + readStream.on('end', () => { + writeStream.end(); + zipfile.readEntry(); + }); + readStream.pipe(writeStream); + }); + } catch (err) { + reject(err); + } + } + }); + + zipfile.on('end', resolve); + zipfile.on('error', reject); + }); + }); + } + +export function getTotalFolderSize(folderPath: string): number { + let totalSize = 0; + const files = fs.readdirSync(folderPath); + + for (const file of files) { + const filePath = path.join(folderPath, file); + const stats = fs.statSync(filePath); + + if (stats.isFile()) { + totalSize += stats.size; + } else if (stats.isDirectory()) { + totalSize += getTotalFolderSize(filePath); + } + } + + return totalSize; + } + diff --git a/src/main/utils/imessage.ts b/src/main/utils/imessage.ts new file mode 100644 index 00000000..cf9c67df --- /dev/null +++ b/src/main/utils/imessage.ts @@ -0,0 +1,84 @@ +import { getTotalFolderSize } from './helpers'; +import path from 'path'; +import { PythonUtils } from './python'; +import fs from 'fs'; +import { dialog } from 'electron'; +import { mainWindow } from '../main'; + +const pythonUtils = new PythonUtils(); + + +export async function getImessageData(event: any, company: string, name: string, id: string) { + //if (process.platform === 'win32') { + const username = process.env.USERNAME || process.env.USER; + const defaultPath = + process.platform === 'win32' + ? path.join('C:', 'Users', username, 'Apple', 'MobileSync', 'Backup') + : path.join( + '/Users', + username, + 'Library', + 'Application Support', + 'MobileSync', + 'Backup', + ); + + if (!fs.existsSync(defaultPath)) { + console.log('NEED TO BACKUP YOUR IMESSAGE FOLDER!'); + return null; + } + + const result = await dialog.showOpenDialog({ + properties: ['openDirectory'], + title: 'Select iMessages Folder', + buttonLabel: 'Select', + defaultPath: defaultPath, + }); + + if (result.filePaths.length > 0) { + const selectedFolder = result.filePaths[0]; + mainWindow?.webContents.send( + 'console-log', + id, + 'Got folder, now exporting iMessages (will take a few minutes)', + ); + + try { + const scriptOutput = await pythonUtils.iMessageScript( + process.platform, + selectedFolder, + company, + name, + id, + ); + mainWindow?.webContents.send( + 'console-log', + id, + 'iMessage export complete!', + ); + // Assuming the last line of the output is the JSON file path + const folderPath = scriptOutput.split('\n').pop()?.trim(); + console.log('JSON file path:', folderPath); + mainWindow?.webContents.send( + 'export-complete', + company, + name, + id, + folderPath, + getTotalFolderSize(folderPath), + ); + return folderPath; + } catch (error) { + console.error('Error running iMessage script:', error); + return null; + } + } + // } else if (process.platform === 'darwin') { + // console.log('Mac is being added soon!'); + // return null; + // } else { + // console.log('Unsupported platform:', process.platform); + // return null; + // } +} + diff --git a/src/main/utils/network.ts b/src/main/utils/network.ts new file mode 100644 index 00000000..c80fc600 --- /dev/null +++ b/src/main/utils/network.ts @@ -0,0 +1,69 @@ +import { app } from 'electron'; +import path from 'path'; +import fs from 'fs'; +import { session } from 'electron'; + +export async function getTwitterCredentials(company: string, name: string) { + const userData = app.getPath('userData'); + const twitterCredentialsPath = path.join( + userData, + 'surfer_data', + company, + name, + 'twitterCredentials.json', + ); + return new Promise((resolve) => { + session.defaultSession.webRequest.onBeforeSendHeaders( + { urls: ['*://*.twitter.com/*', '*://*.x.com/*'] }, + (details: any, callback) => { + if (details.url.includes('/Bookmarks?variables')) { + const bookmarksUrlPattern = + /https:\/\/x\.com\/i\/api\/graphql\/([^/]+)\/Bookmarks\?/; + const match = details.url.match(bookmarksUrlPattern); + + let result = { + bookmarksApiId: null as string | null, + auth: null as string | null, + cookie: null as string | null, + csrf: null as string | null, + }; + + if (match) { + result.bookmarksApiId = match[1]; + } + + result.auth = details.requestHeaders['authorization'] || null; + result.cookie = details.requestHeaders['Cookie'] || null; + result.csrf = details.requestHeaders['x-csrf-token'] || null; + + if ( + result.bookmarksApiId && + result.auth && + result.cookie && + result.csrf + ) { + console.log('got twitter credentials!'); + + // Create the directory if it doesn't exist + fs.mkdirSync(path.dirname(twitterCredentialsPath), { + recursive: true, + }); + + // Write the bigData to the file + fs.writeFileSync( + twitterCredentialsPath, + JSON.stringify(result, null, 2), + ); + resolve(result); + } + } + + callback({ requestHeaders: details.requestHeaders }); + }, + ); + }); +} + +export async function getNotionCredentials(company: string, name: string) { + +} diff --git a/src/renderer/components/profile/DataExtractionTable.jsx b/src/renderer/components/profile/DataExtractionTable.jsx index a22a4d43..ca49d72a 100644 --- a/src/renderer/components/profile/DataExtractionTable.jsx +++ b/src/renderer/components/profile/DataExtractionTable.jsx @@ -43,6 +43,22 @@ const DataExtractionTable = ({ onPlatformClick, webviewRef }) => { const LOGO_SIZE = 24; // Set a consistent size for all logos + + +useEffect(() => { + // Listen for runs request from main process + const handleGetRunsRequest = () => { + // Get runs from IndexedDB + window.electron.ipcRenderer.send('get-runs-response', runs); + }; + + window.electron.ipcRenderer.on('get-runs-request', handleGetRunsRequest); + + // Cleanup listener + return () => { + window.electron.ipcRenderer.removeAllListeners('get-runs-request', handleGetRunsRequest); + }; +}, [runs]); const loadRuns = useCallback(async () => { const db = await openDB('dataExtractionDB', 1, { diff --git a/src/renderer/components/profile/WebviewManager.tsx b/src/renderer/components/profile/WebviewManager.tsx index ff277673..ac46fae5 100644 --- a/src/renderer/components/profile/WebviewManager.tsx +++ b/src/renderer/components/profile/WebviewManager.tsx @@ -194,6 +194,7 @@ const WebviewManager: React.FC = ({ }, [dispatch, runs]); const handleLogs = useCallback((runId: string, ...logs: any[]) => { + console.log('these are the logs: ', logs) const run = runs.find((run) => run.id === runId); if (!run) return; dispatch(updateRunLogs(runId, logs)); @@ -332,6 +333,7 @@ const WebviewManager: React.FC = ({ activeRun && (activeRun.status === 'pending' || activeRun.status === 'running') ) { + window.electron.ipcRenderer.send('run-finished', activeRun.company, activeRun.name, activeRun.id.toString(), ''); dispatch(stopRun(activeRun.id)); console.log('Stopping run:', activeRun.id); @@ -377,6 +379,10 @@ const WebviewManager: React.FC = ({ } }; + // useEffect(() => { + // handleOpenDevTools(); + // }, [webviewRefs.length]); + function modifyUserAgent(userAgent) { // Regular expression to match the Chrome version const chromeVersionRegex = /(Chrome\/)\d+(\.\d+){3}/;