Skip to content

Commit

Permalink
index db working, auth is okay
Browse files Browse the repository at this point in the history
  • Loading branch information
@sahil-lalani
sahil-lalani committed Sep 15, 2024
1 parent ecd4a3c commit 1460ca4
Show file tree
Hide file tree
Showing 12 changed files with 271 additions and 176 deletions.
4 changes: 2 additions & 2 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ logs
*.log

# chrome data
src/firebase.js

config.json
edge_data.json
chrome_data.json
Expand All @@ -16,7 +16,7 @@ screenshot.png
before_interaction.png
after_interaction.png

src/firebase.js
src/firebase.js

# Runtime data
pids
Expand Down
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ Project Link: [https://github.com/CEREBRUS-MAXIMUS/Surfer-Data](https://github.c
## Acknowledgements

- [Electron React Boilerplate](https://github.com/electron-react-boilerplate/electron-react-boilerplate)
- [Vector Storage](https://github.com/nitaiaharoni1/vector-storage)

[contributors-shield]: https://img.shields.io/github/contributors/CEREBRUS-MAXIMUS/Surfer-Data.svg?style=for-the-badge
[contributors-url]: https://github.com/CEREBRUS-MAXIMUS/Surfer-Data/graphs/contributors
Expand Down
189 changes: 66 additions & 123 deletions src/main/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,10 @@ ipcMain.handle('get-openai-api-key', async () => {
return process.env.OPENAI_API_KEY;
});

ipcMain.handle('get-firebase-api-key', async () => {
return process.env.FIREBASE_API_KEY;
});

ipcMain.handle('get-similar-data', async (event, query: string) => {
try {
const embedding = await createEmbedding(query);
Expand Down Expand Up @@ -1381,131 +1385,70 @@ if (!existingData) {
// return;
// }

// try {

// await ensureTableStructure(db);
// fullJSON = JSON.parse(fullContent);

// if (Array.isArray(fullJSON.content)) {
// // JSON with content array
// contentToProcess = fullJSON.content;
// } else if (Array.isArray(fullJSON)) {
// // Regular JSON array
// contentToProcess = fullJSON;
// } else {
// // Treat as single item
// contentToProcess = [fullJSON];
// }
// } catch (error) {
// // Raw text (txt, markdown, etc.)
// contentToProcess = [fullContent];
// }

// function jsonToString(obj: any): string {
// let result = '';
// for (const [key, value] of Object.entries(obj)) {
// if (typeof value === 'string') {
// // Check if the value can be converted to a valid Date
// const date = new Date(value);
// if (isNaN(date.getTime())) {
// // If it's not a valid timestamp, add it to the result
// result += `The ${key} is ${value}. `;
// }
// } else if (typeof value === 'object' && value !== null) {
// result += jsonToString(value);
// }
// }
// return result.trim();
// }

// for (const item of contentToProcess) {
// let textToChunk: string;
// if (typeof item === 'string') {
// textToChunk = item;
// } else if (typeof item === 'object' && item !== null) {
// textToChunk = jsonToString(item);
// } else {
// textToChunk = String(item);
// }

// if (textToChunk.trim().length === 0) {
// console.log('Skipping empty content');
// continue;
// }

// // batch add for chunks?

// const chunks = chunkText(textToChunk);

// try {
// for (const chunk of chunks) {
// const embedding = await createEmbedding(chunk);

// const columns = Object.keys(tableStructure)
// .filter((col) => col !== 'id')
// .join(', ');
// const placeholders = Object.keys(tableStructure)
// .filter((col) => col !== 'id')
// .map(() => '?')
// .join(', ');

// const values = Object.keys(tableStructure)
// .filter((col) => col !== 'id')
// .map((col) => {
// switch (col) {
// case 'company':
// return company;
// case 'timestamp':
// return Date.now();
// case 'name':
// return name;
// case 'runID':
// return runID;
// case 'folderPath':
// return filePath;
// case 'content':
// return chunk;
// case 'embeddings':
// return JSON.stringify(embedding);
// default:
// return null;
// }
// });

// await new Promise((res, rej) => {
// db.run(
// `INSERT INTO db (${columns}) VALUES (${placeholders})`,
// values,
// function (insertErr) {
// if (insertErr) {
// console.error(
// `Error inserting document chunk for ${filePath}:`,
// insertErr,
// );
// rej({ success: false, error: insertErr.message });
// } else {
// console.log(
// 'Inserted document chunk for: ',
// filePath,
// );
// res({ success: true, id: this.lastID });
// }
// },
// );
// });
// }
// } catch (error) {
// console.error('Error in chunking:', error);
// continue;
// }


try {
fullJSON = JSON.parse(fullContent);

if (Array.isArray(fullJSON.content)) {
contentToProcess = fullJSON.content;
} else if (Array.isArray(fullJSON)) {
contentToProcess = fullJSON;
} else {
contentToProcess = [fullJSON];
}
} catch (error) {
contentToProcess = [fullContent];
}

function jsonToString(obj: any): string {
let result = '';
for (const [key, value] of Object.entries(obj)) {
if (typeof value === 'string') {
result += `The ${key} is ${value}. `;
} else if (typeof value === 'object' && value !== null) {
result += jsonToString(value);
}
}
return result.trim();
}

for (const item of contentToProcess) {
let textToChunk: string;
if (typeof item === 'string') {
textToChunk = item;
} else if (typeof item === 'object' && item !== null) {
textToChunk = jsonToString(item);
} else {
textToChunk = String(item);
}

if (textToChunk.trim().length === 0) {
console.log('Skipping empty content');
continue;
}

const chunks: string[] = chunkText(textToChunk);

try {
// send this to renderer to execute addTexts method
mainWindow?.webContents.send(
'add-texts',
chunks,
company,
name,
runID,
filePath, // might change to folderPath / parent later on!
);
} catch (error) {
console.error('Error in chunking:', error);
continue;
}

// }
// }
// )
// console.log(`Data appended to: ${filePath}`);
// })
}

})

});

ipcMain.on('handle-update-complete', (event, runID, platformId, company, name, customFilePath = null) => {
const filePath = customFilePath ? customFilePath : path.join(app.getPath('userData'), 'surfer_data', company, name, platformId, `${platformId}.json`)
Expand Down
7 changes: 4 additions & 3 deletions src/renderer/App.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { Provider } from 'react-redux';
import store from './state/store';
import { ThemeProvider } from './components/ui/theme-provider';
import { PhotoProvider } from 'react-photo-view';
import { AuthProvider } from './auth/FirebaseAuth';

import './styles/globals.css';
import Surfer from './Surfer';
Expand All @@ -11,9 +12,9 @@ export default function App() {
return (
<Provider store={store}>
<ThemeProvider defaultTheme="light" storageKey="app-theme">
<PhotoProvider>
<Surfer />
</PhotoProvider>
<PhotoProvider>
<Surfer />
</PhotoProvider>
</ThemeProvider>
</Provider>
);
Expand Down
30 changes: 30 additions & 0 deletions src/renderer/Surfer.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import { setContentScale, setCurrentRoute, updateBreadcrumb, stopAllJobs, update
import { Alert, AlertTitle, AlertDescription } from './components/ui/alert';
import { Toaster } from './components/ui/toaster';
import { Progress } from './components/ui/progress';
import { addDocuments } from './vector_db';
// import { useAuth } from './auth/FirebaseAuth';

function Surfer() {
const dispatch = useDispatch();
Expand All @@ -26,6 +28,7 @@ function Surfer() {
const [showNotConnectedAlert, setShowNotConnectedAlert] = useState(false);
const [updateProgress, setUpdateProgress] = useState<number | null>(null);
const [content, setContent] = useState<React.ReactNode | null>(null);
// const { currentUser } = useAuth();

useEffect(() => {
const handleUpdateDownloadProgress = (progress: number) => {
Expand All @@ -40,6 +43,33 @@ function Surfer() {
};
}, []);

useEffect(() => {
const handleAddTexts = async (chunks: string[], company: string, name: string, runID: string, folderPath: string) => {
console.log('Received chunks:', chunks);
await addDocuments(chunks, company, name, runID, folderPath);
console.log('finished vectorizing!')
};

window.electron.ipcRenderer.on('add-texts', handleAddTexts);

return () => {
window.electron.ipcRenderer.removeAllListeners('add-texts');
};
}, [])

// useEffect(() => {
// if (currentUser) {
// console.log("current user: ", currentUser)
// }
// }, [currentUser]);

useEffect(() => {
console.log('opening dev tools')
window.electron.ipcRenderer.send('show-dev-tools')
}, [])



useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
// Handle content zooming
Expand Down
4 changes: 0 additions & 4 deletions src/renderer/components/profile/DataExtractionTable.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,6 @@ const DataExtractionTable = ({ onPlatformClick, webviewRef }) => {
};
}, []);

// useEffect(() => {
// console.log('opening dev tools')
// window.electron.ipcRenderer.send('show-dev-tools')
// }, [])



Expand Down
11 changes: 2 additions & 9 deletions src/renderer/components/profile/WebviewManager.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import {
import { useTheme } from '../ui/theme-provider';
import { openDB } from 'idb'; // Import openDB for IndexedDB operations
import { Button } from '../ui/button';
import { addDocuments } from '../../../main/utils/vector_db'
import { addDocuments } from '../../vector_db';
import { VectorStorage } from 'vector-storage';

const FullScreenOverlay = styled.div<{ isVisible: boolean }>`
Expand Down Expand Up @@ -312,18 +312,11 @@ const WebviewManager: React.FC<WebviewManagerProps> = ({

if (!isUpdate) {

// ... existing code ...

const texts = await window.electron.ipcRenderer.invoke('get-texts', folderPath, runToVectorize.filesToVectorize ? runToVectorize.filesToVectorize : []);
console.log('got texts!');

const uniqueTexts : string[] = Array.from(new Set(texts));

const metadataArray = uniqueTexts.map(() => ({ company, name, runID, folderPath }));
const documents = await addDocuments(texts, company, name, runID, folderPath);

const apiKey = await window.electron.ipcRenderer.invoke('get-openai-api-key');
const vectorStore = new VectorStorage({ openAIApiKey: apiKey, openaiModel: 'text-embedding-3-small' });
const documents = await vectorStore.addTexts(uniqueTexts, metadataArray);
console.log('finished vectorizing! ', documents);

}
Expand Down
24 changes: 22 additions & 2 deletions src/renderer/components/subscribe/SubscribeCard.jsx
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,32 @@
import React, { useState } from 'react';
import React, { useState, useEffect } from 'react';
import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "../ui/card";
import { Check } from 'lucide-react';
import { Button } from "../ui/button";
import SignInModal from './SignInModal';
import { useAuth } from '../../auth/FirebaseAuth';
import { signInWithCredential, getAuth, GoogleAuthProvider } from 'firebase/auth';

const SubscribeCard = () => {
const [isSignInModalOpen, setIsSignInModalOpen] = useState(false);
//const { currentUser } = useAuth();

useEffect(() => {

const handleToken = async (token) => {
console.log('Received token, sign in with credential now:', token);
const auth = getAuth();
const credential = GoogleAuthProvider.credential(token);
const result = await signInWithCredential(auth, credential);
console.log('Sign-in result:', result);
};

window.electron.ipcRenderer.on('token', handleToken);

return () => {
window.electron.ipcRenderer.removeAllListeners('token', handleToken);
};
}, []);


return (
<>
Expand Down Expand Up @@ -56,7 +76,7 @@ const SubscribeCard = () => {
<CardFooter>
<Button
className="w-full"
onClick={() => setIsSignInModalOpen(true)}
onClick={() => window.electron.ipcRenderer.send('open-external', 'https://surfsup.ai/signin')}
>
Subscribe
</Button>
Expand Down
1 change: 0 additions & 1 deletion src/renderer/config/initialStates.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,5 @@ export const initialAppState: IAppState = {
isRunLayerVisible: false,
breadcrumb: [{ text: 'Home', link: '/' }],
runs: [],
isSubscribed: true
},
};
Loading

0 comments on commit 1460ca4

Please sign in to comment.