Added

Basic Account system
More manager functions
Reliability system
Censorship (reliability) system

account/create.py

@@ -0,0 +1,37 @@
+import smtplib
+import time
+import streamlit as st
+from account.loader import account_database_loader
+
+conn = account_database_loader()
+cursor = conn.cursor()
+
+def add_user(email, username, password):
+    cursor.execute('''INSERT INTO users (email, username, password) VALUES (?, ?, ?)''', (email, username, password))
+    conn.commit()
+
+def check_existing_email(email):
+    cursor.execute("SELECT * FROM users WHERE email=?", (email,))
+    return cursor.fetchone() is not None
+
+def check_existing_username(username):
+    cursor.execute("SELECT * FROM users WHERE username=?", (username,))
+    return cursor.fetchone() is not None
+
+st.title('User Registration')
+
+email = st.text_input('Email:')
+username = st.text_input('Username:')
+password = st.text_input('Password:', type='password')
+confirm_button = st.button('Register')
+
+if confirm_button:
+    if check_existing_email(email):
+        st.error('This email is already registered. Please use a different email.')
+    elif check_existing_username(username):
+        st.error('This user name already in use. Please use another username.')
+    else:
+        with st.spinner('Checking the given information...'):
+            time.sleep(1)
+            add_user(email, username, password)
+            st.success('Your account has been successfully created.')

account/database.py

@@ -0,0 +1,15 @@
+import sqlite3
+
+def create_users_database():
+    conn = sqlite3.connect('./database/users-account.db')
+    cursor = conn.cursor()
+
+    cursor.execute('''CREATE TABLE IF NOT EXISTS users (
+                        id INTEGER PRIMARY KEY,
+                        email TEXT,
+                        username TEXT,
+                        password TEXT,
+                        reliability INTGER DEFAULT 0
+                    )''')
+    conn.commit()
+    conn.close()

account/loader.py

@@ -0,0 +1,5 @@
+import sqlite3
+
+def account_database_loader():
+    conn = sqlite3.connect('./database/users-account.db')
+    return conn

account/reliability.py

@@ -0,0 +1,20 @@
+from account.loader import account_database_loader
+
+def get_user_reliability(cursor, username, password):
+    cursor.execute('SELECT password FROM users WHERE username = ?', (username,))
+    row = cursor.fetchone()
+
+    if row:
+        stored_password = row[0]
+        if password == stored_password:
+            cursor.execute('SELECT reliability FROM users WHERE username = ?', (username,))
+            reliability_row = cursor.fetchone()
+            if reliability_row:
+                reliability_value = reliability_row[0]
+                return reliability_value
+            else:
+                return None
+        else:
+            return None
+    else:
+        return None

initializer/database.py

@@ -1,6 +1,19 @@
 import sqlite3
 from datetime import datetime
 
+from account.database import create_users_database
+
+def Initializer_Censorship_Database():
+    source_conn = sqlite3.connect('./database/search-index.db')
+
+    destination_conn = sqlite3.connect('./database/censorship.db')
+
+    source_conn.backup(destination_conn)
+
+    source_conn.close()
+    destination_conn.close()
+
+
 def Initializer_Database():
     conn = sqlite3.connect('./database/search-index.db')
     cursor = conn.cursor()
@@ -24,4 +37,7 @@ def Initializer_Database():
     conn.commit()
     conn.close()
 
+    Initializer_Censorship_Database()
+    create_users_database()
+
     return conn

initializer/loader.py

@@ -3,3 +3,7 @@
 def database_loader():
     conn = sqlite3.connect('./database/search-index.db')
     return conn
+
+def censorship_database_loader():
+    conn = sqlite3.connect('./database/censorship.db')
+    return conn

main.py

@@ -2,7 +2,7 @@
 import time
 
 from initializer.loader import database_loader
-from manager.insert import insert_data
+from manager.manager import *
 from search.index import Search_Data
 from initializer.database import Initializer_Database
 
@@ -15,8 +15,8 @@
 st.session_state.setdefault('add_state', True)
 
 with st.form('Input_Form'):
-    col1, col2, col3 = st.columns([3, 0.6, 0.4])
-    AddForm = st.session_state.add_state
+    col1, col2, col3, col4, col5 = st.columns([3, 0.8, 0.6, 0.6, 0.8])
+    AForm = st.session_state.add_state
 
     with col1:
         keyword = st.text_input('Try to search something!', value='Try to search something!', placeholder='Try to search something!', label_visibility='collapsed')
@@ -27,22 +27,65 @@
     with col3:
         submitted2 = st.form_submit_button('Add')
 
+    with col4:
+        submitted3 = st.form_submit_button('Edit')
+
+    with col5:
+        submitted4 = st.form_submit_button('Remove')
+
     if keyword and submitted1:
         Search_Data(conn, keyword)
 
-    if submitted2 and AddForm == True:
+    if submitted2 and AForm == True:
+        username = st.text_input('Username: ')
+        password = st.text_input('Password: ', type='password')
+
+        link = st.text_input('Link (Should not contain a "/" at the end, use only "http" and "https"): ')
+        title = st.text_input('Title: ')
+        text = st.text_input('Text: ')
+        description = st.text_input('Description: ')
+        keywords = st.text_input('Keywords: ')
+        shorttext = st.text_input('Short Text: ')
+
+        if username and password and link and title and text and description and keywords and shorttext:
+            with st.spinner('Checking the given information...'):
+                time.sleep(1)
+                manager_insert_data(conn, username, password, link, title, text, description, keywords, shorttext)
+                st.session_state.add_state = False
+    elif submitted2 and not AForm:
+        st.session_state.add_state = True
+
+    if submitted3 and AForm == True:
+        username = st.text_input('Username: ')
+        password = st.text_input('Password: ', type='password')
+
+        site_id = st.text_input('Site ID: ')
         link = st.text_input('Link (Should not contain a "/" at the end, use only "http" and "https"): ')
         title = st.text_input('Title: ')
         text = st.text_input('Text: ')
         description = st.text_input('Description: ')
         keywords = st.text_input('Keywords: ')
         shorttext = st.text_input('Short Text: ')
 
-        if link and title and text and description and keywords and shorttext:
+        if username and password and site_id and link and title and text and description and keywords and shorttext:
+            with st.spinner('Checking the given information...'):
+                time.sleep(1)
+                manager_edit_data(conn, username, password, site_id, link, title, text, description, keywords, shorttext)
+                st.session_state.add_state = False
+    elif submitted3 and not AForm:
+        st.session_state.add_state = True
+
+    if submitted4 and AForm == True:
+        username = st.text_input('Username: ')
+        password = st.text_input('Password: ', type='password')
+
+        site_id = st.text_input('Site ID: ')
+
+        if username and password and site_id:
             with st.spinner('Checking the given information...'):
                 time.sleep(1)
-                insert_data(conn, link, title, text, description, keywords, shorttext)
+                manager_remove_data(conn, username, password, site_id)
                 st.session_state.add_state = False
-    elif submitted2 and not AddForm:
+    elif submitted4 and not AForm:
         st.session_state.add_state = True
 

manager/edit.py

@@ -0,0 +1,90 @@
+import streamlit as st
+import sqlite3
+import requests
+from bs4 import BeautifulSoup
+import json
+from datetime import datetime
+from urllib.parse import urlparse
+from os.path import splitext
+
+GOOGLE_SAFE_BROWSING_API_KEY = 'API_KEY'
+
+allowed_extensions = {"http", "https"}
+
+def normalize(link):
+    parsed_url = urlparse(link)
+
+    if (splitext(parsed_url.path)[1][1:] not in allowed_extensions) and parsed_url.path:
+        return None
+
+    final_link = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path
+    if parsed_url.port != None and parsed_url.port != -1:
+        final_link += ":" + str(parsed_url.port)
+    if not final_link.endswith("/") and "." not in final_link:
+        final_link += "/"
+
+    return final_link
+
+def content_exists(conn, link):
+    with conn:
+        cursor = conn.cursor()
+        cursor.execute('''SELECT COUNT(*) FROM information WHERE link = ?''', (link,))
+        count = cursor.fetchone()[0]
+        return count > 0
+
+def is_content_safe(link):
+    url = 'https://safebrowsing.googleapis.com/v4/threatMatches:find?key=' + GOOGLE_SAFE_BROWSING_API_KEY
+    payload = {
+        "client": {
+            "clientId": "your-client-id",
+            "clientVersion": "1.5.2"
+        },
+        "threatInfo": {
+            "threatTypes": ["MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION"],
+            "platformTypes": ["ANY_PLATFORM"],
+            "threatEntryTypes": ["URL"],
+            "threatEntries": [{"url": link}]
+        }
+    }
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    response = requests.post(url, headers=headers, data=json.dumps(payload))
+    if response.status_code == 200:
+        data = response.json()
+        if 'matches' in data and data['matches']:
+            return False
+    return True
+
+def edit_data(conn, site_id, link, title, text, description, keywords, shorttext):
+    added = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    normalize_link = normalize(link)
+
+    try:
+        response = requests.get(normalize_link)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, 'html.parser')
+        text_content = "\n".join([p.text for p in soup.find_all('p')])
+    except requests.RequestException as e:
+        st.error("Error accessing or parsing the website.")
+        return
+
+    if not is_content_safe(normalize_link):
+        st.warning("Unsafe content detected. Not editing the database.")
+        return
+
+    with conn:
+        cursor = conn.cursor()
+        try:
+            cursor.execute('''UPDATE information 
+                              SET link=?, title=?, text=?, description=?, keywords=?, shorttext=?, added=?
+                              WHERE site_id=?''', 
+                           (normalize_link, title, text, description, keywords, shorttext, added, site_id))
+            conn.commit()
+            st.success("Data edited successfully.")
+        except sqlite3.Error as e:
+            st.error("Error editing data in the database:", e)
+
+    cursor.close()

manager/insert.py

@@ -98,3 +98,5 @@ def insert_data(conn, link, title, text, description, keywords, shorttext):
             st.success("Data inserted successfully.")
         except sqlite3.Error as e:
             st.error("Error inserting data into the database:", e)
+
+    cursor.close()

manager/manager.py

@@ -0,0 +1,59 @@
+import streamlit as st
+from account.loader import account_database_loader
+from account.reliability import get_user_reliability
+from initializer.loader import censorship_database_loader
+from manager.edit import edit_data
+from manager.insert import insert_data
+from manager.remove import remove_data
+
+def manager_insert_data(conn, username, password, link, title, text, description, keywords, shorttext):
+    account_conn = account_database_loader()
+    cursor = account_conn.cursor()
+
+    reliability = get_user_reliability(cursor, username, password)
+
+    if reliability == 0: 
+        censorship_conn = censorship_database_loader()
+        insert_data(censorship_conn, link, title, text, description, keywords, shorttext)
+        st.success("Your add request has been sent to the administrator.")
+        censorship_conn.close()
+    elif reliability == 1:
+        insert_data(conn, link, title, text, description, keywords, shorttext)
+    else: 
+        st.error("The user's reliability cannot be determined.")
+
+    cursor.close()
+    account_conn.close()
+
+def manager_edit_data(conn, username, password, site_id, link, title, text, description, keywords, shorttext):
+    account_conn = account_database_loader()
+    cursor = account_conn.cursor()
+
+    reliability = get_user_reliability(cursor, username, password)
+
+    if reliability == 0: 
+        censorship_conn = censorship_database_loader()
+        edit_data(censorship_conn, site_id, link, title, text, description, keywords, shorttext)
+        st.success("Your edit request has been sent to the administrator.")
+        censorship_conn.close()
+    elif reliability == 1:
+        edit_data(conn, site_id, link, title, text, description, keywords, shorttext)
+    else: 
+        st.error("The user's reliability cannot be determined.")
+
+    cursor.close()
+    account_conn.close()
+
+def manager_remove_data(conn, username, password, site_id):
+    account_conn = account_database_loader()
+    cursor = account_conn.cursor()
+
+    reliability = get_user_reliability(cursor, username, password)
+
+    if reliability == 1:
+        remove_data(conn, site_id)
+        st.success("Data removed successfully.")
+    account_conn.close()
+    cursor.close()
+
+

manager/remove.py

@@ -0,0 +1,11 @@
+import sqlite3
+
+def remove_data(conn, site_id):
+    cursor = conn.cursor()
+
+    cursor.execute("DELETE FROM information WHERE site_id = ?", (site_id,))
+    cursor.execute("UPDATE information SET site_id = site_id - 1 WHERE site_id > ?", (site_id,))
+
+    conn.commit()
+
+    cursor.close()

requirements.txt

@@ -1,3 +1,4 @@
+email_validator
 streamlit
 requests
 bs4