diff --git a/FTS/initializer.py b/FTS/initializer.py index abc6aa3..8a3f9e3 100644 --- a/FTS/initializer.py +++ b/FTS/initializer.py @@ -3,11 +3,29 @@ from log.write import sys_log def Initializer_Virtual_Table(): - conn = sqlite3.connect('./database/search-index.db') + conn = sqlite3.connect('./database/search-index0.db') conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS information_fts USING FTS5(site_id, link, title, text, description, keywords, shorttext, added)''') conn.close() - sys_log("Initializer Virtual Table", "search-index.db") + sys_log("Initializer Virtual Table", "search-index0.db") + + conn = sqlite3.connect('./database/search-index1.db') + + conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS information_fts + USING FTS5(site_id, link, title, text, description, keywords, shorttext, added)''') + + conn.close() + + sys_log("Initializer Virtual Table", "search-index1.db") + + conn = sqlite3.connect('./database/search-index2.db') + + conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS information_fts + USING FTS5(site_id, link, title, text, description, keywords, shorttext, added)''') + + conn.close() + + sys_log("Initializer Virtual Table", "search-index2.db") diff --git a/FTS/update.py b/FTS/update.py index b3dbcbb..5106ae5 100644 --- a/FTS/update.py +++ b/FTS/update.py @@ -7,4 +7,4 @@ def Update_Virtual_Table(conn): conn.commit() - sys_log("Update Virtual Table", "search-index.db") + sys_log("Update Virtual Table", "search-index012.db") diff --git a/adpn.py b/adpn.py index 9952cb0..5812a95 100644 --- a/adpn.py +++ b/adpn.py @@ -15,53 +15,138 @@ print('Welcome to Neutron Administrator Panel') -def compare_databases(): - try: - conn1 = sqlite3.connect("./database/search-index.db") - conn2 = sqlite3.connect("./database/censorship.db") +def compare_databases(num): + if num == 0: + try: + conn1 = sqlite3.connect("./database/search-index0.db") + conn2 = sqlite3.connect("./database/censorship0.db") - cur1 = conn1.cursor() - cur2 = conn2.cursor() + cur1 = conn1.cursor() + cur2 = conn2.cursor() - table_name = "information" + table_name = "information" - cur1.execute(f"SELECT * FROM {table_name}") - cur2.execute(f"SELECT * FROM {table_name}") + cur1.execute(f"SELECT * FROM {table_name}") + cur2.execute(f"SELECT * FROM {table_name}") - data1 = cur1.fetchall() - data2 = cur2.fetchall() + data1 = cur1.fetchall() + data2 = cur2.fetchall() - set1 = set(data1) - set2 = set(data2) + set1 = set(data1) + set2 = set(data2) - different_list = list(set2 - set1) + different_list = list(set2 - set1) - result = { - "Censorship:": different_list, - "Search Index:": list(set1-set2) - } + result = { + "Censorship:": different_list, + "Search Index:": list(set1-set2) + } - print(result) + print(result) - conn1.close() - conn2.close() + conn1.close() + conn2.close() - if different_list: + if different_list: + return False + else: + return True + except Exception as e: + print("Error comparing databases:", str(e)) + return False + elif num == 1: + try: + conn1 = sqlite3.connect("./database/search-index1.db") + conn2 = sqlite3.connect("./database/censorship1.db") + + cur1 = conn1.cursor() + cur2 = conn2.cursor() + + table_name = "information" + + cur1.execute(f"SELECT * FROM {table_name}") + cur2.execute(f"SELECT * FROM {table_name}") + + data1 = cur1.fetchall() + data2 = cur2.fetchall() + + set1 = set(data1) + set2 = set(data2) + + different_list = list(set2 - set1) + + result = { + "Censorship:": different_list, + "Search Index:": list(set1-set2) + } + + print(result) + + conn1.close() + conn2.close() + + if different_list: + return False + else: + return True + except Exception as e: + print("Error comparing databases:", str(e)) + return False + elif num == 2: + try: + conn1 = sqlite3.connect("./database/search-index2.db") + conn2 = sqlite3.connect("./database/censorship2.db") + + cur1 = conn1.cursor() + cur2 = conn2.cursor() + + table_name = "information" + + cur1.execute(f"SELECT * FROM {table_name}") + cur2.execute(f"SELECT * FROM {table_name}") + + data1 = cur1.fetchall() + data2 = cur2.fetchall() + + set1 = set(data1) + set2 = set(data2) + + different_list = list(set2 - set1) + + result = { + "Censorship:": different_list, + "Search Index:": list(set1-set2) + } + + print(result) + + conn1.close() + conn2.close() + + if different_list: + return False + else: + return True + except Exception as e: + print("Error comparing databases:", str(e)) return False - else: - return True - except Exception as e: - print("Error comparing databases:", str(e)) - return False def synchronization_databases(): try: - if not compare_databases(): + if not compare_databases(0) and not compare_databases(1) and not compare_databases(2): print("Databases cannot be synchronized when there are differences between them.") return else: - os.remove("./database/censorship.db") - shutil.copy("./database/search-index.db", "./database/censorship.db") + os.remove("./database/censorship0.db") + shutil.copy("./database/search-index0.db", "./database/censorship0.db") + print("Synchronization successful.") + + os.remove("./database/censorship1.db") + shutil.copy("./database/search-index1.db", "./database/censorship1.db") + print("Synchronization successful.") + + os.remove("./database/censorship2.db") + shutil.copy("./database/search-index2.db", "./database/censorship2.db") print("Synchronization successful.") except Exception as e: print("Error synchronizing databases:", str(e)) @@ -122,9 +207,17 @@ def change_reliability_by_user_id(user_id, new_reliability): if (yn != 'n'): Initializer_Database() Initializer_Virtual_Table() - vt_conn = database_loader() + + vt_conn = database_loader(0) + Update_Virtual_Table(vt_conn) + vt_conn.close() + vt_conn = database_loader(1) + Update_Virtual_Table(vt_conn) + vt_conn.close() + vt_conn = database_loader(2) Update_Virtual_Table(vt_conn) vt_conn.close() + subprocess.call("start server1", shell=True) subprocess.call("start server2", shell=True) print('The server has been started successfully.') @@ -133,12 +226,19 @@ def change_reliability_by_user_id(user_id, new_reliability): keyword = input('Keyword: ') ATMT_STRT(keyword) elif command == "check": - compare_databases() + compare_databases_num = input('Text(0), Image(1), Video(3): ') + compare_databases(compare_databases_num) elif command == "sync": synchronization_databases() print("Successful data synchronization.") elif command == "sync-fts": - vt_conn = database_loader() + vt_conn = database_loader(0) + Update_Virtual_Table(vt_conn) + vt_conn.close() + vt_conn = database_loader(1) + Update_Virtual_Table(vt_conn) + vt_conn.close() + vt_conn = database_loader(2) Update_Virtual_Table(vt_conn) vt_conn.close() print("Successful data synchronization.") diff --git a/database/censorship.db b/database/censorship.db deleted file mode 100644 index 977fe0d..0000000 Binary files a/database/censorship.db and /dev/null differ diff --git a/database/search-index.db b/database/search-index.db deleted file mode 100644 index 95ba983..0000000 Binary files a/database/search-index.db and /dev/null differ diff --git a/initializer/database.py b/initializer/database.py index ea08965..fa812ea 100644 --- a/initializer/database.py +++ b/initializer/database.py @@ -1,22 +1,38 @@ import sqlite3 from log.write import sys_log -from datetime import datetime from account.database import create_users_database def Initializer_Censorship_Database(): - source_conn = sqlite3.connect('./database/search-index.db') + source_conn1 = sqlite3.connect('./database/search-index0.db') - destination_conn = sqlite3.connect('./database/censorship.db') + destination_conn1 = sqlite3.connect('./database/censorship0.db') - source_conn.backup(destination_conn) + source_conn1.backup(destination_conn1) - source_conn.close() - destination_conn.close() + source_conn1.close() + destination_conn1.close() + source_conn2 = sqlite3.connect('./database/search-index1.db') + + destination_conn2 = sqlite3.connect('./database/censorship1.db') + + source_conn2.backup(destination_conn2) + + source_conn2.close() + destination_conn2.close() + + source_conn3 = sqlite3.connect('./database/search-index2.db') + + destination_conn3 = sqlite3.connect('./database/censorship2.db') + + source_conn3.backup(destination_conn3) + + source_conn3.close() + destination_conn3.close() def Initializer_Database(): - conn = sqlite3.connect('./database/search-index.db') + conn = sqlite3.connect('./database/search-index0.db') cursor = conn.cursor() cursor.execute('''CREATE TABLE IF NOT EXISTS information @@ -36,11 +52,22 @@ def Initializer_Database(): cursor.execute('''CREATE INDEX IF NOT EXISTS idx_shorttext ON information (shorttext)''') conn.commit() + + conn1 = sqlite3.connect('./database/search-index1.db') + conn2 = sqlite3.connect('./database/search-index2.db') + + conn.backup(conn1) + conn.backup(conn2) + conn.close() + conn1.close() + conn2.close() Initializer_Censorship_Database() create_users_database() - sys_log("Initializer Database", "search-index.db") + sys_log("Initializer Database", "search-index0.db") + sys_log("Initializer Database", "search-index1.db") + sys_log("Initializer Database", "search-index2.db") sys_log("Initializer Censorship Database", "censorship.db") sys_log("Create Users Account Database", "users-account.db") diff --git a/initializer/loader.py b/initializer/loader.py index 0604486..51b8238 100644 --- a/initializer/loader.py +++ b/initializer/loader.py @@ -1,12 +1,30 @@ import sqlite3 from log.write import sys_log -def database_loader(): - conn = sqlite3.connect('./database/search-index.db', check_same_thread=False) - sys_log("Loaded", "search-index.db") - return conn +def database_loader(num): + if num == 0: + conn = sqlite3.connect('./database/search-index0.db', check_same_thread=False) + sys_log("Loaded", "search-index0.db") + return conn + elif num == 1: + conn = sqlite3.connect('./database/search-index1.db', check_same_thread=False) + sys_log("Loaded", "search-index1.db") + return conn + elif num == 2: + conn = sqlite3.connect('./database/search-index2.db', check_same_thread=False) + sys_log("Loaded", "search-index2.db") + return conn -def censorship_database_loader(): - conn = sqlite3.connect('./database/censorship.db') - sys_log("Loaded", "censorship.db") - return conn +def censorship_database_loader(num): + if num == 0: + conn = sqlite3.connect('./database/censorship0.db') + sys_log("Loaded", "censorship0.db") + return conn + if num == 1: + conn = sqlite3.connect('./database/censorship1.db') + sys_log("Loaded", "censorship1.db") + return conn + if num == 2: + conn = sqlite3.connect('./database/censorship2.db') + sys_log("Loaded", "censorship2.db") + return conn diff --git a/log.txt b/log.txt index 6ee03b7..e69de29 100644 --- a/log.txt +++ b/log.txt @@ -1,17 +0,0 @@ -SYS: Loaded: users-account.db -SYS: Initializer Database: search-index.db -SYS: Initializer Censorship Database: censorship.db -SYS: Create Users Account Database: users-account.db -SYS: Initializer Virtual Table: search-index.db -SYS: Loaded: search-index.db -SYS: Update Virtual Table: search-index.db -SYS: Start Server: 2024-02-12 07:20:55 -SYS: Loaded: search-index.db -SYS: Loaded: users-account.db -SYS: Loaded: search-index.db -SYS: Loaded: search-index.db -SYS: Loaded: search-index.db -SYS: Update Virtual Table: search-index.db -SYS: Loaded: search-index.db -SYS: Loaded: search-index.db -SYS: Loaded: search-index.db diff --git a/main.py b/main.py index 4367e40..e44f1f4 100644 --- a/main.py +++ b/main.py @@ -5,9 +5,17 @@ from search.index import Search_Data from search.safe import return_special_characters -if 'conn' not in st.session_state: - st.session_state.conn = database_loader() -conn = st.session_state.conn +if 'conn0' not in st.session_state: + st.session_state.conn0 = database_loader(0) +conn0 = st.session_state.conn0 + +if 'conn1' not in st.session_state: + st.session_state.conn1 = database_loader(1) +conn1 = st.session_state.conn1 + +if 'conn2' not in st.session_state: + st.session_state.conn2 = database_loader(2) +conn2 = st.session_state.conn2 st.title('Neutron') st.session_state.setdefault('form_state', True) @@ -26,7 +34,17 @@ submitted3 = st.form_submit_button('Edit') with col5: submitted4 = st.form_submit_button('Remove') - + + options_types = ['Text', 'Image', 'Video'] + search_type = st.radio('', options_types, index=0) + + if search_type == 'Text': + conn = conn0 + elif search_type == 'Image': + conn = conn1 + elif search_type == 'Video': + conn = conn2 + if keyword and submitted1: Search_Result = Search_Data(conn, keyword) if submitted2 and AForm == True: @@ -74,14 +92,34 @@ st.session_state.add_state = False elif submitted4 and not AForm: st.session_state.add_state = True -for row in Search_Result: - row2 = return_special_characters(row[2]) - row6 = return_special_characters(row[6]) - st.markdown('```' + str(row[0]) + '``` ```' + row[1] + '```') - row_title = row2.replace('\n', ' ') - row_title = row_title.replace(':', ' ') - st.markdown("### [" + row_title + ']' + '(' + row[1] + ')') - row_shorttext = row6.replace('\n', ' ') - row_shorttext = row_shorttext.replace('```', ' ') - st.markdown("```" + row_shorttext + "```") - st.markdown("   ") + +if search_type == 'Text': + for row in Search_Result: + row2 = return_special_characters(row[2]) + row6 = return_special_characters(row[6]) + st.markdown('```' + str(row[0]) + '``` ```' + row[1] + '```') + row_title = row2.replace('\n', ' ') + row_title = row_title.replace(':', ' ') + st.markdown("### [" + row_title + ']' + '(' + row[1] + ')') + row_shorttext = row6.replace('\n', ' ') + row_shorttext = row_shorttext.replace('```', ' ') + st.markdown("```" + row_shorttext + "```") + st.markdown("   ") +elif search_type == 'Image': + for row in Search_Result: + st.image(image=row[1]) + row2 = return_special_characters(row[2]) + st.markdown('```' + str(row[0]) + '``` ```' + row[1] + '```') + row_title = row2.replace('\n', ' ') + row_title = row_title.replace(':', ' ') + st.markdown("### [" + row_title + ']' + '(' + row[1] + ')') + st.markdown("   ") +elif search_type == 'Video': + for row in Search_Result: + st.video(row[1]) + row2 = return_special_characters(row[2]) + st.markdown('```' + str(row[0]) + '``` ```' + row[1] + '```') + row_title = row2.replace('\n', ' ') + row_title = row_title.replace(':', ' ') + st.markdown("### [" + row_title + ']' + '(' + row[1] + ')') + st.markdown("   ") \ No newline at end of file diff --git a/search/index.py b/search/index.py index 31cd4ea..6824b62 100644 --- a/search/index.py +++ b/search/index.py @@ -15,6 +15,6 @@ def Search_Data(conn, keyword): if len(rows) == 0: st.write("No results found") - ATMT_STRT(keyword) + #ATMT_STRT(keyword) else: return rows