-
Notifications
You must be signed in to change notification settings - Fork 15
/
search_engine.py
56 lines (44 loc) · 1.7 KB
/
search_engine.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
from reader import ReadFile
from configuration import ConfigClass
from parser_module import Parse
from indexer import Indexer
from searcher import Searcher
import utils
def run_engine():
"""
:return:
"""
number_of_documents = 0
config = ConfigClass()
r = ReadFile(corpus_path=config.get__corpusPath())
p = Parse()
indexer = Indexer(config)
documents_list = r.read_file(file_name='sample3.parquet')
# Iterate over every document in the file
for idx, document in enumerate(documents_list):
# parse the document
parsed_document = p.parse_doc(document)
number_of_documents += 1
# index the document data
indexer.add_new_doc(parsed_document)
print('Finished parsing and indexing. Starting to export files')
utils.save_obj(indexer.inverted_idx, "inverted_idx")
utils.save_obj(indexer.postingDict, "posting")
def load_index():
print('Load inverted index')
inverted_index = utils.load_obj("inverted_idx")
return inverted_index
def search_and_rank_query(query, inverted_index, k):
p = Parse()
query_as_list = p.parse_sentence(query)
searcher = Searcher(inverted_index)
relevant_docs = searcher.relevant_docs_from_posting(query_as_list)
ranked_docs = searcher.ranker.rank_relevant_doc(relevant_docs)
return searcher.ranker.retrieve_top_k(ranked_docs, k)
def main():
run_engine()
query = input("Please enter a query: ")
k = int(input("Please enter number of docs to retrieve: "))
inverted_index = load_index()
for doc_tuple in search_and_rank_query(query, inverted_index, k):
print('tweet id: {}, score (unique common words with query): {}'.format(doc_tuple[0], doc_tuple[1]))