-
Notifications
You must be signed in to change notification settings - Fork 0
/
cosine_sim.py
51 lines (40 loc) · 1.28 KB
/
cosine_sim.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import numpy as np
import re
from sklearn.metrics.pairwise import cosine_similarity
text1 = "AI is our friend and it has been friendly"
text2 = "AI and humans have always been friendly"
def extract_words(sentence):
ignore_words = ['a']
words = re.sub("[^\w]", " ",sentence ).split()
words_cleaned = [w.lower() for w in words if w not in ignore_words]
return words_cleaned
def tokenize_sentences(sentences):
words = []
for sentence in sentences:
w = extract_words(sentence)
words.extend(w)
words = sorted(list(set(words)))
return words
def bag_of_words(sentence, words):
sentence_words = extract_words(sentence)
bag = np.zeros(len(words))
for sw in sentence_words:
for i, word in enumerate(words):
if word == sw:
bag[i] += 1
return np.array(bag)
sentences = [text1, text2]
vocabulary = tokenize_sentences(sentences)
# print(bag_of_words(sentences[0], vocabulary))
vec = []
vec.append(bag_of_words(sentences[0], vocabulary))
vec.append(bag_of_words(sentences[1], vocabulary))
print(vocabulary)
print(extract_words(sentences[0]))
print(vec[0])
print(extract_words(sentences[1]))
print(vec[1])
vec[0] = vec[0].reshape(1, -1)
vec[1] = vec[1].reshape(1, -1)
sim = cosine_similarity(vec[0], vec[1])
print(sim)