-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathformat_alignments_with_training.py
101 lines (77 loc) · 3.91 KB
/
format_alignments_with_training.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import pandas as pd
import processing
import re
import copy
# paths to students andsewrs database
studentAnswers1_path = "test_goldStandard/student/STSint.testinput.answers-students.sent1.txt"
studentAnswers2_path = "test_goldStandard/student/STSint.testinput.answers-students.sent2.txt"
studentAnsewrs_chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt"
studentAnsewrs_chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt"
studentsAnsewrs_alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa"
# load data
studentAnserws = processing.load_sentences(studentAnswers1_path, studentAnswers1_path)
goldstandard_chunked = processing.load_chunked(studentAnsewrs_chunked_path1, studentAnsewrs_chunked_path2)
goldstandard_alignment = processing.load_alignment(studentsAnsewrs_alignment_path)
# get a nice anwser-student table
data = pd.merge(goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True)
file_path = "alignments_with_training_student_K.wa"
responses = processing.load_alignment(file_path)
responses = responses["alignment_text"].to_list()
for i, r in enumerate(responses):
print("\nresponse number " + str(i))
print(r)
unformatted = copy.deepcopy(responses)
for i, response in enumerate(responses):
temp = response.lstrip("\n")
temp = temp.rstrip("\n")
temp = re.sub(r'(?<==)>', '> ', temp) # add space after >
temp = re.sub(r'(?<!\n) +(?!\n)', ' ', temp) # remove double space
temp = temp.replace("]", "")
temp = temp.replace("[", "")
temp = temp.replace(" ==> ", " <==> ")
temp = temp.replace(" => ", " <==> ")
temp = temp.replace(" <=> ", " <==> ")
temp = temp.replace(" <== ", " <==> ")
temp = temp.replace("<==> //", " <==> 0 //")
temp = temp.replace("<==> //", " <==> 0 //")
temp = temp.replace("NOALI <==>", "0 <==>")
temp = temp.replace("<==> NOALI", "<==> 0")
temp = temp.replace("\n// NOALI", "\n0 <==> 0 // NOALI")
temp = re.sub(r'(^|[^<])(==>+)', r' <==>', temp)
temp = re.sub(r'^(<==>)', r'0 \1', temp)
temp = re.sub(r'(?<!\n) +(?!\n)', ' ', temp) # remove double space
temp = temp.split("\n")
for k, t in enumerate(temp):
temp[k] = temp[k].split("<==>")
temp[k][1] = temp[k][1].split("//")
#for j, chunk in enumerate(data.iloc[i]["chunked_sentance1"].sorted(key=len, reverse=True)):
for j, chunk in enumerate(sorted(data.iloc[i]["chunked_sentance1"], key=lambda x: len(x), reverse=True)):
pattern = re.compile(chunk, re.IGNORECASE)
temp[k][0] = pattern.sub(str(j+1), temp[k][0])
#temp[k][0] = temp[k][0].replace(chunk, str(j+1))
for j, chunk in enumerate(sorted(data.iloc[i]["chunked_sentance2"], key=lambda x: len(x), reverse=True)):
pattern = re.compile(chunk, re.IGNORECASE)
temp[k][1][0] = pattern.sub(str(j+1), temp[k][1][0])
#temp[k][1][0] = temp[k][1][0].replace(chunk, str(j+1))
if len(temp[k][1]) >= 3:
temp[k] = temp[k][0] + " <==> " + temp[k][1][0] + " // " + temp[k][1][1] + " // " + temp[k][1][2]
elif len(temp[k][1]) == 2:
temp[k] = temp[k][0] + " <==> " + temp[k][1][0] + " // " + temp[k][1][1] + " // 0"
temp[k] = re.sub(r'\s{2,}', ' ', temp[k]) # remove double space
responses[i] = "\n".join(temp)
print("\nafter formatting\n")
for i, r in enumerate(responses):
print("\nresponse number " + str(i))
print("FORMATTED\n")
print(r)
print("\nUNFORMATTED\n")
print(unformatted[i])
# write to file
file_path = "student_fixed_format_with_training.txt"
with open(file_path, 'w') as file:
for i, r in enumerate(responses):
file.write("<sentence id=\"" + str(i+1) + "\" status=\"\">\n")
file.write("<alignment>\n")
file.write(r)
file.write("\n</alignment>\n")
file.write("</sentence>\n\n")