-
Notifications
You must be signed in to change notification settings - Fork 1
/
scix_eke-1.0.py
69 lines (60 loc) · 2.4 KB
/
scix_eke-1.0.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#!/usr/bin/python3
import argparse
import logging
import glob
import os
import re
import io
from gen_keyphrase_core import gen_term_ctr
def extract_keyph(fpath):
logger = logging.getLogger("extract_keyph")
keyphs = []
contents = io.open(fpath,encoding="utf-8").read()
for term,ctr in gen_term_ctr(contents,1).items():
keyphs.append((term,[0,1]))
return keyphs
def main(confs):
logger = logging.getLogger("main")
testdir = confs['testdir']
outdir = confs['outdir']
n_processed = 0
# loop over all .txt files under testdir
files = glob.glob(os.path.join(testdir,'*.txt'))
for f in files:
logger.info("processing file: %(1)s"%{"1":f})
keyphs = extract_keyph(f)
outfile = re.sub(r"txt$","ann",f).replace(testdir,outdir)
outf = open(outfile,"w")
# keyphs is a list of tuples ("keyphrase",[boundarys])
for keyph in keyphs:
line = "T0\t%(1)s %(2)d %(3)d\t%(4)s\n"%{"1":"PROCESS","2":keyph[1][0],"3":keyph[1][1],"4":keyph[0]}
outf.write(line)
outf.close()
n_processed += 1
logger.info("%(1)d out of %(2)d files processed"%{"1":n_processed,"2":len(files)})
logger.info("end")
if __name__ == "__main__":
# accept input parameters
parser = argparse.ArgumentParser()
parser.add_argument("testdir",metavar='TESTDIR',type=str,help="A directory containing .txt files to be tested")
parser.add_argument("outdir",metavar='OUTDIR',type=str,help="A directory containing the output .ann files")
parser.add_argument("-v","--verbose",help="increase output verbosity",action="store_true",dest="verbose",default=False)
args = parser.parse_args()
if args.verbose:
logging_level = logging.DEBUG
else:
logging_level = logging.INFO
# logging configurations
logging.basicConfig(level=logging_level,\
filename="logs/scix_eke.log",\
format="%(asctime)s %(name)-10s %(levelname)-8s %(message)s",
filemode="w")
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter("%(name)-10s %(levelname)-8s %(message)s")
console.setFormatter(formatter)
logging.getLogger("").addHandler(console)
logging.info("Logging configuration done")
# pass command line configuration to main program
confs = {"testdir":args.testdir,"outdir":args.outdir}
main(confs)