File size: 2,659 Bytes
eb57aa1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
import nltk
from nltk.chunk import conlltags2tree, tree2conlltags
#from sklearn.model_selection import train_test_split
#from collections import Iterable
from nltk import ChunkParserI, ClassifierBasedTagger
class ClassifierChunkParser(ChunkParserI):
def __init__(self):
self.tagger = None
pass
def parse(self, tagged_sent):
chunks = self.tagger.tag(tagged_sent)
iob_triplets = [(w, t, c) for ((w, t), c) in chunks]
# Transform the list of triplets to nltk.Tree format
return conlltags2tree(iob_triplets)
def train_merger(self, train_file_path, test_split=0.1):
print("Loading Data...")
file = open(train_file_path, "r", encoding='utf-8')
file_content = file.read()
file_content = file_content.split("\n\n")
data_list = []
for line in file_content:
line = nltk.chunk.util.conllstr2tree(line, chunk_types=('NP',), root_label='S')
if (len(line) > 0):
data_list.append(line)
# train_sents, test_sents = train_test_split(data_list, test_size=test_split, random_state=91)
train_sents = data_list
test_sents = []
print("Training the model ...")
# Transform the trees in IOB annotated sentences [(word, pos, chunk), ...]
chunked_sents = [tree2conlltags(sent) for sent in train_sents]
# Transform the triplets in pairs, make it compatible with the tagger interface [((word, pos), chunk), ...]
def triplets2tagged_pairs(iob_sent):
return [((word, pos), chunk) for word, pos, chunk in iob_sent]
chunked_sents = [triplets2tagged_pairs(sent) for sent in chunked_sents]
self.feature_detector = self.features
self.tagger = ClassifierBasedTagger(
train=chunked_sents,
feature_detector=self.features)
token_merger_model = self.tagger
if len(test_sents) > 0:
print("evaluating...")
print(token_merger_model.evaluate(test_sents))
return token_merger_model
def nestedtree_to_list(self, tree, separator_char, d=0):
s = ''
for item in tree:
if isinstance(item, tuple):
s += item[0] + separator_char
elif d >= 1:
news = self.nestedtree_to_list(item, separator_char, d + 1)
s += news + separator_char
else:
news = self.nestedtree_to_list(item, separator_char, d + 1) + '\t'
s += news + separator_char
return s.strip(separator_char)
|