EventEntityExtractor/CoNLLDocumentReader.cpp at master · veces/EventEntityExtractor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
/*
 * CoNLLDocumentReader.cpp
 *
 *  Created on: Sep 11, 2013
 *      Author: bishan
 */

#include "CoNLLDocumentReader.h"
#include <fstream>
#include <assert.h>
#include <stack>

CoNLLDocumentReader::CoNLLDocumentReader(string filename) {
	// TODO Auto-generated constructor stub
	infile.open(filename.c_str(), ios::in);
}

CoNLLDocumentReader::~CoNLLDocumentReader() {
	// TODO Auto-generated destructor stub
}

bool CoNLLDocumentReader::ReadDocument(Document *doc) {
	string line;
	while (getline(infile, line) && line.find("#begin document") == string::npos) {
	}

	if (line == "") return false;

	assert(doc != NULL);

	int start_index = line.find('(');
	int end_index = line.find(')');
	doc->doc_id = line.substr(start_index+1, end_index-start_index-1);

	// Read sentences.
	int sent_id = 0;
	string sent_str = "";
	while (getline(infile, line) && line.find("#end document") == string::npos) {
		if (line == "") {
			Sentence *sent = new Sentence();
			if(ReadSentence(sent_str, sent)) {
				sent->docid = doc->doc_id;
				doc->sentences.push_back(sent);
			} else {
				delete sent;
				sent = NULL;
			}
			sent_str = "";
		} else {
			sent_str += line + "\n";
		}
	}

	return true;
}

bool CoNLLDocumentReader::ReadSentence(string sent_str, Sentence* sent) {
	assert(sent != NULL);

	vector<string> lines;
	Utils::Split(sent_str, '\n', lines);
	//cout<<lines.size()<<endl;

	string parse_tree = "";
	for (int i = 0; i < lines.size(); ++i) {
		vector<string> fields;
		Utils::Split(lines[i], '\t', fields);

		if (fields.size() != 12) {
			cout<<lines[i]<<endl;
			continue;
		}

		sent->sent_id = atoi(fields[1].c_str());

		string word = "";
		string pos = "";
		string lemma = "";
		word = fields[3];
		pos = fields[4];

		lemma = fields[6];

		sent->words.push_back(word);
		sent->postags.push_back(pos);
		sent->lemmas.push_back(lemma);

		// Read parse tree info.
		string parse_tag = fields[5];
		int index = parse_tag.find('*');
		if (index >= 0) {
			parse_tree += parse_tag.substr(0, index) +
					"(" + pos + "[" + word + "/" + pos + "]" + " " + word + ")"
					+ parse_tag.substr(index+1);
		}

		// read token tags
		string tag = fields[fields.size()-1];

		sent->token_labels.push_back(tag);
		sent->pred_token_labels.push_back("O");
	}

	//cout<<sent->sent_id<<" "<<sent->token_labels.size()<<endl;

	// Build parse tree.
	if (parse_tree != "")
		sent->buildPennTree(parse_tree);

	return true;
}