Skip to content

Commit ca10ad9

Browse files
author
John Gilmer
committed
Made FeatureGeneration a configurable object and added more configuration for gzip streaming
1 parent dce783b commit ca10ad9

4 files changed

Lines changed: 69 additions & 61 deletions

File tree

‎src/main/java/edu/washington/multir/development/RunDistantSupervision.java‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,8 @@
11
package edu.washington.multir.development;
22

33
import java.io.IOException;
4-
import java.io.PrintWriter;
54
import java.sql.SQLException;
6-
import java.util.ArrayList;
7-
import java.util.Collections;
8-
import java.util.Iterator;
9-
import java.util.List;
10-
import java.util.Map;
115

12-
import edu.stanford.nlp.ling.CoreAnnotations;
13-
import edu.stanford.nlp.pipeline.Annotation;
14-
import edu.stanford.nlp.util.CoreMap;
15-
import edu.stanford.nlp.util.Pair;
16-
import edu.stanford.nlp.util.Triple;
176
import edu.washington.multir.argumentidentification.ArgumentIdentification;
187
import edu.washington.multir.argumentidentification.NERArgumentIdentification;
198
import edu.washington.multir.argumentidentification.NERRelationMatching;
@@ -23,9 +12,6 @@
2312
import edu.washington.multir.corpus.Corpus;
2413
import edu.washington.multir.corpus.CorpusInformationSpecification;
2514
import edu.washington.multir.corpus.DefaultCorpusInformationSpecification;
26-
import edu.washington.multir.corpus.CorpusInformationSpecification.SentGlobalIDInformation.SentGlobalID;
27-
import edu.washington.multir.data.Argument;
28-
import edu.washington.multir.data.KBArgument;
2915
import edu.washington.multir.distantsupervision.DistantSupervision;
3016
import edu.washington.multir.knowledgebase.KnowledgeBase;
3117

@@ -51,8 +37,6 @@ public class RunDistantSupervision {
5137

5238

5339
public static void main(String[] args) throws SQLException, IOException{
54-
long start = System.currentTimeMillis();
55-
5640
//initialize variables
5741
CorpusInformationSpecification cis = new DefaultCorpusInformationSpecification();
5842
Corpus c = new Corpus(args[0],cis,true);

‎src/main/java/edu/washington/multir/development/RunFeatureGeneration.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package edu.washington.multir.development;
2+
3+
4+
import java.io.IOException;
5+
import java.sql.SQLException;
6+
7+
import edu.washington.multir.corpus.Corpus;
8+
import edu.washington.multir.corpus.CorpusInformationSpecification;
9+
import edu.washington.multir.corpus.DefaultCorpusInformationSpecification;
10+
import edu.washington.multir.featuregeneration.DefaultFeatureGenerator;
11+
import edu.washington.multir.featuregeneration.FeatureGeneration;
12+
import edu.washington.multir.featuregeneration.FeatureGenerator;
13+
14+
/**
15+
* App for doing feature generation. Before this is run
16+
* DistantSupervision and AddNegativeExamples should have
17+
* been run.
18+
* @author jgilme1
19+
*
20+
*/
21+
public class RunFeatureGeneration {
22+
/**
23+
*
24+
* @param args
25+
* args[0] is path to DB file
26+
* args[1] is path to Distant Supervision file
27+
* args[2] is path to output features file
28+
* @throws SQLException
29+
* @throws IOException
30+
*/
31+
public static void main(String[] args) throws SQLException, IOException{
32+
//initialize variables
33+
CorpusInformationSpecification cis = new DefaultCorpusInformationSpecification();
34+
FeatureGenerator fg = new DefaultFeatureGenerator();
35+
Corpus c = new Corpus(args[0],cis,true);
36+
37+
FeatureGeneration featureGeneration = new FeatureGeneration(fg);
38+
featureGeneration.run(args[1],args[2],c,cis);
39+
}
40+
}

src/main/java/edu/washington/multir/development/FeatureGeneration.java renamed to src/main/java/edu/washington/multir/featuregeneration/FeatureGeneration.java

Lines changed: 16 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
package edu.washington.multir.development;
1+
package edu.washington.multir.featuregeneration;
22

33
import java.io.BufferedReader;
44
import java.io.BufferedWriter;
55
import java.io.File;
6-
import java.io.FileReader;
7-
import java.io.FileWriter;
6+
import java.io.FileNotFoundException;
87
import java.io.IOException;
98
import java.sql.SQLException;
109
import java.util.ArrayList;
@@ -18,44 +17,26 @@
1817
import edu.stanford.nlp.util.Pair;
1918
import edu.washington.multir.corpus.Corpus;
2019
import edu.washington.multir.corpus.CorpusInformationSpecification;
21-
import edu.washington.multir.corpus.DefaultCorpusInformationSpecification;
22-
import edu.washington.multir.featuregeneration.DefaultFeatureGenerator;
23-
import edu.washington.multir.featuregeneration.FeatureGenerator;
20+
import edu.washington.multir.util.BufferedIOUtils;
2421

25-
/**
26-
* App for doing feature generation. Before this is run
27-
* DistantSupervision and AddNegativeExamples should have
28-
* been run.
29-
* @author jgilme1
30-
*
31-
*/
3222
public class FeatureGeneration {
3323

24+
private FeatureGenerator fg;
25+
private Corpus c;
3426

35-
private static FeatureGenerator fg;
36-
private static CorpusInformationSpecification cis;
37-
private static Corpus c;
38-
/**
39-
*
40-
* @param args
41-
* args[0] is name of DB
42-
* @throws SQLException
43-
* @throws IOException
44-
*/
45-
public static void main(String[] args) throws SQLException, IOException{
27+
public FeatureGeneration(FeatureGenerator fg){
28+
this.fg = fg;
29+
}
30+
31+
public void run(String dsFileName, String featureFileName, Corpus c, CorpusInformationSpecification cis) throws FileNotFoundException, IOException, SQLException{
4632
long start = System.currentTimeMillis();
33+
this.c = c;
4734

4835
//initialize variables
49-
cis = new DefaultCorpusInformationSpecification();
50-
fg = new DefaultFeatureGenerator();
51-
String dbName = args[0];
52-
String dsName = args[0]+"DS";
53-
String featuresName = args[0]+"features";
54-
c = new Corpus(dbName,cis,true);
5536
BufferedReader in;
5637
BufferedWriter bw;
57-
in = new BufferedReader(new FileReader(new File(dsName)));
58-
bw = new BufferedWriter(new FileWriter(new File(featuresName)));
38+
in = BufferedIOUtils.getBufferedReader(new File(dsFileName));
39+
bw = BufferedIOUtils.getBufferedWriter(new File(featureFileName));
5940

6041
String nextLine = in.readLine();
6142
List<SententialArgumentPair> saps = new ArrayList<>();
@@ -92,7 +73,7 @@ public static void main(String[] args) throws SQLException, IOException{
9273
System.out.println("Feature Generation took " + (end-start) + " millisseconds");
9374
}
9475

95-
private static void processSaps(List<SententialArgumentPair> saps,
76+
private void processSaps(List<SententialArgumentPair> saps,
9677
BufferedWriter bw) throws SQLException, IOException {
9778

9879
Map<Integer,Pair<CoreMap,Annotation>> sentAnnotationMap = getSentAnnotationsMap(saps);
@@ -107,7 +88,7 @@ private static void processSaps(List<SententialArgumentPair> saps,
10788

10889
}
10990

110-
private static String makeFeatureString(SententialArgumentPair sap,
91+
private String makeFeatureString(SententialArgumentPair sap,
11192
List<String> features) {
11293
StringBuilder sb = new StringBuilder();
11394
sb.append(String.valueOf(sap.getSentID()));
@@ -129,7 +110,7 @@ private static String makeFeatureString(SententialArgumentPair sap,
129110
return sb.toString().trim();
130111
}
131112

132-
private static Map<Integer, Pair<CoreMap,Annotation>> getSentAnnotationsMap(
113+
private Map<Integer, Pair<CoreMap,Annotation>> getSentAnnotationsMap(
133114
List<SententialArgumentPair> saps) throws SQLException {
134115

135116
Set<Integer> sentIds = new HashSet<Integer>();

‎src/main/java/edu/washington/multir/knowledgebase/KnowledgeBase.java‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package edu.washington.multir.knowledgebase;
22

3+
import java.io.BufferedReader;
34
import java.io.File;
45
import java.io.IOException;
56
import java.util.ArrayList;
@@ -12,6 +13,8 @@
1213
import org.apache.commons.io.FileUtils;
1314
import org.apache.commons.io.LineIterator;
1415

16+
import edu.washington.multir.util.BufferedIOUtils;
17+
1518
/**
1619
* The KnowledgeBase consists of a flat KB representation
1720
* that has
@@ -67,11 +70,11 @@ public KnowledgeBase(String relationKBFilePath, String entityKBFilePath, String
6770
//create a map of the filtered KB and silmultaneously
6871
//keep track of the relevant entities
6972
long start = System.currentTimeMillis();
70-
LineIterator li = FileUtils.lineIterator(new File(relationKBFilePath));
73+
BufferedReader relationReader = BufferedIOUtils.getBufferedReader(new File(relationKBFilePath));
7174
int index =0;
72-
while(li.hasNext()){
73-
String line = li.nextLine();
74-
String[] lineValues = line.split("\t");
75+
String relationLine;
76+
while((relationLine = relationReader.readLine())!=null){
77+
String[] lineValues = relationLine.split("\t");
7578
String e1 = lineValues[0];
7679
String e2 = lineValues[1];
7780
String rel = lineValues[2];
@@ -94,19 +97,19 @@ public KnowledgeBase(String relationKBFilePath, String entityKBFilePath, String
9497
}
9598
index ++;
9699
}
97-
li.close();
100+
relationReader.close();
98101
long end = System.currentTimeMillis();
99102

100103

101104
//load the name to ids entity map
102105
File inputFile = new File(entityKBFilePath);
103-
LineIterator entityli = FileUtils.lineIterator(inputFile);
106+
BufferedReader entityReader = BufferedIOUtils.getBufferedReader(inputFile);
104107
Map<String,List<String>> entityMap= new HashMap<String,List<String>>();
105108

106109
int lineNumber =0;
107-
while(entityli.hasNext()){
108-
String line = entityli.nextLine();
109-
String[] vals = line.split("\t");
110+
String entityLine;
111+
while((entityLine = entityReader.readLine())!=null){
112+
String[] vals = entityLine.split("\t");
110113
String entityId = vals[0];
111114
String entityName = vals[1];
112115

@@ -129,7 +132,7 @@ public KnowledgeBase(String relationKBFilePath, String entityKBFilePath, String
129132
}
130133
lineNumber++;
131134
}
132-
entityli.close();
135+
entityReader.close();
133136

134137

135138
System.out.println("Time took = " + (end- start) + " milliseconds");

0 commit comments

Comments
 (0)