Skip to content

Commit baee738

Browse files
vish-csvish-cs
authored
[Differ] Use dcid instead of Node for schema nodes (#427)
1 parent 37db192 commit baee738

File tree

6 files changed

+28
-29
lines changed

6 files changed

+28
-29
lines changed

‎pipeline/differ/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Data Differ Pipeline
1+
# Import Differ Pipeline
22

33
This module implements a Dataflow pipeline that generates a summary of changes between two versions (current and previous) of the same dataset. It does that by performing an outer join on the datasets. It supports both MCF and TFRecord file formats as input. The generated output (CSV) includes a summary of the changes in the dataset.
44

‎pipeline/differ/src/main/java/org/datacommons/pipeline/differ/DifferUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public void process(ProcessContext c) {
7474
Collections.sort(keys);
7575
String value =
7676
keys.stream()
77-
.filter(key -> !key.equals("Node"))
77+
.filter(key -> !key.equals(Property.dcid.name()))
7878
.map(key -> key + ":" + GraphUtils.getPropertyValue(pv, key))
7979
.collect(Collectors.joining(";"));
8080

‎pipeline/differ/src/test/resources/current/schema.mcf‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,68 @@
1-
Node: dcid:InterestRate_TreasuryBill_6Month
1+
dcid: "InterestRate_TreasuryBill_6Month"
22
name: "InterestRate_TreasuryBill_6Month"
33
typeOf: dcs:StatisticalVariable
44
measuredProperty: dcs:interestRate
55
populationType: dcs:TreasuryBill
66
maturity: [6 Month]
77
statType: dcs:measuredValue
88

9-
Node: dcid:InterestRate_TreasuryBill_1Year
9+
dcid: "InterestRate_TreasuryBill_1Year"
1010
name: "InterestRate_TreasuryBill_1Year"
1111
typeOf: dcs:StatisticalVariable
1212
measuredProperty: dcs:interestRate
1313
populationType: dcs:TreasuryBill
1414
maturity: [1 Year]
1515
statType: dcs:measuredValue
1616

17-
Node: dcid:InterestRate_TreasuryNote_2Year
17+
dcid: "InterestRate_TreasuryNote_2Year"
1818
name: "InterestRate_TreasuryNote_2Year"
1919
typeOf: dcs:StatisticalVariable
2020
measuredProperty: dcs:interestRate
2121
populationType: dcs:TreasuryNote
2222
maturity: [2 Year]
2323
statType: dcs:measuredValue
2424

25-
Node: dcid:InterestRate_TreasuryNote_3Year
25+
dcid: "InterestRate_TreasuryNote_3Year"
2626
name: "InterestRate_TreasuryNote_3Year"
2727
typeOf: dcs:StatisticalVariable
2828
measuredProperty: dcs:interestRate
2929
populationType: dcs:TreasuryNote
3030
maturity: [3 Year]
3131
statType: dcs:measuredValue
3232

33-
Node: dcid:InterestRate_TreasuryNote_5Year
33+
dcid: "InterestRate_TreasuryNote_5Year"
3434
name: "InterestRate_TreasuryNote_5Year"
3535
typeOf: dcs:StatisticalVariable
3636
measuredProperty: dcs:interestRate
3737
populationType: dcs:TreasuryNote
3838
maturity: [5 Year]
3939
statType: dcs:measuredValue
4040

41-
Node: dcid:InterestRate_TreasuryNote_7Year
41+
dcid: "InterestRate_TreasuryNote_7Year"
4242
name: "InterestRate_TreasuryNote_7Year"
4343
typeOf: dcs:StatisticalVariable
4444
measuredProperty: dcs:interestRate
4545
populationType: dcs:TreasuryNote
4646
maturity: [7 Year]
4747
statType: dcs:measuredValue
4848

49-
Node: dcid:InterestRate_TreasuryNote_10Year
49+
dcid: "InterestRate_TreasuryNote_10Year"
5050
name: "InterestRate_TreasuryNote_10Year"
5151
typeOf: dcs:StatisticalVariable
5252
measuredProperty: dcs:interestRate
5353
populationType: dcs:TreasuryNote
5454
maturity: [10 Year]
5555
statType: dcs:measuredValue
5656

57-
Node: dcid:InterestRate_TreasuryBond_20Year
57+
dcid: "InterestRate_TreasuryBond_20Year"
5858
name: "InterestRate_TreasuryBond_20Year"
5959
typeOf: dcs:StatisticalVariable
6060
measuredProperty: dcs:interestRate
6161
populationType: dcs:TreasuryBond
6262
maturity: [20 Year]
6363
statType: dcs:measuredValue
6464

65-
Node: dcid:InterestRate_TreasuryBond_30Year
65+
dcid: "InterestRate_TreasuryBond_30Year"
6666
name: "InterestRate_TreasuryBond_30Year"
6767
typeOf: dcs:StatisticalVariable
6868
measuredProperty: dcs:interestRate

‎pipeline/differ/src/test/resources/previous/schema.mcf‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,63 @@
1-
Node: dcid:InterestRate_TreasuryBill_1Month
1+
dcid: "InterestRate_TreasuryBill_1Month"
22
name: "InterestRate_TreasuryBill_1Month"
33
typeOf: dcs:StatisticalVariable
44
measuredProperty: dcs:interestRate
55
populationType: dcs:TreasuryBill
66
maturity: [1 Month]
77
statType: dcs:measuredValue
88

9-
Node: dcid:InterestRate_TreasuryBill_3Month
9+
dcid: "InterestRate_TreasuryBill_3Month"
1010
name: "InterestRate_TreasuryBill_3Month"
1111
typeOf: dcs:StatisticalVariable
1212
measuredProperty: dcs:interestRate
1313
populationType: dcs:TreasuryBill
1414
maturity: [3 Month]
1515
statType: dcs:measuredValue
1616

17-
Node: dcid:InterestRate_TreasuryBill_6Month
17+
dcid: "InterestRate_TreasuryBill_6Month"
1818
name: "InterestRate_TreasuryBill_6Month"
1919
typeOf: dcs:StatisticalVariable
2020
measuredProperty: dcs:interestRate
2121
populationType: dcs:TreasuryBill
2222
maturity: [6 Month]
2323
statType: dcs:measuredValue
2424

25-
Node: dcid:InterestRate_TreasuryBill_1Year
25+
dcid: "InterestRate_TreasuryBill_1Year"
2626
name: "InterestRate_TreasuryBill_01Year"
2727
typeOf: dcs:StatisticalVariable
2828
measuredProperty: dcs:interestRate
2929
populationType: dcs:TreasuryBill
3030
maturity: [1 Year]
3131
statType: dcs:measuredValue
3232

33-
Node: dcid:InterestRate_TreasuryNote_2Year
33+
dcid: "InterestRate_TreasuryNote_2Year"
3434
name: "InterestRate_TreasuryNote_02Year"
3535
typeOf: dcs:StatisticalVariable
3636
measuredProperty: dcs:interestRate
3737
populationType: dcs:TreasuryNote
3838
maturity: [2 Year]
3939
statType: dcs:measuredValue
4040

41-
Node: dcid:InterestRate_TreasuryNote_10Year
41+
dcid: "InterestRate_TreasuryNote_10Year"
4242
name: "InterestRate_TreasuryNote_10Year"
4343
typeOf: dcs:StatisticalVariable
4444
measuredProperty: dcs:interestRate
4545
populationType: dcs:TreasuryNote
4646
maturity: [10 Year]
4747
statType: dcs:measuredValue
4848

49-
Node: dcid:InterestRate_TreasuryBond_20Year
49+
dcid: "InterestRate_TreasuryBond_20Year"
5050
name: "InterestRate_TreasuryBond_20Year"
5151
typeOf: dcs:StatisticalVariable
5252
measuredProperty: dcs:interestRate
5353
populationType: dcs:TreasuryBond
5454
maturity: [20 Year]
5555
statType: dcs:measuredValue
5656

57-
Node: dcid:InterestRate_TreasuryBond_30Year
57+
dcid: "InterestRate_TreasuryBond_30Year"
5858
name: "InterestRate_TreasuryBond_30Year"
5959
typeOf: dcs:StatisticalVariable
6060
measuredProperty: dcs:interestRate
6161
populationType: dcs:TreasuryBond
6262
maturity: [30 Year]
63-
statType: dcs:measuredValue
64-
63+
statType: dcs:measuredValue

‎pipeline/differ/src/test/resources/schema-diff.csv‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
dcid:InterestRate_TreasuryBill_1Year,maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_01Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED
2-
dcid:InterestRate_TreasuryBill_3Month,,maturity:[3 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_3Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
3-
dcid:InterestRate_TreasuryNote_3Year,maturity:[3 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_3Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
4-
dcid:InterestRate_TreasuryNote_7Year,maturity:[7 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_7Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
5-
dcid:InterestRate_TreasuryNote_5Year,maturity:[5 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_5Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
6-
dcid:InterestRate_TreasuryBill_1Month,,maturity:[1 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
7-
dcid:InterestRate_TreasuryNote_2Year,maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_2Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_02Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED
1+
"InterestRate_TreasuryBill_1Year",maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[1 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_01Year";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED
2+
"InterestRate_TreasuryBill_3Month",,maturity:[3 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_3Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
3+
"InterestRate_TreasuryNote_3Year",maturity:[3 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_3Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
4+
"InterestRate_TreasuryNote_7Year",maturity:[7 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_7Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
5+
"InterestRate_TreasuryNote_5Year",maturity:[5 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_5Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,,ADDED
6+
"InterestRate_TreasuryBill_1Month",,maturity:[1 Month];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryBill_1Month";populationType:dcs:TreasuryBill;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,DELETED
7+
"InterestRate_TreasuryNote_2Year",maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_2Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,maturity:[2 Year];measuredProperty:dcs:interestRate;name:"InterestRate_TreasuryNote_02Year";populationType:dcs:TreasuryNote;statType:dcs:measuredValue;typeOf:dcs:StatisticalVariable,MODIFIED

‎pipeline/differ/template.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ OPERATION=$1
55
if [ "$OPERATION" == "deploy" ]; then
66
echo "Deploying Dataflow Flex Template..."
77
gcloud dataflow flex-template build \
8-
"gs://vishg-dataflow/templates/flex/differ.json" \
8+
"gs://datcom-dataflow/templates/flex/differ.json" \
99
--image-gcr-path "gcr.io/datcom-ci/dataflow-templates/differ:latest" \
1010
--sdk-language "JAVA" \
1111
--flex-template-base-image JAVA17 \

0 commit comments

Comments
 (0)