From 13281ca4c64ab65fb21a1827a31f28a0efee7615 Mon Sep 17 00:00:00 2001 From: Harish Chandrashekar Date: Tue, 10 Mar 2026 10:11:25 +0000 Subject: [PATCH 1/3] fixed pvmap --- .../us_bls/us_cpi/us_cpi_metadata.csv | 2 +- .../us_bls/us_cpi/us_cpi_pvmap.csv | 68 ++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/statvar_imports/us_bls/us_cpi/us_cpi_metadata.csv b/statvar_imports/us_bls/us_cpi/us_cpi_metadata.csv index 79f3efb290..c184983e7e 100644 --- a/statvar_imports/us_bls/us_cpi/us_cpi_metadata.csv +++ b/statvar_imports/us_bls/us_cpi/us_cpi_metadata.csv @@ -6,4 +6,4 @@ output_columns,"observationAbout,observationDate,value,variableMeasured" mapped_rows,6 mapped_columns,3 header_rows,6 -places_resolved_csv,us_cpi_places_resolved.csv +#places_resolved_csv,us_cpi_places_resolved.csv diff --git a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv index ddffa2d910..39c5a03c9a 100644 --- a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv +++ b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv @@ -1,7 +1,7 @@ key,p1,v1,p2,v2,p3,v3,,,, Place,observationAbout,{Data},populationType,ConsumerGoodsAndServices,measuredProperty,consumerPriceIndex,,,, #Relative importance,#ignore,#ignore,,,,,,,, -Consumer Price Index,observationAbout,country/USA,,,,,,,, +#Consumer Price Index,observationAbout,country/USA,,,,,,,, Unadjusted indexes,value,{Number},measurementQualifier,BLSSeasonallyUnadjusted,observationDate,{Year}-{Month},#Eval,"consumerGoodsCategory = str_to_camel_case(re.sub(""['\\-]"", """", Category.replace(' less ', ' except ')), '[^A-Za-z]')",#Aggregate,last 2010,Year,2010,,,,,,,, 2011,Year,2011,,,,,,,, @@ -52,8 +52,70 @@ Dec,Month,12,observationDate,{Year}-{Month},,,,,, Jan.,Month,1,observationDate,{Year}-{Month},,,,,, #Unadjusted percent,#ignore,#ignore,,,,,,,, #Seasonally adjusted percent,#ignore,#ignore,,,,,,,, -Expenditure category,Category,{Data},populationType,ConsumerGoodsAndServices,measuredProperty,consumerPriceIndex,,,, -Area,observationAbout,{Data},populationType,ConsumerGoodsAndServices,measuredProperty,consumerPriceIndex,consumer,UrbanConsumer,consumerGoodsCategory,AllItems +Expenditure category,Category,{Data},populationType,ConsumerGoodsAndServices,measuredProperty,consumerPriceIndex,observationAbout,country/USA,, +Area,populationType,ConsumerGoodsAndServices,measuredProperty,consumerPriceIndex,consumer,UrbanConsumer,consumerGoodsCategory,AllItems CPI-W,consumer,UrbanWageEarnerAndClericalWorker,,,,,,,, CPI-U,consumer,UrbanConsumer,,,,,,, C-CPI-U,consumer,UrbanConsumer,,,,,,, +Northeast,observationAbout,usc/NortheastRegion +Midwest,observationAbout,usc/MidwestRegion +South,observationAbout,usc/SouthRegion +West,observationAbout,usc/WestRegion +New England(4),observationAbout,usc/NewEnglandDivision +Middle Atlantic(4),observationAbout,usc/MiddleAtlanticDivision +East North Central(4),observationAbout,usc/EastNorthCentralDivision +West North Central(4),observationAbout,usc/WestNorthCentralDivision +South Atlantic(4),observationAbout,usc/SouthAtlanticDivision +East South Central(4),observationAbout,usc/EastSouthCentralDivision +West South Central(4),observationAbout,usc/WestSouthCentralDivision +Mountain(4),observationAbout,usc/MountainDivision +Pacific(4),observationAbout,usc/PacificDivision +, +Atlanta-Sandy Springs-Roswell,observationAbout,geoId/C12060 +"Atlanta-Sandy Springs-Roswell, GA",observationAbout,geoId/C12060 +Baltimore-Columbia-Towson,observationAbout,geoId/C12580 +"Baltimore-Columbia-Towson, MD(6)",observationAbout,geoId/C12580 +Houston-The Woodlands-Sugar Land,observationAbout,geoId/C26420 +"Houston-The Woodlands-Sugar Land, TX",observationAbout,geoId/C26420 +Miami-Fort Lauderdale-West Palm Beach,observationAbout,geoId/C33100 +"Miami-Fort Lauderdale-West Palm Beach, FL",observationAbout,geoId/C33100 +Philadelphia-Camden-Wilmington,observationAbout,geoId/C37980 +"Philadelphia-Camden-Wilmington, PA-NJ-DE-MD",observationAbout,geoId/C37980 +Phoenix-Mesa-Scottsdale,observationAbout,geoId/C38060 +"Phoenix-Mesa-Scottsdale, AZ(7)",observationAbout,geoId/C38060 +San Francisco-Oakland-Hayward,observationAbout,geoId/C41860 +"San Francisco-Oakland-Hayward, CA",observationAbout,geoId/C41860 +Seattle-Tacoma-Bellevue,observationAbout,geoId/C42660 +"Seattle-Tacoma-Bellevue, WA",observationAbout,geoId/C42660 +St. Louis,observationAbout,geoId/C41180 +"St. Louis, MO-IL",observationAbout,geoId/C41180 +Boston-Cambridge-Newton,observationAbout,geoId/C14460 +"Boston-Cambridge-Newton, MA-NH",observationAbout,geoId/C14460 +Dallas-Fort Worth-Arlington,observationAbout,geoId/C19100 +"Dallas-Fort Worth-Arlington, TX",observationAbout,geoId/C19100 +Denver-Aurora-Lakewood,observationAbout,geoId/C19740 +"Denver-Aurora-Lakewood, CO",observationAbout,geoId/C19740 +Minneapolis-St.Paul-Bloomington,observationAbout,geoId/C33460 +"Minneapolis-St.Paul-Bloomington, MN-WI",observationAbout,geoId/C33460 +Riverside-San Bernardino-Ontario,observationAbout,geoId/C40140 +"Riverside-San Bernardino-Ontario, CA(4)",observationAbout,geoId/C40140 +San Diego-Carlsbad,observationAbout,geoId/C41740 +"San Diego-Carlsbad, CA",observationAbout,geoId/C41740 +Tampa-St. Petersburg-Clearwater,observationAbout,geoId/C45300 +"Tampa-St. Petersburg-Clearwater, FL(8)",observationAbout,geoId/C45300 +Washington-Arlington-Alexandria,observationAbout,geoId/C47894 +"Washington-Arlington-Alexandria, DC-VA-MD-WV(6)",observationAbout,geoId/C47894 +Northeast urban,observationAbout,usc/NortheastRegion +Midwest urban,observationAbout,usc/MidwestRegion +South urban,observationAbout,usc/SouthRegion +West urban,observationAbout,usc/WestRegion +Atlanta,observationAbout,geoId/1304000 +"Atlanta, GA",observationAbout,geoId/1304000 +Detroit-Warren-Dearborn,observationAbout,geoId/C19820 +"Detroit-Warren-Dearborn, MI",observationAbout,geoId/C19820 +Chicago-Naperville-Elgin,observationAbout,geoId/C16980 +"Chicago-Naperville-Elgin, IL-IN-WI", +Los Angeles-Long Beach-Anaheim,observationAbout,geoId/C31080 +"Los Angeles-Long Beach-Anaheim, CA",observationAbout,geoId/C31080 +New York-Newark-Jersey City,observationAbout,geoId/C35620 +"New York-Newark-Jersey City, NY-NJ-PA",observationAbout,geoId/C35620 From 2216c769ac0bb589ebeeea90e34c29e445357ab3 Mon Sep 17 00:00:00 2001 From: Harish Chandrashekar Date: Tue, 10 Mar 2026 11:38:59 +0000 Subject: [PATCH 2/3] fixed pvmap --- statvar_imports/us_bls/us_cpi/README.md | 4 +++- statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv | 4 +--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/statvar_imports/us_bls/us_cpi/README.md b/statvar_imports/us_bls/us_cpi/README.md index c606f1bb44..8a96d2ee3e 100644 --- a/statvar_imports/us_bls/us_cpi/README.md +++ b/statvar_imports/us_bls/us_cpi/README.md @@ -26,7 +26,9 @@ python3 ../cpi_category/cpi_category_download.py --output_folder=../us_cpi/input After downloading the data, use the `stat_var_processor.py` script to process the files and generate the final statistical variables. #### Note : -Currently we are taking the consumerGoodsCategory directly from the input data using a #Eval in the pvmap, so if there is a new entry for goods category or any change in the existing goods category it will cause the addition of a new schema. +- Currently we are taking the consumerGoodsCategory directly from the input data using a #Eval in the pvmap, so if there is a new entry for goods category or any change in the existing goods category it will cause the addition of a new schema. + +- Anytime a new place comes up in the input file and has a place mapping in DataCommons, make sure the whole string of the place is added in the pv_map as is from source. #### Example diff --git a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv index 39c5a03c9a..e392e8f4da 100644 --- a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv +++ b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv @@ -70,7 +70,6 @@ East South Central(4),observationAbout,usc/EastSouthCentralDivision West South Central(4),observationAbout,usc/WestSouthCentralDivision Mountain(4),observationAbout,usc/MountainDivision Pacific(4),observationAbout,usc/PacificDivision -, Atlanta-Sandy Springs-Roswell,observationAbout,geoId/C12060 "Atlanta-Sandy Springs-Roswell, GA",observationAbout,geoId/C12060 Baltimore-Columbia-Towson,observationAbout,geoId/C12580 @@ -113,8 +112,7 @@ Atlanta,observationAbout,geoId/1304000 "Atlanta, GA",observationAbout,geoId/1304000 Detroit-Warren-Dearborn,observationAbout,geoId/C19820 "Detroit-Warren-Dearborn, MI",observationAbout,geoId/C19820 -Chicago-Naperville-Elgin,observationAbout,geoId/C16980 -"Chicago-Naperville-Elgin, IL-IN-WI", +"Chicago-Naperville-Elgin, IL-IN-WI",observationAbout,geoId/C16980 Los Angeles-Long Beach-Anaheim,observationAbout,geoId/C31080 "Los Angeles-Long Beach-Anaheim, CA",observationAbout,geoId/C31080 New York-Newark-Jersey City,observationAbout,geoId/C35620 From ae1e1b2aa983e82e9923384e00e62a2514a944c6 Mon Sep 17 00:00:00 2001 From: Harish Chandrashekar Date: Tue, 10 Mar 2026 11:44:00 +0000 Subject: [PATCH 3/3] resolved internal comments --- .../us_bls/us_cpi/us_cpi_pvmap.csv | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv index e392e8f4da..817a9ecc9f 100644 --- a/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv +++ b/statvar_imports/us_bls/us_cpi/us_cpi_pvmap.csv @@ -70,50 +70,29 @@ East South Central(4),observationAbout,usc/EastSouthCentralDivision West South Central(4),observationAbout,usc/WestSouthCentralDivision Mountain(4),observationAbout,usc/MountainDivision Pacific(4),observationAbout,usc/PacificDivision -Atlanta-Sandy Springs-Roswell,observationAbout,geoId/C12060 "Atlanta-Sandy Springs-Roswell, GA",observationAbout,geoId/C12060 -Baltimore-Columbia-Towson,observationAbout,geoId/C12580 "Baltimore-Columbia-Towson, MD(6)",observationAbout,geoId/C12580 -Houston-The Woodlands-Sugar Land,observationAbout,geoId/C26420 "Houston-The Woodlands-Sugar Land, TX",observationAbout,geoId/C26420 -Miami-Fort Lauderdale-West Palm Beach,observationAbout,geoId/C33100 "Miami-Fort Lauderdale-West Palm Beach, FL",observationAbout,geoId/C33100 -Philadelphia-Camden-Wilmington,observationAbout,geoId/C37980 "Philadelphia-Camden-Wilmington, PA-NJ-DE-MD",observationAbout,geoId/C37980 -Phoenix-Mesa-Scottsdale,observationAbout,geoId/C38060 "Phoenix-Mesa-Scottsdale, AZ(7)",observationAbout,geoId/C38060 -San Francisco-Oakland-Hayward,observationAbout,geoId/C41860 "San Francisco-Oakland-Hayward, CA",observationAbout,geoId/C41860 -Seattle-Tacoma-Bellevue,observationAbout,geoId/C42660 "Seattle-Tacoma-Bellevue, WA",observationAbout,geoId/C42660 -St. Louis,observationAbout,geoId/C41180 "St. Louis, MO-IL",observationAbout,geoId/C41180 -Boston-Cambridge-Newton,observationAbout,geoId/C14460 "Boston-Cambridge-Newton, MA-NH",observationAbout,geoId/C14460 -Dallas-Fort Worth-Arlington,observationAbout,geoId/C19100 "Dallas-Fort Worth-Arlington, TX",observationAbout,geoId/C19100 -Denver-Aurora-Lakewood,observationAbout,geoId/C19740 "Denver-Aurora-Lakewood, CO",observationAbout,geoId/C19740 -Minneapolis-St.Paul-Bloomington,observationAbout,geoId/C33460 "Minneapolis-St.Paul-Bloomington, MN-WI",observationAbout,geoId/C33460 -Riverside-San Bernardino-Ontario,observationAbout,geoId/C40140 "Riverside-San Bernardino-Ontario, CA(4)",observationAbout,geoId/C40140 -San Diego-Carlsbad,observationAbout,geoId/C41740 "San Diego-Carlsbad, CA",observationAbout,geoId/C41740 -Tampa-St. Petersburg-Clearwater,observationAbout,geoId/C45300 "Tampa-St. Petersburg-Clearwater, FL(8)",observationAbout,geoId/C45300 -Washington-Arlington-Alexandria,observationAbout,geoId/C47894 "Washington-Arlington-Alexandria, DC-VA-MD-WV(6)",observationAbout,geoId/C47894 Northeast urban,observationAbout,usc/NortheastRegion Midwest urban,observationAbout,usc/MidwestRegion South urban,observationAbout,usc/SouthRegion West urban,observationAbout,usc/WestRegion -Atlanta,observationAbout,geoId/1304000 "Atlanta, GA",observationAbout,geoId/1304000 -Detroit-Warren-Dearborn,observationAbout,geoId/C19820 "Detroit-Warren-Dearborn, MI",observationAbout,geoId/C19820 "Chicago-Naperville-Elgin, IL-IN-WI",observationAbout,geoId/C16980 -Los Angeles-Long Beach-Anaheim,observationAbout,geoId/C31080 "Los Angeles-Long Beach-Anaheim, CA",observationAbout,geoId/C31080 -New York-Newark-Jersey City,observationAbout,geoId/C35620 "New York-Newark-Jersey City, NY-NJ-PA",observationAbout,geoId/C35620