Refactor doctor_visits: Load source file only once

doctor_visits/delphi_doctor_visits/config.py

@@ -19,18 +19,37 @@ class Config:
     # data columns
     CLI_COLS = ["Covid_like", "Flu_like", "Mixed"]
     FLU1_COL = ["Flu1"]
-    COUNT_COLS = CLI_COLS + FLU1_COL + ["Denominator"]
+    COUNT_COLS = ["Denominator"] + FLU1_COL + CLI_COLS
     DATE_COL = "ServiceDate"
     GEO_COL = "PatCountyFIPS"
     AGE_COL = "PatAgeGroup"
     HRR_COLS = ["Pat HRR Name", "Pat HRR ID"]
-    ID_COLS = [DATE_COL] + [GEO_COL] + [AGE_COL] + HRR_COLS
+    ID_COLS = [DATE_COL] + [GEO_COL] + HRR_COLS + [AGE_COL]
     FILT_COLS = ID_COLS + COUNT_COLS
-    DTYPES = {"ServiceDate": str, "PatCountyFIPS": str,
-              "Denominator": int, "Flu1": int,
-              "Covid_like": int, "Flu_like": int,
-              "Mixed": int, "PatAgeGroup": str,
-              "Pat HRR Name": str, "Pat HRR ID": float}
+    DTYPES = {
+        "ServiceDate": str,
+        "PatCountyFIPS": str,
+        "Denominator": int,
+        "Flu1": int,
+        "Covid_like": int,
+        "Flu_like": int,
+        "Mixed": int,
+        "PatAgeGroup": str,
+        "Pat HRR Name": str,
+        "Pat HRR ID": float,
+        "servicedate": str,
+        "patCountyFIPS": str,
+        "patAgeGroup": str,
+        "patHRRname": str,
+        "patHRRid": float,
+    }
+    DEVIANT_COLS_MAP = {
+        "servicedate": "ServiceDate",
+        "patCountyFIPS": "PatCountyFIPS",
+        "patHRRname": "Pat HRR Name",
+        "patAgeGroup": "PatAgeGroup",
+        "patHRRid": "Pat HRR ID",
+    }
 
     SMOOTHER_BANDWIDTH = 100  # bandwidth for the linear left Gaussian filter
     MAX_BACKFILL_WINDOW = 7  # maximum number of days used to average a backfill correction

doctor_visits/delphi_doctor_visits/run.py

@@ -16,7 +16,6 @@
 # first party
 from .update_sensor import update_sensor, write_to_csv
 from .download_claims_ftp_files import download
-from .modify_claims_drops import modify_and_write
 from .get_latest_claims_name import get_latest_filename
 
 
@@ -55,9 +54,6 @@ def run_module(params):  # pylint: disable=too-many-statements
     # find the latest files (these have timestamps)
     claims_file = get_latest_filename(params["indicator"]["input_dir"], logger)
 
-    # modify data
-    modify_and_write(claims_file, logger)
-
     ## get end date from input file
     # the filename is expected to be in the format:
     # "EDI_AGG_OUTPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz"

doctor_visits/delphi_doctor_visits/update_sensor.py

@@ -11,6 +11,7 @@
 # standard packages
 from datetime import timedelta
 from multiprocessing import Pool, cpu_count
+from pathlib import Path
 
 # third party
 import numpy as np
@@ -86,12 +87,16 @@ def update_sensor(
     # as of 2020-05-11, input file expected to have 10 columns
     # id cols: ServiceDate, PatCountyFIPS, PatAgeGroup, Pat HRR ID/Pat HRR Name
     # value cols: Denominator, Covid_like, Flu_like, Flu1, Mixed
+    filename = Path(filepath).name
     data = pd.read_csv(
         filepath,
-        usecols=Config.FILT_COLS,
         dtype=Config.DTYPES,
-        parse_dates=[Config.DATE_COL],
     )
+    logger.info(f"Starting processing {filename} ")
+    data.rename(columns=Config.DEVIANT_COLS_MAP, inplace=True)
+    data = data[Config.FILT_COLS]
+    data[Config.DATE_COL] = data[Config.DATE_COL].apply(pd.to_datetime)
+    logger.info(f"finished processing {filename} ")
     assert (
             np.sum(data.duplicated(subset=Config.ID_COLS)) == 0
     ), "Duplicated data! Check the input file"

doctor_visits/tests/test_geomap.py

@@ -3,14 +3,14 @@
 from delphi_doctor_visits.geo_maps import GeoMaps
 from delphi_doctor_visits.config import Config
 
-CONFIG = Config()
 DATA = pd.read_csv(
     "test_data/SYNEDI_AGG_OUTPATIENT_07022020_1455CDT.csv.gz",
-    usecols=CONFIG.FILT_COLS,
-    dtype=CONFIG.DTYPES,
-    parse_dates=[CONFIG.DATE_COL],
+    dtype=Config.DTYPES,
     nrows=9,
 )
+DATA.rename(columns=Config.DEVIANT_COLS_MAP, inplace=True)
+DATA = DATA[Config.FILT_COLS]
+DATA[Config.DATE_COL] = DATA[Config.DATE_COL].apply(pd.to_datetime)
 
 GM = GeoMaps()