Flexible time zone handling (#301)

* First pass at flexible time zone handling
* Add timestamp test data
* Write timestamp information to metadata
* Update documentation

Author: bpbond

pipeline/L1.qmd

@@ -11,6 +11,10 @@ params:
   L1_METADATA: "L1_metadata/"
   METADATA_VARS_TABLE: "L1_metadata_variables.csv"
   METADATA_COLUMNS_TABLE: "L1_metadata_columns.csv"
+  # We use "Etc/GMT+5" rather than e.g. "America/New_York" for
+  # L1_DATA_TIMEZONE because outputs should always be in STANDARD time
+  # See https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+  L1_DATA_TIMEZONE: "Etc/GMT+5"
   L1_VERSION: "???"
   L1_RELEASE_DATE: "???"
   write_plots: true
@@ -174,6 +178,7 @@ readme <- gsub("[VERSION]", params$L1_VERSION, readme, fixed = TRUE)
 readme <- gsub("[DATESTAMP]", params$L1_RELEASE_DATE, readme, fixed = TRUE)
 readme <- gsub("[OBSERVATIONS]", n_obs, readme, fixed = TRUE)
 readme <- gsub("[GIT_COMMIT]", GIT_COMMIT, readme, fixed = TRUE)
+readme <- gsub("[TIMEZONE]", params$L1_DATA_TIMEZONE, readme, fixed = TRUE)
 readme_outfn <- file.path(L1, basename(readme_fn))
 message("Writing overall README ", readme_outfn, "...")
 writeLines(readme, readme_outfn)
@@ -244,6 +249,8 @@ for(dd in data_dirs) {
     md <- md[-col_info_pos]
     # The NA code is an in-line replacement
     md <- gsub("[NA_STRING_L1]", NA_STRING_L1, md, fixed = TRUE)
+    # The time zone is an in-line replacement
+    md <- gsub("[TIMEZONE]", params$L1_DATA_TIMEZONE, md, fixed = TRUE)
 
     # Insert variable metadata
     var_info_pos <- grep("[VARIABLE_INFO]", md, fixed = TRUE)

pipeline/L1_normalize.qmd

@@ -5,10 +5,15 @@ title-block-banner: true
 params:
   html_outfile: "L1_normalize.html"
   DATA_ROOT: "data_TEST/"
-  METADATA_ROOT: "metadata/"
   L0: "L0/"
   L1_NORMALIZE: "L1_normalize/"
   DESIGN_TABLE: "design_table.csv"
+  # We use "Etc/GMT+5" rather than e.g. "America/New_York" for
+  # L1_DATA_TIMEZONE because outputs should always be in STANDARD time
+  # See https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+  L1_DATA_TIMEZONE: "Etc/GMT+5"
+  METADATA_ROOT: "metadata/"
+  METADATA_TIMEZONES_TABLE: "L1_metadata/L1_metadata_timezones.csv"
   METADATA_VARS_TABLE: "L1_metadata/L1_metadata_variables.csv"
   OOS: "out-of-service/"
   logfile: ""
@@ -73,6 +78,10 @@ if(any(duplicated(chk))) {
 }
 DESIGN_TABLE_ENTRIES <- with(dt_ex, paste(Logger, Table, loggernet_variable)) # save for later
 
+# Read the site time zone table
+METADATA_TZ_TABLE <- file.path(params$METADATA_ROOT, params$METADATA_TIMEZONES_TABLE)
+tzt <- read_csv(METADATA_TZ_TABLE, col_types = "ccc")
+
 # Read the variable metadata table
 METADATA_VARS_TABLE <- file.path(params$METADATA_ROOT, params$METADATA_VARS_TABLE)
 mt <- read_csv(METADATA_VARS_TABLE, col_types = "ccccccddc")
@@ -162,7 +171,7 @@ f <- function(fn, out_dir, design_table) {
                                 times = substr(dat$TIMESTAMP, 1, 10), # isolate YYYY-MM-DD
                                 valid_through = dat$valid_through)
 
-    message("\tDropping ", sum(!dat_retain), " out-of-date design links")
+    if(sum(!dat_retain)) message("\tDropping ", sum(!dat_retain), " out-of-date design links")
     dat <- dat[dat_retain,]
     dat$valid_through <- NULL
     
@@ -232,13 +241,31 @@ f <- function(fn, out_dir, design_table) {
             # all `oos_data` entries, OR'ing the results as we go, 
             # instead of breaking out
             dat$F_OOS <- as.integer(oos(oos_data[[tbl]], dat))
-            message("\tAdded ", sum(dat$F_OOS), " OOS flags for ", tbl)
+            if(sum(dat$F_OOS)) message("\tAdded ", sum(dat$F_OOS), " OOS flags for ", tbl)
             oos_data_used[tbl] <<- TRUE # won't work when parallelized
             break
         }
     }
     smry$`OOS%` <- round(sum(dat$F_OOS) / nrow(dat) * 100, 1)
 
+    # ------------- Time zone change, if needed
+    
+    site <- dat$Site[1]
+    if(site %in% tzt$Site) {
+        site_tz <- tzt$Datalogger_time_zone[which(site == tzt$Site)]
+        if(site_tz != params$L1_DATA_TIMEZONE) {
+            message("\tChanging timestamps from ", site_tz, " to ", params$L1_DATA_TIMEZONE)
+            # The timestamps were recorded in the site's time zone, which differs
+            # from the L1 timezone, so convert to POSIXct and change tz
+            timestamps <- ymd_hms(dat$TIMESTAMP, tz = site_tz)
+            timestamps <- with_tz(timestamps, tzone = params$L1_DATA_TIMEZONE)
+            dat$TIMESTAMP <- format(timestamps, "%Y-%m-%d %H:%M:%S")
+        }
+    } else {
+        stop("This site ", site, " is not in the time zone table ", 
+             params$METADATA_TIMEZONES_TABLE)
+    }
+    
     # ------------- Write output files and clean up
 
     write_to_folders(dat, 

pipeline/data_TEST/Raw/Raw_original/_TZTEST_TR_999_CheckTable_20230607000015_short.dat

@@ -0,0 +1,10 @@
+"TOA5","Compass_TZTEST_TR_999","CR1000X","28733","CR1000X.Std.06.00","CPU:COMPASS_v3.3.CR1X","12050","CheckTable"
+"TIMESTAMP","RECORD","BattV","SolarV","Batt_CHECK","Solar_CHECK","Flag(1)","Flag(2)","Flag(3)","Flag(4)","Flag(5)","Flag(6)","Flag(7)","Flag(8)","Flag(9)","Flag(10)","Flag(11)","Flag(12)","Flag(13)","Flag(14)","Statname","PB"
+"TS","RN","Volts","Volts","","","","","","","","","","","","","","","","","",""
+"","","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp","Smp"
+"2023-05-08 00:00:00",1598,11.93,25.42,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999
+"2023-05-08 00:15:00",1599,11.93,25.41,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999
+"2023-05-08 00:30:00",1600,11.93,25.4,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999
+"2023-05-08 00:45:00",1601,11.93,25.38,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999
+"2023-05-08 01:00:00",1602,11.93,25.37,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999
+"2023-05-08 01:15:00",1603,11.93,25.36,"LOW","OKAY",0,0,-1,0,-1,0,0,-1,0,0,0,0,0,0,"Compass_TZTEST_TR",999

pipeline/docs/making-a-new-release.md

@@ -64,18 +64,22 @@ site-specific contact information is correct and, if needed, update the
 various site files in `./pipeline/metadata/L1_metadata`. Also update the
 key publications for each site.
 
+9. Confirm that the datalogger time zone information listed in
+`./metadata/L1_metadata/L1_metadata_timezones.csv` is correct, and that
+the output time zone (set in `driver.R` or the individual Quarto files)
+is what you want.
 
 ## Run the pipeline
 
-9. Commit all your changes. Now set the `ROOT` variable in `driver.R`
+10. Commit all your changes. Now set the `ROOT` variable in `driver.R`
 (if that's what you're using) to "./data" instead of "./data_TEST".
 **This change does NOT get committed**, however, because you want GitHub
 Actions to continue to use the _test_ data.
 
-10. From the `./pipeline` folder, run `reset("data/")` (sourced from 
+11. From the `./pipeline` folder, run `reset("data/")` (sourced from 
 `helpers.R`) to remove any previous files.
 
-11. Run the processing pipeline. If you use `driver.R` it will be
+12. Run the processing pipeline. If you use `driver.R` it will be
 relatively fast, because highly parallelized, but you don't get
 informative html logs, because the parallel processes can't write to
 output. If you want the full, detailed logs, run without parallelism
@@ -89,14 +93,14 @@ for examples of how to resolve these.
 
 ## Check, clean up, upload
 
-12. Double-check the final release README file.
+13. Double-check the final release README file.
 
-13. You may want to clean up the resulting L1 folder; for example,
+14. You may want to clean up the resulting L1 folder; for example,
 remove unwanted hidden files (`find ./ -name ".DS_Store" | xargs rm`) or
 'stub' data (`find ./ -name "*202407*" | xargs rm`). (Before doing this,
 use find's `-print` option to make sure you know what you're deleting!)
 
-14. Push the data (including `L1`, `Raw`, `L0`, and `Logs`) to the
+15. Push the data (including `L1`, `Raw`, `L0`, and `Logs`) to the
 COMPASS HPC. For example:
 
 ```
@@ -110,9 +114,9 @@ rsync -av --chown=:compass-fme-data --perms --chmod=g+rx --exclude=".*" L1/ <use
 # Homebrew (https://brew.sh) to install an up-to-date version of rsync.
 ```
 
-15. Upload to the Google Drive, renaming the folder to the correct
+16. Upload to the Google Drive, renaming the folder to the correct
 version number, for `L1` and `Raw`.
 
-16. Make a Git release corresponding to the version number.
+17. Make a Git release corresponding to the version number.
 
-17. Let everyone know!
+18. Let everyone know!

pipeline/docs/system-files.md

@@ -88,6 +88,7 @@ File | Description
 `GWI.txt` | Site description file for the GWI site
 `L1_metadata_columns.csv` | This specifies the names and ordering of the L1 data columns. Upon being generated, L1 files are checked against this list and the process will error if there's a discrepancy
 `L1_metadata_template.txt` | Template for the various metadata files in each site-year folder. Information placeholders in square brackets are replaced in the L1 metadata-generation step
+`L1_metadata_timezones.csv` | Time zone that the dataloggers are set to at each site; used by `L1_normalize.qmd`
 `L1_metadata_variables.csv` | Along with the design table, this is a key 'information center' for the system. Also known as the 'bounds and units table', it specifies unit conversions, expected bands, variable metadata, etc. Every output variable must have an entry here, or the L1_normalize step will error
 `MSM.txt` | Site description file for the MSM site
 `OWC.txt` | Site description file for the OWC site

pipeline/driver.R

@@ -24,6 +24,10 @@ source("helpers.R")
 ROOT <- "./data_TEST"
 VERSION <- "1-2"
 RELEASE_DATE <- "2024-02-21"
+# We use "Etc/GMT+5" rather than e.g. "America/New_York" because
+# outputs should always be in STANDARD time
+# See https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+L1_DATA_TZ <- "Etc/GMT+5"
 
 # Log file ----------------------------------------------------
 
@@ -87,6 +91,7 @@ begin <- Sys.time()
 driver_try(
     quarto_render("L1_normalize.qmd",
                   execute_params = list(DATA_ROOT = ROOT,
+                                        L1_DATA_TIMEZONE = L1_DATA_TZ,
                                         html_outfile = outfile,
                                         logfile = LOGFILE,
                                         run_parallel = TRUE))
@@ -113,6 +118,7 @@ driver_try(
                   execute_params = list(DATA_ROOT = ROOT,
                                         L1_VERSION = VERSION,
                                         L1_RELEASE_DATE = RELEASE_DATE,
+                                        L1_DATA_TIMEZONE = L1_DATA_TZ,
                                         html_outfile = outfile,
                                         logfile = LOGFILE,
                                         run_parallel = FALSE))

pipeline/metadata/L1_metadata/L1_metadata_columns.csv

@@ -2,7 +2,7 @@
 Column,Description
 Site,Site code; see site info above (character)
 Plot,Plot code; see site info above (character)
-TIMESTAMP,Datalogger timestamp in EST (POSIXct)
+TIMESTAMP,Datalogger timestamp in [TIMEZONE] (POSIXct)
 Instrument,Name of measurement instrument (character)
 Instrument_ID,Identifier of instrument within plot (character)
 Sensor_ID,"Identifier of individual sensor, tree, etc. being measured (character)"

pipeline/metadata/L1_metadata/L1_metadata_timezones.csv

@@ -0,0 +1,10 @@
+Site,Datalogger_time_zone,Comment
+TZTEST,Etc/GMT+6,TEST ONLY
+CRC,Etc/GMT+5,This is EST
+PTR,Etc/GMT+5,This is EST
+OWC,Etc/GMT+5,This is EST
+GWI,Etc/GMT+5,This is EST
+MSM,Etc/GMT+5,This is EST
+GCW,Etc/GMT+5,This is EST
+SWH,Etc/GMT+5,This is EST
+TMP,Etc/GMT+5,This is EST

pipeline/metadata/L1_metadata/README_v???.txt

@@ -2,6 +2,7 @@ COMPASS-FME Level 1 data
 Version: [VERSION]
 Date: [DATESTAMP]
 Observations: [OBSERVATIONS]
+Data timestamps: [TIMEZONE]
 Git commit: [GIT_COMMIT]
 
 DESCRIPTION

pipeline/metadata/L1_metadata/TZTEST.txt

@@ -0,0 +1,2 @@
+THIS IS A DUMMY FILE FOR THE "TZTEST" SITE USED IN TIME ZONE TESTING.
+