Merge pull request #183 from mountainMath/v0.5.1

V0.5.1

Author: mountainMath

.Rbuildignore

@@ -14,6 +14,7 @@ R/hex_sticker.R
 images
 images/*
 card_fix.sh
+lastMiKTeXException
 ^_pkgdown\.yml$
 ^CRAN-RELEASE$
 ^\.github$
@@ -22,3 +23,7 @@ card_fix.sh
 ^\.renviron
 ^\.rprofile
 
+^doc$
+
+^R/geo_suite.R
+^CRAN-SUBMISSION$

.github/workflows/R-CMD-check.yaml

@@ -1,18 +1,10 @@
-# NOTE: This workflow is overkill for most R packages
-# check-standard.yaml is likely a better choice
-# usethis::use_github_action("check-standard") will install it.
-#
-# For help debugging build failures open an issue on the RStudio community with the 'github-actions' tag.
-# https://community.rstudio.com/new-topic?category=Package%20development&tags=github-actions
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches:
-      - main
-      - master
+    branches: [main, master]
   pull_request:
-    branches:
-      - main
-      - master
+    branches: [main, master]
   schedule:
     - cron: "15 15 * * *"
 
@@ -30,100 +22,32 @@ jobs:
         config:
           - {os: macOS-latest,   r: 'release'}
           - {os: windows-latest, r: 'release'}
-          - {os: windows-latest, r: '3.6'}
-          - {os: ubuntu-20.04, r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
-          - {os: ubuntu-20.04, r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest"}
-          # - {os: ubuntu-16.04,   r: 'devel', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest", http-user-agent: "R/4.0.0 (ubuntu-16.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
-          # - {os: ubuntu-16.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          # - {os: ubuntu-16.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          # - {os: ubuntu-16.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          #- {os: ubuntu-16.04,   r: '3.4',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          #- {os: ubuntu-16.04,   r: '3.3',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
+          - {os: ubuntu-latest,   r: 'release'}
+          - {os: ubuntu-latest,   r: 'oldrel-1'}
 
     env:
-      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      RSPM: ${{ matrix.config.rspm }}
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       CM_API_KEY: ${{ secrets.CM_API_KEY }}
       COMPILE_VIG: ${{ secrets.COMPILE_VIG }}
+      R_KEEP_PKG_SOURCE: yes
 
     steps:
       - uses: actions/checkout@v2
 
-      - uses: r-lib/actions/setup-r@v1
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
         with:
           r-version: ${{ matrix.config.r }}
           http-user-agent: ${{ matrix.config.http-user-agent }}
+          use-public-rspm: true
 
-      - uses: r-lib/actions/setup-pandoc@v1
-      - name: "[Custom block] [macOS] Install libraries"
-        if: runner.os == 'macOS'
-        run: |
-          # conflicts with gfortran from r-lib/actions when linking gcc
-          # rm '/usr/local/bin/gfortran'
-          brew install  pkg-config gdal proj geos udunits
-          # brew install pkg-config gdal proj geos udunits
-
-      - name: Query dependencies
-        run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE, type = "binary"), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
-
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v2
+      - uses: r-lib/actions/setup-r-dependencies@v2
         with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-2-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-2-
-
-      - name: Install system dependencies
-        if: runner.os == 'Linux'
-        run: |
-          while read -r cmd
-          do
-            eval sudo $cmd
-          done < <(Rscript -e 'writeLines(remotes::system_requirements("ubuntu", "20.04"))')
-
-      - name: Install dependencies on Windows
-        if: runner.os == 'Windows'
-        run: |
-          remotes::install_deps(dependencies = TRUE, type = "binary")
-          remotes::install_cran("lwgeom", dependencies = TRUE, type="binary")
-          remotes::install_cran("rcmdcheck")
-        shell: Rscript {0}
-
-      - name: Install dependencies on not Windows
-        if: runner.os != 'Windows'
-        run: |
-          remotes::install_deps(dependencies = TRUE)
-          remotes::install_cran("rcmdcheck")
-          remotes::install_cran("sf", force=TRUE)
-        shell: Rscript {0}
-
-      - name: Session info
-        run: |
-          options(width = 100)
-          pkgs <- installed.packages()[, "Package"]
-          sessioninfo::session_info(pkgs, include_base = TRUE)
-        shell: Rscript {0}
-
-      - name: Check
-        env:
-          _R_CHECK_CRAN_INCOMING_: false
-        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
-        shell: Rscript {0}
-
-      - name: Show testthat output
-        if: always()
-        run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true
-        shell: bash
+          extra-packages: any::rcmdcheck
+          needs: check
 
-      - name: Upload check results
-        if: failure()
-        uses: actions/upload-artifact@main
+      - uses: r-lib/actions/check-r-package@v2
         with:
-          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
-          path: check
+          upload-snapshots: true

.gitignore

@@ -14,3 +14,5 @@ Meta
 CRAN-RELEASE
 CRAN-SUBMISSION
 .DS_Store
+/doc/
+/Meta/

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: cancensus
 Type: Package
 Title: Access, Retrieve, and Work with Canadian Census Data and Geography
-Version: 0.5.0
+Version: 0.5.3
 Authors@R: c(
     person("Jens", "von Bergmann", email = "jens@mountainmath.ca", role = c("aut"), comment = "API creator and maintainer"),
     person("Dmitry", "Shkolnik", email = "shkolnikd@gmail.com", role = c("aut", "cre"), comment = "Package maintainer, responsible for correspondence"),
@@ -24,7 +24,7 @@ Imports: digest (>= 0.1),
     httr (>= 1.0.0),
     jsonlite (>= 1.0),
     rlang
-RoxygenNote: 7.1.2
+RoxygenNote: 7.2.1
 Suggests: knitr,
     ggplot2,
     leaflet,

NAMESPACE

@@ -15,14 +15,16 @@ export(list_cancensus_cache)
 export(list_census_datasets)
 export(list_census_regions)
 export(list_census_vectors)
+export(list_recalled_cached_data)
 export(parent_census_vectors)
 export(remove_from_cancensus_cache)
+export(remove_recalled_cached_data)
 export(search_census_regions)
 export(search_census_vectors)
-export(set_api_key)
-export(set_cache_path)
-export(show_api_key)
-export(show_cache_path)
+export(set_cancensus_api_key)
+export(set_cancensus_cache_path)
+export(show_cancensus_api_key)
+export(show_cancensus_cache_path)
 importFrom(dplyr,"%>%")
 importFrom(rlang,.data)
 importFrom(stats,setNames)

NEWS.md

@@ -1,3 +1,18 @@
+# cancensus - 0.5.3
+- Added a check and context menu to install `sf` package when user requests spatial data but does not have the required package installed as opposed to stopping with an error. 
+- fixes a bug in the local data recall check
+
+# cancensus - 0.5.2
+- resolved broken and relocated links picked up in CRAN pretesting
+
+# cancensus - 0.5.1
+- Added functionality for users to detect and remove locally cached data that has been recalled by Statistics Canada
+- added a check on initial `get_census` call that produces a warning if locally cached data has been recalled by Statistics Canada
+- added option to retrieve higher-resolution geometries in `get_census`
+
+## Minor changes
+- renamed get/set cache/api_key functions to avoid name conflicts with related packages
+
 # cancensus - 0.5.0
 
 ## Minor changes
@@ -81,7 +96,7 @@
 
 ## Major changes
 - Added a `max_leaves_option` for the `search_census_vectors` function. Adds functionality to set maximum depth for child census vectors, i.e. `max_level=NA` as an additional parameter. Then e.g. `max_level=1` would only get direct children and no grandchildren.
-- Resolved issues with mislabelled CD UID for CSD level data
+- Resolved issues with mislabeled CD UID for CSD level data
 - Improve reliability of character/numeric alignments for geographic UIDS
 - `get_census_geography` is now soft deprecated and rolled into the standard `get_census` with parameters
 

R/cancensus.R

@@ -11,7 +11,11 @@
 #' @param regions A named list of census regions to retrieve. Names must be valid census aggregation levels.
 #' @param level The census aggregation level to retrieve, defaults to \code{"Regions"}. One of \code{"Regions"}, \code{"PR"}, \code{"CMA"}, \code{"CD"}, \code{"CSD"}, \code{"CT"}, \code{"DA"}, \code{"EA"} (for 1996), or \code{"DB"} (for 2001-2016).
 #' @param vectors An R vector containing the CensusMapper variable names of the census variables to download. If no vectors are specified only geographic data will get downloaded.
-#' @param geo_format By default is set to \code{NA} and appends no geographic information. To include geographic information with census data, specify one of either \code{"sf"} to return an \code{\link[sf]{sf}} object (requires the \code{sf} package) or \code{"sp"} to return a \code{\link[sp]{SpatialPolygonsDataFrame-class}} object (requires the \code{rgdal} package).
+#' @param geo_format By default is set to \code{NA} and appends no geographic information. To include geographic information with census data, specify one of either \code{"sf"} to return an \code{\link[sf]{sf}} object (requires the \code{sf} package) or \code{"sp"} to return a \code{\link[sp]{SpatialPolygonsDataFrame-class}} object (requires the \code{rgdal} package). If user requests geo-spatial data and neither package is available, a context menu will prompt to install the \code{sf} package.
+#' @param resolution Resolution of the geographic data. {cancensus} will download simplified geometries by default. For lower level geometries like DB or DA this will be very close to the high resolution data.
+#' Simplification generally increases as the geographic aggregation level increases.
+#' If high resolution geometries are required
+#' then this option can be set to 'high'. By default this setting is set to \code{'simplified'}.
 #' @param labels Set to "detailed" by default, but truncated Census variable names can be selected by setting labels = "short". Use \code{label_vectors(...)} to return variable label information in detail.
 #' @param use_cache If set to TRUE (the default) data will be read from the local cache if available.
 #' @param quiet When TRUE, suppress messages and warnings.
@@ -44,16 +48,26 @@
 #' # Get details for truncated vectors:
 #' label_vectors(census_data)
 #'}
-get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA, labels = "detailed",
-                        use_cache=TRUE, quiet=FALSE, api_key=Sys.getenv("CM_API_KEY")) {
+get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA,
+                        resolution = 'simplified',
+                        labels = "detailed",
+                        use_cache=TRUE, quiet=FALSE, api_key=Sys.getenv("CM_API_KEY"))
+  {
+
+  # API and data recall checks
+  first_run_checks()
   api_key <- robust_api_key(api_key)
   have_api_key <- valid_api_key(api_key)
   result <- NULL
   data_version<-NULL
   geo_version<-NULL
 
+  # Check region selection validity
   if (is.na(level)) level="Regions"
 
+  # Check spatial resolution
+  if (is.na(geo_format) && !(resolution %in% c("simplified","high"))) stop("The resolution paramerter needs to be either 'simplified' or 'high'.")
+
   # Turn the region list into a valid JSON dictionary.
   if (is.character(regions)) {
     if (!quiet) warning(paste("Passing `regions` as a character vector is",
@@ -65,7 +79,6 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
     regions <- jsonlite::toJSON(lapply(regions,as.character)) # cast to character in case regions are supplied as numeric/interger
   }
 
-
   # Check if the aggregation level is valid.
   if (!level %in% VALID_LEVELS) {
     stop("the `level` parameter must be one of 'Regions', 'PR', 'CMA', 'CD', 'CSD', 'CT', 'DA', 'EA' or 'DB'")
@@ -76,7 +89,18 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
   } else if (!geo_format %in% c("sf", "sp")) {
     stop("the `geo_format` parameter must be one of 'sf', 'sp', or NA")
   } else if (!is.na(geo_format) && !requireNamespace("sf", quietly = TRUE)) {
-    stop("the `sf` package is required to return geographies.")
+    stop("The `sf` package is required to return geographies.")
+  }
+
+  # Check if SF is installed when asking for spatial data
+  if(geo_format == "sf" && !("sf" %in% utils::installed.packages())) {
+    if (utils::menu(c("Yes", "No"),
+             title= paste("The `sf` package is required to return geographies. Would you like to install?")) == "1") {
+      utils::install.packages('sf')
+    } else {
+      print("Cancelling installation and retrieving tabular data only.")
+      geo_format <- NA
+    }
   }
 
   base_url=paste0(cancensus_base_url(),"/api/v1/")
@@ -118,6 +142,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
           readr::read_csv(na = cancensus_na_strings,
                           col_types = list(.default = "c"))
       } else {
+        check_recalled_data_and_warn(meta_file,params)
         httr::content(response, type = "text", encoding = "UTF-8") %>%
           textConnection %>%
           utils::read.csv(colClasses = "character", stringsAsFactors = FALSE, check.names = FALSE) %>%
@@ -133,7 +158,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
       attr(result, "last_updated") <- Sys.time()
       save(result, file = data_file)
       file_info <- file.info(data_file)
-      metadata <- dplyr::tibble(dataset=dataset,regions=as.character(jsonlite::toJSON(regions)),
+      metadata <- dplyr::tibble(dataset=dataset,regions=as.character(regions),
                                  level=level,
                                  vectors=jsonlite::toJSON(as.character(vectors))  %>% as.character(),
                                  created_at=Sys.time(),
@@ -144,16 +169,19 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
       # Load `result` object from cache.
       load(file = data_file)
     }
-    touch_metadata(meta_file)
+    touch_metadata(meta_file,params)
   }
 
   if (!is.na(geo_format)) {
     params <- list(regions=regions,
                    level=level,
                    dataset=dataset,
+                   resolution=resolution,
                    api_key=api_key)
-    param_string <- paste0("regions=", regions, "&level=", level, "&dataset=",
-                           dataset)
+    param_string <- paste0("regions=", regions,
+                           "&level=", level,
+                           "&dataset=", dataset)
+    if (resolution !="simplified") param_string <- paste0(param_string,"&resolution=",resolution)
     geo_base_name <- paste0("CM_geo_", digest::digest(param_string, algo = "md5"))
     geo_file <- cache_path(geo_base_name, ".geojson")
     meta_file <- paste0(geo_file, ".meta")
@@ -172,11 +200,14 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
       geo_version <- response$headers$`data-version`
       write(httr::content(response, type = "text", encoding = "UTF-8"), file = geo_file) # cache result
       file_info <- file.info(geo_file)
-      metadata <- dplyr::tibble(dataset=dataset,regions=as.character(regions),level=level,created_at=Sys.time(),
-                         version=data_version,size=file_info$size)
+      metadata <- dplyr::tibble(dataset=dataset,regions=as.character(regions),
+                                level=level,created_at=Sys.time(),
+                                resolution=resolution,
+                         version=geo_version,size=file_info$size)
       saveRDS(metadata, file = meta_file)
     } else {
       if (!quiet) message("Reading geo data from local cache.")
+      check_recalled_data_and_warn(meta_file,params)
     }
     geos <- geojsonsf::geojson_sf(geo_file) %>%
       sf::st_sf(agr="constant") %>% # need to set this, otherwise sf complains
@@ -192,7 +223,7 @@ get_census <- function (dataset, regions, level=NA, vectors=c(), geo_format = NA
       dplyr::select(result, -dplyr::one_of(to_remove)) %>%
         dplyr::inner_join(geos, ., by = "GeoUID")
     }
-    touch_metadata(meta_file)
+    touch_metadata(meta_file,params)
   }
 
   # ensure sf format even if library not loaded and set agr columns
@@ -268,7 +299,7 @@ VALID_LEVELS <- c("Regions","C","PR", "CMA", "CD", "CSD", "ADA","CT", "DA", 'EA'
 
 #' Query the CensusMapper API for available datasets.
 #'
-#' @param use_cache If set to TRUE (the dfault), data will be read from a temporary local cache for the
+#' @param use_cache If set to TRUE (the default), data will be read from a temporary local cache for the
 #'   duration of the R session, if
 #'   available. If set to FALSE, query the API for the data, and
 #'   refresh the temporary cache with the result.
@@ -279,7 +310,7 @@ VALID_LEVELS <- c("Regions","C","PR", "CMA", "CD", "CSD", "ADA","CT", "DA", 'EA'
 #' Returns a data frame with a column \code{dataset} containing the code for the
 #' dataset, a column \code{description} describing it, a \code{geo_dataset} column
 #' identifying the geography dataset the data is based on, a \code{attribution} column
-#' with an attribtuion string, a \code{reference} column with a reference identifier, and
+#' with an attribution string, a \code{reference} column with a reference identifier, and
 #' a \code{reference_url} column with a link to reference materials.
 #'
 #' @export
@@ -400,6 +431,7 @@ label_vectors <-  function(x) {
 #' # Get details for truncated vectors:
 #' census_vectors(census_data)
 #' }
+#' @keywords internal
 #' @export
 census_vectors <-  function(x) {
   warning("census_vectors() is deprecated. Please use label_vectors() to view details for truncated variable labels.")