🌱 Generate fake descriptors file for testing

Author: znatty22

d3b_api_client_cli/cli/__init__.py

@@ -7,6 +7,14 @@
 import click
 from d3b_api_client_cli.cli.dewrangle import *
 from d3b_api_client_cli.cli.postgres import *
+from d3b_api_client_cli.cli.faker import *
+
+
+@click.group()
+def faker():
+    """
+    Group of lower level CLI commands related to generating fake data 
+    """
 
 
 @click.group()
@@ -35,6 +43,9 @@ def main():
     """
 
 
+# Fake data commands
+faker.add_command(generate_global_id_file)
+
 # Postgres API commands
 postgres.add_command(save_file_to_db)
 
@@ -61,3 +72,4 @@ def main():
 # Add command groups to the root CLI
 main.add_command(dewrangle)
 main.add_command(postgres)
+main.add_command(faker)

d3b_api_client_cli/cli/dewrangle/__init__.py

@@ -10,3 +10,4 @@
 from d3b_api_client_cli.cli.dewrangle.volume_commands import *
 from d3b_api_client_cli.cli.dewrangle.job_commands import *
 from d3b_api_client_cli.cli.dewrangle.billing_group_commands import *
+from d3b_api_client_cli.cli.dewrangle.global_id_commands import *

d3b_api_client_cli/cli/faker/__init__.py

@@ -0,0 +1,5 @@
+"""
+Package containing commands for fake data generation
+"""
+
+from d3b_api_client_cli.cli.faker.global_id_commands import *

d3b_api_client_cli/cli/faker/global_id_commands.py

@@ -0,0 +1,63 @@
+"""
+Commands to generate fake global ID descriptors
+"""
+import os
+import logging
+import click
+
+from d3b_api_client_cli.config import (
+    log, FHIR_RESOURCE_TYPES, FhirResourceType
+)
+from d3b_api_client_cli.faker.global_id import (
+    generate_global_id_file as _generate_global_id_file
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_FHIR_RESOURCE_TYPE: FhirResourceType = FHIR_RESOURCE_TYPES["DocumentReference"]
+
+
+@click.command()
+@click.option(
+    "--output-dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    help="Where the output file will be written"
+)
+@click.option(
+    "--fhir-resource-type",
+    default=DEFAULT_FHIR_RESOURCE_TYPE.resource_type,
+    type=click.Choice(rt for rt in FHIR_RESOURCE_TYPES.keys()),
+    help="What the fhirResourceType column will be populated with"
+)
+@click.option(
+    "--with-global-ids",
+    default=True,
+    is_flag=True,
+    help="Whether or not to generate a globalId column"
+)
+@click.option(
+    "--total-rows",
+    type=int,
+    default=10,
+    help="Total number of rows to generate"
+)
+def generate_global_id_file(
+    total_rows, with_global_ids, fhir_resource_type, output_dir
+):
+    """
+    Send request to upsert global ID descriptors in Dewrangle and 
+    download the resulting global ID descriptors.
+
+    In order to create new global IDs provide a CSV file with the columns:
+    descriptor, fhirResourceType
+
+    In order to update existing global IDs provide a CSV file with the columns:
+    descriptor, fhirResourceType, globalId
+    """
+
+    log.init_logger()
+
+    return _generate_global_id_file(
+        fhir_resource_type, total_rows=total_rows,
+        with_global_ids=with_global_ids, output_dir=output_dir
+    )

d3b_api_client_cli/config/__init__.py

@@ -3,12 +3,14 @@
 """
 
 import os
+from dataclasses import dataclass
 
 from dotenv import find_dotenv, load_dotenv
 
 # File paths and directories
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname((__file__))))
 ROOT_DATA_DIR = os.path.join(ROOT_DIR, "data")
+ROOT_FAKE_DATA_DIR = os.path.join(ROOT_DATA_DIR, "fake_data")
 LOG_DIR = os.path.join(ROOT_DATA_DIR, "logs")
 
 DOTENV_PATH = find_dotenv()
@@ -27,6 +29,23 @@
 DB_USER_PW = os.environ.get("DB_USER_PW")
 
 
+@dataclass
+class FhirResourceType:
+    """
+    Wrapper class to define a FHIR resource type along with a global ID 
+    prefix
+    """
+    resource_type: str
+    id_prefix: str
+
+
+FHIR_RESOURCE_TYPES: dict = {
+    resource_type: FhirResourceType(resource_type, prefix)
+    for resource_type, prefix in
+    [("DocumentReference", "dr")]
+}
+
+
 class SECRETS:
     """
     Used in logger initialization to obfuscate sensitive env variables
@@ -69,6 +88,12 @@ def check_dewrangle_http_config():
         "credential_type": "AWS",
         "billing_group_id": os.environ.get("CAVATICA_BILLING_GROUP_ID"),
     },
+    "faker": {
+        "global_id": {
+            "fhir_resource_types": FHIR_RESOURCE_TYPES
+        }
+
+    },
     "aws": {
         "region": os.environ.get("AWS_DEFAULT_REGION") or "us-east-1",
         "s3": {

d3b_api_client_cli/faker/__init__.py

@@ -0,0 +1,4 @@
+"""
+Package dedicated to generating fake data needed for development and testing
+"""
+

d3b_api_client_cli/faker/global_id.py

@@ -0,0 +1,76 @@
+"""
+Generate files of global ID descriptors for testing and development
+"""
+
+import os
+from typing import Optional
+from pprint import pformat
+import logging
+
+import pandas
+
+from d3b_api_client_cli.config import (
+    config, FhirResourceType, ROOT_FAKE_DATA_DIR
+)
+
+FHIR_RESOURCE_TYPES: dict = config["faker"]["global_id"]["fhir_resource_types"]
+DEFAULT_FHIR_RESOURCE_TYPE: str = "DocumentReference"
+
+logger = logging.getLogger(__name__)
+
+
+def generate_global_id_file(
+    fhir_resource_type: Optional[str] = DEFAULT_FHIR_RESOURCE_TYPE,
+    with_global_ids: Optional[bool] = True,
+    total_rows: Optional[int] = 10,
+    output_dir: Optional[str] = None
+) -> str:
+    """
+    Generate a csv file with global IDs and descriptors
+
+    Options:
+        - fhir_resource_type: the FHIR resource type and global ID prefix
+        to populate the file with
+
+        - with_global_ids: Whether or not to include a column for global IDs 
+        if global IDs are not included and this file is used in
+        upsert_global_descriptors, then new global IDs will be created by 
+        Dewrangle
+
+        - total_rows: Number of rows to generate
+
+    Returns:
+        Path to file
+    """
+    logger.info(
+        "🏭 Generating %s rows for fake global ID descriptors file",
+        total_rows
+    )
+    if not output_dir:
+        output_dir = ROOT_FAKE_DATA_DIR
+        os.makedirs(output_dir, exist_ok=True)
+
+    fhir_resource_type = FHIR_RESOURCE_TYPES.get(fhir_resource_type)
+
+    data = []
+    for i in range(total_rows):
+        row = {
+            "fhirResourceType": fhir_resource_type.resource_type,
+            "descriptor": f"{fhir_resource_type.resource_type}-{i}"
+        }
+        if with_global_ids:
+            row["globalId"] = f"{fhir_resource_type.id_prefix}-{i}000"
+        data.append(row)
+
+        logger.info("Wrote %s to file", pformat(row))
+
+    df = pandas.DataFrame(data)
+
+    filepath = os.path.join(output_dir, "fake_global_descriptors.csv")
+    df.to_csv(filepath, index=False)
+
+    logger.info(
+        "✅ Completed writing global ID descriptors to %s", filepath
+    )
+
+    return filepath