🚚 Migrate Dewrangle-related files and integration tests, replacing existing implementations

.env.ci

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# ************* BEGIN: Required CLI Settings ***********************
+
+# ** DELETE SAFETY CHECK ** 
+# We highly recommend leaving this set to True in production
+# When set to True, delete operations will only be allowed on service URLs that 
+# are on localhost. This includes KF dataservice, Dewrangle, and FHIR service
+# When this is set to False, delete operations will be allowed on any 
+# service the CLI interacts with
+
+# CI needs this to be false
+DWDS_DELETE_SAFETY_CHECK=False
+
+# Dewrangle API variables
+export DEWRANGLE_BASE_URL="https://dewrangle.com/"
+
+# CI overwrites this at runtime, leave commented
+# export DEWRANGLE_DEV_PAT=
+
+# ************* END: Required CLI Settings ***********************

.github/workflows/ci.yml

@@ -2,36 +2,37 @@ name: ✅ CI
 
 on:
   pull_request:
-    types: [opened, reopened, edited, synchronize, closed]
+    types: [opened, reopened, synchronize, ready_for_review]
 
 concurrency:
   group: ci-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 env:
-  DEWRANGLE_DEV_PAT: ${{ secrets.DEWRANGLE_DEV_PAT }} 
-  DEWRANGLE_BASE_URL: ${{ secrets.DEWRANGLE_BASE_URL }} 
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 
-  AWS_BUCKET_DATA_TRANSFER_TEST: ${{ secrets.AWS_BUCKET_DATA_TRANSFER_TEST }} 
-  CAVATICA_BILLING_GROUP_ID: ${{ secrets.CAVATICA_BILLING_GROUP_ID }} 
+  DEWRANGLE_DEV_PAT: ${{ secrets.DEWRANGLE_DEV_PAT }}
+  DEWRANGLE_BASE_URL: ${{ secrets.DEWRANGLE_BASE_URL }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_BUCKET_DATA_TRANSFER_TEST: ${{ secrets.AWS_BUCKET_DATA_TRANSFER_TEST }}
+  CAVATICA_BILLING_GROUP_ID: ${{ secrets.CAVATICA_BILLING_GROUP_ID }}
+  DWDS_DELETE_SAFETY_CHECK: ${{ secrets.DWDS_DELETE_SAFETY_CHECK }}
 
 jobs:
   lint:
     name: 🚨 Lint code
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - name: 👩‍💻 Checkout code
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: 🐍 Setup Python
-        uses: actions/setup-python@v3
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: pip
 
       - name: 📦 Install deps
         run: |
+          python -m pip install --upgrade pip
           pip install black==24.10.0
 
       - name: 🚨 Lint code
@@ -40,55 +41,58 @@ jobs:
 
   unit-test:
     name: ✅ Unit test
-    runs-on: ubuntu-20.04
-    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
-      - name: 👩‍💻 Checkout code
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: 🐍 Setup Python
-        uses: actions/setup-python@v3
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: pip
 
       - name: 📦 Install deps
         run: |
-          pip install --upgrade virtualenv
-          virtualenv -p python3 venv
-          source venv/bin/activate
-          pip install -e .
-          pip install .[dev]
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
 
       - name: ✅ Test
         run: |
-          source venv/bin/activate
-          pytest --show-capture={no,stdout,stderr} ./tests/unit
-
+          pytest --show-capture={no,stdout,stderr} tests/unit
 
   integration-test:
     name: ✅ Integration test
-    runs-on: ubuntu-20.04
-    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    # IMPORTANT: secrets are not exposed to fork PRs
+    if: ${{ github.event.pull_request.head.repo.fork == false }}
+
     steps:
-      - name: 👩‍💻 Checkout code
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: 🐍 Setup Python
-        uses: actions/setup-python@v3
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: pip
+
+      - name: 🐳 Sanity check Docker
+        run: |
+          docker version
+          docker info
 
       - name: 📦 Install deps
         run: |
-          pip install --upgrade virtualenv
-          virtualenv -p python3 venv
-          source venv/bin/activate
-          pip install -e .
-          pip install .[dev]
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
           d3b-clients --help
 
       - name: ✅ Test
+        env:
+          # Ensure these are present in the test process
+          DEWRANGLE_DEV_PAT: ${{ secrets.DEWRANGLE_DEV_PAT }}
+          DEWRANGLE_BASE_URL: ${{ secrets.DEWRANGLE_BASE_URL }}
         run: |
-          source venv/bin/activate
-          pytest --show-capture={no,stdout,stderr} ./tests/integration
-
+          # Optional: hard-fail early with a clear message instead of confusing KeyErrors
+          python -c "import os; assert os.getenv('DEWRANGLE_DEV_PAT') and os.getenv('DEWRANGLE_BASE_URL'), 'Missing Dewrangle secrets'"
+          pytest --show-capture={no,stdout,stderr} tests/integration

.gitignore

@@ -182,7 +182,8 @@ cython_debug/
 *.python-version
 
 data/
-!d3b-api-clients/tests/data
+!tests/data/
+!tests/data/**
 pyvenv.cfg
 
 .DS_Store

d3b_api_client_cli/cli/__init__.py

@@ -4,8 +4,11 @@
 All commands are initialized here
 """
 
-import click
-from d3b_api_client_cli.cli.dewrangle import *
+from d3b_api_client_cli.cli.dewrangle.graphql_commands import *
+from d3b_api_client_cli.cli.dewrangle.ingest_commands import *
+from d3b_api_client_cli.cli.dewrangle.setup_commands import *
+from d3b_api_client_cli.cli.dewrangle.global_id_commands import *
+from d3b_api_client_cli.cli.fhir.commands import *
 from d3b_api_client_cli.cli.postgres import *
 from d3b_api_client_cli.cli.faker import *
 
@@ -27,9 +30,17 @@ def postgres():
 @click.group()
 def dewrangle():
     """
-    Group of lower level CLI commands relating to working directly with the
-    Dewrangle API
+    Group of CLI commands relating to working with the Dewrangle
     """
+    pass
+
+
+@click.group()
+def fhir():
+    """
+    Group of CLI commands relating to FHIR service operations
+    """
+    pass
 
 
 @click.group()
@@ -49,31 +60,37 @@ def main():
 # Postgres API commands
 postgres.add_command(save_file_to_db)
 
-# Dewrangle API commands
+# CRUD Graphql commands
 dewrangle.add_command(upsert_organization)
 dewrangle.add_command(delete_organization)
 dewrangle.add_command(read_organizations)
 dewrangle.add_command(upsert_study)
 dewrangle.add_command(delete_study)
 dewrangle.add_command(read_studies)
-dewrangle.add_command(upsert_credential)
-dewrangle.add_command(delete_credential)
-dewrangle.add_command(read_credentials)
-dewrangle.add_command(upsert_volume)
-dewrangle.add_command(delete_volume)
-dewrangle.add_command(read_volumes)
-dewrangle.add_command(list_and_hash_volume)
-dewrangle.add_command(hash_volume_and_wait)
-dewrangle.add_command(read_job)
-dewrangle.add_command(create_billing_group)
-dewrangle.add_command(delete_billing_group)
-dewrangle.add_command(read_billing_groups)
-dewrangle.add_command(upsert_global_descriptors)
-dewrangle.add_command(download_global_descriptors)
-dewrangle.add_command(upsert_and_download_global_descriptors)
-dewrangle.add_command(upsert_and_download_global_descriptor)
-
-# Add command groups to the root CLI
-main.add_command(dewrangle)
-main.add_command(postgres)
-main.add_command(faker)
+dewrangle.add_command(get_study)
+dewrangle.add_command(read_fhir_ingest_job)
+dewrangle.add_command(read_fhir_servers)
+dewrangle.add_command(upsert_fhir_server)
+dewrangle.add_command(delete_fhir_server)
+
+# Ingest into Dewrangle commands
+dewrangle.add_command(upload_study_file)
+dewrangle.add_command(ingest_study_file)
+dewrangle.add_command(ingest_study_files)
+
+# Setup commands
+dewrangle.add_command(setup_dewrangle_org)
+dewrangle.add_command(setup_dewrangle_study)
+dewrangle.add_command(setup_all_studies)
+
+# Global ID commands
+dewrangle.add_command(upsert_global_ids)
+dewrangle.add_command(download_global_ids)
+
+# FHIR commands
+fhir.add_command(build)
+fhir.add_command(load_fhir)
+fhir.add_command(delete_from_file)
+fhir.add_command(delete_all)
+fhir.add_command(delete_fhir_study)
+fhir.add_command(total_counts)

d3b_api_client_cli/cli/common.py

@@ -0,0 +1,72 @@
+"""
+Common functions for CLI commands. Mostly parameter validators
+"""
+
+from pprint import pformat
+from typing import List
+
+from urllib.parse import urlparse
+import click
+
+from d3b_api_client_cli import utils
+from d3b_api_client_cli.config import (
+    KidsFirstFhirEntity,
+    ETL_STAGES,
+    config,
+)
+
+
+def validate_kids_first_types(types: List[str]):
+    """
+    Validate Kids First Dataservice types
+    """
+    endpoints = config["dataservice"]["endpoints"]
+
+    kids_first_types = set(types)
+    default = set(utils.camel_to_snake(e) for e in endpoints)
+
+    if not kids_first_types <= default:
+        invalid = kids_first_types - default
+        raise click.BadParameter(
+            f"Invalid Kids First Dataservice Type: {pformat(invalid)}."
+            f" Each type must be one of {pformat(default)}"
+        )
+
+
+def validate_url(ctx, param, url: str):
+    """
+    Ensure url is valid
+    """
+    try:
+        result = urlparse(url)
+        if not (result.scheme and result.netloc):
+            raise click.BadParameter(f"{url} is not a valid URL")
+        return url
+    except Exception as exc:
+        raise click.BadParameter(f"{url} is not a valid URL") from exc
+
+
+def validate_kids_first_fhir_types(types: List[str]):
+    """
+    Validate kids_first_fhir_types
+    """
+    kids_first_fhir_types = set(types)
+    default = {v.value for v in KidsFirstFhirEntity}
+
+    if not kids_first_fhir_types <= default:
+        invalid = kids_first_fhir_types - default
+        raise click.BadParameter(
+            f"Invalid Kids First FHIR Type: {pformat(invalid)}."
+            f" Each type must be one of {pformat(default)}"
+        )
+
+
+def validate_stages(stages):
+    """
+    Validate stages CLI option
+    """
+    if not all(s in set(ETL_STAGES) for s in stages):
+        raise click.BadParameter(
+            f"Invalid stages value {stages}. Must one or more chars in:"
+            f" '{ETL_STAGES}'"
+        )

d3b_api_client_cli/cli/dewrangle/__init__.py

@@ -1,13 +0,0 @@
-"""
-Dewrangle CLI commands
-
-Functions for interacting the Dewrangle's GraphQL and REST APIs
-"""
-
-from d3b_api_client_cli.cli.dewrangle.organization_commands import *
-from d3b_api_client_cli.cli.dewrangle.study_commands import *
-from d3b_api_client_cli.cli.dewrangle.credential_commands import *
-from d3b_api_client_cli.cli.dewrangle.volume_commands import *
-from d3b_api_client_cli.cli.dewrangle.job_commands import *
-from d3b_api_client_cli.cli.dewrangle.billing_group_commands import *
-from d3b_api_client_cli.cli.dewrangle.global_id_commands import *