Skip to content

Commit 57709c7

Browse files
krista-skylightKrista ChanKrista Chan
authored
APP-543 Convert QA07 (#3301)
Checklist for adding a library: - [x] Copy this checklist into your (draft) PR description - [x] Check the existing description for the library in `apps/modernization-api/src/main/resources/db/changelog/report/execution/03_ODSE_Data_Report_Library_Init.sql` - [ ] If the report execution service is not yet in use by any STLT (even beta), you should update this file - [ ] Update the description in the library-specific migration - [x] Add a migration to `apps/modernization-api/src/main/resources/db/report/execution/libraries` named `<your library name>.sql` - [ ] Copy another migration and update the variables at the top of the file - [x] Add the migration to `apps/modernization-api/src/main/resources/db/report-execution-changelog.yml`. It should be added to the latest `changeSet` since the last release - this could require a new `changeSet` if there isn't one since last release - [x] Check the migration works by building/running the containerized dev setup - [x] Add a python library file to `apps/report-execution/src/libraries` named in lowercase, but generally following the naming convention of SAS (needs to be human recognizable as the same library) - [x] Follow the pattern established in existing libraries to create your library. The `execute` function is the required method and its signature will (someday) be checked for validity - [ ] Make sure pertinent text embedded in the reports is included in the description of the report - [ ] Note in the doc string any deviation from the SAS - [x] Keep an eye out for things in the sas library that are/could be centralized to pre- or post-processing or a util would be helpful for (e.g. common formatting needs) - [x] Add integration tests in a `apps/report-execution/tests/libraries/<your_library_here>py` file following the conventions established for other libraries - [ ] The `subset_sql` can be assumed to be well tested by the modernization-api, so focus on any logic and additional joins/queries/analysis that is added in the library - [ ] The database used in integration testing is our standard golden db + seed used in other testing - if additional data is needed to test the report, add it to the seeding process - [x] Add e2e test coverage for the report - [ ] If this step is not set up, create a ticket to track this as follow on work needed for this report - [x] Check that the report has parity with the NBS 6 version using (https://cdc-nbs.atlassian.net/wiki/spaces/NE/pages/2372272139/Comparison+Testing+Libraries) - [ ] Keep an eye out for opportunities to: - [ ] Consolidate reports by using parameters (parameter design tbd) - [ ] Improve the report spec or result expectations - [x] Update the [SAS to Python tracking spreadsheet](https://docs.google.com/spreadsheets/d/1xpl-znNQIql_tii39i5WYPIrO6vsLRqLBG1dbtuq8-4/edit?usp=sharing) with information about the newly converted library - [ ] Update/clarify these instructions in the epic based on your experience --------- Co-authored-by: Krista Chan <kristachan@Kristas-MacBook-Pro.local> Co-authored-by: Krista Chan <kristachan@ip-10-0-0-12.us-east-2.compute.internal>
1 parent f7e6caf commit 57709c7

16 files changed

Lines changed: 9604 additions & 9274 deletions

File tree

‎apps/modernization-api/src/main/resources/db/changelog/report-execution-changelog.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,15 @@ databaseChangeLog:
6666
- sqlFile:
6767
path: db/report/execution/libraries/qa_06.sql
6868
splitStatements: false
69+
- sqlFile:
70+
path: db/report/execution/libraries/qa_07_30.sql
71+
splitStatements: false
72+
- sqlFile:
73+
path: db/report/execution/libraries/qa_07_60.sql
74+
splitStatements: false
75+
- sqlFile:
76+
path: db/report/execution/libraries/qa_07_90.sql
77+
splitStatements: false
6978
- sqlFile:
7079
path: db/report/execution/libraries/nbs_sr_18.sql
7180
splitStatements: false

‎apps/modernization-api/src/main/resources/db/report/execution/libraries/qa_07_30.sql‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
-- Migrate the QA06.SAS library to the qa_06 python library
2+
3+
USE [NBS_ODSE]
4+
5+
DECLARE @pyLib VARCHAR(50) = 'qa_07'
6+
DECLARE @sasLib VARCHAR(50) = 'QA07_30.SAS'
7+
DECLARE @desc VARCHAR(300) = 'QA07: Duplicate Cases - 30 Days'
8+
DECLARE @libraryParams VARCHAR(300) = '{"days_value": 30}'
9+
10+
IF EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE UPPER(library_name) = @sasLib)
11+
BEGIN
12+
UPDATE [dbo].[Report_Library]
13+
SET
14+
library_params = @libraryParams,
15+
library_name = @pyLib,
16+
runner = 'python',
17+
desc_txt = @desc,
18+
last_chg_time = CURRENT_TIMESTAMP,
19+
last_chg_user_id = 99999999
20+
WHERE
21+
UPPER(library_name) = @sasLib;
22+
END
23+
ELSE IF NOT EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE library_name = @pyLib AND library_params = @libraryParams)
24+
BEGIN
25+
-- Create a row for this library
26+
INSERT INTO [dbo].[Report_Library] (
27+
library_name,
28+
desc_txt,
29+
runner,
30+
column_select_ind,
31+
is_builtin_ind,
32+
library_params,
33+
add_time,
34+
add_user_id,
35+
last_chg_time,
36+
last_chg_user_id
37+
) VALUES (
38+
@pyLib,
39+
@desc,
40+
'python',
41+
'N',
42+
'Y',
43+
@libraryParams,
44+
CURRENT_TIMESTAMP,
45+
99999999,
46+
CURRENT_TIMESTAMP,
47+
99999999
48+
);
49+
END

‎apps/modernization-api/src/main/resources/db/report/execution/libraries/qa_07_60.sql‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
-- Migrate the QA06.SAS library to the qa_06 python library
2+
3+
USE [NBS_ODSE]
4+
5+
DECLARE @pyLib VARCHAR(50) = 'qa_07'
6+
DECLARE @sasLib VARCHAR(50) = 'QA07_60.SAS'
7+
DECLARE @desc VARCHAR(300) = 'QA07: Duplicate Cases - 60 Days'
8+
DECLARE @libraryParams VARCHAR(300) = '{"days_value": 60}'
9+
10+
IF EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE UPPER(library_name) = @sasLib)
11+
BEGIN
12+
UPDATE [dbo].[Report_Library]
13+
SET
14+
library_params = @libraryParams,
15+
library_name = @pyLib,
16+
runner = 'python',
17+
desc_txt = @desc,
18+
last_chg_time = CURRENT_TIMESTAMP,
19+
last_chg_user_id = 99999999
20+
WHERE
21+
UPPER(library_name) = @sasLib;
22+
END
23+
ELSE IF NOT EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE library_name = @pyLib AND library_params = @libraryParams)
24+
BEGIN
25+
-- Create a row for this library
26+
INSERT INTO [dbo].[Report_Library] (
27+
library_name,
28+
desc_txt,
29+
runner,
30+
column_select_ind,
31+
is_builtin_ind,
32+
library_params,
33+
add_time,
34+
add_user_id,
35+
last_chg_time,
36+
last_chg_user_id
37+
) VALUES (
38+
@pyLib,
39+
@desc,
40+
'python',
41+
'N',
42+
'Y',
43+
@libraryParams,
44+
CURRENT_TIMESTAMP,
45+
99999999,
46+
CURRENT_TIMESTAMP,
47+
99999999
48+
);
49+
END

‎apps/modernization-api/src/main/resources/db/report/execution/libraries/qa_07_90.sql‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
-- Migrate the QA06.SAS library to the qa_06 python library
2+
3+
USE [NBS_ODSE]
4+
5+
DECLARE @pyLib VARCHAR(50) = 'qa_07'
6+
DECLARE @sasLib VARCHAR(50) = 'QA07_90.SAS'
7+
DECLARE @desc VARCHAR(300) = 'QA07: Duplicate Cases - 90 Days'
8+
DECLARE @libraryParams VARCHAR(300) = '{"days_value": 90}'
9+
10+
IF EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE UPPER(library_name) = @sasLib)
11+
BEGIN
12+
UPDATE [dbo].[Report_Library]
13+
SET
14+
library_params = @libraryParams,
15+
library_name = @pyLib,
16+
runner = 'python',
17+
desc_txt = @desc,
18+
last_chg_time = CURRENT_TIMESTAMP,
19+
last_chg_user_id = 99999999
20+
WHERE
21+
UPPER(library_name) = @sasLib;
22+
END
23+
ELSE IF NOT EXISTS (SELECT * FROM [dbo].[Report_Library] WHERE library_name = @pyLib AND library_params = @libraryParams)
24+
BEGIN
25+
-- Create a row for this library
26+
INSERT INTO [dbo].[Report_Library] (
27+
library_name,
28+
desc_txt,
29+
runner,
30+
column_select_ind,
31+
is_builtin_ind,
32+
library_params,
33+
add_time,
34+
add_user_id,
35+
last_chg_time,
36+
last_chg_user_id
37+
) VALUES (
38+
@pyLib,
39+
@desc,
40+
'python',
41+
'N',
42+
'Y',
43+
@libraryParams,
44+
CURRENT_TIMESTAMP,
45+
99999999,
46+
CURRENT_TIMESTAMP,
47+
99999999
48+
);
49+
END

‎apps/report-execution/src/execute_report.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def execute_report(report_spec: models.ReportSpec):
2525
data_source_name=report_spec.data_source_name,
2626
days_value=report_spec.days_value,
2727
column_map=report_spec.column_map,
28+
library_params=report_spec.library_params,
2829
)
2930

3031
check_valid_result(result, report_spec)

‎apps/report-execution/src/libraries/qa_07.py‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
from src.db_transaction import Transaction
2+
from src.models import ReportResult
3+
4+
5+
def execute(
6+
trx: Transaction,
7+
subset_query: str,
8+
data_source_name: str,
9+
library_params: dict,
10+
**kwargs,
11+
):
12+
"""QA07: Duplicate Cases. The Report will consist of 3 reports, identical
13+
except for the number of days used, and will display in the NBS Report Module as:
14+
• QA07 - Duplicate Cases (30 Days)
15+
• QA07 - Duplicate Cases (60 Days)
16+
• QA07 - Duplicate Cases (90 Days)
17+
This report generates a list, by name, of individuals that have possible duplicate
18+
case incidents.
19+
User filtering includes a time period (based on case confirmation date), diagnoses
20+
and range between occurrences. “Cases” only include investigations with Case Status
21+
of Probable or Confirmed.
22+
23+
Conversion notes:
24+
* The original SAS code had the days value hard coded in the library, but we made
25+
it a parameter that can be passed in from the API.
26+
* The output is sorted by PATIENT_NAME, DIAGNOSIS, FL_FUP_EXAM_DT, and
27+
INVESTIGATION_KEY (the last is used only as a tie‑breaker and does not appear
28+
in the final output). SAS does not have a tiebreaker value.
29+
"""
30+
if not isinstance(library_params, dict):
31+
raise ValueError(f"""
32+
library_params must be a dictionary containing 'days_value' \
33+
(e.g., 30, 60, 90), got {library_params}
34+
""")
35+
days = library_params.get('days_value')
36+
if days is None:
37+
raise ValueError(f"""
38+
library_params must contain 'days_value' \
39+
(e.g., 30, 60, 90), got {library_params}
40+
""")
41+
42+
sql = f"""
43+
WITH subset AS (
44+
{subset_query}
45+
),
46+
inv AS (
47+
SELECT
48+
i.INVESTIGATION_KEY,
49+
i.INV_CASE_STATUS,
50+
i.REFERRAL_BASIS,
51+
e.ADD_USER_ID,
52+
up.PROVIDER_QUICK_CODE
53+
FROM RDB.dbo.INVESTIGATION i
54+
INNER JOIN RDB.dbo.EVENT_METRIC e ON i.CASE_UID = e.EVENT_UID
55+
INNER JOIN RDB.dbo.USER_PROFILE up ON e.ADD_USER_ID = up.NEDSS_ENTRY_ID
56+
),
57+
lab_max AS (
58+
SELECT
59+
b.INVESTIGATION_KEY,
60+
MAX(d.SPECIMEN_COLLECTION_DT) AS SPECIMEN_COLLECTION_DT
61+
FROM RDB.dbo.LAB_TEST_RESULT b
62+
INNER JOIN RDB.dbo.LAB_TEST d ON b.LAB_TEST_KEY = d.LAB_TEST_KEY
63+
GROUP BY b.INVESTIGATION_KEY
64+
),
65+
derived AS (
66+
SELECT
67+
s.INVESTIGATION_KEY,
68+
s.PATIENT_NAME,
69+
s.PATIENT_LOCAL_ID,
70+
s.INV_LOCAL_ID,
71+
s.DIAGNOSIS,
72+
s.CONFIRMATION_DT,
73+
s.FL_FUP_EXAM_DT AS ORIG_FL_FUP_EXAM_DT,
74+
inv.REFERRAL_BASIS,
75+
inv.INV_CASE_STATUS,
76+
lab.SPECIMEN_COLLECTION_DT,
77+
mr.DIAGNOSIS_DT,
78+
CAST(SUBSTRING(s.PATIENT_LOCAL_ID, 4, 8) AS BIGINT) - 10000000 AS PATIENTID
79+
FROM subset s
80+
INNER JOIN inv ON s.INVESTIGATION_KEY = inv.INVESTIGATION_KEY
81+
LEFT JOIN lab_max lab ON s.INVESTIGATION_KEY = lab.INVESTIGATION_KEY
82+
LEFT JOIN RDB.dbo.MORBIDITY_REPORT_EVENT mre
83+
ON s.INVESTIGATION_KEY = mre.INVESTIGATION_KEY
84+
LEFT JOIN RDB.dbo.MORBIDITY_REPORT mr ON mre.MORB_RPT_KEY = mr.MORB_RPT_KEY
85+
WHERE s.INV_LOCAL_ID IS NOT NULL
86+
AND s.DIAGNOSIS IS NOT NULL
87+
AND inv.INV_CASE_STATUS IN ('Probable', 'Confirmed')
88+
),
89+
with_fup_dt AS (
90+
SELECT
91+
INVESTIGATION_KEY,
92+
PATIENT_NAME,
93+
PATIENT_LOCAL_ID,
94+
INV_LOCAL_ID,
95+
DIAGNOSIS,
96+
CONFIRMATION_DT,
97+
PATIENTID,
98+
COALESCE(
99+
ORIG_FL_FUP_EXAM_DT,
100+
CASE
101+
WHEN REFERRAL_BASIS = 'T1 - Positive Test'
102+
THEN SPECIMEN_COLLECTION_DT
103+
WHEN REFERRAL_BASIS = 'T2 - Morbidity Report'
104+
THEN DIAGNOSIS_DT
105+
END
106+
) AS FL_FUP_EXAM_DT
107+
FROM derived
108+
WHERE ORIG_FL_FUP_EXAM_DT IS NOT NULL
109+
OR (
110+
REFERRAL_BASIS = 'T1 - Positive Test'
111+
AND SPECIMEN_COLLECTION_DT IS NOT NULL
112+
)
113+
OR (REFERRAL_BASIS = 'T2 - Morbidity Report' AND DIAGNOSIS_DT IS NOT NULL)
114+
),
115+
days_desc AS (
116+
SELECT
117+
*,
118+
LAG(FL_FUP_EXAM_DT) OVER (
119+
PARTITION BY PATIENT_LOCAL_ID, DIAGNOSIS
120+
ORDER BY FL_FUP_EXAM_DT DESC, INVESTIGATION_KEY
121+
) AS lag_desc,
122+
DATEDIFF(day, LAG(FL_FUP_EXAM_DT) OVER (
123+
PARTITION BY PATIENT_LOCAL_ID, DIAGNOSIS
124+
ORDER BY FL_FUP_EXAM_DT DESC, INVESTIGATION_KEY
125+
), FL_FUP_EXAM_DT) AS DAYS_raw
126+
FROM with_fup_dt
127+
),
128+
days_asc AS (
129+
SELECT
130+
*,
131+
LAG(FL_FUP_EXAM_DT) OVER (
132+
PARTITION BY PATIENT_LOCAL_ID, DIAGNOSIS
133+
ORDER BY FL_FUP_EXAM_DT ASC, INVESTIGATION_KEY
134+
) AS lag_asc,
135+
DATEDIFF(day, LAG(FL_FUP_EXAM_DT) OVER (
136+
PARTITION BY PATIENT_LOCAL_ID, DIAGNOSIS
137+
ORDER BY FL_FUP_EXAM_DT ASC, INVESTIGATION_KEY
138+
), FL_FUP_EXAM_DT) AS DAYS1_raw
139+
FROM with_fup_dt
140+
),
141+
merged AS (
142+
SELECT
143+
d.INVESTIGATION_KEY,
144+
d.PATIENT_NAME,
145+
d.PATIENT_LOCAL_ID,
146+
d.INV_LOCAL_ID,
147+
d.DIAGNOSIS,
148+
d.CONFIRMATION_DT,
149+
d.FL_FUP_EXAM_DT,
150+
d.PATIENTID,
151+
ISNULL(d.DAYS_raw, 0) AS DAYS,
152+
ISNULL(a.DAYS1_raw, 0) AS DAYS1
153+
FROM days_desc d
154+
INNER JOIN days_asc a ON d.INVESTIGATION_KEY = a.INVESTIGATION_KEY
155+
),
156+
filtered AS (
157+
SELECT *,
158+
CASE
159+
WHEN DAYS >= -{days} AND DAYS1 <= {days}
160+
THEN 1 ELSE 0
161+
END AS meets_condition
162+
FROM merged
163+
),
164+
counts AS (
165+
SELECT PATIENT_LOCAL_ID, DIAGNOSIS, SUM(meets_condition) AS cnt
166+
FROM filtered
167+
GROUP BY PATIENT_LOCAL_ID, DIAGNOSIS
168+
)
169+
SELECT
170+
f.PATIENT_NAME,
171+
f.PATIENT_LOCAL_ID,
172+
f.INV_LOCAL_ID,
173+
f.DIAGNOSIS,
174+
CONVERT(varchar, f.CONFIRMATION_DT, 101) AS CONFIRMATION_DT,
175+
CONVERT(varchar, f.FL_FUP_EXAM_DT, 101) AS FL_FUP_EXAM_DT,
176+
f.PATIENTID,
177+
f.DAYS,
178+
f.DAYS1,
179+
c.cnt AS COUNT
180+
FROM filtered f
181+
INNER JOIN counts c
182+
ON f.PATIENT_LOCAL_ID = c.PATIENT_LOCAL_ID
183+
AND f.DIAGNOSIS = c.DIAGNOSIS
184+
WHERE f.meets_condition = 1 AND c.cnt > 1
185+
ORDER BY f.PATIENT_NAME, f.DIAGNOSIS, f.FL_FUP_EXAM_DT, f.INVESTIGATION_KEY
186+
"""
187+
content = trx.query(sql)
188+
return ReportResult(content_type='table', content=content)

0 commit comments

Comments
 (0)