Skip to content

feat: Retain per-outlier detail from the DHIS2 API response #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
KatherineWyers wants to merge 1 commit into
base: main
Choose a base branch
from
+134 −11
Open

feat: Retain per-outlier detail from the DHIS2 API response #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions app/analyzers/outlier_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
'destination_data_element': params['destination_data_element'],
'destination_dataset': params.get('destination_dataset'),
'lower_bound': params.get('lower_bound', 0)
# set 'replacement_method': True to call return_raw from _process_outlier_results()
}
#Optionally add date offsets if provided
if 'start_date_offset' in params:
Expand All @@ -50,28 +51,29 @@
]

results_nested = await asyncio.gather(*tasks, return_exceptions=True)
results = []
data_values = []
errors = []

for i, result in enumerate(results_nested):
for i, data_value in enumerate(results_nested['data_values']):
ou = ous[i]
if isinstance(result, Exception):
msg = f"Outlier detection failed for OU '{ou}': {str(result)}"
if isinstance(data_value, Exception):
msg = f"Outlier detection failed for OU '{ou}': {str(data_value)}"
logging.error(msg)
errors.append(msg)
elif isinstance(result, list):
results.extend(result)
elif isinstance(data_value, list):
data_values.extend(data_value)
else:
msg = f"Unexpected result type for OU '{ou}': {type(result)}"
msg = f"Unexpected result type for OU '{ou}': {type(data_value)}"
logging.warning(msg)
errors.append(msg)
return {
'dataValues': results,
'dataValues': data_values,
'rawOutliers': results_nested['raw_outliers'],
'errors': errors
}

except Exception as e:
logging.error(f"Error running outlier stage '{stage['name']}': {e}")

Check failure on line 76 in app/analyzers/outlier_analyzer.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use "logging.exception()" instead. 


See more on https://sonarcloud.io/project/issues?id=dhis2_tool-validation-monitor&issues=AZ5qYvU5K6TeCn1vHM2S&open=AZ5qYvU5K6TeCn1vHM2S&pullRequest=14
return []

async def _run_outlier_dataset_stage_async(self, session, params, semaphore):
Expand Down Expand Up @@ -104,12 +106,12 @@
outlier_json = await response.json()

return self._process_outlier_results(outlier_json, params['destination_data_element'],
params['lower_bound'], params.get('destination_dataset'))
params['lower_bound'], params.get('destination_dataset') , params.get('replacement_method', False))

except Exception as e:
return e

def _process_outlier_results(self, results, destination_data_element, lower_bound, destination_dataset=None):
def _process_outlier_results(self, results, destination_data_element, lower_bound, destination_dataset=None, return_raw: bool = False):
outliers_by_ou_and_period = {}

for outlier in results.get('outlierValues', []):
Expand All @@ -131,4 +133,10 @@
for dv in data_values:
dv['_dataset'] = destination_dataset

return data_values
raw_outliers = [outlier for outlier in results.get('outlierValues', []) if
float(outlier['value']) > lower_bound] if return_raw else []

return {
'data_values': data_values,
'raw_outliers': raw_outliers
}
115 changes: 115 additions & 0 deletions tests/test_outlier_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import unittest
from app.analyzers.outlier_analyzer import OutlierAnalyzer
import logging

class TestOutlierAnalyzer(unittest.TestCase):

OUTLIER_JSON = {"metadata": {"algorithm": "Z_SCORE", "threshold": 3.0, "orderBy": "MEAN_ABS_DEV", "maxResults": 500,
"count": 114}, "outlierValues": [
{"de": "ldGXl6SEdqf", "deName": "Weight for age between middle and lower line (yellow)", "pe": "202007",
"ou": "cgqkFdShPzg", "ouName": "Loreto Clinic", "coc": "psbwp3CQEhs", "cocName": "Fixed, >1y",
"aoc": "HllvX50cXC0", "value": 1634.0, "mean": 328.22222222222223, "stdDev": 422.73992384192496,
"absDev": 1305.7777777777778, "zScore": 3.0888442376359206, "lowerBound": 9.9975493035527,
"upperBound": 1596.441993747997, "aocName": "default", "followup": False},
{"de": "NLnXLV5YpZF", "deName": "Weight for age on or above middle line (green)", "pe": "202007",
"ou": "cgqkFdShPzg", "ouName": "Loreto Clinic", "coc": "psbwp3CQEhs", "cocName": "Fixed, >1y",
"aoc": "HllvX50cXC0", "value": -1582.0, "mean": 519.6386554621848, "stdDev": 350.6662262083907,
"absDev": 1062.361344537815, "zScore": 3.0295513657665016, "lowerBound": -6.3600231629872,
"upperBound": 1571.637334087357, "aocName": "default", "followup": False},
{"de": "pikOziyCXbM", "deName": "OPV1 doses given", "pe": "202011", "ou": "tSBcgrTDdB8",
"ouName": "Paramedical CHC", "coc": "Prlt0C1RF0s", "cocName": "Fixed, <1y", "aoc": "HllvX50cXC0",
"value": -808.0, "mean": 113.37984496124031, "stdDev": 220.18865039850954, "absDev": 694.6201550387597,
"zScore": 3.1546592150939565, "lowerBound": 80.1861062342883, "upperBound": 773.945796156769,
"aocName": "default", "followup": False},
{"de": "I78gJm4KBo7", "deName": "Penta2 doses given", "pe": "202010", "ou": "mzsOsz0NwNY",
"ouName": "New Police Barracks CHC", "coc": "Prlt0C1RF0s", "cocName": "Fixed, <1y", "aoc": "HllvX50cXC0",
"value": 751.0, "mean": 73.68217054263566, "stdDev": 194.02057688748857, "absDev": 677.3178294573644,
"zScore": 3.490958744288948, "lowerBound": 508.37956011983005, "upperBound": 655.7439012051013,
"aocName": "default", "followup": False},
{"de": "tU7GixyHhsv", "deName": "Vitamin A given to < 5y", "pe": "202010", "ou": "cgqkFdShPzg",
"ouName": "Loreto Clinic", "coc": "Prlt0C1RF0s", "cocName": "Fixed, <1y", "aoc": "HllvX50cXC0",
"value": 830.0, "mean": 197.12605042016807, "stdDev": 204.18382534240962, "absDev": 632.8739495798319,
"zScore": 3.099530281198929, "lowerBound": 15.4254256070608, "upperBound": 809.677526447397,
"aocName": "default", "followup": False},
]}

OUTLIER_JSON_FILTERED_VALUE_0 = {
"metadata": {"algorithm": "Z_SCORE", "threshold": 3.0, "orderBy": "MEAN_ABS_DEV", "maxResults": 500,
"count": 114}, "outlierValues": [
{"de": "ldGXl6SEdqf", "deName": "Weight for age between middle and lower line (yellow)", "pe": "202007",
"ou": "cgqkFdShPzg", "ouName": "Loreto Clinic", "coc": "psbwp3CQEhs", "cocName": "Fixed, >1y",
"aoc": "HllvX50cXC0", "value": 1634.0, "mean": 328.22222222222223, "stdDev": 422.73992384192496,
"absDev": 1305.7777777777778, "zScore": 3.0888442376359206, "lowerBound": 9.9975493035527,
"upperBound": 1596.441993747997, "aocName": "default", "followup": False},
{"de": "I78gJm4KBo7", "deName": "Penta2 doses given", "pe": "202010", "ou": "mzsOsz0NwNY",
"ouName": "New Police Barracks CHC", "coc": "Prlt0C1RF0s", "cocName": "Fixed, <1y",
"aoc": "HllvX50cXC0",
"value": 751.0, "mean": 73.68217054263566, "stdDev": 194.02057688748857, "absDev": 677.3178294573644,
"zScore": 3.490958744288948, "lowerBound": 508.37956011983005, "upperBound": 655.7439012051013,
"aocName": "default", "followup": False},
{"de": "tU7GixyHhsv", "deName": "Vitamin A given to < 5y", "pe": "202010", "ou": "cgqkFdShPzg",
"ouName": "Loreto Clinic", "coc": "Prlt0C1RF0s", "cocName": "Fixed, <1y", "aoc": "HllvX50cXC0",
"value": 830.0, "mean": 197.12605042016807, "stdDev": 204.18382534240962, "absDev": 632.8739495798319,
"zScore": 3.099530281198929, "lowerBound": 15.4254256070608, "upperBound": 809.677526447397,
"aocName": "default", "followup": False},
]}

OUTLIER_JSON_FILTERED_VALUE_1000 = {
"metadata": {"algorithm": "Z_SCORE", "threshold": 3.0, "orderBy": "MEAN_ABS_DEV", "maxResults": 500,
"count": 114}, "outlierValues": [
{"de": "ldGXl6SEdqf", "deName": "Weight for age between middle and lower line (yellow)", "pe": "202007",
"ou": "cgqkFdShPzg", "ouName": "Loreto Clinic", "coc": "psbwp3CQEhs", "cocName": "Fixed, >1y",
"aoc": "HllvX50cXC0", "value": 1634.0, "mean": 328.22222222222223, "stdDev": 422.73992384192496,
"absDev": 1305.7777777777778, "zScore": 3.0888442376359206, "lowerBound": 9.9975493035527,
"upperBound": 1596.441993747997, "aocName": "default", "followup": False},
]}

def setUp(self):
self.analyzer = OutlierAnalyzer(
config={
'server': {
'base_url': 'http://localhost',
'd2_token': 'fake-token',
'default_coc': 'HllvX50cXC0'
}
},
base_url='http://localhost',
headers={}
)

def _process(self, outlier_json, lower_bound, return_raw=None):
kwargs = {} if return_raw is None else {'return_raw': return_raw}

return self.analyzer._process_outlier_results(
outlier_json, "vaYRah9aFHM", lower_bound, "PKEP8aBjv8Q", **kwargs
)

def test_false_and_blank_are_equivalent(self):
for lower_bound in [0, 500, 2000]:
with self.subTest(lower_bound=lower_bound):
self.assertEqual(
self._process(self.OUTLIER_JSON, lower_bound),
self._process(self.OUTLIER_JSON, lower_bound, return_raw=False)
)

def test_data_values_unchanged_by_return_raw(self):
for lower_bound in [0, 1000]:
with self.subTest(lower_bound=lower_bound):
self.assertEqual(
self._process(self.OUTLIER_JSON, lower_bound)['data_values'],
self._process(self.OUTLIER_JSON, lower_bound, return_raw=True)['data_values']
)

def test_raw_outliers_empty_when_not_returning_raw(self):
for lower_bound in [0, 1000]:
with self.subTest(lower_bound=lower_bound):
self.assertEqual(self._process(self.OUTLIER_JSON, lower_bound)['raw_outliers'], [])

def test_raw_outliers_filtered_by_lower_bound_0(self):
self.assertEqual(self._process(self.OUTLIER_JSON, 0, return_raw=True)['raw_outliers'], self.OUTLIER_JSON_FILTERED_VALUE_0['outlierValues'])

def test_raw_outliers_filtered_by_lower_bound_1000(self):
self.assertEqual(self._process(self.OUTLIER_JSON, 1000, return_raw=True)['raw_outliers'], self.OUTLIER_JSON_FILTERED_VALUE_1000['outlierValues'])

if __name__ == "__main__":
unittest.main()