Skip to content

Pull request for all the changes in update_descriptives_files #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
donghwanHwang wants to merge 11 commits into
base: master
Choose a base branch
from
Open

Pull request for all the changes in update_descriptives_files #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
6da2693
testing before changing descriptives table
Aug 22, 2023
420574f
moved first hist_1_amount update into next update where clause
Aug 22, 2023
094d4f4
removed redundancy in a_is_current, a_lapsed_days descriptives flag
Aug 22, 2023
045eff8
Simplifed isnull(a_is_extra,0)=0 check into a_is_extra is null check
Aug 23, 2023
13982eb
Temporarily fixed NULL(Integer -> datetime) conversion error
Aug 24, 2023
74eafa4
Merged git pull
Aug 24, 2023
290a293
removed redundant check in merge_catchup condition
Aug 24, 2023
835e65c
merge_adjacent change (dh1 ntl gift setting while aggregating tmp_mer…
Aug 24, 2023
7ea75b0
Fixed a_is_current and a_lapsed_days first to limit 1 in order to mat…
Aug 25, 2023
52e0347
Commit before rollback
Aug 25, 2023
8750ef9
Updated all the comments
donghwanHwang Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,14 @@ update_descriptives_merge_adjacent "system/query"

delete collection tmp_merge_aggregates;


-- Step One: Locate adjacent segments
-- compare the combined segments average to the
-- longer segment's average.

-- dh1 is the adjacent giving with the same amount
-- dh2 is the regular giving.

update
identity collection a_descriptives_hist dh1,
collection a_descriptives_hist dh2
Expand All @@ -25,11 +30,20 @@ update_descriptives_merge_adjacent "system/query"
:dh2:p_donor_partner_key = :dh1:p_donor_partner_key and
:dh2:a_amount = :dh1:a_amount and
:dh1:a_hist_id < :dh2:a_hist_id and
--adjacent giving more recent than regular
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount)))
;
-- Extra recurssions to ensure a merge does not occur
-- into a history that is also getting merged away

-- If you don't perform these redundant merges, rows with a non-null a_merged_id will be deleted after the query runs.
-- Consider a chain 1->2->3->4->5->6. After one query run, the chain updates to 1->3, 2->4, 3->5, 4->6, 5->6.
-- Merges occur only if a_merged_id of the recipient (dh2) is null. In this example, only 4->6 and 5->6 merge, because 6 has a_merged_id == null.
-- The next step, "Delete Merged," removes all rows with a non-null a_merged_id. So, 1, 2, 3, 4, 5 are deleted.
-- Though 4 and 5 are safely merged into 6, data from 1, 2, and 3 are lost as they're deleted without merging.
-- This issue could have be avoided with two more merges, hence the redundant query.

update
identity collection a_descriptives_hist dh1,
collection a_descriptives_hist dh2
Expand All @@ -56,8 +70,9 @@ update_descriptives_merge_adjacent "system/query"
:dh2:p_donor_partner_key = :dh1:p_donor_partner_key and
:dh1:a_merged_id = :dh2:a_hist_id
;

-- Step Two : Aggregate fields that are going to be merged into the same history


-- Step Two : Aggregate fields that are going to be merged into the same history
insert
collection tmp_merge_aggregates
select
Expand All @@ -72,7 +87,7 @@ update_descriptives_merge_adjacent "system/query"
next_start_dh1 = condition(first(isnull(:dh1:a_next_start, 'null')) = 'null', null, first(:dh1:a_next_start)),
first_dh1_gift = min(:dh1:a_first_gift),
last_dh1_gift = max(:dh1:a_last_gift),
ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_first_gift, 2)),
ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_last_gift, 2)),
merge_next = :dh2:a_merged_id
from
collection a_descriptives_hist dh1,
Expand Down
3 changes: 3 additions & 0 deletions kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ update_descriptives_merge_adjacent_2 "system/query"
-- check how far out you are merging in
(datediff(day, condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh2:a_last_gift, :dh1:a_last_gift), condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) / 30.5 <= (:dh1:a_amount / :dh2:a_amount * 2 + 2)) and
-- don't merge if both are established giving patterns of different amounts with nice intervals
-- checking if at least one of the merging sets is not regular giving
(:dh2:a_amount = :dh1:a_amount or :dh2:a_count <= 5 or :dh1:a_count <= 5 or :dh2:a_act_average_months <= 5 or :dh1:a_act_average_months <= 5 or abs(round(:dh2:a_act_average_interval) - :dh2:a_act_average_interval) >= 0.15 or abs(round(:dh1:a_act_average_interval) - :dh1:a_act_average_interval) >= 0.15)
;
-- Extra recurssions to ensure a merge does not occur
Expand Down Expand Up @@ -114,11 +115,13 @@ update_descriptives_merge_adjacent_2 "system/query"
:dh2:a_total = :dh2:a_total + :t:sum_dh1_a_total,
:dh2:a_prev_end = condition(
:t:prev_end_dh1 is null,
-- t is the first
null,
condition(:t:prev_end_dh1 < :dh2:a_prev_end, :t:prev_end_dh1, :dh2:a_prev_end)
),
:dh2:a_next_start = condition(
:t:next_start_dh1 is null,
-- t is the last
null,
condition(:t:next_start_dh1 > :dh2:a_next_start, :t:next_start_dh1, :dh2:a_next_start)
),
Expand Down
1 change: 1 addition & 0 deletions kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ update_descriptives_merge_adjacent_singles "system/query"
:dh2:p_donor_partner_key = :dh1:p_donor_partner_key and
:dh2:a_amount = :dh1:a_amount and
:dh1:a_hist_id < :dh2:a_hist_id and
--dh1 is more recent than dh2
:dh1:a_count = 1 and
:dh2:a_count = 1 and
datediff(month, :dh2:a_first_gift, :dh1:a_first_gift) < 37
Expand Down
36 changes: 27 additions & 9 deletions kardia-app/modules/rcpt/update_descriptives_merge_approx.qy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,20 @@ update_descriptives_merge_approx "system/query"
:dh:a_is_approximate is null and
(:dh:a_count < 5 or :dh:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated
--(:dh:a_prev_end is not null or :dh:a_count != 1) and -- first gifts are special
(:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and-- don't merge most recent extra gifts, as there isn't enough info about them yet
(:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and --exclude (most recent + extra gift)
-- don't merge most recent extra gifts, as there isn't enough info about them yet
not datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 334 and -- this prevents annual gifts from being merged together
(select
count(1)
from collection a_descriptives_hist dh2
where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and (:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and (not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0))
) >= 1,
2,
where :dh2:a_ledger_number = :dh:a_ledger_number and
:dh2:a_fund = :dh:a_fund and
:dh2:p_donor_partner_key = :dh:p_donor_partner_key and
(:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and --gift history adjacent to each other?
(:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated
(not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0))
) >= 1,
2,--intermediate value to distinguish(different conditions) testing purpose, being able to identify the descriptive
0
)
where
Expand Down Expand Up @@ -64,6 +70,7 @@ update_descriptives_merge_approx "system/query"
-- Step Three: Aggregate Each Segment
insert
collection tmp_aggregated_segments
--merge into dh2 where earliest giving segments and isapproximate
select
ledger = :t1:ledger,
fund = :t1:fund,
Expand All @@ -74,7 +81,7 @@ update_descriptives_merge_approx "system/query"
amount = null,
first = min(:t1:first),
last = max(:t1:last),
ntl = null,
ntl = convert(datetime, null),
count = sum(:t1:count),
total = sum(:t1:total),
prev_end = condition(last(isnull(:t1:prev_end, 'null')) = 'null', null, last(:t1:prev_end)),
Expand All @@ -101,9 +108,13 @@ update_descriptives_merge_approx "system/query"
-- This is so that other descriptives come out cleaner and
-- because a real value for the amount provides better descriptives than
-- an averged value. In case of a tie, this instead uses the avg of the amounts

--select mode or average for amount based on total count
update
collection tmp_aggregated_segments t
set
-- if the mode of giving count is 1 , select average amount.
-- else, select mode.
:t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)),
:t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc)
where
Expand All @@ -119,10 +130,12 @@ update_descriptives_merge_approx "system/query"
:dh2:a_amount = :t:amount,
:dh2:a_first_gift = :t:first,
:dh2:a_last_gift = :t:last,
:dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, null),
:dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, convert(datetime, null)),
:dh2:a_count = :t:count,
:dh2:a_total = :t:total,
:dh2:a_is_approximate = condition(:t:merge_count > 1, 1, 0),
--filtering from isapproximate=2 to find where merge_count bigger than 1
--if merge_count =1 , set it to 0
:dh2:a_prev_end = :t:prev_end,
:dh2:a_next_start = :t:next_start,
:dh2:a_act_average_amount = condition(
Expand All @@ -144,8 +157,7 @@ update_descriptives_merge_approx "system/query"
:t:fund = :dh2:a_fund and
:t:donor = :dh2:p_donor_partner_key and
:t:merged_id = :dh2:a_hist_id
;

;
-- Step Five: Delete Merged
delete
collection a_descriptives_hist dh1
Expand All @@ -154,9 +166,13 @@ update_descriptives_merge_approx "system/query"
(:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and
:dh1:a_is_approximate = 2
;

-- Merge extra gifts if there are only extras and an approximate history
-- Step One : Identify

-- From descriptives_hist dh,
-- set dh1:merged_id into dh2:hist:id where dh1 is extra and dh2 is approximate count >1,
-- only if there isn't anywhere dh is both not extra and is_approximate count =1
update
identity collection a_descriptives_hist dh1,
collection a_descriptives_hist dh2
Expand Down Expand Up @@ -217,6 +233,8 @@ update_descriptives_merge_approx "system/query"
update
collection tmp_merge_aggregates t
set
-- if the mode of giving count is 1 , select average amount.
-- else, select mode.
:t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)),
:t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc)
where
Expand Down
19 changes: 10 additions & 9 deletions kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ update_descriptives_merge_catchup "system/query"

sql = " declare collection tmp_gift_items scope application;

-- Merge in 'catch up' giving. dh1 is the larger catch up amount, and dh2 is
-- the regular giving.

-- Merge in 'catch up' giving.
-- dh1 is the larger catch up amount, and dh2 is the regular giving.
-- Step One: Locate
update
-- identity refers to the collection that is going to be updated
identity collection a_descriptives_hist dh1,
collection a_descriptives_hist dh2
set
Expand All @@ -21,19 +22,19 @@ update_descriptives_merge_catchup "system/query"
:dh2:a_ledger_number = :dh1:a_ledger_number and
:dh2:a_fund = :dh1:a_fund and
:dh2:p_donor_partner_key = :dh1:p_donor_partner_key and
:dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason
:dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason, arbitrary number for analysis
(:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular
round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup
:dh2:a_count > :dh1:a_count and -- regular has more gifts than catchup
:dh2:a_count >= 2 and -- regular has at least 2 gifts
:dh2:a_first_gift < :dh2:a_last_gift and -- regular gifts are not on the same day
round(:dh2:a_act_average_interval, 0) != 0 and -- regular has an established interval (non-zero)
(:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent
(:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent
datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and
-- checks that the average giving stays similar after the merge would occur
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount)))
;
--checks that the average giving stays similar after the merge would occur
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and
(:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount)
;

-- Step Two: Merge
update
Expand Down
Loading