diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy index 91fb5def..6e07fdbe 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent.qy @@ -9,9 +9,14 @@ update_descriptives_merge_adjacent "system/query" delete collection tmp_merge_aggregates; + -- Step One: Locate adjacent segments -- compare the combined segments average to the -- longer segment's average. + + -- dh1 is the adjacent giving with the same amount + -- dh2 is the regular giving. + update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -25,11 +30,20 @@ update_descriptives_merge_adjacent "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_amount = :dh1:a_amount and :dh1:a_hist_id < :dh2:a_hist_id and + --adjacent giving more recent than regular (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) ; -- Extra recurssions to ensure a merge does not occur -- into a history that is also getting merged away + + -- If you don't perform these redundant merges, rows with a non-null a_merged_id will be deleted after the query runs. + -- Consider a chain 1->2->3->4->5->6. After one query run, the chain updates to 1->3, 2->4, 3->5, 4->6, 5->6. + -- Merges occur only if a_merged_id of the recipient (dh2) is null. In this example, only 4->6 and 5->6 merge, because 6 has a_merged_id == null. + -- The next step, "Delete Merged," removes all rows with a non-null a_merged_id. So, 1, 2, 3, 4, 5 are deleted. + -- Though 4 and 5 are safely merged into 6, data from 1, 2, and 3 are lost as they're deleted without merging. + -- This issue could have be avoided with two more merges, hence the redundant query. + update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -56,8 +70,9 @@ update_descriptives_merge_adjacent "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_merged_id = :dh2:a_hist_id ; - - -- Step Two : Aggregate fields that are going to be merged into the same history + + + -- Step Two : Aggregate fields that are going to be merged into the same history insert collection tmp_merge_aggregates select @@ -72,7 +87,7 @@ update_descriptives_merge_adjacent "system/query" next_start_dh1 = condition(first(isnull(:dh1:a_next_start, 'null')) = 'null', null, first(:dh1:a_next_start)), first_dh1_gift = min(:dh1:a_first_gift), last_dh1_gift = max(:dh1:a_last_gift), - ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_first_gift, 2)), + ntl_dh1_gift = condition(first(:dh1:a_count) >=2, first(:dh1:a_ntl_gift), nth(:dh1:a_last_gift, 2)), merge_next = :dh2:a_merged_id from collection a_descriptives_hist dh1, diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy index 0f6d2313..a29b099f 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_2.qy @@ -33,6 +33,7 @@ update_descriptives_merge_adjacent_2 "system/query" -- check how far out you are merging in (datediff(day, condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh2:a_last_gift, :dh1:a_last_gift), condition(:dh2:a_last_gift < :dh1:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) / 30.5 <= (:dh1:a_amount / :dh2:a_amount * 2 + 2)) and -- don't merge if both are established giving patterns of different amounts with nice intervals + -- checking if at least one of the merging sets is not regular giving (:dh2:a_amount = :dh1:a_amount or :dh2:a_count <= 5 or :dh1:a_count <= 5 or :dh2:a_act_average_months <= 5 or :dh1:a_act_average_months <= 5 or abs(round(:dh2:a_act_average_interval) - :dh2:a_act_average_interval) >= 0.15 or abs(round(:dh1:a_act_average_interval) - :dh1:a_act_average_interval) >= 0.15) ; -- Extra recurssions to ensure a merge does not occur @@ -114,11 +115,13 @@ update_descriptives_merge_adjacent_2 "system/query" :dh2:a_total = :dh2:a_total + :t:sum_dh1_a_total, :dh2:a_prev_end = condition( :t:prev_end_dh1 is null, + -- t is the first null, condition(:t:prev_end_dh1 < :dh2:a_prev_end, :t:prev_end_dh1, :dh2:a_prev_end) ), :dh2:a_next_start = condition( :t:next_start_dh1 is null, + -- t is the last null, condition(:t:next_start_dh1 > :dh2:a_next_start, :t:next_start_dh1, :dh2:a_next_start) ), diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy index b008c17e..df29a205 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_adjacent_singles.qy @@ -24,6 +24,7 @@ update_descriptives_merge_adjacent_singles "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_amount = :dh1:a_amount and :dh1:a_hist_id < :dh2:a_hist_id and + --dh1 is more recent than dh2 :dh1:a_count = 1 and :dh2:a_count = 1 and datediff(month, :dh2:a_first_gift, :dh1:a_first_gift) < 37 diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy index 9cb851a7..b1222503 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_approx.qy @@ -21,14 +21,20 @@ update_descriptives_merge_approx "system/query" :dh:a_is_approximate is null and (:dh:a_count < 5 or :dh:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated --(:dh:a_prev_end is not null or :dh:a_count != 1) and -- first gifts are special - (:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and-- don't merge most recent extra gifts, as there isn't enough info about them yet + (:dh:a_hist_id != 1 or :dh:a_is_extra != 1) and --exclude (most recent + extra gift) + -- don't merge most recent extra gifts, as there isn't enough info about them yet not datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 334 and -- this prevents annual gifts from being merged together (select count(1) from collection a_descriptives_hist dh2 - where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and (:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and (:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and (not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0)) - ) >= 1, - 2, + where :dh2:a_ledger_number = :dh:a_ledger_number and + :dh2:a_fund = :dh:a_fund and + :dh2:p_donor_partner_key = :dh:p_donor_partner_key and + (:dh2:a_hist_id = :dh:a_hist_id + 1 or :dh2:a_hist_id = :dh:a_hist_id - 1) and --gift history adjacent to each other? + (:dh2:a_count < 5 or :dh2:a_act_average_months < 5) and -- these try to discount regular giving patterns from being approximated + (not isnull(:dh:a_is_extra, 0) or not isnull(:dh2:a_is_extra, 0)) + ) >= 1, + 2,--intermediate value to distinguish(different conditions) testing purpose, being able to identify the descriptive 0 ) where @@ -64,6 +70,7 @@ update_descriptives_merge_approx "system/query" -- Step Three: Aggregate Each Segment insert collection tmp_aggregated_segments + --merge into dh2 where earliest giving segments and isapproximate select ledger = :t1:ledger, fund = :t1:fund, @@ -74,7 +81,7 @@ update_descriptives_merge_approx "system/query" amount = null, first = min(:t1:first), last = max(:t1:last), - ntl = null, + ntl = convert(datetime, null), count = sum(:t1:count), total = sum(:t1:total), prev_end = condition(last(isnull(:t1:prev_end, 'null')) = 'null', null, last(:t1:prev_end)), @@ -101,9 +108,13 @@ update_descriptives_merge_approx "system/query" -- This is so that other descriptives come out cleaner and -- because a real value for the amount provides better descriptives than -- an averged value. In case of a tie, this instead uses the avg of the amounts + + --select mode or average for amount based on total count update collection tmp_aggregated_segments t set + -- if the mode of giving count is 1 , select average amount. + -- else, select mode. :t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)), :t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc) where @@ -119,10 +130,12 @@ update_descriptives_merge_approx "system/query" :dh2:a_amount = :t:amount, :dh2:a_first_gift = :t:first, :dh2:a_last_gift = :t:last, - :dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, null), + :dh2:a_ntl_gift = condition(:t:count > 1, :t:ntl, convert(datetime, null)), :dh2:a_count = :t:count, :dh2:a_total = :t:total, :dh2:a_is_approximate = condition(:t:merge_count > 1, 1, 0), + --filtering from isapproximate=2 to find where merge_count bigger than 1 + --if merge_count =1 , set it to 0 :dh2:a_prev_end = :t:prev_end, :dh2:a_next_start = :t:next_start, :dh2:a_act_average_amount = condition( @@ -144,8 +157,7 @@ update_descriptives_merge_approx "system/query" :t:fund = :dh2:a_fund and :t:donor = :dh2:p_donor_partner_key and :t:merged_id = :dh2:a_hist_id - ; - + ; -- Step Five: Delete Merged delete collection a_descriptives_hist dh1 @@ -154,9 +166,13 @@ update_descriptives_merge_approx "system/query" (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and :dh1:a_is_approximate = 2 ; - + -- Merge extra gifts if there are only extras and an approximate history -- Step One : Identify + + -- From descriptives_hist dh, + -- set dh1:merged_id into dh2:hist:id where dh1 is extra and dh2 is approximate count >1, + -- only if there isn't anywhere dh is both not extra and is_approximate count =1 update identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 @@ -217,6 +233,8 @@ update_descriptives_merge_approx "system/query" update collection tmp_merge_aggregates t set + -- if the mode of giving count is 1 , select average amount. + -- else, select mode. :t:amount = condition((select sum(:count) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1) = 1, (select avg(:amount) from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg), (select :amount from collection tmp_segments where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :seg = :t:seg group by :amount order by sum(:count) desc, :amount desc limit 1)), :t:ntl = (select nth(:giftdate, 2) from collection tmp_gift_items where :ledger = :t:ledger and :fund = :t:fund and :donor = :t:donor and :giftdate <= :t:last order by :giftdate desc) where diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy index 0b4ee853..aa8953be 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup.qy @@ -6,11 +6,12 @@ update_descriptives_merge_catchup "system/query" sql = " declare collection tmp_gift_items scope application; - -- Merge in 'catch up' giving. dh1 is the larger catch up amount, and dh2 is - -- the regular giving. - + -- Merge in 'catch up' giving. + -- dh1 is the larger catch up amount, and dh2 is the regular giving. + -- Step One: Locate update + -- identity refers to the collection that is going to be updated identity collection a_descriptives_hist dh1, collection a_descriptives_hist dh2 set @@ -21,19 +22,19 @@ update_descriptives_merge_catchup "system/query" :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and - :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason + :dh1:a_amount <= 8 * :dh2:a_amount and -- catchup more than regular within reason, arbitrary number for analysis (:dh1:a_amount / :dh2:a_amount) = round(:dh1:a_amount / :dh2:a_amount, 0) and -- even multiple of regular round(:dh2:a_act_average_interval) <= round(isnull(:dh1:a_act_average_interval, 100)) and -- regular is more often than catchup :dh2:a_count > :dh1:a_count and -- regular has more gifts than catchup :dh2:a_count >= 2 and -- regular has at least 2 gifts :dh2:a_first_gift < :dh2:a_last_gift and -- regular gifts are not on the same day round(:dh2:a_act_average_interval, 0) != 0 and -- regular has an established interval (non-zero) - (:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent + (:dh1:a_last_gift = :dh2:a_prev_end or :dh2:a_last_gift = :dh1:a_prev_end) and -- gifts are adjacent datediff(month, condition(:dh1:a_last_gift < :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift), condition(:dh1:a_first_gift > :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift)) <= 14 and - -- checks that the average giving stays similar after the merge would occur - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) - ; + --checks that the average giving stays similar after the merge would occur + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) and + (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 1.15 * isnull(:dh2:a_act_average_amount, :dh2:a_amount) + ; -- Step Two: Merge update diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy index 81f23880..8c3e4857 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy @@ -40,7 +40,8 @@ update_descriptives_merge_catchup_with_extra "system/query" -- how much is a part of regular giving (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) > 0.85 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and (:dh1:a_total + :dh2:a_total - :dh2:a_amount) / round(datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh2:a_first_gift), condition(:dh1:a_last_gift > :dh2:a_last_gift, :dh1:a_last_gift, :dh2:a_last_gift)) / 30.5) < 3.00 * (condition(:dh1:a_count > :dh2:a_count, isnull(:dh1:a_act_average_amount, :dh1:a_amount), isnull(:dh2:a_act_average_amount, :dh2:a_amount))) and - (select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift)) = 1 --and -- only 1 gift within +/- half an interval + (select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift)) = 1 + -- check if only 1 gift within +/- half an interval of extra giving --((:dh1:a_amount / :dh2:a_amount) != round(:dh1:a_amount / :dh2:a_amount, 0) or :dh1:a_hist_id != 1) -- first history value could be a pay-in-advance gift ; @@ -53,12 +54,17 @@ update_descriptives_merge_catchup_with_extra "system/query" donor = :dh1:p_donor_partner_key, merged_id = :dh1:a_hist_id, regular_id = :dh2:a_hist_id, + --this catchup_amount logic is not fully functional. + --It either needs improvement on the logic or needs different method to check for catchup_amount inside extra giving. + --changing "<=" in front of "truncate(:dh1:a_amount / :dh2:a_amount))" to ">=" in order to improve the logic, merges few more rows. + --However, some interval values shifts greatly after changing the condition + -- needs more analysis on the datasets to improve the logic - -- The interval shift is to compensate for merging into a giving pattern - -- that is monthly, but due to a short timespan, the giving interval deviates - -- greatly from the more appropriate value (See sample donor 100400) - -- This also ensures that the catchup_amount won't cause a segment to go negative + --if the extra gift contain bigger or equal to gift amount expected in the time interval, + --it is considered catch_up gift and truncated as an catchup gift. + --Else, if it is false, it is given a value of 0. catchup_amount = condition(round((datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh1:a_prev_end), condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_next_start, :dh1:a_first_gift)) / 30.5) / ((round(:dh2:a_act_average_interval) + :dh2:a_act_average_interval) / 2) <= truncate(:dh1:a_amount / :dh2:a_amount)), round((datediff(day, condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_first_gift, :dh1:a_prev_end), condition(:dh1:a_first_gift < :dh2:a_first_gift, :dh1:a_next_start, :dh1:a_first_gift)) / 30.5) / ((round(:dh2:a_act_average_interval) + :dh2:a_act_average_interval) / 2) <= truncate(:dh1:a_amount / :dh2:a_amount)), truncate(:dh1:a_amount / :dh2:a_amount)) * :dh2:a_amount, + --use this condition to filter catchup gift from extra giving to update the count for regular giving count = condition( catchup_amount = :dh1:a_amount, :dh1:a_count, @@ -135,6 +141,7 @@ update_descriptives_merge_catchup_with_extra "system/query" :dh1:a_amount = :dh1:a_amount - :t:catchup_amount, :dh1:a_merged_id = null, :dh1:a_is_extra = 1 + --extra giving that has been merged already where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and @@ -152,6 +159,7 @@ update_descriptives_merge_catchup_with_extra "system/query" (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and :dh1:a_is_extra and + --same as a_is_extra =2 :dh1:a_amount = 0 "; } diff --git a/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy b/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy index 39fdfe9d..fb9ec8f2 100644 --- a/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy +++ b/kardia-app/modules/rcpt/update_descriptives_merge_extra.qy @@ -12,6 +12,8 @@ update_descriptives_merge_extra "system/query" -- Step One: Identify -- Identify one-time-extra gifts. We consider a gift as one-time-extra if it is a single -- larger gift occurring within a regular giving pattern. + + -- dh is extra and dh2 is regular giving update identity collection a_descriptives_hist dh, collection a_descriptives d @@ -53,6 +55,7 @@ update_descriptives_merge_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_is_extra = 2 and :dh1:a_hist_id != 1 and + --not most recent :dh2:a_count > 1 and :dh2:a_first_gift <= :dh1:a_first_gift and :dh2:a_last_gift >= :dh1:a_last_gift @@ -72,6 +75,7 @@ update_descriptives_merge_extra "system/query" :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh1:a_is_extra = 2 and :dh1:a_hist_id = 1 and + --it is the most recent gift :dh2:a_count > 1 and :dh2:a_last_gift = :dh1:a_prev_end ; @@ -89,6 +93,8 @@ update_descriptives_merge_extra "system/query" -- The fuzziness here can be tweaked. If there are no other gifts -- within +/- half an interval of the extra gift, the gift is assumed -- to contain the regular giving amount lumped with the gift amount. + + --detecting if the extra gift is all on its own includes_regular_giving = ((select count(1) from collection tmp_gift_items where :donor = :dh1:p_donor_partner_key and :ledger = :dh1:a_ledger_number and :fund = :dh1:a_fund and :giftdate > dateadd(day, convert(integer, round(-0.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :giftdate < dateadd(day, convert(integer, round(.5 * 30.5 * :dh2:a_act_average_interval)), :dh1:a_first_gift) and :dh1:a_amount > :dh2:a_amount) = 1) from collection a_descriptives_hist dh1, @@ -108,6 +114,8 @@ update_descriptives_merge_extra "system/query" collection a_descriptives_hist dh1, collection tmp_intermediate_vals t set + --if we subtract it from some place we have to add it into other place + --moving money from step 3 to step 4 :dh2:a_total = condition( :t:includes_regular_giving, :dh2:a_total + :dh2:a_amount, @@ -193,6 +201,7 @@ update_descriptives_merge_extra "system/query" ), :dh1:a_merged_id = null, :dh1:a_is_extra = 1 + --succeeded in merging where (:parameters:fund is null or :parameters:fund = :dh2:a_fund) and (:parameters:donor is null or :parameters:donor = :dh2:p_donor_partner_key) and @@ -216,5 +225,6 @@ update_descriptives_merge_extra "system/query" (:parameters:fund is null or :parameters:fund = :a_fund) and (:parameters:donor is null or :parameters:donor = :p_donor_partner_key) and :a_is_extra = 2 + --failed in merging "; } diff --git a/kardia-app/modules/rcpt/update_descriptives_new.qy b/kardia-app/modules/rcpt/update_descriptives_new.qy index 7ba384c0..082721c9 100644 --- a/kardia-app/modules/rcpt/update_descriptives_new.qy +++ b/kardia-app/modules/rcpt/update_descriptives_new.qy @@ -37,7 +37,12 @@ update_descriptives_new "system/query" amount = :i:a_amount, fund = :i:a_fund, ledger = :i:a_ledger_number, + --row_number() is an advanced sql function in order to provide structure for rows within a table + --row_number() assigns row_number incrementally throughout each rows id = row_number(), + --row_number() with parameters assigns row_number incrementally only if all the parameters are equal + --Else, it resets to zero and starts the process again + --Therefore, Based on the calculation, seg column outputs same segmentation value for parameter columns with same values seg = row_number() - row_number(:i:a_ledger_number, :i:a_fund, :i:p_dn_donor_partner_id, :i:a_amount) from /apps/kardia/data/Kardia_DB/a_subtrx_gift_item/rows i @@ -54,9 +59,11 @@ update_descriptives_new "system/query" :i:p_dn_donor_partner_id, isnull(:i:a_dn_gift_postmark_date, :i:a_dn_gift_received_date) desc ; - + -- Handling gifts that have been reimbursed by Lightsys -- Delete negative giving values and cancel out a -- corresponding positive gift + -- EX) negative gift : scholarship to donor , cancelled gift: donation gift + -- t1 is the negative gift, t2 is the gift to be -- cancelled out, and t3 is the intermediate value print 'Correcting for gift returns at ' + dateformat(getdate(), 'hh:mm:ss'); @@ -66,6 +73,7 @@ update_descriptives_new "system/query" ledger = :t1:ledger, fund = :t1:fund, donor = :t1:donor, + -- find the minimum date difference among gifts that are zero-sum to each other min_diff = (select min(abs(datediff(day, :t2:giftdate, :t1:giftdate))) from collection tmp_gift_items t2 where :t1:ledger = :t2:ledger and :t1:donor = :t2:donor and :t1:fund = :t2:fund and :t2:amount = -1 * :t1:amount group by :t2:ledger, :t2:donor, :t2:fund), id = null from @@ -74,12 +82,16 @@ update_descriptives_new "system/query" (:parameters:fund is null or :parameters:fund = :t1:fund) and (:parameters:donor is null or :parameters:donor = :t1:donor) and :t1:amount < 0 + -- t1:amount < 0 means gift given back to donor ; + --setting id if date difference among gifts are minimum and zero-sum to each other update identity collection tmp_intermediate_vals t3, collection tmp_gift_items t1 set :t3:id = (select :t2:id from collection tmp_gift_items t2 where :t1:ledger = :t2:ledger and :t1:donor = :t2:donor and :t1:fund = :t2:fund and :t2:amount = -1 * :t1:amount and :t3:min_diff = datediff(day, :t2:giftdate, :t1:giftdate) limit 1) + --checking if minimum date difference among gifts are zero-sum to each other + --this condition used since gifts and reimbursements are paired to each other consequently. where (:parameters:fund is null or :parameters:fund = :t1:fund) and (:parameters:donor is null or :parameters:donor = :t1:donor) and @@ -124,7 +136,7 @@ update_descriptives_new "system/query" a_first_gift_amount = last(:amount), a_last_gift = first(:giftdate), a_last_gift_amount = first(:amount), - a_ntl_gift = nth(:giftdate, 2), + a_ntl_gift = (nth(:giftdate, 2)), a_ntl_gift_amount = nth(:amount, 2), -- Bootstrap the analysis process using the data we have. @@ -167,30 +179,39 @@ update_descriptives_new "system/query" select a_ledger_number = :ledger, p_donor_partner_key = :donor, + --needed to manually generate a_hist_id column since we are using temporally collection + a_hist_id = row_number(), a_fund = :fund, a_amount = :amount, a_first_gift = last(:giftdate), a_last_gift = first(:giftdate), - a_ntl_gift = nth(:giftdate, 2), + -- arbitrary datetime value to represent null '1900-00-00 00:00:00' + a_ntl_gift = isnull(nth(:giftdate, 2), convert(datetime, '1900-00-00 00:00:00')), a_count = count(1), a_total = sum(:amount), a_act_average_months = convert(integer, round(datediff(day, last(:giftdate), first(:giftdate)) / 30.5)), + -- ( 366 (days in year) / 12 (months in year) ) = 30.5 a_act_average_interval = condition( count(1) < 2 or datediff(day, last(:giftdate), first(:giftdate)) = 0, convert(double, null), (datediff(day, last(:giftdate), first(:giftdate)) / 30.5) / convert(double, count(1) - 1) ), + -- total months / interval == interval month average + -- we subtract 1 from (convert(double, count(1)) in order to match interval denominator and numerator + -- it is same as :amount * (count(1) - 1) a_act_average_amount = condition( datediff(day, last(:giftdate), first(:giftdate)) < 16, convert(money, null), (sum(:amount) - :amount) / round(datediff(day, last(:giftdate), first(:giftdate)) / 30.5) ), + -- in centralix SQL , when all the amounts are equal, we do not have to specify a row for a column s_date_created = getdate(), s_created_by = user_name(), s_date_modified = getdate(), s_modified_by = user_name() from collection tmp_gift_items + --segmenting based on duplicate gifts for descriptives_hist group by :ledger, :fund, @@ -203,14 +224,27 @@ update_descriptives_new "system/query" :seg, :giftdate desc ; + -- manually updating null dates into datetime value by using intermediate date value + -- convert(datetime, null) is not working for some reason + update + collection a_descriptives_hist + set + :a_ntl_gift = null + where + :a_ntl_gift = convert(datetime, '1900-00-00 00:00:00') + ; + -- Update previous/next interval start/end dates print 'Updating segment start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); update collection a_descriptives_hist dh1 set + --pick last gift from closest segment (amount) :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift order by :dh2:a_last_gift desc limit 1), + --pick first gift from closest segment (amount) :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift order by :dh2:a_first_gift asc limit 1) + --linking previous and next to every descriptives_hist to link all the segments together where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) @@ -221,104 +255,110 @@ update_descriptives_new "system/query" -- known limitation. These repeats provide good data, but are finite -- and may need to be repeated more for larger data sets. - -- Merge in 'catch up' giving - print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy + -- Merge in 'catch up' giving + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy + ; + + -- Merge in adjacent giving segments + print 'Merge adjacent at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in 'catch up' giving that includes extra gifts - print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy + ; + -- Merge in 'catch up' giving that includes extra gifts + print 'Merge catch-up at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_catchup_with_extra.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in extra giving - print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy + ; + -- Merge in extra giving + print 'Merge extra amounts at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_extra.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in adjacent single-gift segments - print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy + ; + -- Merge in adjacent single-gift segments + print 'Merge adjacent singles at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_singles.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Merge in adjacent giving segments - print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy + ; + -- Merge in adjacent giving segments + print 'Merge adjacent(2) at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_adjacent_2.qy fund=:parameters:fund, donor=:parameters:donor - ; - -- Approximate segments that did not fit in anywhere else - print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); - exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy + ; + -- Approximate segments that did not fit in anywhere else + print 'Merge approximates at ' + dateformat(getdate(), 'hh:mm:ss'); + exec /apps/kardia/modules/rcpt/update_descriptives_merge_approx.qy fund=:parameters:fund, donor=:parameters:donor - ; + ; - -- Correct previous/next interval start/end dates - -- Extra gifts are all linked together, and regular gifts are - -- linked together. - print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Correct previous/next interval start/end dates + -- Extra gifts are all linked together, and regular gifts are + -- linked together. + print 'Correcting start/end dates at ' + dateformat(getdate(), 'hh:mm:ss'); + + --link Regular Giving that are segmented based on same gift amount to track time sequence between segments + update collection a_descriptives_hist dh1 - set + set :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and not :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and not :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - where + where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - not :dh1:a_is_extra - ; - update + not :dh1:a_is_extra + ; + + --link Extra Giving that are segmented based on same gift amount to track time sequence between segments + + update collection a_descriptives_hist dh1 - set + set :dh1:a_prev_end = (select :a_last_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_last_gift < :dh1:a_first_gift and :dh2:a_is_extra order by :dh2:a_last_gift desc limit 1), :dh1:a_next_start = (select :a_first_gift from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh1:a_ledger_number and :dh2:a_fund = :dh1:a_fund and :dh2:p_donor_partner_key = :dh1:p_donor_partner_key and :dh2:a_first_gift > :dh1:a_last_gift and :dh2:a_is_extra order by :dh2:a_first_gift asc limit 1) - where + where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - :dh1:a_is_extra - ; + :dh1:a_is_extra + ; - -- Correct NTL Tracking - print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Correct NTL Tracking + print 'Fixing next-to-last tracking at ' + dateformat(getdate(), 'hh:mm:ss'); + update collection a_descriptives_hist dh1 - set + set :dh1:a_ntl_gift = (select nth(:t:giftdate, 2) from collection tmp_gift_items t where :t:ledger = :dh1:a_ledger_number and :t:fund = :dh1:a_fund and :t:donor = :dh1:p_donor_partner_key and :t:giftdate <= :dh1:a_last_gift order by :t:giftdate desc) - where + where (:parameters:fund is null or :parameters:fund = :dh1:a_fund) and (:parameters:donor is null or :parameters:donor = :dh1:p_donor_partner_key) and - :dh1:a_ntl_gift is null and - :dh1:a_count > 1 - ; + :dh1:a_ntl_gift is null and + :dh1:a_count > 1 + ; - -- First history value - print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); - update + ---- First history value + print 'Propagating history #1 at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set :a_hist_1_amount = (select first(:dh:a_amount) from @@ -330,7 +370,7 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ), - :a_hist_1_last = (select + :a_hist_1_last = (select first(:dh:a_last_gift) from collection a_descriptives_hist dh @@ -341,32 +381,32 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ) - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update + ; + update identity collection a_descriptives d, collection a_descriptives_hist dh - set + set :d:a_hist_1_count = :dh:a_count, :d:a_hist_1_first = :dh:a_first_gift, :d:a_hist_1_last = :dh:a_last_gift - where + where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and :d:a_hist_1_amount = :dh:a_amount and - :d:a_hist_1_last = :dh:a_last_gift and + :d:a_hist_1_last = :dh:a_last_gift and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Second history value - print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Second history value + print 'Propagating history #2 at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set :a_hist_2_amount = (select first(:dh:a_amount) from @@ -379,7 +419,7 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ), - :a_hist_2_last = (select + :a_hist_2_last = (select first(:dh:a_last_gift) from collection a_descriptives_hist dh @@ -391,31 +431,31 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ) - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update + ; + update identity collection a_descriptives d, collection a_descriptives_hist dh - set + set :d:a_hist_2_count = :dh:a_count, :d:a_hist_2_first = :dh:a_first_gift - where + where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and :d:a_hist_2_amount = :dh:a_amount and - :d:a_hist_2_last = :dh:a_last_gift and + :d:a_hist_2_last = :dh:a_last_gift and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Third history value - print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Third history value + print 'Propagating history #3 at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set :a_hist_3_amount = (select first(:dh:a_amount) from @@ -429,7 +469,7 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ), - :a_hist_3_last = (select + :a_hist_3_last = (select first(:dh:a_last_gift) from collection a_descriptives_hist dh @@ -442,74 +482,78 @@ update_descriptives_new "system/query" order by :dh:a_last_gift desc ) - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update + ; + update identity collection a_descriptives d, collection a_descriptives_hist dh - set + set :d:a_hist_3_count = :dh:a_count, :d:a_hist_3_first = :dh:a_first_gift - where + where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and :d:a_hist_3_amount = :dh:a_amount and - :d:a_hist_3_last = :dh:a_last_gift and + :d:a_hist_3_last = :dh:a_last_gift and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Overall adjustments to a_act_average_xyz for a_descriptives - -- if the interval is biannual or less, lookback slightly less than a year from the most recent gift - -- the lookahead date is half an interval past the most recent gift - print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Overall adjustments to a_act_average_xyz for a_descriptives + -- the lookahead date is half an interval past the most recent gift + -- if the interval is biannual or less, lookback date slightly less than a year from the most recent gift + -- else the lookback date is half an interval past the oldest gift + print 'Setting general stats at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, -366, dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - :a_act_average_interval = convert(double, null), + :a_act_average_interval = convert(double, null), :a_act_average_months = null, :a_act_average_amount = convert(money, null) - where + where :a_act_average_interval < 7 and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - -- if the interval is greater than bi-annually, check back the last three intervals - update + ; + -- if the interval is greater than bi-annually, check three intervals before last_gift date + -- if first_gift date more recent than three intervals before last_gift_date, just use first_gift date + update identity collection a_descriptives d - set + set :a_act_lookahead_date = dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift), :a_act_lookback_date = condition(dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift) < dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(-3.0 * 30.5 * :a_act_average_interval)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_last_gift)), dateadd(day, convert(integer, round(0.5 * 30.5 * :a_act_average_interval)), :a_first_gift)), - :a_act_average_interval = convert(double, null), - :a_act_average_months = null, + -- between half interval past the first_gift date and three intervals before (half interval past the last_gift date), choose the bigger date + -- between first_gift date and three interval before last_gift date, choose more recent date + :a_act_average_interval = convert(double, null), + :a_act_average_months = null, :a_act_average_amount = convert(money, null) - where + where :a_act_average_interval >= 7 and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - -- this handles one time gifts (and giving that falls all within half a month) - update + ; + -- this handles one time gifts (and giving that falls all within half a month) + update identity collection a_descriptives d - set + set :a_act_average_interval = convert(double, null), :a_act_average_months = null, :a_act_average_amount = convert(money, null), :a_act_lookahead_date = condition(getdate() < dateadd(day, 366, :a_last_gift), getdate(), dateadd(day, 366, :a_last_gift)), :a_act_lookback_date = :a_first_gift - where + where :a_act_average_months = 0 and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - update + ; + update identity collection a_descriptives d - set + set :a_act_average_months = convert(integer, round(datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5)), :a_act_average_amount = isnull((select condition( count(1) = 0 or datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) < 16, @@ -521,45 +565,45 @@ update_descriptives_new "system/query" convert(double, null), (datediff(day, :d:a_act_lookback_date, :d:a_act_lookahead_date) / 30.5) / convert(double, count(1)) ) from collection tmp_gift_items i where :i:giftdate >= :d:a_act_lookback_date and :i:giftdate <= :d:a_act_lookahead_date and :i:fund = :d:a_fund and :i:ledger = :d:a_ledger_number and :i:donor = :d:p_donor_partner_key) - where + where :d:a_act_average_interval is null and (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Update a_act_average_xyz for a_descriptives_hist - update - identity collection a_descriptives_hist dh - set - :a_act_average_amount = condition( - datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, - convert(money, null), - (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) - ), - :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), - :a_act_average_interval = condition( - :dh:a_count = 1, - convert(double, null), - (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) - ) - where - (:parameters:fund is null or :parameters:fund = :dh:a_fund) and - (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - :dh:is_approximate = 0 - ; + -- Update a_act_average_xyz for a_descriptives_hist + update + identity collection a_descriptives_hist dh + set + :a_act_average_amount = condition( + datediff(day, :dh:a_first_gift, :dh:a_last_gift) < 16, + convert(money, null), + (:dh:a_total - :dh:a_amount) / round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) + ), + :a_act_average_months = convert(integer, round(datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5)), + :a_act_average_interval = condition( + :dh:a_count = 1, + convert(double, null), + (datediff(day, :dh:a_first_gift, :dh:a_last_gift) / 30.5) / (:dh:a_total / :dh:a_amount - 1) + ) + where + (:parameters:fund is null or :parameters:fund = :dh:a_fund) and + (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and + :dh:is_approximate = 0 + ; - -- Update past/current flags on histories and on descriptives. For a history entry with - -- more than one gift, we base the current/lapsed indicator on the gift interval compared to - -- the most recent gift and current date. For a one-gift history entry, we look back 13 - -- months, considering the gift a onetime/annual gift. - -- - -- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current - -- a_lapsed_days Values: negative = days before the statistical next gift date, - -- positive = days after the next statistical gift date. - print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Update past/current flags on histories and on descriptives. For a history entry with + -- more than one gift, we base the current/lapsed indicator on the gift interval compared to + -- the most recent gift and current date. For a one-gift history entry, we look back 13 + -- months, considering the gift a onetime/annual gift. + -- + -- a_is_current Values: 0 = lapsed, 1 = due / possibly current, 2 = current + -- a_lapsed_days Values: negative = days before the statistical next gift date, + -- positive = days after the next statistical gift date. + print 'Setting lapsed stats at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives_hist dh - set + set :a_is_current = condition(:a_count >= 2, condition(:a_act_average_interval < 7.0, condition(:a_last_gift > dateadd(day, -7 - convert(integer, round(30.5 * :a_act_average_interval)), getdate()), @@ -586,161 +630,174 @@ update_descriptives_new "system/query" datediff(day, dateadd(day, convert(integer, round(30.5 * :a_act_average_interval)), :a_last_gift), getdate()), datediff(day, dateadd(day, 365, :a_last_gift), getdate()) ) - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ; - -- For the past/current flag for the overall giving to a fund, we use the history entry - -- with the most recent gift in it, ignoring other history entries even if they seem to - -- still be current. - update + ---- For the past/current flag for the overall giving to a fund, we use the history entry + ---- with the most recent gift in it, ignoring other history entries even if they seem to + ---- still be current. + update identity collection a_descriptives d - set - :a_is_current = (select first(:dh:a_is_current) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), - :a_lapsed_days = (select first(:dh:a_lapsed_days) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) - where + set + --filtering to appropriate range + :a_is_current = (select :dh:a_is_current from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1), + --filtering to appropriate range + :a_lapsed_days = (select :dh:a_lapsed_days from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 1) + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Identify increases in giving. An increase is when the history entry has a larger monthly - -- average than the immediately previous history entry that is not a one-time-extra gift. We - -- also require that the history entry and its predecessor both have at least two gifts, - -- unless the giving interval is greater than 7 (effectively yearly), in which case any - -- number of gifts for either produces useful results. - print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Identify increases in giving. An increase is when the history entry has a larger monthly + -- average than the immediately previous history entry that is not a one-time-extra gift. We + -- also require that the history entry and its predecessor both have at least two gifts, + -- unless the giving interval is greater than 7 (effectively yearly), in which case any + -- number of gifts for either produces useful results. + + --calculating the increased percentage from most recent previous gift to current gift + + print 'Identifying increases at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives_hist dh - set + set :dh:a_increase_pct = condition( - :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, - condition( - :dh:a_count > 1, - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, - null - ) - ) - where + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)) - 1.0, + condition( + :dh:a_count > 1, + convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select :dh2:a_amount / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))) - 1.0, + null + ) + ) + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - not isnull(:dh:a_is_extra, 0) != 0 - ; - update + not isnull(:dh:a_is_extra, 0) != 0 + ; + ---removing meaningless descriptives_pct value for history segments + update identity collection a_descriptives_hist dh, collection a_descriptives d - set + set + --filtering to appropriate range :dh:a_increase_pct = condition((:dh:a_count > 1 or :d:a_act_average_interval > 1.8) and :dh:a_increase_pct > 0.001, :dh:a_increase_pct, null) - where + where :d:a_ledger_number = :dh:a_ledger_number and :d:a_fund = :dh:a_fund and :d:p_donor_partner_key = :dh:p_donor_partner_key and (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ; - -- Identify decreases in giving. A decrease is when the history entry has a smaller monthly - -- average than the immediately previous history entry that is not a one-time-extra gift. We - -- also require that the history entry and its predecessor both have at least two gifts, - -- unless the giving interval is greater than 7 (effectively yearly), in which case any - -- number of gifts for either produces useful results. - print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); - update + -- Identify decreases in giving. A decrease is when the history entry has a smaller monthly + -- average than the immediately previous history entry that is not a one-time-extra gift. We + -- also require that the history entry and its predecessor both have at least two gifts, + -- unless the giving interval is greater than 7 (effectively yearly), in which case any + -- number of gifts for either produces useful results. + print 'Identifying decreases at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives_hist dh - set + set :dh:a_decrease_pct = condition( - :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), - 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), - condition( - :dh:a_count > 1, - 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), - null - ) - ) - where + :dh:a_act_average_interval > 7 or (datediff(day, :dh:a_prev_end, :dh:a_first_gift) > 212 and :dh:a_count = 1), + 1.0 - convert(double, :dh:a_amount) / convert(double, isnull((select :dh2:a_amount from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift order by :dh2:a_last_gift desc limit 1), :dh:a_amount)), + condition( + :dh:a_count > 1, + 1.0 - convert(double, :dh:a_amount) / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)) / convert(double, isnull((select convert(double, :dh2:a_amount) / condition(round(:dh2:a_act_average_interval) < 1, 1, round(:dh2:a_act_average_interval)) from collection a_descriptives_hist dh2 where :dh2:a_ledger_number = :dh:a_ledger_number and :dh2:a_fund = :dh:a_fund and :dh2:p_donor_partner_key = :dh:p_donor_partner_key and isnull(:dh2:a_is_extra, 0) = 0 and :dh2:a_last_gift < :dh:a_first_gift and :dh2:a_count > 1 order by :dh2:a_last_gift desc limit 1), :dh:a_amount / condition(round(:dh:a_act_average_interval) < 1, 1, round(:dh:a_act_average_interval)))), + null + ) + ) + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) and - not isnull(:dh:a_is_extra, 0) != 0 - ; - update + not isnull(:dh:a_is_extra, 0) != 0 + ; + update identity collection a_descriptives_hist dh - set + set + --filtering to appropriate range :dh:a_decrease_pct = condition(:dh:a_decrease_pct > 0.001, :dh:a_decrease_pct, null) - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; - - -- Propagate informational flags/data to the overall giving descriptives - print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); - update + ; + -- Propagate informational flags/data to the overall giving descriptives + print 'Setting informational flags at ' + dateformat(getdate(), 'hh:mm:ss'); + update identity collection a_descriptives d - set + set + --getting increase pct of most recent gift :a_increase_pct = (select :dh:a_increase_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + --getting the most recent increase date :a_increase_date = (select condition(:dh:a_increase_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), :a_decrease_pct = (select :dh:a_decrease_pct from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), :a_decrease_date = (select condition(:dh:a_decrease_pct is not null, :dh:a_first_gift, convert(datetime, null)) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key where :a_is_extra = 0 order by :a_last_gift desc limit 1), + --tracking the extra from most recent three last gifts :a_is_extra = (select max(:dh:a_is_extra) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3), - :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) - where + --tracking the approximate from most recent three last gifts + :a_is_approximate = (select max(:dh:a_is_approximate) from collection a_descriptives_hist dh where :dh:a_ledger_number = :d:a_ledger_number and :dh:a_fund = :d:a_fund and :dh:p_donor_partner_key = :d:p_donor_partner_key order by :a_last_gift desc limit 3) + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; + ; - -- Clear out any old descriptives - print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); - delete + --- Clear out any old descriptives + print 'Clearing old DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); + delete /apps/kardia/data/Kardia_DB/a_descriptives/rows d - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - delete + ; + delete /apps/kardia/data/Kardia_DB/a_descriptives_hist/rows dh - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:ledger is null or :parameters:ledger = :dh:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; - - -- Save to database - print 'Saving DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); - insert + ; + --- Save to database + print 'Saving DB descriptives at ' + dateformat(getdate(), 'hh:mm:ss'); + insert /apps/kardia/data/Kardia_DB/a_descriptives/rows - select + select * - from + from collection a_descriptives d - where + where (:parameters:fund is null or :parameters:fund = :d:a_fund) and (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - ; - insert + ; + + print ' Saving Descriptives Hist Table'+ dateformat(getdate(), 'hh:mm:ss'); + insert /apps/kardia/data/Kardia_DB/a_descriptives_hist/rows dh - select + select * - from + from collection a_descriptives_hist dh - where + where (:parameters:fund is null or :parameters:fund = :dh:a_fund) and (:parameters:ledger is null or :parameters:ledger = :dh:a_ledger_number) and (:parameters:donor is null or :parameters:donor = :dh:p_donor_partner_key) - ; + ; + ---- Return the dataset to the caller + --- This gets printed on the console (optional) - -- Return the dataset to the caller - print 'Done at ' + dateformat(getdate(), 'hh:mm:ss'); - select - * - from - collection a_descriptives_hist d - where - (:parameters:fund is null or :parameters:fund = :d:a_fund) and - (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and - (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) - order by - :a_hist_id desc + --print 'Done at ' + dateformat(getdate(), 'hh:mm:ss'); + --select + -- * + --from + -- collection a_descriptives_hist d + --where + -- (:parameters:fund is null or :parameters:fund = :d:a_fund) and + -- (:parameters:ledger is null or :parameters:ledger = :d:a_ledger_number) and + -- (:parameters:donor is null or :parameters:donor = :d:p_donor_partner_key) + --order by + -- :a_hist_id desc "; } diff --git a/limit b/limit new file mode 100644 index 00000000..e69de29b