[COST-7114] Distribute GPU unallocated at start of month (#5846)

Author: myersCody

koku/masu/database/ocp_report_db_accessor.py

@@ -35,6 +35,7 @@
 from masu.processor import OCP_GPU_COST_MODEL_UNLEASH_FLAG
 from masu.util.common import filter_dictionary
 from masu.util.common import source_in_trino_table
+from masu.util.common import SummaryRangeConfig
 from masu.util.common import trino_table_exists
 from masu.util.ocp.common import DistributionConfig
 from masu.util.ocp.common import get_cluster_alias_from_cluster_id
@@ -468,7 +469,9 @@ def populate_markup_cost(self, markup, start_date, end_date, cluster_id):
             ),
         )
 
-    def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, distribution_info):
+    def populate_distributed_cost_sql(
+        self, summary_range: SummaryRangeConfig, provider_uuid: uuid.UUID, distribution_info: dict
+    ) -> None:
         """
         Populate the distribution cost model options.
 
@@ -503,27 +506,50 @@ def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, dis
                 cost_model_rate_type="gpu_distributed",
                 query_type="trino",
                 required_table="openshift_gpu_usage_line_items_daily",
+                requires_full_month=True,
             ),
         }
 
         table_name = self._table_map["line_item_daily_summary"]
-        report_period = self.report_periods_for_provider_uuid(provider_uuid, start_date)
-        if not report_period:
-            msg = "no report period for OCP provider, skipping distribution update"
-            context = {"schema": self.schema, "provider_uuid": provider_uuid, "start_date": start_date}
-            LOG.info(log_json(msg=msg, context=context))
-            return
-
-        report_period_id = report_period.id
+        dh = DateHelper()
         for cost_model_key, config in distribution_configs.items():
             sql_params = {
-                "start_date": start_date,
-                "end_date": end_date,
+                "start_date": summary_range.start_date,
+                "end_date": summary_range.end_date,
                 "schema": self.schema,
-                "report_period_id": report_period_id,
                 "source_uuid": provider_uuid,
                 "cost_model_rate_type": config.cost_model_rate_type,
             }
+            # Handle distributions that require full month data
+            if config.requires_full_month:
+                # Skip full-month distributions on subsequent days when iterating day-by-day
+                if summary_range.skip_full_month:
+                    continue
+                sql_params["start_date"] = summary_range.start_of_month
+                sql_params["end_date"] = summary_range.end_of_month
+                if summary_range.is_current_month:
+                    # Trigger distribution for previous month during a window of the current
+                    # month
+                    if dh.now_utc.day in [1, 2, 3]:
+                        sql_params["start_date"] = summary_range.start_of_previous_month
+                        sql_params["end_date"] = summary_range.end_of_previous_month
+                    else:
+                        msg = f"Skipping {cost_model_key} distribution requires full month"
+                        LOG.info(log_json(msg=msg, context={"schema": self.schema, "cost_model_key": cost_model_key}))
+                        continue
+
+            report_period = self.report_periods_for_provider_uuid(provider_uuid, sql_params["start_date"])
+            if not report_period:
+                msg = f"no report period for OCP provider, skipping {cost_model_key} distribution update"
+                context = {
+                    "schema": self.schema,
+                    "provider_uuid": provider_uuid,
+                    "start_date": sql_params["start_date"],
+                }
+                LOG.info(log_json(msg=msg, context=context))
+                continue
+            sql_params["report_period_id"] = report_period.id
+
             self._delete_monthly_cost_model_rate_type_data(sql_params, cost_model_key)
             populate = distribution_info.get(cost_model_key, config.distribute_by_default)
             if not populate:
@@ -551,6 +577,8 @@ def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, dis
             else:
                 self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params, operation=f"INSERT: {log_msg}")
 
+        return summary_range
+
     def _delete_monthly_cost_model_rate_type_data(self, sql_params, cost_model_key):
         delete_sql = pkgutil.get_data(
             "masu.database", "sql/openshift/cost_model/delete_monthly_cost_model_rate_type.sql"

koku/masu/database/trino_sql/openshift/cost_model/monthly_cost_gpu.sql

@@ -117,6 +117,8 @@ WITH cte_unutilized_uptime_hours AS (
         ON gpu.node = node_ut.node
         AND gpu.interval_date = DATE(node_ut.interval_start)
     where node_labels like '%"nvidia_com_gpu_present": "True"%'
+        AND date(node_ut.interval_start) >= DATE({{start_date}})
+        AND date(node_ut.interval_start) <= DATE({{end_date}})
         AND node_ut.month = {{month}}
         AND node_ut.year = {{year}}
         AND node_ut.source = {{source_uuid}}

koku/masu/processor/aws/aws_cost_model_cost_updater.py

@@ -13,6 +13,7 @@
 from masu.database.aws_report_db_accessor import AWSReportDBAccessor
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.util.aws.common import get_bills_from_provider
+from masu.util.common import SummaryRangeConfig
 from reporting.provider.aws.models import UI_SUMMARY_TABLES_MARKUP_SUBSET
 
 
@@ -54,12 +55,11 @@ def _update_markup_cost(self, start_date, end_date):
                 exc_info=error,
             )
 
-    def update_summary_cost_model_costs(self, start_date=None, end_date=None):
+    def update_summary_cost_model_costs(self, summary_range: SummaryRangeConfig) -> None:
         """Update the AWS summary table with the charge information.
 
         Args:
-            start_date (str, Optional) - Start date of range to update derived cost.
-            end_date (str, Optional) - End date of range to update derived cost.
+            summary_range (SummaryRangeConfig) - Date range configuration for cost model updates.
 
         Returns
             None
@@ -69,28 +69,31 @@ def update_summary_cost_model_costs(self, start_date=None, end_date=None):
             log_json(
                 msg="starting charge calculation updates",
                 schema=self._schema,
-                start_date=start_date,
-                end_date=end_date,
+                start_date=summary_range.start_date,
+                end_date=summary_range.end_date,
                 provider_uuid=self._provider.uuid,
             )
         )
 
-        self._update_markup_cost(start_date, end_date)
+        self._update_markup_cost(summary_range.start_date, summary_range.end_date)
 
         with AWSReportDBAccessor(self._schema) as accessor:
             LOG.debug(
                 log_json(
                     msg="updating AWS derived cost summary",
                     schema=self._schema,
                     provider_uuid=self._provider.uuid,
-                    start_date=start_date,
-                    end_date=end_date,
+                    start_date=summary_range.start_date,
+                    end_date=summary_range.end_date,
                 )
             )
             accessor.populate_ui_summary_tables(
-                start_date, end_date, self._provider.uuid, UI_SUMMARY_TABLES_MARKUP_SUBSET
+                summary_range.start_date,
+                summary_range.end_date,
+                self._provider.uuid,
+                UI_SUMMARY_TABLES_MARKUP_SUBSET,
             )
-            bills = accessor.bills_for_provider_uuid(self._provider.uuid, start_date)
+            bills = accessor.bills_for_provider_uuid(self._provider.uuid, summary_range.start_date)
             with schema_context(self._schema):
                 for bill in bills:
                     bill.derived_cost_datetime = timezone.now()

koku/masu/processor/azure/azure_cost_model_cost_updater.py

@@ -13,6 +13,7 @@
 from masu.database.azure_report_db_accessor import AzureReportDBAccessor
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.util.azure.common import get_bills_from_provider
+from masu.util.common import SummaryRangeConfig
 
 
 LOG = logging.getLogger(__name__)
@@ -54,12 +55,11 @@ def _update_markup_cost(self, start_date, end_date):
         except AzureCostModelCostUpdaterError as error:
             LOG.error(log_json(msg="unable to update markup costs", context=self._context), exc_info=error)
 
-    def update_summary_cost_model_costs(self, start_date=None, end_date=None):
+    def update_summary_cost_model_costs(self, summary_range: SummaryRangeConfig) -> None:
         """Update the Azure summary table with the charge information.
 
         Args:
-            start_date (str, Optional) - Start date of range to update derived cost.
-            end_date (str, Optional) - End date of range to update derived cost.
+            summary_range (SummaryRangeConfig) - Date range configuration for cost model updates.
 
         Returns
             None
@@ -69,17 +69,17 @@ def update_summary_cost_model_costs(self, start_date=None, end_date=None):
             log_json(
                 msg="starting charge calculation updates",
                 context=self._context,
-                start_date=start_date,
-                end_date=end_date,
+                start_date=summary_range.start_date,
+                end_date=summary_range.end_date,
             )
         )
 
-        self._update_markup_cost(start_date, end_date)
+        self._update_markup_cost(summary_range.start_date, summary_range.end_date)
 
         with AzureReportDBAccessor(self._schema) as accessor:
             LOG.debug(log_json(msg="updating Azure derived cost summary", context=self._context))
-            accessor.populate_ui_summary_tables(start_date, end_date, self._provider.uuid)
-            bills = accessor.bills_for_provider_uuid(self._provider.uuid, start_date)
+            accessor.populate_ui_summary_tables(summary_range.start_date, summary_range.end_date, self._provider.uuid)
+            bills = accessor.bills_for_provider_uuid(self._provider.uuid, summary_range.start_date)
             with schema_context(self._schema):
                 for bill in bills:
                     bill.derived_cost_datetime = timezone.now()

koku/masu/processor/cost_model_cost_updater.py

@@ -3,11 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 """Update Cost Model Cost info for report summary tables."""
-import datetime
 import logging
 
-import ciso8601
-
 from api.models import Provider
 from api.utils import DateHelper
 from koku.cache import invalidate_cache_for_tenant_and_cache_key
@@ -18,6 +15,7 @@
 from masu.processor.azure.azure_cost_model_cost_updater import AzureCostModelCostUpdater
 from masu.processor.gcp.gcp_cost_model_cost_updater import GCPCostModelCostUpdater
 from masu.processor.ocp.ocp_cost_model_cost_updater import OCPCostModelCostUpdater
+from masu.util.common import SummaryRangeConfig
 
 LOG = logging.getLogger(__name__)
 
@@ -76,18 +74,6 @@ def _set_updater(self):
         if self._provider.type in (Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL):
             return GCPCostModelCostUpdater(self._schema, self._provider)
 
-    def _format_dates(self, start_date, end_date):
-        """Convert dates to strings for use in the updater."""
-        if isinstance(start_date, datetime.date):
-            start_date = start_date.strftime("%Y-%m-%d")
-        elif isinstance(start_date, str):
-            start_date = ciso8601.parse_datetime(start_date).date()
-        if isinstance(end_date, datetime.date):
-            end_date = end_date.strftime("%Y-%m-%d")
-        elif isinstance(end_date, str):
-            end_date = ciso8601.parse_datetime(end_date).date()
-        return start_date, end_date
-
     def update_cost_model_costs(self, start_date=None, end_date=None):
         """
         Update usage charge information.
@@ -100,14 +86,18 @@ def update_cost_model_costs(self, start_date=None, end_date=None):
             None
 
         """
+        dh = DateHelper()
+        if start_date is None:
+            start_date = dh.this_month_start.date()
+        if end_date is None:
+            end_date = dh.today.date()
+        summary_range = SummaryRangeConfig(start_date=start_date, end_date=end_date)
         if self._updater:
             if is_customer_cost_model_large(self._schema):
-                for day_date in DateHelper().list_days(start_date, end_date):
-                    start, end = self._format_dates(day_date, day_date)
-                    self._updater.update_summary_cost_model_costs(start, end)
+                for day_range in summary_range.iter_days():
+                    self._updater.update_summary_cost_model_costs(day_range)
             else:
-                start_date, end_date = self._format_dates(start_date, end_date)
-                self._updater.update_summary_cost_model_costs(start_date, end_date)
+                self._updater.update_summary_cost_model_costs(summary_range)
             invalidate_view_cache_for_tenant_and_source_type(self._schema, self._provider.type)
             # Invalidate the tag_rate_map for tag mapping
             invalidate_cache_for_tenant_and_cache_key(self._schema, TAG_MAPPING_PREFIX)

koku/masu/processor/gcp/gcp_cost_model_cost_updater.py

@@ -12,6 +12,7 @@
 from api.common import log_json
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.database.gcp_report_db_accessor import GCPReportDBAccessor
+from masu.util.common import SummaryRangeConfig
 from masu.util.gcp.common import get_bills_from_provider
 from reporting.provider.gcp.models import UI_SUMMARY_TABLES
 
@@ -50,12 +51,11 @@ def _update_markup_cost(self, start_date, end_date):
         except GCPCostModelCostUpdaterError as error:
             LOG.error(log_json(msg="unable to update markup costs"), exc_info=error)
 
-    def update_summary_cost_model_costs(self, start_date=None, end_date=None):
+    def update_summary_cost_model_costs(self, summary_range: SummaryRangeConfig) -> None:
         """Update the GCP summary table with the charge information.
 
         Args:
-            start_date (str, Optional) - Start date of range to update derived cost.
-            end_date (str, Optional) - End date of range to update derived cost.
+            summary_range (SummaryRangeConfig) - Date range configuration for cost model updates.
 
         Returns
             None
@@ -66,12 +66,12 @@ def update_summary_cost_model_costs(self, start_date=None, end_date=None):
                 msg="starting charge calculation updates",
                 schema=self._schema,
                 provider_uuid=self._provider.uuid,
-                start_date=start_date,
-                end_date=end_date,
+                start_date=summary_range.start_date,
+                end_date=summary_range.end_date,
             )
         )
 
-        self._update_markup_cost(start_date, end_date)
+        self._update_markup_cost(summary_range.start_date, summary_range.end_date)
 
         with GCPReportDBAccessor(self._schema) as accessor:
             LOG.debug(
@@ -81,9 +81,9 @@ def update_summary_cost_model_costs(self, start_date=None, end_date=None):
                     provider_uuid=self._provider.uuid,
                 )
             )
-            invoice_month = start_date.strftime("%Y%m")
+            invoice_month = summary_range.start_date.strftime("%Y%m")
             invoice_dates = accessor.fetch_invoice_month_dates(
-                start_date, end_date, invoice_month, self._provider.uuid
+                summary_range.start_date, summary_range.end_date, invoice_month, self._provider.uuid
             )
             invoice_start, invoice_end = invoice_dates[0]
             accessor.populate_ui_summary_tables(