update sectors data and validation (#35)

Author: jm-rivera

src/data/analysis_tools/sub_sectors.json

@@ -6,7 +6,7 @@
   "4": "Health, General",
   "5": "Post-Secondary Education",
   "6": "Secondary Education",
-  "7": "Unallocated/ Unspecificed",
+  "7": "Unallocated/ Unspecified",
   "8": "Business & Other Services",
   "9": "Democratic participation and civil society",
   "10": "Emergency Response",
@@ -63,5 +63,8 @@
   "61": "Public procurement",
   "62": "Hybrid Energy Plants",
   "63": "Non-communicable diseases (NCDs)",
-  "64": "Unallocated/unspecificed"
+  "64": "Unallocated/unspecified",
+  "65": "General Environment Protection",
+  "66": "Energy",
+  "67": "Unallocated/unspecificed"
 }

src/data/partners/seek/sectors.py

@@ -10,6 +10,7 @@
 from oda_data.clean_data.channels import add_channel_names
 
 from src.data import config
+from src.data.config import PATHS
 
 set_data_path(config.PATHS.DATA)
 
@@ -228,24 +229,51 @@ def pipeline(
     currency: str = "USD",
     by_recipient: bool = False,
     base_year: int | None = None,
+    include_bilateral: bool = False,
 ) -> pd.DataFrame:
-    bilateral = get_bilateral_disbursements_by_sector(
-        start_year=start_year,
-        end_year=end_year,
-        currency=currency,
-        by_recipient=by_recipient,
-        base_year=base_year,
-    ).assign(indicator="Bilateral")
-    multilateral = get_imputed_multilateral_disbursements_by_sector(
-        start_year=start_year,
-        end_year=end_year,
-        currency=currency,
-        by_recipient=by_recipient,
-        base_year=base_year,
-    ).assign(indicator="Multilateral")
+    if include_bilateral:
+        bilateral = get_bilateral_disbursements_by_sector(
+            start_year=start_year,
+            end_year=end_year,
+            currency=currency,
+            by_recipient=by_recipient,
+            base_year=base_year,
+        ).assign(indicator="bilateral_flow_disbursement_gross")
+    else:
+        bilateral = pd.DataFrame()
 
-    return pd.concat([bilateral, multilateral], ignore_index=True)
+    data = (
+        get_imputed_multilateral_disbursements_by_sector(
+            start_year=start_year,
+            end_year=end_year,
+            currency=currency,
+            by_recipient=by_recipient,
+            base_year=base_year,
+        )
+        .assign(indicator="imputed_multi_flow_disbursement_gross")
+        .pipe(add_names_columns, ["purpose_code"])
+    )
+
+    if include_bilateral:
+        data = pd.concat([bilateral, data], ignore_index=True)
+
+    return data.filter(
+        [
+            "year",
+            "indicator",
+            "donor_code",
+            "donor_name",
+            "purpose_code",
+            "purpose_name",
+            "channel_code",
+            "mapped_name",
+            "currency",
+            "prices",
+            "value",
+        ]
+    )
 
 
 if __name__ == "__main__":
-    df = pipeline()
+    df = pipeline(base_year=2024)
+    df.to_csv(PATHS.TOPIC_PAGE / "seek_imputed_2024_constant.csv", index=False)

src/data/scripts/sectors_view.py

@@ -1,7 +1,7 @@
 import json
 import pandas as pd
 
-from oda_data import CRSData
+from oda_data import CRSData, add_sectors
 from oda_data.tools import sector_lists
 from oda_data.indicators.research.sector_imputations import (
     imputed_multilateral_by_purpose,
@@ -148,7 +148,11 @@ def combined_sectors():
     sectors["sub_sector_code"] = sectors["sub_sector"].map(subsector_mapping)
     sectors = sectors.rename(columns={"sub_sector": "sub_sector_name"})
     sector_mapping = sector_lists.get_broad_sector_groups()
-    sectors["sector_name"] = sectors["sub_sector_name"].map(sector_mapping)
+    sectors["sector_name"] = (
+        sectors["sub_sector_name"]
+        .map(sector_mapping)
+        .fillna("Unallocated/ Unspecified")
+    )
 
     logger.info("Pivoting to wide format...")
     sectors = widen_currency_price(