HDXDSYS-742 global coverage

src/hapi/pipelines/app/__main__.py

@@ -220,7 +220,6 @@ def main(
         "conflict_event.yaml",
         "core.yaml",
         "food_security.yaml",
-        "funding.yaml",
         "national_risk.yaml",
         "operational_presence.yaml",
         "population.yaml",

src/hapi/pipelines/app/pipelines.py

@@ -43,6 +43,7 @@ def __init__(
         scrapers_to_run: Optional[ListTuple[str]] = None,
         errors_on_exit: Optional[ErrorsOnExit] = None,
         use_live: bool = True,
+        countries_to_run: Optional[ListTuple[str]] = None,
     ):
         self.configuration = configuration
         self.session = session
@@ -51,8 +52,9 @@ def __init__(
             configuration=configuration,
             session=session,
             use_live=use_live,
+            countries=countries_to_run,
         )
-        self.countries = configuration["HAPI_countries"]
+        self.countries = self.locations.hapi_countries
         libhxl_dataset = AdminLevel.get_libhxl_dataset().cache()
         self.admins = Admins(
             configuration, session, self.locations, libhxl_dataset
@@ -61,9 +63,9 @@ def __init__(
         self.adminone = AdminLevel(admin_config=admin1_config, admin_level=1)
         admin2_config = configuration["admin2"]
         self.admintwo = AdminLevel(admin_config=admin2_config, admin_level=2)
-        self.adminone.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.adminone.setup_from_libhxl_dataset(libhxl_dataset)
         self.adminone.load_pcode_formats()
-        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset)
         self.admintwo.load_pcode_formats()
         self.admintwo.set_parent_admins_from_adminlevels([self.adminone])
         logger.info("Admin one name mappings:")
@@ -172,7 +174,6 @@ def _create_configurable_scrapers(
         )
         _create_configurable_scrapers("operational_presence", "national")
         _create_configurable_scrapers("national_risk", "national")
-        _create_configurable_scrapers("funding", "national")
         _create_configurable_scrapers("refugees", "national")
         _create_configurable_scrapers("poverty_rate", "national")
         _create_configurable_scrapers("conflict_event", "national")
@@ -272,14 +273,11 @@ def output_refugees(self):
 
     def output_funding(self):
         if not self.themes_to_run or "funding" in self.themes_to_run:
-            results = self.runner.get_hapi_results(
-                self.configurable_scrapers["funding"]
-            )
             funding = Funding(
                 session=self.session,
                 metadata=self.metadata,
                 locations=self.locations,
-                results=results,
+                configuration=self.configuration,
             )
             funding.populate()
 
@@ -325,6 +323,7 @@ def output_food_prices(self):
                 admins=self.admins,
                 adminone=self.adminone,
                 admintwo=self.admintwo,
+                configuration=self.configuration,
             )
             wfp_market.populate()
             food_price = FoodPrice(

src/hapi/pipelines/configs/conflict_event.yaml

@@ -26,12 +26,6 @@ conflict_event_default:
     - "conflict_event_eth_political_violence"
     - "conflict_event_eth_civilian_targeting"
     - "conflict_event_eth_demonstration"
-    - "conflict_event_gtm_political_violence"
-    - "conflict_event_gtm_civilian_targeting"
-    - "conflict_event_gtm_demonstration"
-    - "conflict_event_hnd_political_violence"
-    - "conflict_event_hnd_civilian_targeting"
-    - "conflict_event_hnd_demonstration"
     - "conflict_event_hti_political_violence"
     - "conflict_event_hti_civilian_targeting"
     - "conflict_event_hti_demonstration"
@@ -56,9 +50,6 @@ conflict_event_default:
     - "conflict_event_sdn_political_violence"
     - "conflict_event_sdn_civilian_targeting"
     - "conflict_event_sdn_demonstration"
-    - "conflict_event_slv_political_violence"
-    - "conflict_event_slv_civilian_targeting"
-    - "conflict_event_slv_demonstration"
     - "conflict_event_som_political_violence"
     - "conflict_event_som_civilian_targeting"
     - "conflict_event_som_demonstration"
@@ -1181,73 +1172,23 @@ conflict_event_admintwo:
       - "#event+num+demonstration"
 
 conflict_event_national:
-  conflict_event_gtm_political_violence:
-    dataset: "guatemala-acled-conflict-data"
-    resource: "guatemala_political_violence_events_and_fatalities_by_month-year"
+  conflict_event_political_violence:
+    dataset: "political-violence-events-and-fatalities"
+    format: "xlsx"
+    xlsx2csv: True
+    sheet: "Non_HRP"
     admin:
-      - ~
-    admin_single: "GTM"
-    input:
-      - "Fatalities"
-    list:
-      - "Fatalities"
-    output:
-      - "Fatalities political_violence"
-      - "Month political_violence"
-      - "Year political_violence"
-      - "Events political_violence"
-    output_hxl:
-      - "#fatality+num+political_violence"
-      - "#date+month+political_violence"
-      - "#date+year+political_violence"
-      - "#event+num+political_violence"
-
-  conflict_event_gtm_civilian_targeting:
-    dataset: "guatemala-acled-conflict-data"
-    resource: "guatemala_civilian_targeting_events_and_fatalities_by_month-year"
-    admin:
-      - ~
-    admin_single: "GTM"
-    input:
-      - "Fatalities"
-    list:
-      - "Fatalities"
-    output:
-      - "Fatalities civilian_targeting"
-      - "Month civilian_targeting"
-      - "Year civilian_targeting"
-      - "Events civilian_targeting"
-    output_hxl:
-      - "#fatality+num+civilian_targeting"
-      - "#date+month+civilian_targeting"
-      - "#date+year+civilian_targeting"
-      - "#event+num+civilian_targeting"
-
-  conflict_event_gtm_demonstration:
-    dataset: "guatemala-acled-conflict-data"
-    resource: "guatemala_demonstration_events_by_month-year"
-    admin:
-      - ~
-    admin_single: "GTM"
-    output:
-      - "Month demonstration"
-      - "Year demonstration"
-      - "Events demonstration"
-    output_hxl:
-      - "#date+month+demonstration"
-      - "#date+year+demonstration"
-      - "#event+num+demonstration"
-
-  conflict_event_hnd_political_violence:
-    dataset: "honduras-acled-conflict-data"
-    resource: "honduras_political_violence_events_and_fatalities_by_month-year"
-    admin:
-      - ~
-    admin_single: "HND"
+      - "Country"
     input:
       - "Fatalities"
+      - "Month"
+      - "Year"
+      - "Events"
     list:
       - "Fatalities"
+      - "Month"
+      - "Year"
+      - "Events"
     output:
       - "Fatalities political_violence"
       - "Month political_violence"
@@ -1259,16 +1200,23 @@ conflict_event_national:
       - "#date+year+political_violence"
       - "#event+num+political_violence"
 
-  conflict_event_hnd_civilian_targeting:
-    dataset: "honduras-acled-conflict-data"
-    resource: "honduras_civilian_targeting_events_and_fatalities_by_month-year"
+  conflict_event_civilian_targeting:
+    dataset: "civilian-targeting-events-and-fatalities"
+    format: "xlsx"
+    xlsx2csv: True
+    sheet: "Non_HRP"
     admin:
-      - ~
-    admin_single: "HND"
+      - "Country"
     input:
       - "Fatalities"
+      - "Month"
+      - "Year"
+      - "Events"
     list:
       - "Fatalities"
+      - "Month"
+      - "Year"
+      - "Events"
     output:
       - "Fatalities civilian_targeting"
       - "Month civilian_targeting"
@@ -1280,69 +1228,21 @@ conflict_event_national:
       - "#date+year+civilian_targeting"
       - "#event+num+civilian_targeting"
 
-  conflict_event_hnd_demonstration:
-    dataset: "honduras-acled-conflict-data"
-    resource: "honduras_demonstration_events_by_month-year"
-    admin:
-      - ~
-    admin_single: "HND"
-    output:
-      - "Month demonstration"
-      - "Year demonstration"
-      - "Events demonstration"
-    output_hxl:
-      - "#date+month+demonstration"
-      - "#date+year+demonstration"
-      - "#event+num+demonstration"
-
-  conflict_event_slv_political_violence:
-    dataset: "el-salvador-acled-conflict-data"
-    resource: "el salvador_political_violence_events_and_fatalities_by_month-year"
-    admin:
-      - ~
-    admin_single: "SLV"
-    input:
-      - "Fatalities"
-    list:
-      - "Fatalities"
-    output:
-      - "Fatalities political_violence"
-      - "Month political_violence"
-      - "Year political_violence"
-      - "Events political_violence"
-    output_hxl:
-      - "#fatality+num+political_violence"
-      - "#date+month+political_violence"
-      - "#date+year+political_violence"
-      - "#event+num+political_violence"
-
-  conflict_event_slv_civilian_targeting:
-    dataset: "el-salvador-acled-conflict-data"
-    resource: "el salvador_civilian_targeting_events_and_fatalities_by_month-year"
+  conflict_event_demonstration:
+    dataset: "demonstration-events"
+    format: "xlsx"
+    xlsx2csv: True
+    sheet: "Non_HRP"
     admin:
-      - ~
-    admin_single: "SLV"
+      - "Country"
     input:
-      - "Fatalities"
+      - "Month"
+      - "Year"
+      - "Events"
     list:
-      - "Fatalities"
-    output:
-      - "Fatalities civilian_targeting"
-      - "Month civilian_targeting"
-      - "Year civilian_targeting"
-      - "Events civilian_targeting"
-    output_hxl:
-      - "#fatality+num+civilian_targeting"
-      - "#date+month+civilian_targeting"
-      - "#date+year+civilian_targeting"
-      - "#event+num+civilian_targeting"
-
-  conflict_event_slv_demonstration:
-    dataset: "el-salvador-acled-conflict-data"
-    resource: "el salvador_demonstration_events_by_month-year"
-    admin:
-      - ~
-    admin_single: "SLV"
+      - "Month"
+      - "Year"
+      - "Events"
     output:
       - "Month demonstration"
       - "Year demonstration"