Update pipelines and fix tests

requirements.txt

@@ -54,7 +54,8 @@ greenlet==3.1.1
     # via sqlalchemy
 gspread==6.1.3
     # via hdx-python-scraper
-hapi-schema@git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@develop
+hapi-schema @ git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@04d402d23d89a5dd489d8255f5462a6b115a5afa
+    # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.4
     # via
     #   hapi-pipelines (pyproject.toml)
@@ -115,7 +116,7 @@ markdown-it-py==3.0.0
     # via rich
 marko==2.1.2
     # via frictionless
-markupsafe==2.1.5
+markupsafe==3.0.0
     # via jinja2
 mdurl==0.1.2
     # via markdown-it-py

src/hapi/pipelines/app/pipelines.py

@@ -45,6 +45,7 @@ def __init__(
         scrapers_to_run: Optional[ListTuple[str]] = None,
         errors_on_exit: Optional[ErrorsOnExit] = None,
         use_live: bool = True,
+        countries_to_run: Optional[ListTuple[str]] = None,
     ):
         self.configuration = configuration
         self.session = session
@@ -53,8 +54,9 @@ def __init__(
             configuration=configuration,
             session=session,
             use_live=use_live,
+            countries=countries_to_run,
         )
-        self.countries = configuration["HAPI_countries"]
+        self.countries = self.locations.hapi_countries
         reader = Read.get_reader("hdx")
         libhxl_dataset = AdminLevel.get_libhxl_dataset(
             retriever=reader
@@ -69,11 +71,11 @@ def __init__(
         self.adminone = AdminLevel(admin_config=admin1_config, admin_level=1)
         admin2_config = configuration["admin2"]
         self.admintwo = AdminLevel(admin_config=admin2_config, admin_level=2)
-        self.adminone.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.adminone.setup_from_libhxl_dataset(libhxl_dataset)
         self.adminone.load_pcode_formats_from_libhxl_dataset(
             libhxl_format_dataset
         )
-        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset, self.countries)
+        self.admintwo.setup_from_libhxl_dataset(libhxl_dataset)
         self.admintwo.load_pcode_formats_from_libhxl_dataset(
             libhxl_format_dataset
         )
@@ -184,7 +186,6 @@ def _create_configurable_scrapers(
         )
         _create_configurable_scrapers("operational_presence", "national")
         _create_configurable_scrapers("national_risk", "national")
-        _create_configurable_scrapers("funding", "national")
         _create_configurable_scrapers("refugees_and_returnees", "national")
         _create_configurable_scrapers("idps", "national")
         _create_configurable_scrapers(
@@ -307,14 +308,11 @@ def output_idps(self):
 
     def output_funding(self):
         if not self.themes_to_run or "funding" in self.themes_to_run:
-            results = self.runner.get_hapi_results(
-                self.configurable_scrapers["funding"]
-            )
             funding = Funding(
                 session=self.session,
                 metadata=self.metadata,
                 locations=self.locations,
-                results=results,
+                configuration=self.configuration,
             )
             funding.populate()
 
@@ -360,6 +358,7 @@ def output_food_prices(self):
                 admins=self.admins,
                 adminone=self.adminone,
                 admintwo=self.admintwo,
+                configuration=self.configuration,
             )
             wfp_market.populate()
             food_price = FoodPrice(

tests/test_main.py

@@ -247,7 +247,7 @@ def test_conflict_event(self, configuration, folder, pipelines):
         count = session.scalar(
             select(func.count(DBConflictEvent.resource_hdx_id))
         )
-        check.equal(count, 71319)
+        check.equal(count, 313455)
 
     @pytest.mark.parametrize(
         "themes_to_run", [{"poverty_rate": ("AFG", "BFA")}]