Merge pull request #115 from OCHA-DAP/HDXDSYS-829-update-3w

Hdxdsys 829 Update 3W

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.18] - 2024-06-24
+
+### Added
+
+- Specify countries per theme when running via the command line
+
+### Fixed
+
+- Thanks to bugfix in `openpyxl`, added 3W data for: BFA, CAF, COD, NER
+- Updated 3W data for: MLI, SOM, ETH, SDN, SSD
+- Avoid overwriting orgs with multiple org types
+
 ## [0.9.17] - 2024-06-10
 
 ### Changed

pyproject.toml

@@ -36,7 +36,7 @@ requires-python = ">=3.8"
 dependencies = [
     "hapi-schema>=0.8.9",
     "hdx-python-api>= 6.2.9",
-    "hdx-python-country>= 3.7.2",
+    "hdx-python-country>= 3.7.2, <3.7.3",
     "hdx-python-database[postgresql]>= 1.3.1",
     "hdx-python-scraper>= 2.3.7",
     "hdx-python-utilities>= 3.6.9",

requirements.txt

@@ -24,7 +24,7 @@ ckanapi==4.8
     # via hdx-python-api
 click==8.1.7
     # via typer
-coverage==7.5.3
+coverage==7.5.4
     # via pytest-cov
 cryptography==42.0.8
     # via pyopenssl
@@ -40,11 +40,11 @@ docopt==0.6.2
     #   num2words
 docutils==0.21.2
     # via defopt
-email-validator==2.1.1
+email-validator==2.2.0
     # via hdx-python-api
 et-xmlfile==1.1.0
     # via openpyxl
-filelock==3.14.0
+filelock==3.15.4
     # via virtualenv
 frictionless==5.17.0
     # via hdx-python-utilities
@@ -71,9 +71,9 @@ hdx-python-country==3.7.2
     #   hdx-python-scraper
 hdx-python-database==1.3.1
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.3.7
+hdx-python-scraper==2.3.8
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.6.9
+hdx-python-utilities==3.7.0
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -89,7 +89,7 @@ idna==3.7
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.2.1
+inflect==7.3.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -118,7 +118,7 @@ makefun==1.15.2
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.3
+marko==2.1.2
     # via frictionless
 markupsafe==2.1.5
     # via jinja2
@@ -134,7 +134,7 @@ num2words==0.5.13
     # via quantulum3
 oauthlib==3.2.2
     # via requests-oauthlib
-openpyxl==3.1.3
+openpyxl==3.1.4
     # via hdx-python-utilities
 packaging==24.1
     # via pytest
@@ -166,7 +166,7 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.3
+pydantic==2.7.4
     # via frictionless
 pydantic-core==2.18.4
     # via pydantic
@@ -190,7 +190,7 @@ pytest-cov==5.0.0
     # via hapi-pipelines (pyproject.toml)
 pytest-mock==3.14.0
     # via hapi-pipelines (pyproject.toml)
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   frictionless
     #   hdx-python-utilities
@@ -242,7 +242,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==70.0.0
+setuptools==70.1.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -259,7 +259,7 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-sqlalchemy==2.0.30
+sqlalchemy==2.0.31
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
@@ -281,7 +281,6 @@ typer==0.12.3
 typing-extensions==4.12.2
     # via
     #   frictionless
-    #   inflect
     #   psycopg
     #   pydantic
     #   pydantic-core
@@ -292,13 +291,13 @@ unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.1
+urllib3==2.2.2
     # via
     #   libhxl
     #   requests
 validators==0.28.3
     # via frictionless
-virtualenv==20.26.2
+virtualenv==20.26.3
     # via pre-commit
 wheel==0.43.0
     # via libhxl

src/hapi/pipelines/app/__main__.py

@@ -54,7 +54,13 @@ def parse_args():
         default=None,
         help="Database connection parameters. Overrides --db-uri.",
     )
-    parser.add_argument("-th", "--themes", default=None, help="Themes to run")
+    th_help = (
+        "Themes to run. Can pass a single theme or multiple themes "
+        "separated by commas. You can also append a colon to the "
+        "theme name and pass ISO3s separated by a pipe. For example: "
+        "population:AFG|COD,poverty_rate:BFA,funding"
+    )
+    parser.add_argument("-th", "--themes", default=None, help=th_help)
     parser.add_argument(
         "-sc", "--scrapers", default=None, help="Scrapers to run"
     )
@@ -181,7 +187,11 @@ def main(
             if len(theme_strs) == 1:
                 themes_to_run[theme_strs[0]] = None
             else:
-                themes_to_run[theme_strs[0]] = theme_strs[1]
+                # Split values by pipe for multiple countries
+                values = theme_strs[1].split("|")
+                themes_to_run[theme_strs[0]] = (
+                    values if len(values) > 1 else values[0]
+                )
     else:
         themes_to_run = None
     if args.scrapers:

src/hapi/pipelines/configs/core.yaml

@@ -219,6 +219,7 @@ org_type:
 org_type_map:
   academy: "431"
   agence un: "447"
+  agence du systeme des nations unies: "447"
   autre type: "443"
   civile society: "501"
   cooperation internationale: "501"
@@ -229,9 +230,11 @@ org_type_map:
   gov: "435"
   govt: "435"
   ingo: "437"
+  institution etatique: "435"
   international institution: "438"
   local ngo: "504"
   mouv cr: "445"
+  mouvement croix rouge: "445"
   mouvement de la croix rouge: "445"
   movimiento de la cruz roja media luna roja: "445"
   nations unies: "447"
@@ -242,6 +245,7 @@ org_type_map:
   ong internacional: "437"
   ong internationale: "437"
   ong nat: "441"
+  ong national: "441"
   ong nacional: "441"
   ong nationale: "441"
   organisation des nations unies: "447"
@@ -262,8 +266,11 @@ org_type_map:
 
 sector_map:
   abna: "SHL"
+  abri: "SHL"
+  abri bna: "SHL"
   abris: "SHL"
   abris ame: "SHL"
+  abris/ame: "SHL"
   abris/bna: "SHL"
   abris/bna/cccm: "SHL"
   abris / nfi: "SHL"
@@ -273,6 +280,7 @@ sector_map:
   agriculture: "FSC"
   agua saneamiento e higiene: "WSH"
   all: "Intersectoral"
+  ame: "SHL"
   alojamiento de emergencia: "SHL"
   alojamiento de emergencia (shelter): "SHL"
   alojamientos y asentamientos: "SHL"
@@ -302,16 +310,20 @@ sector_map:
   epah: "WSH"
   erl: "ERY"
   esnfi: "SHL"
+  explosive hazards: "PRO-MIN"
   food: "FSC"
   food security and agriculture: "FSC"
   food security and livelihoods: "FSC"
   food security and nutrition: "FSC"
   food security livelihood: "FSC"
   fsl: "FSC"
+  gestion des sites d'accueil temporaires: "SHL"
   gbv: "PRO-GBV"
+  hlp: "PRO-HLP"
   humanitaire: "Hum"
   hygiene: "WSH"
   hygiene assainissement: "WSH"
+  intercluster: "Multi"  # From Somalia 3W, hopefully not to be confused with intersectoral
   logement terre et biens: "PRO-HLP"
   logistica: "LOG"
   logistique: "LOG"
@@ -330,11 +342,17 @@ sector_map:
   operational presence protection: "PRO"
   pronna: "PRO-CPN"
   propg: "PRO"
+  proteccion infantil: "PRO-CPN"
   protection: "PRO"
   protection de l'enfance: "PRO-CPN"
   protection de l'enfant: "PRO-CPN"
   protection generale: "PRO"
-  proteccion infantil: "PRO-CPN"
+  protection logement terre et propriete: "PRO-HLP"
+  protection ltb: "PRO-HLP"
+  protection lutte anti mines: "PRO-MIN"
+  protection protection de l'enfant: "PRO-CPN"
+  protection violences basees sur le genre: "PRO-GBV"
+  protection vgb: "PRO-GBV"
   proteccion: "PRO"
   provbg: "PRO-GBV"
   psea: "PRO-GBV"
@@ -368,6 +386,7 @@ sector_map:
   shelter and non food items: "SHL"
   site management: "CCM"
   snfi: "SHL"
+  telecommunications: "TEL"
   telecomunicaciones de emergencia: "TEL"
   telecommunications d'urgence: "TEL"
   vbg: "PRO-GBV"