Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into develop-global
Browse files Browse the repository at this point in the history
  • Loading branch information
b-j-mills committed Oct 7, 2024
2 parents 3ccd4e6 + 0b5781f commit 7165f11
Show file tree
Hide file tree
Showing 7 changed files with 32,052 additions and 18 deletions.
4 changes: 2 additions & 2 deletions .config/pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ repos:
- id: end-of-file-fixer
- id: check-ast
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.5.5
rev: v0.6.9
hooks:
# Run the linter.
- id: ruff
Expand All @@ -17,7 +17,7 @@ repos:
- id: ruff-format
args: [--config, .config/ruff.toml]
- repo: https://github.com/astral-sh/uv-pre-commit
rev: 0.2.31
rev: 0.4.18
hooks:
# Run the pip compile
- id: pip-compile
Expand Down
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

## [0.10.5] - 2024-10-08

### Fixed

- Fix broken database export
- WFP commodity normalised and fuzzy matching

## [0.10.4] - 2024-10-08

### Fixed

- Fix broken tests

## [0.10.3] - 2024-09-24

### Changed
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ requires-python = ">=3.8"
dependencies = [
"hapi-schema@git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@develop",
"hdx-python-api>= 6.3.4",
"hdx-python-country>= 3.7.8",
"hdx-python-country>= 3.8.1",
"hdx-python-database[postgresql]>= 1.3.1",
"hdx-python-scraper>= 2.5.0",
"hdx-python-utilities>= 3.7.3",
"hdx-python-utilities>= 3.7.4",
"libhxl",
"sqlalchemy"
]
Expand Down
26 changes: 13 additions & 13 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ defopt==6.4.0
# via hdx-python-api
distlib==0.3.8
# via virtualenv
dnspython==2.6.1
dnspython==2.7.0
# via email-validator
docopt==0.6.2
# via
Expand All @@ -42,7 +42,7 @@ et-xmlfile==1.1.0
# via openpyxl
filelock==3.16.1
# via virtualenv
frictionless==5.17.1
frictionless==5.18.0
# via hdx-python-utilities
google-auth==2.35.0
# via
Expand All @@ -52,14 +52,14 @@ google-auth-oauthlib==1.2.1
# via gspread
greenlet==3.1.1
# via sqlalchemy
gspread==6.1.2
gspread==6.1.3
# via hdx-python-scraper
hapi-schema@git+https://github.com/OCHA-DAP/hapi-sqlalchemy-schema@develop
hdx-python-api==6.3.4
# via
# hapi-pipelines (pyproject.toml)
# hdx-python-scraper
hdx-python-country==3.7.8
hdx-python-country==3.8.1
# via
# hapi-pipelines (pyproject.toml)
# hdx-python-api
Expand All @@ -68,13 +68,13 @@ hdx-python-database==1.3.3
# via hapi-pipelines (pyproject.toml)
hdx-python-scraper==2.5.0
# via hapi-pipelines (pyproject.toml)
hdx-python-utilities==3.7.3
hdx-python-utilities==3.7.4
# via
# hapi-pipelines (pyproject.toml)
# hdx-python-api
# hdx-python-country
# hdx-python-scraper
humanize==4.10.0
humanize==4.11.0
# via frictionless
identify==2.6.1
# via pre-commit
Expand Down Expand Up @@ -109,7 +109,7 @@ libhxl==5.2.1
# hdx-python-country
loguru==0.7.2
# via hdx-python-utilities
makefun==1.15.4
makefun==1.15.6
# via hdx-python-api
markdown-it-py==3.0.0
# via rich
Expand Down Expand Up @@ -143,11 +143,11 @@ ply==3.11
# libhxl
pockets==0.9.1
# via sphinxcontrib-napoleon
pre-commit==3.8.0
pre-commit==4.0.0
# via hapi-pipelines (pyproject.toml)
psycopg==3.2.2
psycopg==3.2.3
# via hdx-python-database
psycopg-binary==3.2.2
psycopg-binary==3.2.3
# via psycopg
pyasn1==0.6.1
# via
Expand Down Expand Up @@ -215,7 +215,7 @@ requests-oauthlib==2.0.0
# via google-auth-oauthlib
rfc3986==2.0.0
# via frictionless
rich==13.8.1
rich==13.9.2
# via typer
rpds-py==0.20.0
# via
Expand All @@ -231,7 +231,7 @@ setuptools==75.1.0
# via ckanapi
shellingham==1.5.4
# via typer
simpleeval==0.9.13
simpleeval==1.0.0
# via frictionless
simplejson==3.19.3
# via ckanapi
Expand Down Expand Up @@ -282,7 +282,7 @@ urllib3==2.2.3
# requests
validators==0.34.0
# via frictionless
virtualenv==20.26.5
virtualenv==20.26.6
# via pre-commit
wheel==0.44.0
# via libhxl
Expand Down
10 changes: 9 additions & 1 deletion src/hapi/pipelines/database/wfp_commodity.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
from hapi_schema.db_wfp_commodity import DBWFPCommodity
from hapi_schema.utils.enums import CommodityCategory
from hdx.scraper.framework.utilities.reader import Read
from hdx.utilities.text import normalise
from sqlalchemy.orm import Session

from ..utilities.mappings import get_code_from_name
from .base_uploader import BaseUploader

logger = getLogger(__name__)
Expand All @@ -22,6 +24,7 @@ def __init__(
super().__init__(session)
self._datasetinfo = datasetinfo
self.data = {}
self.unmatched = []

def populate(self) -> None:
logger.info("Populating WFP commodity table")
Expand All @@ -38,8 +41,13 @@ def populate(self) -> None:
)
self.data[name] = code
self.data[code] = name
self.data[normalise(name)] = code
self._session.add(commodity_row)
self._session.commit()

def get_commodity_code(self, commodity: str) -> Optional[str]:
return self.data.get(commodity)
return get_code_from_name(
name=commodity,
code_lookup=self.data,
unmatched=self.unmatched,
)
158 changes: 158 additions & 0 deletions tests/fixtures/input/download-global-pcode-lengths.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
Location,Country Length,Admin 1 Length,Admin 2 Length,Admin 3 Length,Admin 4 Length,Admin 5 Length
#country+code,#country+len,#adm1+len,#adm2+len,#adm3+len,#adm4+len,#adm5+len
AFG,2,2,2,,,
AGO,2,2,3,3,,
ALB,2,2,2,2,,
ARE,2,2,,,,
ARG,2,3,3,,,
ARM,2,2,1,3,,
ATG,2,2,,,,
AZE,2,8,,,,
BDI,3,3,3,,,
BEN,2,2,2,,,
BES,3,1,,,,
BFA,2,2,2,2,,
BGD,2,2,2,2,2,
BGR,2,3,3,,,
BLR,2,3,3,,,
BLZ,2,2,,,,
BMU,2,2,1,,,
BOL,2,2,2,2,,
BRA,2,2,5,,,
BRB,2,2,,,,
BTN,2,3,2,,,
BWA,2,2,2,2,,
CAF,2,2,1,1,2,
CHL,2,2,1,2,,
CHN,2,3,3,,,
CIV,2,2,2,2,,
CMR,2,3,3,3,,
COD,2,2,2,,,
COG,2,2,2,,,
COL,2,2,3,,,
COM,2,1,1,1,,
CPV,2,2,2,,,
CRI,2,1,2,2,,
CUB,2,2,2,,,
CUW,2,2,,,,
CYM,2,2,,,,
DJI,2,2,2,,,
DMA,2,2,,,,
DOM,2,2,2,2,2,
DZA,2,3,3,,,
ECU,2,2,2,2,,
EGY,2,2,2,2,,
ERI,2,1,2,,,
ESH,2,2,,,,
ETH,2,2,2,2,,
FJI,2,1,2,2,,
FSM,2,1,2,,,
GAB,3,3,3,,,
GEO,2,2,2,,,
GHA,2,2,2,,,
GIN,2,3,3,2,,
GLP,2,2,2,,,
GMB,2,2,2,2,,
GNB,2,2,2,,,
GNQ,2,3,3,,,
GRD,2,2,,,,
GTM,2,2,2,,,
GUF,2,1,3,,,
GUY,2,2,2,,,
HND,2,2,2,,,
HTI,2,2,2,3,,
HUN,2,3,3,,,
IDN,2,2,2,3,,
IRN,2,3,3,,,
IRQ,2,3,3,3,,
JAM,2,2,2,2,,
KAZ,3,3,3,,,
KEN,2,3,3,,,
KGZ,2,11,0,0,,
KHM,2,2,2,2,,
KIR,2,1,2,,,
KNA,2,2,,,,
KWT,2,2,,,,
LAO,2,2,2,,,
LBN,2,1,1,1,,
LBR,2,2,2,,,
LBY,2,2,2,,,
LCA,2,2,9,,,
LKA,2,1,1,2,3,
LSO,2,1,2,,,
MAR,2,3,3,7,1,
MDA,2,3,,,,
MDG,2,2,7|3,-1|3,3,
MDV,2,3,3,3,,
MEX,2,2,3,,,
MHL,2,2,2,,,
MLI,2,2,2,2,,
MMR,3,3,4,-1,3,3
MNG,2,2,2,,,
MOZ,2,2,2,2,,
MRT,2,2,1,2,,
MSR,2,2,,,,
MTQ,2,2,2,,,
MUS,2,2,,,,
MWI,2,1,2,2,,
MYS,2,2,2,,,
NAM,2,2,2,,,
NER,2,3,3,3,,
NGA,2,3,3,3,,
NIC,2,2,2,,,
NPL,2,2,2,3,,
OMN,2,2,2,,,
PAK,2,1,2,2,,
PAN,2,2,2,2,,
PER,2,2,2,2,,
PHL,2,2,3,2,3,
PNG,2,2,2,2,,
POL,2,3,3,,,
PRI,2,2,,,,
PRK,2,2,2,,,
PRY,2,2,2,,,
PSE,2,2,2,,,
QAT,3,3,3,3,,
ROU,2,3,3,,,
RUS,2,3,3,,,
RWA,2,1,1,2,2,
SAU,2,2,,,,
SDN,2,2,3,,,
SEN,2,2,2,2,,
SLB,2,2,4,4,,
SLE,2,2,2,2,2,
SLV,2,2,3,3,,
SOM,2,2,2,,,
SSD,2,2,2,2,,
STP,2,2,2,,,
SUR,2,2,2,,,
SVK,2,3,3,,,
SWZ,2,1,2,,,
SXM,2,1,,,,
SYC,2,1,1,4,,
SYR,2,2,2,2,,
TCA,2,1,2,,,
TCD,2,2,2,2,,
TGO,2,2,2,2,,
THA,2,2,2,2,,
TJK,0,7,0,,,
TLS,2,2,2,2,,
TON,2,1,1,2,,
TTO,2,2,,,,
TUN,2,1,1,2,2,
TUR,3,3,3,3|9,3,
TZA,2,2,2,3|4,,
UGA,2,1,3,2,2,
UKR,2,2,2,3,3,
URY,2,2,3,,,
UZB,2,2,3,,,
VCT,2,1,2,,,
VEN,2,2,2,2,,
VGB,2,2,,,,
VIR,2,3,5,,,
VNM,2,3,2,,,
VUT,2,2,3,,,
YEM,2,2,2,2,,
ZAF,2,1,2,1,3,
ZMB,2,3,3,3,3,
ZWE,2,2,2,2,,
Loading

0 comments on commit 7165f11

Please sign in to comment.