Skip to content

Commit

Permalink
HDX-9859 adapting patch table and VAT tables
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexandru-m-g
alexandru-m-g committed Jun 5, 2024
1 parent 9b183c8 commit 52b022a
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 40 deletions.
10 changes: 7 additions & 3 deletions src/hapi_schema/db_patch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,20 @@ class DBPatch(Base):
patch_sequence_number: Mapped[int] = mapped_column(
Integer, nullable=False, index=True
)
commit_hash: Mapped[str] = mapped_column(String(48), nullable=False)
commit_hash: Mapped[str] = mapped_column(
String(48), unique=False, nullable=False
)
commit_date: Mapped[datetime] = mapped_column(DateTime, nullable=False)
patch_path: Mapped[str] = mapped_column(
String(512), nullable=False, index=True
)
patch_permalink_url: Mapped[str] = mapped_column(
String(1024), nullable=False, unique=True
String(1024), nullable=False, unique=False
)
patch_target: Mapped[str] = mapped_column(String(128), nullable=False)
patch_hash: Mapped[str] = mapped_column(String(48), nullable=False)
patch_hash: Mapped[str] = mapped_column(
String(48), unique=False, nullable=False
)
state: Mapped[StateEnum] = mapped_column(
Enum(StateEnum), nullable=False, index=True
)
Expand Down
131 changes: 98 additions & 33 deletions src/hapi_schema/db_views_as_tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,20 @@
from sqlalchemy.orm import Mapped, mapped_column

from hapi_schema.utils.base import Base
from hapi_schema.utils.enums import (
CommodityCategory,
DisabledMarker,
EventType,
Gender,
IPCPhase,
IPCType,
PopulationGroup,
PopulationStatus,
PriceFlag,
PriceType,
RiskClass,
build_enum_using_values,
)


class DBAdmin1VAT(Base):
Expand All @@ -25,7 +39,9 @@ class DBAdmin1VAT(Base):
DateTime, index=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand All @@ -43,7 +59,9 @@ class DBAdmin2VAT(Base):
DateTime, nullable=True, index=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, nullable=True, index=True
DateTime,
nullable=True,
index=True,
)
admin1_code: Mapped[str] = mapped_column(String(128))
admin1_name: Mapped[str] = mapped_column(String(512))
Expand All @@ -56,14 +74,18 @@ class DBConflictEventVAT(Base):
__tablename__ = "conflict_event_vat"
resource_hdx_id: Mapped[str] = mapped_column(String(36))
admin2_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
event_type: Mapped[str] = mapped_column(String(18), primary_key=True)
event_type: Mapped[EventType] = mapped_column(
build_enum_using_values(EventType), primary_key=True
)
events: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
fatalities: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
reference_period_start: Mapped[datetime] = mapped_column(
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand Down Expand Up @@ -98,22 +120,28 @@ class DBFoodPriceVAT(Base):
market_code: Mapped[str] = mapped_column(String(32), primary_key=True)
commodity_code: Mapped[str] = mapped_column(String(32), primary_key=True)
currency_code: Mapped[str] = mapped_column(String(32), index=True)
unit: Mapped[str] = mapped_column(String(32))
price_flag: Mapped[str] = mapped_column(String(16), primary_key=True)
price_type: Mapped[str] = mapped_column(String(9), primary_key=True)
unit: Mapped[str] = mapped_column(String(32), primary_key=True)
price_flag: Mapped[PriceFlag] = mapped_column(
build_enum_using_values(PriceFlag), primary_key=True
)
price_type: Mapped[PriceType] = mapped_column(
build_enum_using_values(PriceType), primary_key=True
)
price: Mapped[Decimal] = mapped_column()
reference_period_start: Mapped[datetime] = mapped_column(
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
admin2_ref: Mapped[int] = mapped_column(Integer)
market_name: Mapped[str] = mapped_column(String(512), index=True)
lat: Mapped[float] = mapped_column(Float, index=True)
lon: Mapped[float] = mapped_column(Float, index=True)
commodity_category: Mapped[str] = mapped_column(
String(18), primary_key=True
commodity_category: Mapped[CommodityCategory] = mapped_column(
build_enum_using_values(CommodityCategory)
)
commodity_name: Mapped[str] = mapped_column(String(512), index=True)
location_code: Mapped[str] = mapped_column(String(128), index=True)
Expand All @@ -132,15 +160,21 @@ class DBFoodSecurityVAT(Base):
__tablename__ = "food_security_vat"
resource_hdx_id: Mapped[str] = mapped_column(String(36))
admin2_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
ipc_phase: Mapped[str] = mapped_column(String(12), primary_key=True)
ipc_type: Mapped[str] = mapped_column(String(17), primary_key=True)
ipc_phase: Mapped[IPCPhase] = mapped_column(
build_enum_using_values(IPCPhase), primary_key=True
)
ipc_type: Mapped[IPCType] = mapped_column(
build_enum_using_values(IPCType), primary_key=True
)
population_in_phase: Mapped[int] = mapped_column(Integer, index=True)
population_fraction_in_phase: Mapped[Decimal] = mapped_column(index=True)
reference_period_start: Mapped[datetime] = mapped_column(
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand Down Expand Up @@ -168,7 +202,8 @@ class DBFundingVAT(Base):
DateTime, index=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, nullable=True
DateTime,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand All @@ -178,22 +213,30 @@ class DBHumanitarianNeedsVAT(Base):
__tablename__ = "humanitarian_needs_vat"
resource_hdx_id: Mapped[str] = mapped_column(String(36))
admin2_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
gender: Mapped[str] = mapped_column(String(11), primary_key=True)
gender: Mapped[Gender] = mapped_column(
build_enum_using_values(Gender), primary_key=True
)
age_range: Mapped[str] = mapped_column(String(32), primary_key=True)
min_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
max_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
sector_code: Mapped[str] = mapped_column(String(32))
population_group: Mapped[str] = mapped_column(String(14), primary_key=True)
population_status: Mapped[str] = mapped_column(
String(10), primary_key=True
sector_code: Mapped[str] = mapped_column(String(32), primary_key=True)
population_group: Mapped[PopulationGroup] = mapped_column(
build_enum_using_values(PopulationGroup), primary_key=True
)
population_status: Mapped[PopulationStatus] = mapped_column(
build_enum_using_values(PopulationStatus), primary_key=True
)
disabled_marker: Mapped[str] = mapped_column(String(3), primary_key=True)
population: Mapped[int] = mapped_column(Integer, primary_key=True)
disabled_marker: Mapped[DisabledMarker] = mapped_column(
build_enum_using_values(DisabledMarker), primary_key=True
)
population: Mapped[int] = mapped_column(Integer)
reference_period_start: Mapped[datetime] = mapped_column(
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
sector_name: Mapped[str] = mapped_column(String(512))
location_code: Mapped[str] = mapped_column(String(128), index=True)
Expand All @@ -218,15 +261,19 @@ class DBLocationVAT(Base):
DateTime, index=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)


class DBNationalRiskVAT(Base):
__tablename__ = "national_risk_vat"
resource_hdx_id: Mapped[str] = mapped_column(String(36))
location_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
risk_class: Mapped[str] = mapped_column(String(9))
risk_class: Mapped[RiskClass] = mapped_column(
build_enum_using_values(RiskClass)
)
global_rank: Mapped[int] = mapped_column(Integer)
overall_risk: Mapped[Decimal] = mapped_column()
hazard_exposure_risk: Mapped[Decimal] = mapped_column()
Expand All @@ -238,7 +285,9 @@ class DBNationalRiskVAT(Base):
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand All @@ -255,7 +304,9 @@ class DBOperationalPresenceVAT(Base):
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
org_type_code: Mapped[str] = mapped_column(String(32))
org_type_description: Mapped[str] = mapped_column(String(512), index=True)
Expand Down Expand Up @@ -292,7 +343,9 @@ class DBPopulationVAT(Base):
__tablename__ = "population_vat"
resource_hdx_id: Mapped[str] = mapped_column(String(36))
admin2_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
gender: Mapped[str] = mapped_column(String(11), primary_key=True)
gender: Mapped[Gender] = mapped_column(
build_enum_using_values(Gender), primary_key=True
)
age_range: Mapped[str] = mapped_column(String(32), primary_key=True)
min_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
max_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
Expand All @@ -301,7 +354,9 @@ class DBPopulationVAT(Base):
DateTime, index=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand Down Expand Up @@ -331,7 +386,9 @@ class DBPovertyRateVAT(Base):
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
location_code: Mapped[str] = mapped_column(String(128), index=True)
location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand All @@ -345,8 +402,12 @@ class DBRefugeesVAT(Base):
resource_hdx_id: Mapped[str] = mapped_column(String(36))
origin_location_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
asylum_location_ref: Mapped[int] = mapped_column(Integer, primary_key=True)
population_group: Mapped[str] = mapped_column(String(14))
gender: Mapped[str] = mapped_column(String(11), primary_key=True)
population_group: Mapped[PopulationGroup] = mapped_column(
build_enum_using_values(PopulationGroup), primary_key=True
)
gender: Mapped[Gender] = mapped_column(
build_enum_using_values(Gender), primary_key=True
)
age_range: Mapped[str] = mapped_column(String(32), primary_key=True)
min_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
max_age: Mapped[int] = mapped_column(Integer, nullable=True, index=True)
Expand All @@ -355,7 +416,9 @@ class DBRefugeesVAT(Base):
DateTime, primary_key=True
)
reference_period_end: Mapped[datetime] = mapped_column(
DateTime, index=True
DateTime,
index=True,
nullable=True,
)
origin_location_code: Mapped[str] = mapped_column(String(128), index=True)
origin_location_name: Mapped[str] = mapped_column(String(512), index=True)
Expand Down Expand Up @@ -392,7 +455,9 @@ class DBSectorVAT(Base):
class DBWfpCommodityVAT(Base):
__tablename__ = "wfp_commodity_vat"
code: Mapped[str] = mapped_column(String(32), primary_key=True)
category: Mapped[str] = mapped_column(String(18), index=True)
category: Mapped[CommodityCategory] = mapped_column(
build_enum_using_values(CommodityCategory), index=True
)
name: Mapped[str] = mapped_column(String(512), index=True)


Expand Down
3 changes: 1 addition & 2 deletions tests/test_patch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ def test_db_patch(capfd):
patch_hash VARCHAR(48) NOT NULL,
state VARCHAR(10) NOT NULL,
execution_date DATETIME,
PRIMARY KEY (id),
UNIQUE (patch_permalink_url)
PRIMARY KEY (id)
)
"""
sql_patch_sequence_number_index_creation = """
Expand Down
5 changes: 3 additions & 2 deletions tests/test_views_as_tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def test_food_price_vat(
expected_primary_keys = [
"market_code",
"commodity_code",
"commodity_category",
"unit",
"price_flag",
"price_type",
"reference_period_start",
Expand Down Expand Up @@ -192,10 +192,10 @@ def test_humanitarian_needs_vat(
"admin2_ref",
"gender",
"age_range",
"sector_code",
"population_group",
"population_status",
"disabled_marker",
"population",
"reference_period_start",
]
expected_indexes = [
Expand Down Expand Up @@ -370,6 +370,7 @@ def test_refugees_vat(
expected_primary_keys = [
"origin_location_ref",
"asylum_location_ref",
"population_group",
"gender",
"age_range",
"reference_period_start",
Expand Down

0 comments on commit 52b022a

Please sign in to comment.