Skip to content

Commit

Permalink
Update satellite and fix sql execute
Browse files Browse the repository at this point in the history
  • Loading branch information
luabida committed Mar 7, 2024
1 parent 49bbe1b commit bea590a
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 9 deletions.
12 changes: 4 additions & 8 deletions alertflow/dags/satellite-weather/brasil.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,16 @@ def extract_transform_load(
from satellite import downloader as sat_d
from satellite import weather as sat_w
from satellite.weather.brazil.extract_latlons import MUNICIPALITIES
from sqlalchemy import create_engine
from sqlalchemy import create_engine, text

start_date = parser.parse(str(date))
max_update_delay = start_date - timedelta(days=6)

with create_engine(psql_uri["PSQL_MAIN_URI"]).connect() as conn:
cur = conn.execute(
cur = conn.execute(text(
"SELECT geocodigo FROM weather.copernicus_brasil"
f" WHERE date = '{str(max_update_delay.date())}'"
)
))
table_geocodes = set(chain(*cur.fetchall()))

all_geocodes = set([mun["geocodigo"] for mun in MUNICIPALITIES])
Expand All @@ -116,17 +116,13 @@ def extract_transform_load(
# Reads the NetCDF4 file using Xarray
ds = sat_w.load_dataset(netcdf_file)

try:
conn = create_engine(psql_uri["PSQL_MAIN_URI"]).raw_connection()

with create_engine(psql_uri["PSQL_MAIN_URI"]).connect() as conn:
ds.copebr.to_sql(
tablename="copernicus_brasil",
schema="weather",
geocodes=list(geocodes),
con=conn,
)
finally:
conn.close()

# Deletes the NetCDF4 file
Path(netcdf_file).unlink(missing_ok=True)
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ RUN source /home/airflow/mambaforge/bin/activate \
&& sed -i "s/include-system-site-packages = false/include-system-site-packages = true/" /opt/envs/py310/pyvenv.cfg \
&& source /opt/envs/py310/bin/activate \
&& pip install \
"satellite-weather-downloader >= 1.9.3" \
"satellite-weather-downloader >= 1.9.4" \
psycopg2

RUN echo "alias activate_episcanner='source /home/airflow/mambaforge/bin/activate episcanner-downloader'" >> /home/airflow/.bashrc
Expand Down

0 comments on commit bea590a

Please sign in to comment.