Skip to content

Commit

Permalink
Update satellite and fix sql execute
Browse files Browse the repository at this point in the history
  • Loading branch information
luabida committed Mar 7, 2024
1 parent 49bbe1b commit 162dd19
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 12 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
cancel-in-progress: true

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- uses: actions/setup-node@v3
with:
Expand Down Expand Up @@ -65,7 +65,7 @@ jobs:
convention-name: conventionalcommits

- name: Install Conda Dependencies
uses: conda-incubator/setup-miniconda@v2
uses: conda-incubator/setup-miniconda@v3
with:
miniconda-version: "latest"
mamba-version: "*"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
shell: bash -l {0}

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive
Expand Down
12 changes: 4 additions & 8 deletions alertflow/dags/satellite-weather/brasil.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,16 @@ def extract_transform_load(
from satellite import downloader as sat_d
from satellite import weather as sat_w
from satellite.weather.brazil.extract_latlons import MUNICIPALITIES
from sqlalchemy import create_engine
from sqlalchemy import create_engine, text

start_date = parser.parse(str(date))
max_update_delay = start_date - timedelta(days=6)

with create_engine(psql_uri["PSQL_MAIN_URI"]).connect() as conn:
cur = conn.execute(
cur = conn.execute(text(
"SELECT geocodigo FROM weather.copernicus_brasil"
f" WHERE date = '{str(max_update_delay.date())}'"
)
))
table_geocodes = set(chain(*cur.fetchall()))

all_geocodes = set([mun["geocodigo"] for mun in MUNICIPALITIES])
Expand All @@ -116,17 +116,13 @@ def extract_transform_load(
# Reads the NetCDF4 file using Xarray
ds = sat_w.load_dataset(netcdf_file)

try:
conn = create_engine(psql_uri["PSQL_MAIN_URI"]).raw_connection()

with create_engine(psql_uri["PSQL_MAIN_URI"]).connect() as conn:
ds.copebr.to_sql(
tablename="copernicus_brasil",
schema="weather",
geocodes=list(geocodes),
con=conn,
)
finally:
conn.close()

# Deletes the NetCDF4 file
Path(netcdf_file).unlink(missing_ok=True)
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ RUN source /home/airflow/mambaforge/bin/activate \
&& sed -i "s/include-system-site-packages = false/include-system-site-packages = true/" /opt/envs/py310/pyvenv.cfg \
&& source /opt/envs/py310/bin/activate \
&& pip install \
"satellite-weather-downloader >= 1.9.3" \
"satellite-weather-downloader >= 1.9.4" \
psycopg2

RUN echo "alias activate_episcanner='source /home/airflow/mambaforge/bin/activate episcanner-downloader'" >> /home/airflow/.bashrc
Expand Down

0 comments on commit 162dd19

Please sign in to comment.