-
Notifications
You must be signed in to change notification settings - Fork 0
/
export_benevalibre_data.py
105 lines (78 loc) · 3.54 KB
/
export_benevalibre_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#! /usr/bin/env python
from playwright.async_api import async_playwright
import asyncio
import tkinter
from tkinter import filedialog
import pandas as pd
from skimpy import clean_columns
from datetime import date
import os
from dotenv import load_dotenv, find_dotenv
import sys
async def export_data(folder_path: str, mail: str, password: str, id_organisation: str) -> None:
"""Download the data of all volunteer actions from Bénévalibre.
The data of volunteer actions are exported from Bénévalibre
(https://app.benevalibre.org/) to a JSON file, then it is converted to an
Excel file.
:param folder_path: Output folder path.
:type folder_path: str
:param mail: Email to login to Bénévalibre.
:type mail: str
:param password: Password to login to Bénévalibre.
:type password: str
:param id_organisation: Bénévalibre ID of the organisation.
:type id_organisation: str
"""
# Create the output folder.
today = date.isoformat(date.today())
if not os.path.exists(f"{folder_path}/{today}"):
os.makedirs(f"{folder_path}/{today}")
file = f"{folder_path}/{today}/benevalibre_{today}"
print(f'The Excel file will be downloaded in the folder "{folder_path}/{today}/".')
print("Loading, please wait...")
# Go to the Bénévalibre web app then download the data.
async with async_playwright() as p:
print("Launching browser...")
browser = await p.firefox.launch(headless=True)
context = await browser.new_context(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0")
page = await context.new_page()
print("Going to the Bénévalibre web app to login...")
await page.goto('https://app.benevalibre.org/account/login/')
await page.wait_for_timeout(100)
await page.get_by_label("Adresse mail").fill(mail)
await page.get_by_label("Mot de passe").fill(password)
await page.get_by_role("button", name="Se connecter").click()
print("Successful login. Download is starting...")
async with page.expect_download() as download_info:
try:
await page.goto(f"https://app.benevalibre.org/associations/{id_organisation}/benevalo/?_export=json")
except:
pass
download = await download_info.value
await download.save_as(f"{file}.json")
await browser.close()
# Import the JSON file then convert it to an Excel file.
data = clean_columns(pd.read_json(f"{file}.json").convert_dtypes())
data["date_de_fin"] = pd.to_datetime(data["date_de_fin"], format="%Y-%m-%d")
data.to_excel(f"{file}.xlsx", index=False)
os.remove(f"{file}.json")
def get_inputs() -> list:
"""Get credentials from the ``.env`` file and the output folder path from user input.
:returns: The output folder path, the email, the password ans the organisation ID.
:rtype: list
"""
load_dotenv(find_dotenv(raise_error_if_not_found=True))
MAIL = os.environ.get("MAIL")
PASSWORD = os.environ.get("PASSWORD")
ID_ORGANISATION = os.environ.get("ID_ORGANISATION")
tkinter.Tk().withdraw()
folder_path = filedialog.askdirectory(title="Select output folder")
if folder_path:
return [folder_path, MAIL, PASSWORD, ID_ORGANISATION]
else:
sys.exit("No folder selected.")
if __name__ == '__main__':
print("Loading, please wait...")
print("Please select the folder where to export the Excel file.")
inputs = get_inputs()
asyncio.run(export_data(*inputs))