Set http caching for tests

.github/workflows/pkgTests.yml

@@ -9,8 +9,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 1
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.12"]
 
     steps:
     - uses: actions/checkout@v2
@@ -46,10 +47,15 @@ jobs:
         # run all py files
         # for f in *.py; do python "$f"; done
         cd ../..
+    - name: Add requests-cache - s3fs - py7zr
+      run: pip install requests-cache s3fs py7zr # Used to monkeypatch requests and store artifacts
     - name: Test with pytest
       env:      
         insee_key: ${{ secrets.INSEE_KEY }}
         insee_secret: ${{ secrets.INSEE_SECRET }}
+        s3_token: ${{ secrets.S3_TOKEN }}
+        s3_secret: ${{ secrets.S3_SECRET }}
+        s3_key: ${{ secrets.S3_KEY }}
       run: |
         pytest -v --cov
     - name: "Upload coverage to Codecov"

.github/workflows/pkgTests_pull_requests.yml

@@ -9,8 +9,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 1
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -48,10 +49,15 @@ jobs:
         # run all py files
         # for f in *.py; do python "$f"; done
         cd ../..
+    - name: Add requests-cache - s3fs - py7zr
+      run: pip install requests-cache s3fs py7zr # Used to monkeypatch requests and store artifacts
     - name: Test with pytest
       env:      
         insee_key: ${{ secrets.INSEE_KEY }}
         insee_secret: ${{ secrets.INSEE_SECRET }}
+        s3_token: ${{ secrets.S3_TOKEN }}
+        s3_secret: ${{ secrets.S3_SECRET }}
+        s3_key: ${{ secrets.S3_KEY }}
       run: |
         pytest -v --cov
     - name: "Upload coverage to Codecov"

.gitignore

@@ -23,4 +23,5 @@ docs/examples/.jupyter_cache/*
 .spyproject/
 poetry.lock
 docs/examples/*.csv
-docs/examples/*.svg
+docs/examples/*.svg
+.env

pynsee/utils/_get_credentials.py

@@ -1,81 +1,82 @@
-# -*- coding: utf-8 -*-
-# Copyright : INSEE, 2021
-
-import json
-import logging
-import os
-
-from functools import lru_cache
-from typing import Dict
-
-from platformdirs import user_config_dir
-
-
-logger = logging.getLogger(__name__)
-
-
-def _get_credentials() -> Dict[str, str]:
-    '''
-    Try to load credentials.
-
-    If the environment variables `insee_key` and `insee_secret` are set, use
-    these.
-    Otherwise try to load them from the config file.
-
-    Returns a dict containing at least an "insee_key" and an "insee_secret"
-    entry.
-    '''
-    key_dict: Dict[str, str] = {}
-
-    try:
-        key_dict["insee_key"] = os.environ["insee_key"]
-        key_dict["insee_secret"] = os.environ["insee_secret"]
-
-        envir_var_used = True
-    except KeyError:
-        envir_var_used = False
-
-        config_file = os.path.join(
-            user_config_dir("pynsee", ensure_exists=True), "config.json")
-
-        try:
-            with open(config_file, "r") as f:
-                key_dict = json.load(f)
-
-            http_proxy = key_dict["http_proxy"]
-            https_proxy = key_dict["https_proxy"]
-
-            if (http_proxy is None) or (not isinstance(http_proxy, str)):
-                http_proxy = ""
-            if (https_proxy is None) or (not isinstance(https_proxy, str)):
-                https_proxy = ""
-
-            os.environ["http_proxy"] = http_proxy
-            os.environ["https_proxy"] = https_proxy
-        except Exception:
-            _missing_credentials()
-
-    if envir_var_used:
-        _warn_env_credentials()
-
-    return key_dict
-
-
-@lru_cache(maxsize=None)
-def _missing_credentials() -> None:
-    logger.critical(
-        "INSEE API credentials have not been found: please try to reuse "
-        "pynsee.utils.init_conn to save them locally.\n"
-        "Otherwise, you can still use environment variables as follow:\n\n"
-        "import os\n"
-        "os.environ['insee_key'] = 'my_insee_key'\n"
-        "os.environ['insee_secret'] = 'my_insee_secret'"
-    )
-
-
-@lru_cache(maxsize=None)
-def _warn_env_credentials() -> None:
-    logger.warning(
-        "Existing environment variables used, instead of locally "
-        "saved credentials"
-    )
+# -*- coding: utf-8 -*-
+# Copyright : INSEE, 2021
+
+import json
+import logging
+import os
+
+from functools import lru_cache
+from typing import Dict
+
+from platformdirs import user_config_dir
+
+
+logger = logging.getLogger(__name__)
+
+
+def _get_credentials() -> Dict[str, str]:
+    """
+    Try to load credentials.
+
+    If the environment variables `insee_key` and `insee_secret` are set, use
+    these.
+    Otherwise try to load them from the config file.
+
+    Returns a dict containing at least an "insee_key" and an "insee_secret"
+    entry.
+    """
+    key_dict: Dict[str, str] = {}
+
+    try:
+        key_dict["insee_key"] = os.environ["insee_key"]
+        key_dict["insee_secret"] = os.environ["insee_secret"]
+
+        envir_var_used = True
+    except KeyError:
+        envir_var_used = False
+
+        config_file = os.path.join(
+            user_config_dir("pynsee", ensure_exists=True), "config.json"
+        )
+
+        try:
+            with open(config_file, "r") as f:
+                key_dict = json.load(f)
+
+            http_proxy = key_dict["http_proxy"]
+            https_proxy = key_dict["https_proxy"]
+
+            if (http_proxy is None) or (not isinstance(http_proxy, str)):
+                http_proxy = ""
+            if (https_proxy is None) or (not isinstance(https_proxy, str)):
+                https_proxy = ""
+
+            os.environ["http_proxy"] = http_proxy
+            os.environ["https_proxy"] = https_proxy
+        except Exception:
+            _missing_credentials()
+
+    if envir_var_used:
+        _warn_env_credentials()
+
+    return key_dict
+
+
+@lru_cache(maxsize=None)
+def _missing_credentials() -> None:
+    logger.critical(
+        "INSEE API credentials have not been found: please try to reuse "
+        "pynsee.utils.init_conn to save them locally.\n"
+        "Otherwise, you can still use environment variables as follow:\n\n"
+        "import os\n"
+        "os.environ['insee_key'] = 'my_insee_key'\n"
+        "os.environ['insee_secret'] = 'my_insee_secret'"
+    )
+
+
+@lru_cache(maxsize=None)
+def _warn_env_credentials() -> None:
+    logger.warning(
+        "Existing environment variables used, instead of locally "
+        "saved credentials"
+    )

pynsee/utils/_request_insee.py

@@ -8,7 +8,11 @@
 
 from pynsee.utils._get_token import _get_token
 from pynsee.utils._get_credentials import _get_credentials
-from pynsee.utils.requests_params import _get_requests_session, _get_requests_headers, _get_requests_proxies
+from pynsee.utils.requests_params import (
+    _get_requests_session,
+    _get_requests_headers,
+    _get_requests_proxies,
+)
 
 import logging
 
@@ -53,7 +57,7 @@ def _request_insee(
             print(api_url)
     except:
         pass
-        
+
     # force sdmx use with a system variable
     try:
         pynsee_use_sdmx = os.environ["pynsee_use_sdmx"]
@@ -80,11 +84,11 @@ def _request_insee(
 
         if token is not None:
             user_agent = _get_requests_headers()
-            
+
             headers = {
                 "Accept": file_format,
                 "Authorization": "Bearer " + token,
-                'User-Agent': user_agent['User-Agent']
+                "User-Agent": user_agent["User-Agent"],
             }
 
             session = _get_requests_session()