201 update dependencies

.github/workflows/pytest.yml

@@ -1,26 +1,48 @@
 name: Unit Tests
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os.host }}
     strategy:
       matrix:
         python-version: ["3.8", "3.9"]
+        os:
+          - name: ubuntu
+            host: ubuntu-latest
+          - name: windows
+            host: windows-latest
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
+
+      - name: (ubuntu) Install dependencies
+        if: runner.os != 'windows'
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements_dev.txt
           python -m spacy download en_core_web_sm
           pip install -e ."[test]"
-      - name: Test with pytest
+      - name: (ubuntu) Test with pytest
+        if: runner.os != 'windows'
+        run: |
+          pytest --verbose
+
+      - name: (windows) Install dependencies
+        if: runner.os == 'windows'
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+          python -m spacy download en_core_web_sm
+          pip install -e .
+      - name: (windows) Test with pytest
+        if: runner.os == 'windows'
+        shell: bash
         run: |
           pytest --verbose

README.md

@@ -35,10 +35,6 @@ You will also need to download [spaCy's](https://spacy.io/models/en) `en_core_we
 python -m spacy download en_core_web_sm
 ```
 
-### AWS CLI
-
-When the package is first used it will automatically download a folder of neccessary data and models. This file is ~ 1GB. Although you don't need to have AWS credentials for this to work, you will need to download the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html).
-
 ## TL;DR: Using Nesta's Skills Extractor library <a name="usage"></a>
 
 The library supports three key skills extraction functionalities :

ojd_daps_skills/app/requirements_app.txt

@@ -1,4 +1,2 @@
-numpy==1.21.1
 ojd-daps-skills
-awscli==1.27.25
-streamlit== 1.16.0
+streamlit==1.16.0

ojd_daps_skills/getters/download_public_data.py

@@ -1,17 +1,35 @@
 from ojd_daps_skills import PUBLIC_DATA_FOLDER_NAME, PROJECT_DIR
 
 import os
-
+import boto3
+from botocore.exceptions import ClientError
+from botocore import UNSIGNED
+from botocore.config import Config
+from zipfile import ZipFile
 
 def download():
+    """Download public data. Expected to run once on first use."""
+    s3 = boto3.client(
+        "s3", region_name="eu-west-1", config=Config(signature_version=UNSIGNED)
+    )
+
+    bucket_name = "open-jobs-indicators"
+    key = f"escoe_extension/{PUBLIC_DATA_FOLDER_NAME}.zip"
 
     public_data_dir = os.path.join(PROJECT_DIR, PUBLIC_DATA_FOLDER_NAME)
 
-    os.system(
-        f"aws --no-sign-request --region=eu-west-1 s3 cp s3://open-jobs-indicators/escoe_extension/{PUBLIC_DATA_FOLDER_NAME}.zip {public_data_dir}.zip"
-    )
-    os.system(f"unzip {public_data_dir}.zip -d {PROJECT_DIR}")
-    os.system(f"rm {public_data_dir}.zip")
+    try:
+        s3.download_file(bucket_name, key, f"{public_data_dir}.zip")
+
+        with ZipFile(f"{public_data_dir}.zip", "r") as zip_ref:
+            zip_ref.extractall(PROJECT_DIR)
+
+        os.remove(f"{public_data_dir}.zip")
+
+    except ClientError as ce:
+        print(f"Error: {ce}")
+    except FileNotFoundError as fnfe:
+        print(f"Error: {fnfe}")
 
 
 if __name__ == "__main__":

requirements.txt

@@ -1,13 +1,13 @@
-numpy==1.22.4
-scipy==1.8.1
+numpy==1.24.4
+scipy==1.10.1
 pandas==1.3.5
 tqdm==4.64.0
 filelock==3.7.1
 typer==0.4.1
 sh==1.14.2
-transformers==4.20.1
+transformers==4.33.3
 sentence-transformers==2.2.2
-scikit-learn==0.23.2
+scikit-learn==1.3.1
 spacy==3.4.0
 nervaluate==0.1.8
 s3fs==2022.5.0

requirements_dev.txt

@@ -9,7 +9,6 @@ black
 Sphinx
 sphinxcontrib-napoleon
 sphinx-rtd-theme
-awscli==1.27.32
 pre-commit
 pre-commit-hooks
 spacy==3.4.0

setup.py

@@ -5,13 +5,20 @@
 
 import os
 import subprocess
+import platform	
 
+tag_cmd = "git describe --tags --abbrev=0"	
+tag_cmd = tag_cmd if platform.system() == "Windows" else f"echo $({tag_cmd})"
 
-tag_cmd = "echo $(git describe --tags --abbrev=0)"
-tag_version = (
-    subprocess.check_output(tag_cmd, shell=True).decode("ascii").replace("\n", "")
-)
-
+try:
+    tag_version = (
+        subprocess.check_output(tag_cmd, shell=False).decode("ascii").replace("\n", "")
+    )
+except:
+    #an error occurred, potentially an issue with git cli
+    tag_version = "v1.0.0"
+
+print(f"Tag version: {tag_version}")
 
 def read_lines(path):
     """Read lines of `path`."""
@@ -24,7 +31,7 @@ def read_lines(path):
 
 setup(
     name="ojd_daps_skills",
-    long_description=open(os.path.join(BASE_DIR, "README.md")).read(),
+    long_description=open(os.path.join(BASE_DIR, "README.md"), encoding="utf-8").read(),
     long_description_content_type="text/markdown",
     install_requires=read_lines(os.path.join(BASE_DIR, "requirements.txt")),
     extras_require={"dev": read_lines(os.path.join(BASE_DIR, "requirements_dev.txt"))},