From 0456b7a8ce637eeb0eba01941b2d7504c45ac159 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 18 Jun 2024 12:55:47 -0400 Subject: [PATCH 1/6] distro family tests --- .github/workflows/tests.yml | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4c7584280..f601fd23b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,12 +14,12 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable with: options: "--check" - name: Install Python 3 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -27,7 +27,7 @@ jobs: pip install flake8 - name: flake8 run: | - flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" + flake8 test: needs: lint runs-on: ubuntu-latest @@ -35,9 +35,9 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -53,16 +53,35 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./cov.xml verbose: true + test-distros: + needs: lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, debian, archlinux, fedora-latest, gentoo, alpine] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install poetry + poetry install + - name: Run tests + run: | + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . update_docs: needs: test runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }} - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -85,8 +104,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && (github.ref == 'refs/heads/dev') steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV @@ -107,11 +126,11 @@ jobs: if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable') continue-on-error: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install dependencies @@ -160,7 +179,7 @@ jobs: # runs-on: ubuntu-latest # if: github.event_name == 'push' && github.ref == 'refs/heads/stable' # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # with: # ref: ${{ github.head_ref }} # fetch-depth: 0 # Fetch all history for all tags and branches From 5134dfe6847bb8deb3f566456535ced944728fb0 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 18 Jun 2024 12:57:42 -0400 Subject: [PATCH 2/6] add flake8 config --- .flake8 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..759908205 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +select = F,E722 +ignore = F403,F405,F541 +per-file-ignores = + */__init__.py:F401,F403 \ No newline at end of file From e734bd4333856d33c73a251ebbccc8443fb912c5 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 18 Jun 2024 12:58:16 -0400 Subject: [PATCH 3/6] simplified run_tests.sh --- bbot/test/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/run_tests.sh b/bbot/test/run_tests.sh index 0b20a7056..868b2c8c5 100755 --- a/bbot/test/run_tests.sh +++ b/bbot/test/run_tests.sh @@ -10,7 +10,7 @@ echo echo "[+] Linting with flake8" echo "=======================" -flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403" "$bbot_dir" || exit 1 +flake8 "$bbot_dir" || exit 1 echo echo "[+] Testing with pytest" From 1eb35b4667927324704ad7b8f040510967de4992 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 18 Jun 2024 18:43:29 +0100 Subject: [PATCH 4/6] Install unstructured deps on multiple os --- bbot/modules/unstructured.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/bbot/modules/unstructured.py b/bbot/modules/unstructured.py index 96d384b49..d07f81b5e 100644 --- a/bbot/modules/unstructured.py +++ b/bbot/modules/unstructured.py @@ -66,7 +66,26 @@ class unstructured(BaseModule): "ignore_folders": "Subfolders to ignore when crawling downloaded folders", } - deps_apt = ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"] + deps_ansible = [ + { + "name": "Install Deps (Debian/Ubuntu)", + "package": {"name": ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian'", + }, + { + "name": "Install Deps (Arch)", + "package": {"name": ["file", "poppler", "tesseract", "libreoffice-fresh", "pandoc"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Archlinux'", + }, + { + "name": "Install Deps (Fedora)", + "package": {"name": ["file-devel", "poppler-utils", "tesseract", "libreoffice", "pandoc"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Fedora'", + }, + ] deps_pip = ["unstructured[all-docs]"] async def setup(self): From 4a1bdff5003866a348471f6bf8e77ae90871d926 Mon Sep 17 00:00:00 2001 From: Dom Whewell Date: Tue, 18 Jun 2024 18:47:28 +0100 Subject: [PATCH 5/6] blacked --- bbot/modules/unstructured.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bbot/modules/unstructured.py b/bbot/modules/unstructured.py index d07f81b5e..63b3926e6 100644 --- a/bbot/modules/unstructured.py +++ b/bbot/modules/unstructured.py @@ -69,7 +69,10 @@ class unstructured(BaseModule): deps_ansible = [ { "name": "Install Deps (Debian/Ubuntu)", - "package": {"name": ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"], "state": "present"}, + "package": { + "name": ["libmagic-dev", "poppler-utils", "tesseract-ocr", "libreoffice", "pandoc"], + "state": "present", + }, "become": True, "when": "ansible_facts['os_family'] == 'Debian'", }, @@ -81,7 +84,10 @@ class unstructured(BaseModule): }, { "name": "Install Deps (Fedora)", - "package": {"name": ["file-devel", "poppler-utils", "tesseract", "libreoffice", "pandoc"], "state": "present"}, + "package": { + "name": ["file-devel", "poppler-utils", "tesseract", "libreoffice", "pandoc"], + "state": "present", + }, "become": True, "when": "ansible_facts['os_family'] == 'Fedora'", }, From efcdaff5c5a204fdfcc2eca52444b3ef9e105651 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 20 Jun 2024 08:59:55 -0400 Subject: [PATCH 6/6] update dev merge --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f601fd23b..e3041b2cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,7 +46,7 @@ jobs: poetry install - name: Run tests run: | - poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . - name: Upload Code Coverage uses: codecov/codecov-action@v3 with: @@ -64,14 +64,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.x" - name: Install dependencies run: | pip install poetry poetry install - name: Run tests run: | - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . update_docs: needs: test runs-on: ubuntu-latest