Merge pull request #1538 from WithoutPants/package-management

Restructuring and index building

.github/workflows/deploy.yml

@@ -0,0 +1,51 @@
+name: Deploy repository to Github Pages
+
+on:
+  push:
+    branches: [ master, stable ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout main
+      uses: actions/checkout@v2
+      with:
+        path: master
+        ref: master
+        fetch-depth: '0'
+    - run: |
+        cd master
+        ./build_site.sh ../_site/develop
+    # uncomment this once we have a stable branch
+    # - name: Checkout Stable
+    #   uses: actions/checkout@v2
+    #   with:
+    #     path: stable
+    #     ref: stable
+    #     fetch-depth: '0'
+    # - run: |
+    #     cd stable
+    #     ../master/build_site.sh ../_site/stable
+    - uses: actions/upload-pages-artifact@v2
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-22.04
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
+

.gitignore

@@ -5,3 +5,5 @@ yarn-error.log
 # Scraper-generated files
 /scrapers/*.ini
 **/__pycache__/
+
+/_site

build_site.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# builds a repository of scrapers
+# outputs to _site with the following structure:
+# index.yml
+# <scraper_id>.zip
+# Each zip file contains the scraper.yml file and any other files in the same directory
+
+outdir="$1"
+if [ -z "$outdir" ]; then
+    outdir="_site"
+fi
+
+rm -rf "$outdir"
+mkdir -p "$outdir"
+
+buildScraper() 
+{
+    f=$1
+    dir=$(dirname "$f")
+
+    # get the scraper id from the filename
+    scraper_id=$(basename "$f" .yml)
+    versionFile=$f
+    if [ "$scraper_id" == "package" ]; then
+        scraper_id=$(basename "$dir")
+    fi
+
+    if [ "$dir" != "./scrapers" ]; then
+        versionFile="$dir"
+    fi
+
+    echo "Processing $scraper_id"
+
+    # create a directory for the version
+    version=$(git log -n 1 --pretty=format:%h -- "$versionFile")
+    updated=$(git log -n 1 --date="format:%F %T %z" --pretty=format:%ad -- "$versionFile")
+
+    # create the zip file
+    # copy other files
+    zipfile=$(realpath "$outdir/$scraper_id.zip")
+
+    name=$(grep "^name:" "$f" | cut -d' ' -f2- | sed -e 's/\r//' -e 's/^"\(.*\)"$/\1/')
+    ignore=$(grep "^# ignore:" "$f" | cut -c 10- | sed -e 's/\r//')
+    dep=$(grep "^# requires:" "$f" | cut -c 12- | sed -e 's/\r//')
+
+    # always ignore package file
+    ignore="-x $ignore package"
+
+    pushd "$dir" > /dev/null
+    if [ "$dir" != "./scrapers" ]; then
+        zip -r "$zipfile" . ${ignore} > /dev/null
+    else
+        zip "$zipfile" "$scraper_id.yml" > /dev/null
+    fi
+    popd > /dev/null
+
+    # write to spec index
+    echo "- id: $scraper_id
+  name: $name
+  version: $version
+  date: $updated
+  path: $scraper_id.zip
+  sha256: $(sha256sum "$zipfile" | cut -d' ' -f1)" >> "$outdir"/index.yml
+
+    # handle dependencies
+    if [ ! -z "$dep" ]; then
+        echo "  requires:" >> "$outdir"/index.yml
+        for d in ${dep//,/ }; do
+            echo "    - $d" >> "$outdir"/index.yml
+        done
+    fi
+
+    echo "" >> "$outdir"/index.yml
+}
+
+# find all yml files in ./scrapers - these are packages individually
+for f in ./scrapers/*.yml; do 
+    buildScraper "$f"
+done
+
+find ./scrapers/ -mindepth 2 -name *.yml -print0 | while read -d $'\0' f; do
+    buildScraper "$f"
+done
+
+# handle dependency packages
+find ./scrapers/ -mindepth 2 -name package -print0 | while read -d $'\0' f; do
+    buildScraper "$f"
+done

scrapers/Algolia_21Naturals.yml → scrapers/21Naturals/21Naturals.yml

@@ -1,30 +1,31 @@
+# requires: Algolia
 name: "21Naturals"
 sceneByURL:
   - action: script
     url:
       - 21naturals.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21naturals
       - validName
 # Last Updated March 23, 2022

scrapers/Algolia_21Sextreme.yml → scrapers/21Sextreme/21Sextreme.yml

@@ -1,30 +1,31 @@
+# requires: Algolia
 name: "21Sextreme"
 sceneByURL:
   - action: script
     url:
       - 21sextreme.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextreme
       - validName
 # Last Updated March 23, 2022

scrapers/Algolia_21Sextury.yml → scrapers/21Sextury/21Sextury.yml

@@ -1,30 +1,31 @@
+# requires: Algolia
 name: "21Sextury"
 sceneByURL:
   - action: script
     url:
       - 21sextury.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - 21sextury.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - 21sextury
       - gallery
 # Last Updated December 22, 2022

scrapers/ATKGirlfriends.yml → scrapers/ATKGirlfriends/ATKGirlfriends.yml

@@ -1,4 +1,5 @@
 name: ATK Girlfriends
+# requires: py_common
 sceneByFragment:
   action: script
   script:

scrapers/Algolia_ActiveDuty.yml → scrapers/ActiveDuty/ActiveDuty.yml

@@ -1,30 +1,31 @@
+# requires: Algolia
 name: "Active Duty"
 sceneByURL:
   - action: script
     url:
       - activeduty.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - activeduty
 sceneByFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
 sceneByName:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
     - searchName
 sceneByQueryFragment:
   action: script
   script:
     - python
-    - Algolia.py
+    - ../Algolia/Algolia.py
     - activeduty
     - validName
 # Last Updated September 26, 2023

scrapers/Algolia_AddictedToGirls.yml → scrapers/AddictedToGirls/AddictedToGirls.yml

@@ -1,30 +1,31 @@
+# requires: Algolia
 name: "addicted 2 Girls"
 sceneByURL:
   - action: script
     url:
       - addicted2girls.com/en/video
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
 sceneByFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
 sceneByName:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - searchName
 sceneByQueryFragment:
     action: script
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - validName
 galleryByURL:
@@ -33,7 +34,7 @@ galleryByURL:
       - addicted2girls.com/en/photo/
     script:
       - python
-      - Algolia.py
+      - ../Algolia/Algolia.py
       - addicted2girls
       - gallery
 # Last Updated December 22, 2022