Merge pull request #232 from jvlflame/dev

2.4.0

.github/CHANGELOG.md

@@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.4.0]
+
+### Added
+- Added `-OpenModule` parameter to open the Javinizer module directory
+- Added setting `scraper.option.addmaleactors` to scrape avdanyuwiki.com for male actors (#147)
+    -  Updated English names in the thumb csv will be added in a later release
+- Sorting files via the CLI now populates the history csv file (-OpenHistory)
+- GUI: Added ouput and filematcher preview
+- GUI: Added recurse depth selection on sort card
+- GUI: Added genre/tag editor popups
+- GUI: Added stats page that corresponds with the history csv
+- GUI: Added an advanced manual search workflow
+- GUI: Added tooltip descriptions for most GUI actions and settings options
+- GUI: Added better dark/light theme management, refreshed appbar
+- GUI: Added 'not matched' count to sort progress popup
+
+### Changed
+- **Updated settings metadata priority defaults to favor javlibrary over r18**
+- mgstageja scraper no longer includes actress aliases in the JapaneseName
+- jav321 scrapers no longer include actress aliases in the JapaneseName
+- History csv now includes all metadata fields
+- GUI: Installing the GUI no longer requires first-run setup via `-OpenGUI` and Docker
+- GUI: PowerShell Universal version upgraded from 1.4.7 => 1.5.13
+- GUI: Default port changed from 5000 => 8600
+- GUI: GUI now runs in a non-admin scope to allow easier access to network drives
+
+### Fixed
+- Fixed issue where actress images would fail to download if they contained special characters on mgstage (#223)
+- Fixed issue causing update check to never run
+- Fixed issue with javlibraryja and javlibraryzh scrapers not accepting manual cloudflare cookies
+- Fixed issue where R18 series metadata included a blank 'tab' character
+
 ## [2.3.3]
 
 ### Changed

Dockerfile

@@ -1,41 +1,69 @@
+FROM python:3.9.2-buster
 
-FROM ubuntu:18.04
+# Add docker entrypoint script
+ADD docker-entrypoint.sh /
+RUN chmod +x /docker-entrypoint.sh
 
-ADD docker-entrypoint.sh /home
-ADD src/Javinizer/Universal/Repository/javinizergui.ps1 /home
-RUN chmod +x /home/docker-entrypoint.sh
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
-RUN apt-get update -y && apt-get install -y curl unrar wget software-properties-common apt-transport-https
-RUN add-apt-repository multiverse
-RUN mkdir /home/Universal
-WORKDIR /home/Universal
+# Install dependencies
+RUN apt-get update \
+    && apt-get install -y unzip wget
 
-RUN wget https://ftp.jvlflame.net/Universal.linux-x64.1.4.7.rar \
-    && unrar x Universal.linux-x64.1.4.7.rar \
-    && rm Universal.linux-x64.1.4.7.rar
+# Create directories
+RUN mkdir -p /home/data/Repository/.universal \
+    && mkdir -p /home/Universal
 
-RUN chmod +x /home/Universal/Universal.Server
+# Add dashboard files
+ADD src/Javinizer/Universal/Repository/javinizergui.ps1 /home/data/Repository
+ADD src/Javinizer/Universal/Repository/dashboards.ps1 /home/data/Repository/.universal
 
-RUN wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb && dpkg -i packages-microsoft-prod.deb && apt-get update
-RUN add-apt-repository universe
-RUN apt-get install -y powershell
+# Download powershell universal
+WORKDIR /home
+RUN wget https://imsreleases.blob.core.windows.net/universal/production/1.5.13/Universal.linux-x64.1.5.13.zip
+
+# Extract powershell universal to /home/Universal
+RUN unzip -q /home/Universal.linux-x64.1.5.13.zip -d /home/Universal/ \
+    && chmod +x /home/Universal/Universal.Server \
+    && rm /home/Universal.linux-x64.1.5.13.zip
+
+# Install mediainfo
 RUN apt-get install -y mediainfo
-RUN add-apt-repository -y ppa:deadsnakes/ppa
-RUN apt-get update -y
-RUN apt-get install -y python3.8 python3-pip
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10
-RUN pip3 install pillow google_trans_new googletrans==4.0.0rc1
 
-# Add custom UD components
-RUN pwsh -Command "Set-PSRepository -Name 'PSGallery' -InstallationPolicy Trusted"
-RUN pwsh -Command "Install-Module UniversalDashboard.Style; Install-Module UniversalDashboard.UDPlayer; Install-Module UniversalDashboard.UDSpinner; Install-Module UniversalDashboard.UDScrollUp; Install-Module UniversalDashboard.CodeEditor"
-RUN pwsh -Command "Install-Module Javinizer"
+# Install pwsh
+RUN wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb \
+    && dpkg -i packages-microsoft-prod.deb \
+    && apt-get update \
+    && apt-get install -y powershell \
+    && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /home
-EXPOSE 8600
-VOLUME ["/data"]
+# Install pwsh modules
+RUN pwsh -c "Set-PSRepository 'PSGallery' -InstallationPolicy Trusted" \
+    && pwsh -c "Install-Module UniversalDashboard.Style" \
+    && pwsh -c "Install-Module UniversalDashboard.Charts" \
+    && pwsh -c "Install-Module UniversalDashboard.UDPlayer" \
+    && pwsh -c "Install-Module UniversalDashboard.UDSpinner" \
+    && pwsh -c "Install-Module UniversalDashboard.UDScrollUp" \
+    && pwsh -c "Install-Module UniversalDashboard.CodeEditor" \
+    && pwsh -c "Install-Module Javinizer"
+
+# Install python modules
+RUN pip3 install pillow \
+    google_trans_new \
+    googletrans==4.0.0rc1
+
+# Clean up
+#RUN apt-get purge unzip \
+#    wget \
+#    && apt-get autoremove
+
+# Create symlink to module settings file
+RUN pwsh -c "ln -s (Join-Path (Get-InstalledModule Javinizer).InstalledLocation -ChildPath jvSettings.json) /home/jvSettings.json"
+
+# Add powershell universal environmental variables
+ENV Kestrel__Endpoints__HTTP__Url http://*:8600
 ENV Data__RepositoryPath ./data/Repository
 ENV Data__ConnectionString ./data/database.db
 ENV UniversalDashboard__AssetsFolder ./data/UniversalDashboard
 ENV Logging__Path ./data/logs/log.txt
-ENTRYPOINT ["/home/docker-entrypoint.sh"]
+
+EXPOSE 8600
+ENTRYPOINT ["/docker-entrypoint.sh"]

README.md

@@ -81,7 +81,7 @@ To run Javinizer, you will need to install following:
 
 ### Installation
 
-After installing the required prerequisites, run the following command in an administrator PowerShell 7 console to install the Javinizer module. If this is your first time using PowerShell, you may run into some prompts about security policies. Follow the instructions given in the prompts to unrestrict the code.
+After installing the required prerequisites, run the following command in an administrator PowerShell 7 (pwsh.exe) console to install the Javinizer module. If this is your first time using PowerShell, you may run into some prompts about security policies. Follow the instructions given in the prompts to unrestrict the code.
 
 ```powershell
 # Install the module from PowerShell gallery
@@ -99,7 +99,7 @@ Here are some common commands that you can run with Javinizer:
 # Run a command to sort your JAV files using default settings
 > Javinizer -Path "C:\JAV\Unsorted" -DestinationPath "C:\JAV\Sorted"
 
-# Run a command to sort your JAV files while searching folders recursively
+# Run a command to sort your JAV files while searching folders recursively (within the folders)
 > Javinizer -Path "C:\JAV\Unsorted" -DestinationPath "C:\JAV\Sorted" -Recurse
 
 # Run a command to sort a JAV file using direct URLs
@@ -126,28 +126,26 @@ Here are some common commands that you can run with Javinizer:
 
 ### Quick start (GUI)
 
-**NOTE** The GUI version of Javinizer is not updated at the same rate as the CLI version. For a more streamlined experience, I recommend using the CLI version unless you require the visual feedback and customization during your sorts.
-
-After running the GUI, [view the docs](https://docs.jvlflame.net/installation/install-javinizer-web-gui#import-the-javinizer-dashboard) for first-time setup to import the Javinizer dashboard into PowerShell Universal.
-
 #### Windows
 
 ```powershell
-# Install PowerShell Universal to Javinizer module folder
+# Install PowerShell Universal to Javinizer module folder (Run as administrator)
 > Javinizer -InstallGUI
 
-# Runs the PowerShell Universal application and opens the Javinizer GUI
+# Runs the PowerShell Universal application and opens the Javinizer GUI dashboard page
+# Optionally specify a custom port using the -Port parameter
 > Javinizer -OpenGUI
 ```
 
+After running `Javinizer -OpenGUI`, the PowerShell Universal process should run in a separate window and open your browser to 'http://localhost:[PORT]/' which contains the Javinizer dashboard.
+
 #### Docker
 
 ```
-docker run --name javinizer -p 5000:5000 -d jvlflame/javinizer:latest
+docker run --name javinizer -p 8600:8600 -d jvlflame/javinizer:latest
 
 # Optional
--v path/to/appdata:/home/data
--v path/to/settingsFile:/home/Javinizer/src/Javinizer/jvSettings.json
+-v path/to/jvSettings.json:/home/jvSettings.json
 ```
 
 ## Example Output

docker-entrypoint.sh

@@ -1,3 +1,5 @@
-#!/bin/bash
+#!/bin/sh
 umask 000
+
+# Start the powershell universal server
 /home/Universal/Universal.Server

src/Javinizer/Javinizer.psd1

@@ -13,7 +13,7 @@
 
     # Version number of this module.
 
-    ModuleVersion     = '2.3.3'
+    ModuleVersion     = '2.4.0'
 
     # Supported PSEditions
     # CompatiblePSEditions = @('Core')
@@ -73,6 +73,9 @@
     FunctionsToExport = @(
         'Invoke-JVParallel',
         'Javinizer',
+        'Get-AVDanyuData',
+        'Get-AVEntertainmentData',
+        'Get-AVEntertainmentUrl',
         'Get-CfSession',
         'Get-DLgetchuData',
         'Get-DmmData',
@@ -88,9 +91,11 @@
         'Get-JVAggregatedData',
         'Get-JVData',
         'Get-JVItem',
+        'Get-JVMediaInfo',
         'Get-JVNfo',
         'Get-JVSettings',
         'Get-JVSortData',
+        'Get-MaleActors',
         'Get-MgstageData',
         'Get-MgstageUrl',
         'Get-R18Data',
@@ -101,6 +106,7 @@
         'Set-JVMovie',
         'Start-JVGUI',
         'Update-JVModule',
+        'Update-JVNfo',
         'Update-JVThumbCsv',
         'Write-JVWebLog'
     )

src/Javinizer/Private/Convert-HTMLCharacter.ps1

@@ -17,6 +17,7 @@ function Convert-HTMLCharacter {
             -replace '※', '.*.' `
             -replace ''', "'" `
             -replace '&#039', '' `
+            -replace '	', '' `
             -replace '', '' # Seemingly invisible character that appears in mgstage
 
         $newString = $String.Trim()

src/Javinizer/Private/Convert-JVCleanString.ps1

@@ -0,0 +1,29 @@
+function Convert-JVCleanString {
+    param (
+        [Parameter(Mandatory = $true, Position = 0)]
+        [string]$String
+    )
+
+    $invalidSymbols = @(
+        '\',
+        '/',
+        ':',
+        '*',
+        '?',
+        '"',
+        '<',
+        '>',
+        '|',
+        "'"
+    )
+
+    foreach ($symbol in $invalidSymbols) {
+        if ([regex]::Escape($symbol) -eq '/') {
+            $String = $String -replace [regex]::Escape($symbol), '-'
+        } else {
+            $String = $String -replace [regex]::Escape($symbol), ''
+        }
+    }
+
+    Write-Output $String
+}

src/Javinizer/Private/Scraper.Dmm.ps1

@@ -272,7 +272,7 @@ function Get-DmmActress {
         } else {
             try {
                 $actressBlock = (($Webrequest.Content -split '<td align="right" valign="top" class="nw">(出演者：|Performers:)<\/td>')[2] -split '<\/td>')[0]
-                $movieActress = ($actressBlock | Select-String -Pattern '\/article=actress\/id=(\d*)\/">(.*)<\/a>' -AllMatches).Matches
+                $movieActress = ($actressBlock | Select-String -Pattern ' >(.*)</a>' -AllMatches).Matches
             } catch {
                 return
             }
@@ -283,7 +283,7 @@ function Get-DmmActress {
             $firstName = $null
             $engActressUrl = "https://www.dmm.co.jp/en/mono/dvd/-/list/=/article=actress/id=$($actress.Groups[1].Value)/"
             $jaActressUrl = "https://www.dmm.co.jp/mono/dvd/-/list/=/article=actress/id=$($actress.Groups[1].Value)/"
-            $actressName = $actress.Groups[2].Value
+            $actressName = $actress.Groups[1].Value
             if ($actressName -match '[\u3040-\u309f]|[\u30a0-\u30ff]|[\uff66-\uff9f]|[\u4e00-\u9faf]') {
                 if ($ScrapeActress) {
                     $session = New-Object Microsoft.PowerShell.Commands.WebRequestSession

src/Javinizer/Private/Scraper.Jav321.ps1

@@ -119,7 +119,7 @@ function Get-Jav321Genre {
 
         try {
             $genre = ($Webrequest | Select-String '<a href="\/genre\/.+?">(.+?)<\/a>' -AllMatches).Matches |
-                ForEach-Object { $_.Groups[1].Value }
+            ForEach-Object { $_.Groups[1].Value }
         } catch {
             return
         }
@@ -181,7 +181,7 @@ function Get-Jav321Actress {
             $actress | ForEach-Object { $movieActressObject += [PSCustomObject]@{
                     LastName     = $null
                     FirstName    = $null
-                    JapaneseName = ($_.Matches.Groups[2].Value)
+                    JapaneseName = ($_.Matches.Groups[2].Value -replace '（.*）', '')
                     ThumbUrl     = if (($_.Matches.Groups[1].Value -replace "'", '') -ne '.jpg') { 'https://www.jav321.com/mono/actjpgs/' + $_.Matches.Groups[1].Value -replace "'", '' }
                 }
             }

src/Javinizer/Private/Scraper.Javlibrary.ps1

@@ -215,7 +215,7 @@ function Get-JavlibraryActress {
             $enActress = Get-Actress -Webrequest $Webrequest
             $jaActress = Get-Actress -Webrequest $jaWebrequest
         } else {
-            $enWebrequest = Invoke-WebRequest -Uri $enActressUrl -Method Get -Verbose:$false
+            $enWebrequest = Invoke-WebRequest -Uri $enActressUrl -Method Get -WebSession:$Session -UserAgent:$Session.UserAgent -Verbose:$false
             $enActress = Get-Actress -Webrequest $enWebrequest
             $jaActress = Get-Actress -Webrequest $Webrequest
         }

src/Javinizer/Private/Scraper.Mgstage.ps1

@@ -230,7 +230,7 @@ function Get-MgstageActress {
                 $movieActressObject += [PSCustomObject]@{
                     LastName     = $null
                     FirstName    = $null
-                    JapaneseName = $actress
+                    JapaneseName = Convert-JVCleanString -String $actress
                     ThumbUrl     = $null
                 }
             } else {

src/Javinizer/Private/Test-Administrator.ps1

@@ -0,0 +1,4 @@
+function Test-Administrator {
+    $user = [Security.Principal.WindowsIdentity]::GetCurrent();
+    (New-Object Security.Principal.WindowsPrincipal $user).IsInRole([Security.Principal.WindowsBuiltinRole]::Administrator)
+}

src/Javinizer/Private/Test-JVData.ps1

@@ -5,7 +5,13 @@ function Test-JVData {
         [PSObject]$Data,
 
         [Parameter(Mandatory = $true, Position = 1)]
-        [Array]$RequiredFields
+        [Array]$RequiredFields,
+
+        [Parameter(ValueFromPipelineByPropertyName = $true)]
+        [PSObject]$AllData,
+
+        [Parameter(ValueFromPipelineByPropertyName = $true)]
+        [PSObject]$Selected
     )
 
     process {
@@ -28,6 +34,8 @@ function Test-JVData {
 
         $dataObject = [PSCustomObject]@{
             Data       = $Data
+            AllData    = $AllData
+            Selected   = $Selected
             NullFields = $nullFields
         }