diff --git a/src/Javinizer/Public/Get-JVData.ps1 b/src/Javinizer/Public/Get-JVData.ps1 index 22240b81..93341262 100644 --- a/src/Javinizer/Public/Get-JVData.ps1 +++ b/src/Javinizer/Public/Get-JVData.ps1 @@ -98,7 +98,7 @@ function Get-JVData { if ($R18) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - R18] [Url - $R18Url]" - Start-ThreadJob -Name "R18" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-R18" -ScriptBlock { Import-Module $using:jvModulePath if ($using:R18Url) { $using:R18Url | Get-R18Data @@ -110,7 +110,7 @@ function Get-JVData { if ($R18Zh) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - R18Zh] [Url - $R18ZhUrl]" - Start-ThreadJob -Name "R18Zh" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-R18Zh" -ScriptBlock { Import-Module $using:jvModulePath if ($using:R18ZhUrl) { $using:R18ZhUrl | Get-R18Data @@ -122,7 +122,7 @@ function Get-JVData { if ($Javlibrary) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javlibrary] [Url - $JavlibraryUrl]" - Start-ThreadJob -Name "Javlibrary" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Javlibrary" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavlibraryUrl) { $using:JavlibraryUrl | Get-JavlibraryData @@ -134,7 +134,7 @@ function Get-JVData { if ($JavlibraryJa) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavlibraryJa] [Url - $JavlibraryJaUrl]" - Start-ThreadJob -Name "JavlibraryJa" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavlibraryJa" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavlibraryJaUrl) { $using:JavlibraryJaUrl | Get-JavlibraryData @@ -146,7 +146,7 @@ function Get-JVData { if ($JavlibraryZh) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavlibraryZh] [Url - $JavlibraryZhUrl]" - Start-ThreadJob -Name "JavlibraryZh" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavlibraryZh" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavlibraryZhUrl) { $using:JavlibraryZhUrl | Get-JavlibraryData @@ -158,7 +158,7 @@ function Get-JVData { if ($Dmm) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Dmm] [Url - $DmmUrl]" - Start-ThreadJob -Name "Dmm" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Dmm" -ScriptBlock { Import-Module $using:jvModulePath if ($using:DmmUrl) { $using:DmmUrl | Get-DmmData @@ -170,7 +170,7 @@ function Get-JVData { if ($Javbus) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javbus] [Url - $JavbusUrl]" - Start-ThreadJob -Name "Javbus" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Javbus" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavbusUrl) { $using:JavbusUrl | Get-JavbusData @@ -182,7 +182,7 @@ function Get-JVData { if ($JavbusJa) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavbusJa] [Url - $JavbusJaUrl]" - Start-ThreadJob -Name "JavbusJa" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavbusJa" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavbusJaUrl) { $using:JavbusJaUrl | Get-JavbusData @@ -194,7 +194,7 @@ function Get-JVData { if ($JavbusZh) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavbusZh] [Url - $JavbusZhUrl]" - Start-ThreadJob -Name "JavbusZh" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavbusZh" -ScriptBlock { Import-Module $using:jvModulePath if ($using:JavbusZhUrl) { $using:JavbusZhUrl | Get-JavbusData @@ -206,7 +206,7 @@ function Get-JVData { if ($Jav321) { Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Jav321] [$Url - $Jav321Url]" - Start-ThreadJob -Name "Jav321" -ScriptBlock { + Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Jav321" -ScriptBlock { Import-Module $using:jvModulePath if ($using:Jav321Url) { $using:Jav321Url | Get-Jav321Data @@ -216,9 +216,9 @@ function Get-JVData { } | Out-Null } - $jobCount = (Get-Job | Where-Object { $_.Name -notlike 'PowerShell*' }).Count - $jobId = @((Get-Job | Where-Object { $_.Name -notlike "PowerShell*" } | Select-Object Id).Id) - $jobName = @((Get-Job | Where-Object { $_.Name -notlike "PowerShell*" } | Select-Object Name).Name) + $jobCount = (Get-Job | Where-Object { $_.Name -like 'jvdata-*' }).Count + $jobId = @((Get-Job | Where-Object { $_.Name -like "jvdata-*" } | Select-Object Id).Id) + $jobName = @((Get-Job | Where-Object { $_.Name -like "jvdata-*" } | Select-Object Name).Name) if ($jobCount -eq 0) { Write-JVLog -Level Warning -Message "[$Id] No scrapers were run" @@ -226,7 +226,7 @@ function Get-JVData { } else { Write-Debug "[$Id] [$($MyInvocation.MyCommand.Name)] [Waiting - Scraper jobs] [$jobName]" # Wait-Job is used separately rather than in a pipeline due to the PowerShell.Exit job that is being created during the first-run of this function - Wait-Job -Id $jobId | Out-Null + Wait-Job -Id $jobId Write-Debug "[$Id] [$($MyInvocation.MyCommand.Name)] [Completed - Scraper jobs] [$jobName]" $javinizerDataObject = Get-Job -Id $jobId | Receive-Job diff --git a/src/Javinizer/Public/Javinizer.ps1 b/src/Javinizer/Public/Javinizer.ps1 index 51b1bc39..465a16f6 100644 --- a/src/Javinizer/Public/Javinizer.ps1 +++ b/src/Javinizer/Public/Javinizer.ps1 @@ -39,6 +39,11 @@ function Javinizer { [Parameter(ParameterSetName = 'Path')] [Array]$Url, + [Parameter(ParameterSetName = 'Path')] + [ValidateRange(1, 10)] + [AllowEmptyString()] + [Int]$Multi, + [Parameter(ParameterSetName = 'Path')] [System.IO.FileInfo]$SettingsPath, @@ -346,7 +351,7 @@ function Javinizer { try { $javMovies = $Settings | Get-JVItem -Path $Path -Recurse:$Recurse -Strict:$Strict - Write-Host "[$($MyInvocation.MyCommand.Name)] [Path - $Path] [DestinationPath - $DestinationPath] [Files - $($javMovies.Count)]" + # Write-Host "[$($MyInvocation.MyCommand.Name)] [Path - $Path] [DestinationPath - $DestinationPath] [Files - $($javMovies.Count)]" } catch { Write-JVLog -Level Warning -Message "Exiting -- no valid movies detected in [$Path]" return @@ -366,21 +371,65 @@ function Javinizer { } } } else { - $index = 1 - foreach ($movie in $javMovies) { - Write-Host "[$index of $($javMovies.Count)] Sorting [$($movie.FileName)] as [$($movie.Id)]" - $index++ - $javData = Get-JVData -Id $movie.Id -Settings $Settings - if ($null -ne $javData) { - $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield' - if ($null -ne $javAggregatedData) { - $javAggregatedData | Set-JVMovie -Path $movie.FullName -DestinationPath $DestinationPath -Settings $Settings -PartNumber $movie.Partnumber -Force:$Force + + if ($PSBoundParameters.ContainsKey('Multi')) { + try { + $jvModulePath = Join-Path -Path ((Get-Item $PSScriptRoot).Parent) -ChildPath 'Javinizer.psm1' + foreach ($movie in $javMovies) { + Start-ThreadJob -Name "javinizer-$($movie.BaseName)" -ThrottleLimit $Multi -ScriptBlock { + Import-Module $using:jvModulePath + $jvMovie = $using:movie + Javinizer -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference + } -StreamingHost $Host | Out-Null + } + + $waitJobs = Get-Job -IncludeChildJob | Where-Object { $_.PSJobTypeName -eq 'ThreadJob' -and $_.Name -like 'javinizer-*' } + $totalJobs = $waitJobs.Count + $completed = 0 + while ($waitJobs.Count -ne 0) { + $runningJobs = @() + $completedJobs = @() + $otherJobs = @() + + foreach ($job in $waitJobs) { + if ($job.State -eq 'Completed') { + $completedJobs += $job + } elseif ($job.State -eq 'Running') { + $runningJobs += $job + } else { + $otherJobs += $job + } + } + + Write-Progress -Id 1 -Activity 'Javinizer' -Status "Remaining Jobs: $($waitjobs.Count)" -PercentComplete (($completed / $totalJobs) * 100) + Write-Progress -ParentId 1 -Id 2 -Activity "Max threads: [$Multi]" -Status "Sorting: $($runningJobs.Name -replace 'javinizer-', '' -join ', ')" + + $waitJobs = $runningJobs + $otherJobs + $completed += $completedJobs.Count + } + } catch { + Write-Error "[$($MyInvocation.MyCommand.Name)] There was an error starting multi sort for path: [$($getPath.FullName)] with destinationpath: [$DestinationPath] and threads: [$throttleCount]: $PSItem" + } finally { + # Stop all running jobs if script is stopped by user input + Get-Job | Remove-Job -Force + } + } else { + $index = 1 + foreach ($movie in $javMovies) { + # Write-Host "Sorting [$($movie.FileName)] as [$($movie.Id)]" + $index++ + $javData = Get-JVData -Id $movie.Id -Settings $Settings + if ($null -ne $javData) { + $javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield' + if ($null -ne $javAggregatedData) { + $javAggregatedData | Set-JVMovie -Path $movie.FullName -DestinationPath $DestinationPath -Settings $Settings -PartNumber $movie.Partnumber -Force:$Force + } else { + Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- missing required metadata fields" + return + } } else { - Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- missing required metadata fields" - return + Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- not matched" } - } else { - Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- not matched" } } } diff --git a/src/Javinizer/Public/Set-JVMovie.ps1 b/src/Javinizer/Public/Set-JVMovie.ps1 index c63338cd..b82f864d 100644 --- a/src/Javinizer/Public/Set-JVMovie.ps1 +++ b/src/Javinizer/Public/Set-JVMovie.ps1 @@ -327,7 +327,7 @@ function Set-JVMovie { $filePath = Join-Path -Path $folderPath -ChildPath "$fileName$((Get-Item -LiteralPath $Path).Extension)" if ((Get-Item -LiteralPath $DestinationPath).Directory -ne (Get-Item -LiteralPath $Path).Directory) { Move-Item -LiteralPath $Path -Destination $filePath -Force:$Force - Write-JVLog -Level Info "[$($Data.Id)] Completed [$Path] => [$filePath]" + Write-JVLog -Level Info "Completed [$Path] => [$filePath]" } } catch { Write-JVLog -Level Error -Message "[$($Data.Id)] [$($MyInvocation.MyCommand.Name)] Error occurred when renaming and moving file [$Path] to [$filePath]: $PSItem" @@ -337,7 +337,7 @@ function Set-JVMovie { $filePath = Join-Path -Path $folderPath -ChildPath (Get-Item -LiteralPath $Path).Name if ((Get-Item -LiteralPath $DestinationPath).Directory -ne (Get-Item -LiteralPath $Path).Directory) { Move-Item -LiteralPath $Path -Destination $filePath -Force:$Force - Write-JVLog -Level Info "[$($Data.Id)] Completed [$Path] => [$filePath]" + Write-JVLog -Level Info "Completed [$Path] => [$filePath]" } } catch {