Skip to content

Commit

Permalink
Add multi functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@jvlflame
jvlflame committed Sep 2, 2020
1 parent a800817 commit 9e30222
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 30 deletions.
28 changes: 14 additions & 14 deletions src/Javinizer/Public/Get-JVData.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ function Get-JVData {

if ($R18) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - R18] [Url - $R18Url]"
Start-ThreadJob -Name "R18" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-R18" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:R18Url) {
$using:R18Url | Get-R18Data
Expand All @@ -110,7 +110,7 @@ function Get-JVData {

if ($R18Zh) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - R18Zh] [Url - $R18ZhUrl]"
Start-ThreadJob -Name "R18Zh" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-R18Zh" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:R18ZhUrl) {
$using:R18ZhUrl | Get-R18Data
Expand All @@ -122,7 +122,7 @@ function Get-JVData {

if ($Javlibrary) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javlibrary] [Url - $JavlibraryUrl]"
Start-ThreadJob -Name "Javlibrary" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Javlibrary" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavlibraryUrl) {
$using:JavlibraryUrl | Get-JavlibraryData
Expand All @@ -134,7 +134,7 @@ function Get-JVData {

if ($JavlibraryJa) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavlibraryJa] [Url - $JavlibraryJaUrl]"
Start-ThreadJob -Name "JavlibraryJa" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavlibraryJa" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavlibraryJaUrl) {
$using:JavlibraryJaUrl | Get-JavlibraryData
Expand All @@ -146,7 +146,7 @@ function Get-JVData {

if ($JavlibraryZh) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavlibraryZh] [Url - $JavlibraryZhUrl]"
Start-ThreadJob -Name "JavlibraryZh" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavlibraryZh" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavlibraryZhUrl) {
$using:JavlibraryZhUrl | Get-JavlibraryData
Expand All @@ -158,7 +158,7 @@ function Get-JVData {

if ($Dmm) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Dmm] [Url - $DmmUrl]"
Start-ThreadJob -Name "Dmm" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Dmm" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:DmmUrl) {
$using:DmmUrl | Get-DmmData
Expand All @@ -170,7 +170,7 @@ function Get-JVData {

if ($Javbus) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Javbus] [Url - $JavbusUrl]"
Start-ThreadJob -Name "Javbus" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Javbus" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavbusUrl) {
$using:JavbusUrl | Get-JavbusData
Expand All @@ -182,7 +182,7 @@ function Get-JVData {

if ($JavbusJa) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavbusJa] [Url - $JavbusJaUrl]"
Start-ThreadJob -Name "JavbusJa" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavbusJa" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavbusJaUrl) {
$using:JavbusJaUrl | Get-JavbusData
Expand All @@ -194,7 +194,7 @@ function Get-JVData {

if ($JavbusZh) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - JavbusZh] [Url - $JavbusZhUrl]"
Start-ThreadJob -Name "JavbusZh" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-JavbusZh" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:JavbusZhUrl) {
$using:JavbusZhUrl | Get-JavbusData
Expand All @@ -206,7 +206,7 @@ function Get-JVData {

if ($Jav321) {
Write-JVLog -Level Debug -Message "[$Id] [$($MyInvocation.MyCommand.Name)] [Search - Jav321] [$Url - $Jav321Url]"
Start-ThreadJob -Name "Jav321" -ScriptBlock {
Start-ThreadJob -ThrottleLimit 150 -Name "jvdata-Jav321" -ScriptBlock {
Import-Module $using:jvModulePath
if ($using:Jav321Url) {
$using:Jav321Url | Get-Jav321Data
Expand All @@ -216,17 +216,17 @@ function Get-JVData {
} | Out-Null
}

$jobCount = (Get-Job | Where-Object { $_.Name -notlike 'PowerShell*' }).Count
$jobId = @((Get-Job | Where-Object { $_.Name -notlike "PowerShell*" } | Select-Object Id).Id)
$jobName = @((Get-Job | Where-Object { $_.Name -notlike "PowerShell*" } | Select-Object Name).Name)
$jobCount = (Get-Job | Where-Object { $_.Name -like 'jvdata-*' }).Count
$jobId = @((Get-Job | Where-Object { $_.Name -like "jvdata-*" } | Select-Object Id).Id)
$jobName = @((Get-Job | Where-Object { $_.Name -like "jvdata-*" } | Select-Object Name).Name)

if ($jobCount -eq 0) {
Write-JVLog -Level Warning -Message "[$Id] No scrapers were run"
return
} else {
Write-Debug "[$Id] [$($MyInvocation.MyCommand.Name)] [Waiting - Scraper jobs] [$jobName]"
# Wait-Job is used separately rather than in a pipeline due to the PowerShell.Exit job that is being created during the first-run of this function
Wait-Job -Id $jobId | Out-Null
Wait-Job -Id $jobId

Write-Debug "[$Id] [$($MyInvocation.MyCommand.Name)] [Completed - Scraper jobs] [$jobName]"
$javinizerDataObject = Get-Job -Id $jobId | Receive-Job
Expand Down
77 changes: 63 additions & 14 deletions src/Javinizer/Public/Javinizer.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ function Javinizer {
[Parameter(ParameterSetName = 'Path')]
[Array]$Url,

[Parameter(ParameterSetName = 'Path')]
[ValidateRange(1, 10)]
[AllowEmptyString()]
[Int]$Multi,

[Parameter(ParameterSetName = 'Path')]
[System.IO.FileInfo]$SettingsPath,

Expand Down Expand Up @@ -346,7 +351,7 @@ function Javinizer {

try {
$javMovies = $Settings | Get-JVItem -Path $Path -Recurse:$Recurse -Strict:$Strict
Write-Host "[$($MyInvocation.MyCommand.Name)] [Path - $Path] [DestinationPath - $DestinationPath] [Files - $($javMovies.Count)]"
# Write-Host "[$($MyInvocation.MyCommand.Name)] [Path - $Path] [DestinationPath - $DestinationPath] [Files - $($javMovies.Count)]"
} catch {
Write-JVLog -Level Warning -Message "Exiting -- no valid movies detected in [$Path]"
return
Expand All @@ -366,21 +371,65 @@ function Javinizer {
}
}
} else {
$index = 1
foreach ($movie in $javMovies) {
Write-Host "[$index of $($javMovies.Count)] Sorting [$($movie.FileName)] as [$($movie.Id)]"
$index++
$javData = Get-JVData -Id $movie.Id -Settings $Settings
if ($null -ne $javData) {
$javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
if ($null -ne $javAggregatedData) {
$javAggregatedData | Set-JVMovie -Path $movie.FullName -DestinationPath $DestinationPath -Settings $Settings -PartNumber $movie.Partnumber -Force:$Force

if ($PSBoundParameters.ContainsKey('Multi')) {
try {
$jvModulePath = Join-Path -Path ((Get-Item $PSScriptRoot).Parent) -ChildPath 'Javinizer.psm1'
foreach ($movie in $javMovies) {
Start-ThreadJob -Name "javinizer-$($movie.BaseName)" -ThrottleLimit $Multi -ScriptBlock {
Import-Module $using:jvModulePath
$jvMovie = $using:movie
Javinizer -Path $jvMovie.FullName -DestinationPath $using:DestinationPath -Set $using:Set -SettingsPath:$using:SettingsPath -Strict:$using:Strict -Force:$using:Force -Verbose:$using:VerbosePreference -Debug:$using:DebugPreference
} -StreamingHost $Host | Out-Null
}

$waitJobs = Get-Job -IncludeChildJob | Where-Object { $_.PSJobTypeName -eq 'ThreadJob' -and $_.Name -like 'javinizer-*' }
$totalJobs = $waitJobs.Count
$completed = 0
while ($waitJobs.Count -ne 0) {
$runningJobs = @()
$completedJobs = @()
$otherJobs = @()

foreach ($job in $waitJobs) {
if ($job.State -eq 'Completed') {
$completedJobs += $job
} elseif ($job.State -eq 'Running') {
$runningJobs += $job
} else {
$otherJobs += $job
}
}

Write-Progress -Id 1 -Activity 'Javinizer' -Status "Remaining Jobs: $($waitjobs.Count)" -PercentComplete (($completed / $totalJobs) * 100)
Write-Progress -ParentId 1 -Id 2 -Activity "Max threads: [$Multi]" -Status "Sorting: $($runningJobs.Name -replace 'javinizer-', '' -join ', ')"

$waitJobs = $runningJobs + $otherJobs
$completed += $completedJobs.Count
}
} catch {
Write-Error "[$($MyInvocation.MyCommand.Name)] There was an error starting multi sort for path: [$($getPath.FullName)] with destinationpath: [$DestinationPath] and threads: [$throttleCount]: $PSItem"
} finally {
# Stop all running jobs if script is stopped by user input
Get-Job | Remove-Job -Force
}
} else {
$index = 1
foreach ($movie in $javMovies) {
# Write-Host "Sorting [$($movie.FileName)] as [$($movie.Id)]"
$index++
$javData = Get-JVData -Id $movie.Id -Settings $Settings
if ($null -ne $javData) {
$javAggregatedData = $javData | Get-JVAggregatedData -Settings $Settings | Test-JVData -RequiredFields $Settings.'sort.metadata.requiredfield'
if ($null -ne $javAggregatedData) {
$javAggregatedData | Set-JVMovie -Path $movie.FullName -DestinationPath $DestinationPath -Settings $Settings -PartNumber $movie.Partnumber -Force:$Force
} else {
Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- missing required metadata fields"
return
}
} else {
Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- missing required metadata fields"
return
Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- not matched"
}
} else {
Write-JVLog -Level Warning -Message "[$($movie.FileName)] Skipped -- not matched"
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/Javinizer/Public/Set-JVMovie.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ function Set-JVMovie {
$filePath = Join-Path -Path $folderPath -ChildPath "$fileName$((Get-Item -LiteralPath $Path).Extension)"
if ((Get-Item -LiteralPath $DestinationPath).Directory -ne (Get-Item -LiteralPath $Path).Directory) {
Move-Item -LiteralPath $Path -Destination $filePath -Force:$Force
Write-JVLog -Level Info "[$($Data.Id)] Completed [$Path] => [$filePath]"
Write-JVLog -Level Info "Completed [$Path] => [$filePath]"
}
} catch {
Write-JVLog -Level Error -Message "[$($Data.Id)] [$($MyInvocation.MyCommand.Name)] Error occurred when renaming and moving file [$Path] to [$filePath]: $PSItem"
Expand All @@ -337,7 +337,7 @@ function Set-JVMovie {
$filePath = Join-Path -Path $folderPath -ChildPath (Get-Item -LiteralPath $Path).Name
if ((Get-Item -LiteralPath $DestinationPath).Directory -ne (Get-Item -LiteralPath $Path).Directory) {
Move-Item -LiteralPath $Path -Destination $filePath -Force:$Force
Write-JVLog -Level Info "[$($Data.Id)] Completed [$Path] => [$filePath]"
Write-JVLog -Level Info "Completed [$Path] => [$filePath]"

}
} catch {
Expand Down

0 comments on commit 9e30222

Please sign in to comment.