Refactor and Cleanup

.github/workflows/main.yml

@@ -52,7 +52,8 @@ jobs:
         shell: pwsh
         run: |
           $env:psakeDeploy = $true
-          Invoke-psake .\PSParquet.psake.ps1 Build
+          # The build fails in GitHub Actions
+          Invoke-psake .\PSParquet.psake.ps1 Test -Verbose
 
       - name: Upload Build Artifact
         uses: actions/[email protected]

.gitignore

@@ -1,7 +1,11 @@
-src/bin/
-src/obj/
-.vs/
+src/PSParquet/bin/*
+src/PSParquet/obj
+.vs/*
+src/PSParquet/.vs/*
+output/*
+PSParquet/bin/*
 
-!src/bin/Debug/net7.0/IronCompress.dll
-!src/bin/Debug/net7.0/Microsoft.IO.RecyclableMemoryStream.dll
-!src/bin/Debug/net7.0/Parquet.dll
+# Keep the required DLLs
+!src/PSParquet/bin/Debug/net7.0/IronCompress.dll
+!src/PSParquet/bin/Debug/net7.0/Microsoft.IO.RecyclableMemoryStream.dll
+!src/PSParquet/bin/Debug/net7.0/Parquet.dll

PSParquet.psake.ps1

@@ -5,23 +5,26 @@ Properties {
     # Basic naming variables
     $Script:BuildDir = Split-Path $psake.build_script_file
     $Script:ModuleName = $(Split-Path $psake.build_script_file -leaf) -split '\.' | Select-Object -first 1
-    $Script:OutputModule = $BuildDir + '\output\' + $ModuleName
+    $Script:OutputModule = Join-Path (Join-Path $Script:BuildDir 'output') $Script:ModuleName
     # Variables for ps1 files
-    $Script:DevModuleFolder = "$BuildDir\$ModuleName"
+    $Script:DevModuleFolder = Join-Path $Script:BuildDir $Script:ModuleName
     $Script:PublicFiles = @()
     $Script:PrivateFiles = @()
     # Variables for primary 3. party dlls
-    $Script:DevBinfolder = "$Script:DevModuleFolder\bin"
+    $Script:DevBinfolder = Join-String $Script:DevModuleFolder "bin"
     $Script:NestedModules = @()
     $Script:NestedModuleFiles = @()
     # Variables for binary Powershell modules (not done yet)
-    $Script:DevSrcFolder = "$BuildDir\src\$ModuleName"
-    $Script:DevOutputFolder = "$Script:OutputModule\bin"
+    $Script:SrcDir = (Join-Path $Script:BuildDir 'src')
+    $Script:DevSrcFolder = Join-Path $Script:SrcDir $Script:ModuleName
+    $Script:DevOutputFolder = Join-Path $Script:DevSrcFolder "bin"
+    $Script:DebugDllFolder = Join-Path (Join-Path $Script:DevOutputFolder "Debug") "net7.0" 
+    $Script:OutputModuleBin = Join-Path $Script:OutputModule 'bin'
     $Script:CmdletsToExport = @()
     $Script:Cmdlets = @()
     # psm1 and psd1
-    $Script:psm1 = "$DevModuleFolder\$ModuleName.psm1"
-    $Script:psd1 = "$DevModuleFolder\$ModuleName.psd1"
+    $Script:psm1 = Join-Path $Script:DevModuleFolder $($ModuleName + ".psm1")
+    $Script:psd1 = Join-Path $Script:DevModuleFolder $($ModuleName + ".psd1")
 }
 
 Task Default -depends BuildBinaries, Initialize3PartyBinaries, InitializeModuleFile, InitializeManifestFile, UpdateHelp
@@ -39,16 +42,22 @@ Task InitializeBinary {
 
 Task BuildBinaries {
     # Do compilation of binaries, if there are any
-    if (Test-Path "$Script:BuildDir\src") {
-        if (!$(Test-Path $Script:DevSrcFolder)) { New-Item -ItemType Directory -Path $Script:DevSrcFolder -Force }
+    if (Test-Path $Script:DevSrcFolder) {
         dotnet build "$Script:DevSrcFolder"
-        $Script:CmdletsToExport = foreach ($dll in $(Get-ChildItem "$Script:BuildDir\src\$Script:ModuleName\bin\Debug\net7.0" -filter *dll)) {
-            if (!$(Test-Path $Script:DevOutputFolder)) { New-Item -ItemType Directory -Path $Script:DevOutputFolder -Force }
-            Copy-Item -Path $dll -Destination $Script:DevOutputFolder -force
-            $PlacedDll = Copy-Item -Path $dll -Destination $Script:DevBinfolder -force -PassThru
+        if (!$(Test-Path $Script:OutputModuleBin)) {
+            "Creating $Script:OutputModuleBin"
+            $null = New-Item -ItemType Directory -Path $Script:OutputModuleBin -Force
+        }
+        $Script:CmdletsToExport = foreach ($dll in $(Get-ChildItem $Script:DebugDllFolder -filter *dll)) {
+            Write-Verbose "Copying $dll to $Script:OutputModuleBin" 
+            $PlacedDll = Copy-Item -Path $dll -Destination $Script:OutputModuleBin -force -PassThru
             if ($PlacedDll.BaseName -eq $Script:ModuleName) {
-                $PlacedDll.FullName | foreach { Write-Verbose $_ -Verbose }
-                $cs = Start-Job -ScriptBlock { Import-Module $args -Verbose -PassThru | Select-Object -ExpandProperty ExportedCommands } -ArgumentList $dll.FullName | Wait-Job | Receive-Job
+                $PlacedDll.FullName | foreach { Write-Verbose $_ }
+                $cs = Start-Job -ScriptBlock {
+                    $VerbosePreference, $dll = $args
+                    Write-Verbose "Importing $dll"
+                    Import-Module $dll  -PassThru | Select-Object -ExpandProperty ExportedCommands
+                } -ArgumentList $VerbosePreference, $dll.FullName | Wait-Job | Receive-Job
                 Update-ModuleManifest -Path $Script:psd1 -FunctionsToExport $cs.Keys -NestedModules "bin/$($PlacedDll.name)"
             }
         }
@@ -64,8 +73,6 @@ Task Initialize3PartyBinaries {
     }
 }
 
-
-
 Task InitializeModuleFile {
     $Script:ClassFiles = Get-ChildItem $Script:DevModuleFolder\Classes\*.ps1 -ErrorAction SilentlyContinue
     "Class files found: $($Script:ClassFiles.count)"
@@ -83,14 +90,22 @@ Task InitializeModuleFile {
 }
 
 Task UpdateHelp -depends InitializeModuleFile {
-    Start-Job -ScriptBlock {
-        Import-Module $args[0] -Force -Global
-        if (!$(Test-Path $args[1]\docs)) {
-            New-MarkdownHelp -WithModulePage -Module $args[2] -OutputFolder $args[1]\docs
+    $ScriptBlock = {
+        $psm1File, $DevModuleFolder, $Modulename = $args
+        $Docs = Join-Path $DevModuleFolder 'docs'
+        $EnUs = Join-Path $DevModuleFolder 'en-US'
+        Import-Module $psm1File -Force -Global
+        if (!$(Test-Path $Docs)) {
+            New-MarkdownHelp -WithModulePage -Module $Modulename -OutputFolder $Docs
         }
-        Update-MarkdownHelpModule $args[1]\docs -RefreshModulePage -Force
-        New-ExternalHelp $args[1]\docs -OutputPath $args[1]\en-US\ -Force
-    } -ArgumentList $Script:psm1, $Script:DevModuleFolder, $Script:ModuleName | Wait-Job | Remove-Job
+        Update-MarkdownHelpModule $Docs -RefreshModulePage -Force
+        if (!$(Test-Path $EnUs)) {
+            New-Item -ItemType Directory -Path $EnUs
+        }
+        New-ExternalHelp $Docs -OutputPath $EnUs -Force
+    }
+    Start-Job -ScriptBlock $ScriptBlock -ArgumentList $Script:psm1, $Script:DevModuleFolder, $Script:ModuleName | Wait-Job | Receive-Job
+    Get-Job | Remove-Job
 
 }
 
@@ -156,10 +171,11 @@ Task BuildPSSecretsExtension -depends Default {
 
 Task Test -depends Build {
     start-job -scriptblock {
-        $pesterConfig = @{Path = '.\Tests\' }
+        $VerbosePreference = $args
+        $pesterConfig = @{Path = './Tests/' }
         $c = New-PesterContainer @pesterConfig
         Invoke-Pester -Container $c -Output Detailed
-    } | Wait-Job | Receive-job
+    } -ArgumentList $VerbosePreference | Wait-Job | Receive-job
     Get-Job | Remove-Job
 }
 

PSParquet/PSParquet.psd1

@@ -3,7 +3,7 @@
 #
 # Generated by: Axel Bøg Andersen
 #
-# Generated on: 10-10-2023
+# Generated on: 04-11-2023
 #
 
 @{
@@ -12,7 +12,7 @@
 RootModule = 'PSParquet.psm1'
 
 # Version number of this module.
-ModuleVersion = '0.1.0'
+ModuleVersion = '0.2.0'
 
 # Supported PSEditions
 # CompatiblePSEditions = @()

PSParquet/README.md

@@ -1,5 +1,8 @@
 # Versions
 
+0.2.0:
+* Project made Open Source
+
 0.1.0:
 * Parquet.Net updated to 4.16.4
 * Implemented low level API

PSParquet/changelog.txt

@@ -0,0 +1,9 @@
+# Versions
+
+0.2.0:
+* Project made Open Source
+
+0.1.0:
+* Parquet.Net updated to 4.16.4
+* Implemented low level API
+* Export-Parquet InputObject takes values from pipeline

PSParquet/docs/Export-Parquet.md

@@ -13,11 +13,11 @@ Export objects to Parquet file
 ## SYNTAX
 
 ```
-Export-Parquet [-FilePath] <String> [-InputObject] <Object> [-PassThru] [<CommonParameters>]
+Export-Parquet [-FilePath] <FileInfo> [-InputObject] <PSObject[]> [-PassThru] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
-Esports an array of objects to a parquet file
+Exports an array of objects to a parquet file. Make sure your objects are formatted correctly. Array, List and HashTable parameters are not supported. Convert these types to basic types before running Export-Parquet.
 
 ## EXAMPLES
 
@@ -37,13 +37,48 @@ PS C:\> $files | Export-Parquet -FilePath C:\temp\files.parquet
 
 Get a list of files recursively from the current directory and export them tp a Parquet file
 
+### Example 3
+```
+$File = C:\Temp\Test.parquet
+$data = 1..100 | foreach {
+    [pscustomobject]@{
+        Date        = (Get-Date).AddHours($_)
+        Int32       = $_
+        TypedInt    = [int]$_
+        IntWithNull = (($_ % 3 -eq 0) ? $null : $_)
+        Text        = "Iteration $_"
+    }
+}
+$data | Export-Parquet -FilePath $File -Force
+```
+
+Exports the objects to C:\Temp\Test.parquet and overwrites the file if it already exists.
+
+### Example 4
+```
+$File = C:\Temp\Test.parquet
+$data = 1..10 | foreach {
+    [pscustomobject]@{
+        name = $_ 
+        nested=@{
+            nest="blah $_"
+        }
+    }
+}
+$data | Export-Parquet -FilePath $File -Force
+
+WARNING: InputObjects contains unsupported values. Transform the data prior to running Export-Parquet.
+```
+
+The data contains nested object and returns a warning. No data will be exported.
+
 ## PARAMETERS
 
 ### -FilePath
 Path to the Parquet file. Existing file will be overwritten
 
 ```yaml
-Type: String
+Type: FileInfo
 Parameter Sets: (All)
 Aliases:
 
@@ -55,22 +90,22 @@ Accept wildcard characters: False
 ```
 
 ### -InputObject
-An array of objects to export to Parquet
+An array of objects to export to Parquet.
 
 ```yaml
-Type: Object
+Type: PSObject[]
 Parameter Sets: (All)
 Aliases:
 
 Required: True
 Position: 1
 Default value: None
-Accept pipeline input: False
+Accept pipeline input: True (ByValue)
 Accept wildcard characters: False
 ```
 
 ### -PassThru
-Passes the files to the pipeline
+Passes the objects to the pipeline
 
 ```yaml
 Type: SwitchParameter

PSParquet/docs/PSParquet.md

@@ -8,7 +8,7 @@ Locale: {{ Update Locale }}
 
 # PSParquet Module
 ## Description
-This module contains modules to import and export data from and to Parquet files
+This module contains modules to import and export data from and to Parquet files. Export objects must be "flat" objects. Nested objects are not (currently) supported. Strong typed parameters are preserved. PSObject based on Int like values are converted to doubles. This is due to the often large amount of objects in the arrays exported to Parquet.
 
 ## PSParquet Cmdlets
 ### [Export-Parquet](Export-Parquet.md)