diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c190a75..490f702 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,7 +52,8 @@ jobs: shell: pwsh run: | $env:psakeDeploy = $true - Invoke-psake .\PSParquet.psake.ps1 Build + # The build fails in GitHub Actions + Invoke-psake .\PSParquet.psake.ps1 Test -Verbose - name: Upload Build Artifact uses: actions/upload-artifact@v2.2.3 diff --git a/.gitignore b/.gitignore index 1ccc151..2a5a498 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ -src/bin/ -src/obj/ -.vs/ +src/PSParquet/bin/* +src/PSParquet/obj +.vs/* +src/PSParquet/.vs/* +output/* +PSParquet/bin/* -!src/bin/Debug/net7.0/IronCompress.dll -!src/bin/Debug/net7.0/Microsoft.IO.RecyclableMemoryStream.dll -!src/bin/Debug/net7.0/Parquet.dll \ No newline at end of file +# Keep the required DLLs +!src/PSParquet/bin/Debug/net7.0/IronCompress.dll +!src/PSParquet/bin/Debug/net7.0/Microsoft.IO.RecyclableMemoryStream.dll +!src/PSParquet/bin/Debug/net7.0/Parquet.dll \ No newline at end of file diff --git a/.vs/VSWorkspaceState.json b/.vs/VSWorkspaceState.json deleted file mode 100644 index 25b83c1..0000000 --- a/.vs/VSWorkspaceState.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "ExpandedNodes": [ - "", - "\\src", - "\\src\\PSParquet" - ], - "SelectedNode": "\\src\\PSParquet\\Classes", - "PreviewInSolutionExplorer": false -} \ No newline at end of file diff --git a/.vs/slnx.sqlite b/.vs/slnx.sqlite deleted file mode 100644 index 3248e8d..0000000 Binary files a/.vs/slnx.sqlite and /dev/null differ diff --git a/PSParquet.psake.ps1 b/PSParquet.psake.ps1 index 5eb8d4f..a5471d4 100644 --- a/PSParquet.psake.ps1 +++ b/PSParquet.psake.ps1 @@ -5,23 +5,26 @@ Properties { # Basic naming variables $Script:BuildDir = Split-Path $psake.build_script_file $Script:ModuleName = $(Split-Path $psake.build_script_file -leaf) -split '\.' | Select-Object -first 1 - $Script:OutputModule = $BuildDir + '\output\' + $ModuleName + $Script:OutputModule = Join-Path (Join-Path $Script:BuildDir 'output') $Script:ModuleName # Variables for ps1 files - $Script:DevModuleFolder = "$BuildDir\$ModuleName" + $Script:DevModuleFolder = Join-Path $Script:BuildDir $Script:ModuleName $Script:PublicFiles = @() $Script:PrivateFiles = @() # Variables for primary 3. party dlls - $Script:DevBinfolder = "$Script:DevModuleFolder\bin" + $Script:DevBinfolder = Join-String $Script:DevModuleFolder "bin" $Script:NestedModules = @() $Script:NestedModuleFiles = @() # Variables for binary Powershell modules (not done yet) - $Script:DevSrcFolder = "$BuildDir\src\$ModuleName" - $Script:DevOutputFolder = "$Script:OutputModule\bin" + $Script:SrcDir = (Join-Path $Script:BuildDir 'src') + $Script:DevSrcFolder = Join-Path $Script:SrcDir $Script:ModuleName + $Script:DevOutputFolder = Join-Path $Script:DevSrcFolder "bin" + $Script:DebugDllFolder = Join-Path (Join-Path $Script:DevOutputFolder "Debug") "net7.0" + $Script:OutputModuleBin = Join-Path $Script:OutputModule 'bin' $Script:CmdletsToExport = @() $Script:Cmdlets = @() # psm1 and psd1 - $Script:psm1 = "$DevModuleFolder\$ModuleName.psm1" - $Script:psd1 = "$DevModuleFolder\$ModuleName.psd1" + $Script:psm1 = Join-Path $Script:DevModuleFolder $($ModuleName + ".psm1") + $Script:psd1 = Join-Path $Script:DevModuleFolder $($ModuleName + ".psd1") } Task Default -depends BuildBinaries, Initialize3PartyBinaries, InitializeModuleFile, InitializeManifestFile, UpdateHelp @@ -39,16 +42,22 @@ Task InitializeBinary { Task BuildBinaries { # Do compilation of binaries, if there are any - if (Test-Path "$Script:BuildDir\src") { - if (!$(Test-Path $Script:DevSrcFolder)) { New-Item -ItemType Directory -Path $Script:DevSrcFolder -Force } + if (Test-Path $Script:DevSrcFolder) { dotnet build "$Script:DevSrcFolder" - $Script:CmdletsToExport = foreach ($dll in $(Get-ChildItem "$Script:BuildDir\src\$Script:ModuleName\bin\Debug\net7.0" -filter *dll)) { - if (!$(Test-Path $Script:DevOutputFolder)) { New-Item -ItemType Directory -Path $Script:DevOutputFolder -Force } - Copy-Item -Path $dll -Destination $Script:DevOutputFolder -force - $PlacedDll = Copy-Item -Path $dll -Destination $Script:DevBinfolder -force -PassThru + if (!$(Test-Path $Script:OutputModuleBin)) { + "Creating $Script:OutputModuleBin" + $null = New-Item -ItemType Directory -Path $Script:OutputModuleBin -Force + } + $Script:CmdletsToExport = foreach ($dll in $(Get-ChildItem $Script:DebugDllFolder -filter *dll)) { + Write-Verbose "Copying $dll to $Script:OutputModuleBin" + $PlacedDll = Copy-Item -Path $dll -Destination $Script:OutputModuleBin -force -PassThru if ($PlacedDll.BaseName -eq $Script:ModuleName) { - $PlacedDll.FullName | foreach { Write-Verbose $_ -Verbose } - $cs = Start-Job -ScriptBlock { Import-Module $args -Verbose -PassThru | Select-Object -ExpandProperty ExportedCommands } -ArgumentList $dll.FullName | Wait-Job | Receive-Job + $PlacedDll.FullName | foreach { Write-Verbose $_ } + $cs = Start-Job -ScriptBlock { + $VerbosePreference, $dll = $args + Write-Verbose "Importing $dll" + Import-Module $dll -PassThru | Select-Object -ExpandProperty ExportedCommands + } -ArgumentList $VerbosePreference, $dll.FullName | Wait-Job | Receive-Job Update-ModuleManifest -Path $Script:psd1 -FunctionsToExport $cs.Keys -NestedModules "bin/$($PlacedDll.name)" } } @@ -64,8 +73,6 @@ Task Initialize3PartyBinaries { } } - - Task InitializeModuleFile { $Script:ClassFiles = Get-ChildItem $Script:DevModuleFolder\Classes\*.ps1 -ErrorAction SilentlyContinue "Class files found: $($Script:ClassFiles.count)" @@ -83,14 +90,22 @@ Task InitializeModuleFile { } Task UpdateHelp -depends InitializeModuleFile { - Start-Job -ScriptBlock { - Import-Module $args[0] -Force -Global - if (!$(Test-Path $args[1]\docs)) { - New-MarkdownHelp -WithModulePage -Module $args[2] -OutputFolder $args[1]\docs + $ScriptBlock = { + $psm1File, $DevModuleFolder, $Modulename = $args + $Docs = Join-Path $DevModuleFolder 'docs' + $EnUs = Join-Path $DevModuleFolder 'en-US' + Import-Module $psm1File -Force -Global + if (!$(Test-Path $Docs)) { + New-MarkdownHelp -WithModulePage -Module $Modulename -OutputFolder $Docs } - Update-MarkdownHelpModule $args[1]\docs -RefreshModulePage -Force - New-ExternalHelp $args[1]\docs -OutputPath $args[1]\en-US\ -Force - } -ArgumentList $Script:psm1, $Script:DevModuleFolder, $Script:ModuleName | Wait-Job | Remove-Job + Update-MarkdownHelpModule $Docs -RefreshModulePage -Force + if (!$(Test-Path $EnUs)) { + New-Item -ItemType Directory -Path $EnUs + } + New-ExternalHelp $Docs -OutputPath $EnUs -Force + } + Start-Job -ScriptBlock $ScriptBlock -ArgumentList $Script:psm1, $Script:DevModuleFolder, $Script:ModuleName | Wait-Job | Receive-Job + Get-Job | Remove-Job } @@ -156,10 +171,11 @@ Task BuildPSSecretsExtension -depends Default { Task Test -depends Build { start-job -scriptblock { - $pesterConfig = @{Path = '.\Tests\' } + $VerbosePreference = $args + $pesterConfig = @{Path = './Tests/' } $c = New-PesterContainer @pesterConfig Invoke-Pester -Container $c -Output Detailed - } | Wait-Job | Receive-job + } -ArgumentList $VerbosePreference | Wait-Job | Receive-job Get-Job | Remove-Job } diff --git a/PSParquet/PSParquet.psd1 b/PSParquet/PSParquet.psd1 index 12d34dc..b7bc843 100644 --- a/PSParquet/PSParquet.psd1 +++ b/PSParquet/PSParquet.psd1 @@ -3,7 +3,7 @@ # # Generated by: Axel Bøg Andersen # -# Generated on: 10-10-2023 +# Generated on: 04-11-2023 # @{ @@ -12,7 +12,7 @@ RootModule = 'PSParquet.psm1' # Version number of this module. -ModuleVersion = '0.1.0' +ModuleVersion = '0.2.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/PSParquet/README.md b/PSParquet/README.md index a2505eb..c43928a 100644 --- a/PSParquet/README.md +++ b/PSParquet/README.md @@ -1,5 +1,8 @@ # Versions +0.2.0: +* Project made Open Source + 0.1.0: * Parquet.Net updated to 4.16.4 * Implemented low level API diff --git a/PSParquet/bin/IronCompress.dll b/PSParquet/bin/IronCompress.dll deleted file mode 100644 index 330e1e1..0000000 Binary files a/PSParquet/bin/IronCompress.dll and /dev/null differ diff --git a/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll b/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll deleted file mode 100644 index 6d4515a..0000000 Binary files a/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll and /dev/null differ diff --git a/PSParquet/bin/PSParquet.dll b/PSParquet/bin/PSParquet.dll deleted file mode 100644 index 79cc3e1..0000000 Binary files a/PSParquet/bin/PSParquet.dll and /dev/null differ diff --git a/PSParquet/bin/Parquet.dll b/PSParquet/bin/Parquet.dll deleted file mode 100644 index 94225b9..0000000 Binary files a/PSParquet/bin/Parquet.dll and /dev/null differ diff --git a/PSParquet/changelog.txt b/PSParquet/changelog.txt index e69de29..c43928a 100644 --- a/PSParquet/changelog.txt +++ b/PSParquet/changelog.txt @@ -0,0 +1,9 @@ +# Versions + +0.2.0: +* Project made Open Source + +0.1.0: +* Parquet.Net updated to 4.16.4 +* Implemented low level API +* Export-Parquet InputObject takes values from pipeline diff --git a/PSParquet/docs/Export-Parquet.md b/PSParquet/docs/Export-Parquet.md index 408cfde..0d620cb 100644 --- a/PSParquet/docs/Export-Parquet.md +++ b/PSParquet/docs/Export-Parquet.md @@ -13,11 +13,11 @@ Export objects to Parquet file ## SYNTAX ``` -Export-Parquet [-FilePath] [-InputObject] [-PassThru] [] +Export-Parquet [-FilePath] [-InputObject] [-PassThru] [] ``` ## DESCRIPTION -Esports an array of objects to a parquet file +Exports an array of objects to a parquet file. Make sure your objects are formatted correctly. Array, List and HashTable parameters are not supported. Convert these types to basic types before running Export-Parquet. ## EXAMPLES @@ -37,13 +37,48 @@ PS C:\> $files | Export-Parquet -FilePath C:\temp\files.parquet Get a list of files recursively from the current directory and export them tp a Parquet file +### Example 3 +``` +$File = C:\Temp\Test.parquet +$data = 1..100 | foreach { + [pscustomobject]@{ + Date = (Get-Date).AddHours($_) + Int32 = $_ + TypedInt = [int]$_ + IntWithNull = (($_ % 3 -eq 0) ? $null : $_) + Text = "Iteration $_" + } +} +$data | Export-Parquet -FilePath $File -Force +``` + +Exports the objects to C:\Temp\Test.parquet and overwrites the file if it already exists. + +### Example 4 +``` +$File = C:\Temp\Test.parquet +$data = 1..10 | foreach { + [pscustomobject]@{ + name = $_ + nested=@{ + nest="blah $_" + } + } +} +$data | Export-Parquet -FilePath $File -Force + +WARNING: InputObjects contains unsupported values. Transform the data prior to running Export-Parquet. +``` + +The data contains nested object and returns a warning. No data will be exported. + ## PARAMETERS ### -FilePath Path to the Parquet file. Existing file will be overwritten ```yaml -Type: String +Type: FileInfo Parameter Sets: (All) Aliases: @@ -55,22 +90,22 @@ Accept wildcard characters: False ``` ### -InputObject -An array of objects to export to Parquet +An array of objects to export to Parquet. ```yaml -Type: Object +Type: PSObject[] Parameter Sets: (All) Aliases: Required: True Position: 1 Default value: None -Accept pipeline input: False +Accept pipeline input: True (ByValue) Accept wildcard characters: False ``` ### -PassThru -Passes the files to the pipeline +Passes the objects to the pipeline ```yaml Type: SwitchParameter diff --git a/PSParquet/docs/PSParquet.md b/PSParquet/docs/PSParquet.md index 8d9e320..522131a 100644 --- a/PSParquet/docs/PSParquet.md +++ b/PSParquet/docs/PSParquet.md @@ -8,7 +8,7 @@ Locale: {{ Update Locale }} # PSParquet Module ## Description -This module contains modules to import and export data from and to Parquet files +This module contains modules to import and export data from and to Parquet files. Export objects must be "flat" objects. Nested objects are not (currently) supported. Strong typed parameters are preserved. PSObject based on Int like values are converted to doubles. This is due to the often large amount of objects in the arrays exported to Parquet. ## PSParquet Cmdlets ### [Export-Parquet](Export-Parquet.md) diff --git a/PSParquet/en-US/PSParquet.dll-Help.xml b/PSParquet/en-US/PSParquet.dll-Help.xml index acd5b2b..4c6140c 100644 --- a/PSParquet/en-US/PSParquet.dll-Help.xml +++ b/PSParquet/en-US/PSParquet.dll-Help.xml @@ -10,7 +10,7 @@ - Esports an array of objects to a parquet file + Exports an array of objects to a parquet file. Make sure your objects are formatted correctly. Array, List and HashTable parameters are not supported. Convert these types to basic types before running Export-Parquet. @@ -20,21 +20,21 @@ Path to the Parquet file. Existing file will be overwritten - String + FileInfo - String + FileInfo None - + InputObject - An array of objects to export to Parquet + An array of objects to export to Parquet. - Object + PSObject[] - Object + PSObject[] None @@ -42,7 +42,7 @@ PassThru - Passes the files to the pipeline + Passes the objects to the pipeline SwitchParameter @@ -58,21 +58,21 @@ Path to the Parquet file. Existing file will be overwritten - String + FileInfo - String + FileInfo None - + InputObject - An array of objects to export to Parquet + An array of objects to export to Parquet. - Object + PSObject[] - Object + PSObject[] None @@ -80,7 +80,7 @@ PassThru - Passes the files to the pipeline + Passes the objects to the pipeline SwitchParameter @@ -124,6 +124,49 @@ PS C:\> Export-Parquet -InputObject $files -FilePath C:\temp\files.parquetGet a list of files recursively from the current directory and export them tp a Parquet file + + -------------------------- Example 2 -------------------------- + PS C:\> $files = Get-ChildItem -File -Recurse +PS C:\> $files | Export-Parquet -FilePath C:\temp\files.parquet + + Get a list of files recursively from the current directory and export them tp a Parquet file + + + + -------------------------- Example 3 -------------------------- + $File = C:\Temp\Test.parquet +$data = 1..100 | foreach { + [pscustomobject]@{ + Date = (Get-Date).AddHours($_) + Int32 = $_ + TypedInt = [int]$_ + IntWithNull = (($_ % 3 -eq 0) ? $null : $_) + Text = "Iteration $_" + } +} +$data | Export-Parquet -FilePath $File -Force + + Exports the objects to C:\Temp\Test.parquet and overwrites the file if it already exists. + + + + -------------------------- Example 4 -------------------------- + $File = C:\Temp\Test.parquet +$data = 1..10 | foreach { + [pscustomobject]@{ + name = $_ + nested=@{ + nest="blah $_" + } + } +} +$data | Export-Parquet -FilePath $File -Force + +WARNING: InputObjects contains unsupported values. Transform the data prior to running Export-Parquet. + + The data contains nested object and returns a warning. No data will be exported. + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..3cc22c7 --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# PSParquet + +## Import and Export Data to and from Parquet Files + +This PowerShell module is a free and open source tool that allows you to easily import and export data to and from Parquet files directly from the command line. + + +It's built on the [Parquet.Net library](https://github.com/aloneguid/parquet-dotnet) developed by https://github.com/aloneguid + +## Advantages of Using this Module + +- **Efficient Storage:** Parquet is a columnar storage format that is highly optimized for performance and space efficiency. This module makes it easy to store and retrieve very large datasets efficiently. + +- **Open Source and Free:** The module is free and open source, making it accessible to everyone and giving users the ability to contribute and improve the code. + +- **Easy to Use:** With this module, you can easily import and export data to and from Parquet files directly from the command line. This makes it an ideal tool for ETL. + +- **Works with Microsoft Fabric's OneLake Client:** This module is perfect for modern big data implementations using Microsoft products like Microsoft Fabric's OneLake client, as well as other Cloud native tools in Azure. + +## How to Use + +To get started, simply install the module from the PowerShell Gallery: + +```powershell +Install-Module PSParquet +``` + +Once installed, you can use the `Import-Parquet` and `Export-Parquet` cmdlets to import and export data to and from Parquet files. + +For example, to import data from a Parquet file: + +```powershell +Import-Parquet -Path "path/to/file.parquet" +``` + +And to export data to a Parquet file: + +```powershell +Export-Parquet -InputObject $data -Path "path/to/file.parquet" +``` + +Use the `Get-Help` cmdlet for more help. + +## Contributions + +Contributions to this open source module are always welcome! If you find a bug or have an idea for an improvement, you can submit an issue on the GitHub repository. If you'd like to contribute code, you can fork the repository, make your changes, and submit a pull request. The project's README.md file contains information on how to build and test the module locally, which can help you get started. By contributing to this module, you can help make it even more useful and accessible to others in the community. + +## Build and test + +The project consists of 2 parts: + +* A PowerShell module compiler that builds the C# code, wraps the dlls in a PowerShell and handles documentation. +* A C# project for developing the binary PowerShell module + + +### PowerShell module compiler + +The PowerShell module compiler uses the modules psake, Pester and platyPS. Make sure these modules are installed. + +To compile the module, simply run the following command in from the project root folder: + +```powershell +Invoke-psake ./PSParquet.psake.ps1 Test +``` + +Look in the psake file for a details on the tasks it carries out. + +Running the build increments the build version. To avoid that, set the environment variable `$env:psakeDeploy` to true + +### C# project + +* Load the solution in src/PSParquet. The project is a standard binary PowerShell module project. +* Setup The debugger with a Executable profile that launches PowerShell with NoProfile and imports the DLL from the debug folder + +Once the C# code has been updated with the desired changes, the PowerShell module compiler packages and exports the module to the output/PSParquet folder. This folder then have a working PowerShell module with everything required for import or upload to the PowerShell Gallery. + +## License + +This PowerShell module is released under the [MIT license](https://github.com/username/repo/blob/master/LICENSE). This means that it is free to use, modify, and distribute, even for commercial purposes, as long as the original license is included. By using this module, you agree to the terms and conditions of the MIT license. If you plan to contribute to the code, please make sure to read the license carefully to understand your rights and obligations. + +## Donations + +While this module is free and open source, donations are always appreciated to support its development and maintenance. If you find this module useful and would like to say thanks, you can [buy me a coffee](https://www.buymeacoffee.com/axely) or make a donation of your choice. Your contribution will help ensure that this module continues to be maintained and improved for the benefit of the community. Thank you for your support! \ No newline at end of file diff --git a/Tests/CodeDev.jpynb.ipynb b/Tests/CodeDev.jpynb.ipynb deleted file mode 100644 index 5063796..0000000 --- a/Tests/CodeDev.jpynb.ipynb +++ /dev/null @@ -1,609 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Installed Packages
  • IronCompress, 1.5.1
  • Parquet.Net, 4.16.4
  • PowerShellStandard.Library, 5.1.1
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "Loading extensions from `C:\\Users\\Ax\\.nuget\\packages\\microsoft.data.analysis\\0.20.1\\interactive-extensions\\dotnet\\Microsoft.Data.Analysis.Interactive.dll`" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "#r \"nuget: PowerShellStandard.Library, 5.1.1\"\n", - "#r \"nuget: Parquet.Net, 4.16.4\"\n", - "#r \"nuget: IronCompress, 1.5.1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - " public class ParquetData\n", - " {\n", - " public Type Type { get; set; }\n", - " public string Parameter { get; set; }\n", - " public dynamic[] Data { get; set; }\n", - "\n", - " public ParquetData() { }\n", - "\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Submission#3+ParquetData
Types
<null>
Parameters
<null>
Data
<null>
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "var data = new ParquetData();\n", - "return data;" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using System.Management.Automation;" - ] - }, - { - "cell_type": "code", - "execution_count": 120, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "\n", - "List io = new List();\n", - "Random rnd = new Random();\n", - "DateTime start = new DateTime(1995, 1, 1);\n", - "for (int i=0; i<1000;i++)\n", - "{\n", - " int range = ((TimeSpan)(DateTime.Today - start)).Days; \n", - " var obj = new PSObject();\n", - " obj.Members.Add(new PSNoteProperty(\"Id\", (i == 1 ? null : Guid.NewGuid().ToString())));\n", - " obj.Members.Add(new PSNoteProperty(\"NumberInt64\", (i == 1 ? null : rnd.NextInt64())));\n", - " obj.Members.Add(new PSNoteProperty(\"NumberInt\", (i == 1 ? null : rnd.Next())));\n", - " obj.Members.Add(new PSNoteProperty(\"NumberSingle\", (i == 1 ? null : rnd.NextSingle())));\n", - " obj.Members.Add(new PSNoteProperty(\"Date\", (i == 1 ? null : start.AddDays(rnd.Next(range)))));\n", - " io.Add(obj);\n", - "}\n" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
indexvalue
0
string Id=13407869-ad3c-45c1-aa31-7576b992faaa
MemberTypeNoteProperty
IsSettable
True
IsGettable
True
Value
13407869-ad3c-45c1-aa31-7576b992faaa
TypeNameOfValue
System.String
Name
Id
IsInstance
True
1
long NumberInt64=7845629069167213222
MemberTypeNoteProperty
IsSettable
True
IsGettable
True
Value
7845629069167213222
TypeNameOfValue
System.Int64
Name
NumberInt64
IsInstance
True
2
int NumberInt=1183366430
MemberTypeNoteProperty
IsSettable
True
IsGettable
True
Value
1183366430
TypeNameOfValue
System.Int32
Name
NumberInt
IsInstance
True
3
float NumberSingle=0.12362164
MemberTypeNoteProperty
IsSettable
True
IsGettable
True
Value
0.12362164
TypeNameOfValue
System.Single
Name
NumberSingle
IsInstance
True
4
datetime Date=8/29/2006 12:00:00 AM
MemberTypeNoteProperty
IsSettable
True
IsGettable
True
Value2006-08-29 00:00:00Z
TypeNameOfValue
System.DateTime
Name
Date
IsInstance
True
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "var op = io[0];\n", - "var props = io[0].Members.Where(w => w.GetType() == typeof(PSNoteProperty)).ToList() ;\n", - "return props;" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": { - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
indexvalue
0
Submission#67+ParquetData
TypeSystem.String
Parameter
Id
Data
indexvalue
0
13407869-ad3c-45c1-aa31-7576b992faaa
1
<null>
2
d5f5e218-ef1f-46d4-8ea2-dc01e78906fe
3
9eec5c8b-49e0-4b81-9d1b-b247a4e5639e
4
f8a66913-4c8d-4848-8462-1490e6774b98
5
00d8d384-3c09-4de6-afcb-47ed2505c56f
6
ad977730-0e27-462b-b718-85ad3c261d9f
7
83e7efdd-fd81-407a-9aa2-549c5c8006cc
8
eb9f6bf5-1d77-4d10-9a9f-e62666cb802c
9
47af8013-160c-4efb-97ea-0f788859f1bf
10
e29e3a3e-23a4-40a0-9daa-d487b617deb7
11
961b2fce-7816-42d0-8a85-a94913c1efd3
12
eb5790b9-b9fa-46ba-8121-1375da4a0a93
13
6c32cad5-4364-4fef-9d89-7a1e45d3721f
14
b20c4074-0b18-400e-abdb-9f7a333de315
15
9ba942dd-bad4-4d52-aad2-008a91c8864a
16
1c3272d3-831e-433d-9d2f-25a0430c1e65
17
a3517aa3-f8c6-4d73-9857-11e93f855217
18
daab4d01-1b0c-4dbe-b787-d74a61c04e9f
19
60d14592-9666-4462-b0f7-40415d02a3b7
(980 more)
1
Submission#67+ParquetData
TypeSystem.Int64
Parameter
NumberInt64
Data
indexvalue
0
7845629069167213222
1
<null>
2
2571223319784182554
3
7985874446053538612
4
4420816931674320696
5
5602574144345208453
6
7764738873350501996
7
2639691341388613735
8
6773178977122308016
9
4416214264821160745
10
1632346996002094844
11
7416908330797196143
12
8807084430715767040
13
7239457714236281931
14
9216734254666769212
15
5689991950669048508
16
2659217869213947676
17
2239368262774874772
18
7134440804552972918
19
8383528301941172495
(980 more)
2
Submission#67+ParquetData
TypeSystem.Int32
Parameter
NumberInt
Data
indexvalue
0
1183366430
1
<null>
2
1462848545
3
945868203
4
46677557
5
1165821533
6
2130713355
7
640634395
8
428596348
9
106631736
10
1128625980
11
721705016
12
1416489286
13
927904960
14
1973625565
15
69251447
16
243020427
17
680331213
18
1037198219
19
836848704
(980 more)
3
Submission#67+ParquetData
TypeSystem.Single
Parameter
NumberSingle
Data
indexvalue
0
0.12362164
1
<null>
2
0.47886884
3
0.15406251
4
0.84951955
5
0.360834
6
0.1448291
7
0.24469388
8
0.04587066
9
0.4232409
10
0.75893164
11
0.61954206
12
0.8413385
13
0.21290243
14
0.19657022
15
0.54872507
16
0.37734288
17
0.35460144
18
0.7781794
19
0.1444934
(980 more)
4
Submission#67+ParquetData
TypeSystem.DateTime
Parameter
Date
Data
indexvalue
02006-08-29 00:00:00Z
1
<null>
22002-08-13 00:00:00Z
31996-06-01 00:00:00Z
42020-10-13 00:00:00Z
52006-03-12 00:00:00Z
62020-07-06 00:00:00Z
71995-10-27 00:00:00Z
82010-09-17 00:00:00Z
91997-09-13 00:00:00Z
102001-06-07 00:00:00Z
112004-04-30 00:00:00Z
122022-02-07 00:00:00Z
132008-02-09 00:00:00Z
142000-05-22 00:00:00Z
152016-07-17 00:00:00Z
162022-03-03 00:00:00Z
171996-07-28 00:00:00Z
182017-04-13 00:00:00Z
192011-07-02 00:00:00Z
(980 more)
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "List nplist = props.Select(s => new ParquetData {\n", - " Parameter = s.Name,\n", - " Type = Type.GetType(s.TypeNameOfValue),\n", - " Data = (from o in io select o.Properties[s.Name].Value).ToArray()\n", - " }).ToList();\n", - "return nplist;" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using Parquet;\n", - "using Parquet.Data;\n", - "using Parquet.Schema;" - ] - }, - { - "cell_type": "code", - "execution_count": 116, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Id (System.String)\\r\\nNumberInt64 (System.Int64)\\r\\nNumberInt (System.Int32)\\r\\nNumberSingle (System.Single)\\r\\nDate (System.DateTime)
Fields
indexvalue
0
Id (System.String)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.String
ClrNullableIfHasNullsTypeSystem.String
SchemaTypeData
Name
Id
Path
Id
FirstPart
Id
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
1
NumberInt64 (System.Int64)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Int64
ClrNullableIfHasNullsTypeSystem.Int64
SchemaTypeData
Name
NumberInt64
Path
NumberInt64
FirstPart
NumberInt64
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
2
NumberInt (System.Int32)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Int32
ClrNullableIfHasNullsTypeSystem.Int32
SchemaTypeData
Name
NumberInt
Path
NumberInt
FirstPart
NumberInt
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
3
NumberSingle (System.Single)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Single
ClrNullableIfHasNullsTypeSystem.Single
SchemaTypeData
Name
NumberSingle
Path
NumberSingle
FirstPart
NumberSingle
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
4
Date (System.DateTime)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.DateTime
ClrNullableIfHasNullsTypeSystem.DateTime
SchemaTypeData
Name
Date
Path
Date
FirstPart
Date
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
DataFields
indexvalue
0
Id (System.String)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.String
ClrNullableIfHasNullsTypeSystem.String
SchemaTypeData
Name
Id
Path
Id
FirstPart
Id
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
1
NumberInt64 (System.Int64)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Int64
ClrNullableIfHasNullsTypeSystem.Int64
SchemaTypeData
Name
NumberInt64
Path
NumberInt64
FirstPart
NumberInt64
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
2
NumberInt (System.Int32)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Int32
ClrNullableIfHasNullsTypeSystem.Int32
SchemaTypeData
Name
NumberInt
Path
NumberInt
FirstPart
NumberInt
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
3
NumberSingle (System.Single)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.Single
ClrNullableIfHasNullsTypeSystem.Single
SchemaTypeData
Name
NumberSingle
Path
NumberSingle
FirstPart
NumberSingle
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
4
Date (System.DateTime)
IsNullable
False
HasNulls
False
IsArray
False
ClrTypeSystem.DateTime
ClrNullableIfHasNullsTypeSystem.DateTime
SchemaTypeData
Name
Date
Path
Date
FirstPart
Date
Length
1
MaxRepetitionLevel
0
MaxDefinitionLevel
0
SchemaElement
<null>
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "var schema = new ParquetSchema(\n", - " nplist.Select(s => new DataField(s.Parameter, s.Type,false))\n", - ");\n", - "return schema;" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using System;\n", - "using System.IO;\n", - "using System.ComponentModel;" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - " public static object GetTypedNull(Type type)\n", - " {\n", - " dynamic val;\n", - " switch (type.ToString())\n", - " {\n", - " case (\"System.String\"):\n", - " val = \"\";\n", - " break;\n", - " case (\"System.DateTime\"):\n", - " val = DateTime.MinValue;\n", - " break;\n", - " default:\n", - " val = 0;\n", - " break;\n", - "\n", - " }\n", - " return Convert.ChangeType(val, type);\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 118, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using (Stream fileStream = System.IO.File.OpenWrite(\"c:\\\\temp\\\\test.parquet\"))\n", - "{\n", - " using (ParquetWriter parquetWriter = await ParquetWriter.CreateAsync(schema, fileStream))\n", - " {\n", - " parquetWriter.CompressionMethod = CompressionMethod.Gzip;\n", - " parquetWriter.CompressionLevel = System.IO.Compression.CompressionLevel.Optimal;\n", - " // create a new row group in the file\n", - " using (ParquetRowGroupWriter groupWriter = parquetWriter.CreateRowGroup())\n", - " {\n", - " for (int i = 0;i < nplist.Count; i++)\n", - " {\n", - " Array arr = Array.CreateInstance(nplist[i].Type, nplist[i].Data.Count());\n", - " var data = nplist[i].Data.Select(s => s ?? GetTypedNull(nplist[i].Type)).ToArray();\n", - " Array.Copy(data, arr, nplist[i].Data.Count());\n", - " //return arr.GetEnumerator();\n", - " DataColumn dc = new DataColumn(schema.DataFields[i], arr);\n", - " await groupWriter.WriteColumnAsync(dc);\n", - " }\n", - " }\n", - " }\n", - "}\n" - ] - }, - { - "cell_type": "code", - "execution_count": 126, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - " public static async Task WriteParquetFile(List inputObject, string filePath)\n", - " {\n", - " var properties = inputObject[0].Members.Where(w => w.GetType() == typeof(PSNoteProperty)).ToList();\n", - "\n", - " List parquetData = properties.Select(s => new ParquetData\n", - " {\n", - " Parameter = s.Name,\n", - " Type = Type.GetType(s.TypeNameOfValue),\n", - " Data = (from o in inputObject select o.Properties[s.Name].Value).ToArray()\n", - " }).ToList();\n", - "\n", - " var schema = new ParquetSchema(\n", - " parquetData.Select(s => new DataField(s.Parameter, s.Type, false))\n", - " );\n", - "\n", - " using (Stream fileStream = System.IO.File.OpenWrite(filePath))\n", - " {\n", - " using (ParquetWriter parquetWriter = await ParquetWriter.CreateAsync(schema, fileStream))\n", - " {\n", - " parquetWriter.CompressionMethod = CompressionMethod.Gzip;\n", - " parquetWriter.CompressionLevel = System.IO.Compression.CompressionLevel.Optimal;\n", - " // create a new row group in the file\n", - " using (ParquetRowGroupWriter groupWriter = parquetWriter.CreateRowGroup())\n", - " {\n", - " for (int i = 0; i < parquetData.Count; i++)\n", - " {\n", - " Array arr = Array.CreateInstance(parquetData[i].Type, parquetData[i].Data.Count());\n", - " var data = parquetData[i].Data.Select(s => s ?? GetTypedNull(parquetData[i].Type)).ToArray();\n", - " Array.Copy(data, arr, parquetData[i].Data.Count());\n", - " DataColumn dc = new DataColumn(schema.DataFields[i], arr);\n", - " await groupWriter.WriteColumnAsync(dc);\n", - " }\n", - " }\n", - " }\n", - " }\n", - "\n", - " }" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "polyglot_notebook": { - "kernelName": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
System.Threading.Tasks.Task`1[System.Threading.Tasks.VoidTaskResult]
Id
8862
Exception
<null>
StatusRanToCompletion
IsCanceled
False
IsCompleted
True
IsCompletedSuccessfully
True
CreationOptionsNone
AsyncState
<null>
IsFaulted
False
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "var res = WriteParquetFile(io, \"c:\\\\temp\\\\testvscode3.parquet\");\n", - "return res;" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/Tests/PSParquet.Tests.ps1 b/Tests/PSParquet.Tests.ps1 index 488e2b9..8df10f2 100644 --- a/Tests/PSParquet.Tests.ps1 +++ b/Tests/PSParquet.Tests.ps1 @@ -1,21 +1,48 @@ Describe "Module tests" { BeforeAll { $ModulePath = Join-Path (Join-Path $(Split-Path $PSScriptRoot) 'output') 'PSParquet' - Write-Verbose $ModulePath -Verbose + Write-Verbose "Test Path: $ModulePath" Import-Module $ModulePath -Force $tempFile = New-TemporaryFile - $data = Get-ChildItem -Path $(Split-Path $PSScriptRoot) -Recurse -File - Export-Parquet -FilePath $tempFile.FullName -InputObject $data + $data = 1..100 | foreach { + [pscustomobject]@{ + Date = (Get-Date).AddHours($_) + Int32 = $_ + TypedInt = [int]$_ + IntWithNull = (($_ % 3 -eq 0) ? $null : $_) + Text = "Iteration $_" + } + } + Export-Parquet -FilePath $tempFile.FullName -InputObject $data -Force $Content = Import-Parquet -FilePath $tempFile.FullName } it "Export data to file" { $tempFile.Length | Should -BeGreaterThan 1000 } it "Import data from file" { - $Content | Should -Not -Be $Null + $Content.Count | Should -Be 100 } - it "PSParquet.dll is in the import" { - $Content.name | Should -Contain 'PSParquet.dll' + it "Date is a DateTime" { + ($Content[0].Date -is [DateTime]) | Should -Be $true + } + it "Int32 is a double" { + ($Content[0].Int32 -is [double]) | Should -Be $true + } + it "IntWithNull is a double" { + ($Content[2].IntWithNull -is [double]) | Should -Be $true + } + it "IntWithNull has 0" { + ($Content.IntWithNull -contains 0) | Should -Be $true + } + it "TypedInt is an Int" { + ($Content[0].TypedInt -is [Int32]) | Should -Be $true + } + it "Test is a string" { + ($Content[0].Text -is [String]) | Should -Be $true + } + + AfterAll { + Remove-Item $tempFile -Force } } diff --git a/output/PSParquet/PSParquet.psd1 b/output/PSParquet/PSParquet.psd1 deleted file mode 100644 index 12d34dc..0000000 --- a/output/PSParquet/PSParquet.psd1 +++ /dev/null @@ -1,132 +0,0 @@ -# -# Module manifest for module 'PSParquet' -# -# Generated by: Axel Bøg Andersen -# -# Generated on: 10-10-2023 -# - -@{ - -# Script module or binary module file associated with this manifest. -RootModule = 'PSParquet.psm1' - -# Version number of this module. -ModuleVersion = '0.1.0' - -# Supported PSEditions -# CompatiblePSEditions = @() - -# ID used to uniquely identify this module -GUID = '09f2e1e7-475e-4bb3-a02c-6f5f899ebb5c' - -# Author of this module -Author = 'Axel Bøg Andersen' - -# Company or vendor of this module -CompanyName = 'Carnaperium' - -# Copyright statement for this module -Copyright = '(c) Axel Bøg Andersen. All rights reserved.' - -# Description of the functionality provided by this module -Description = 'Import and export objects to parquet format' - -# Minimum version of the PowerShell engine required by this module -# PowerShellVersion = '' - -# Name of the PowerShell host required by this module -# PowerShellHostName = '' - -# Minimum version of the PowerShell host required by this module -# PowerShellHostVersion = '' - -# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# DotNetFrameworkVersion = '' - -# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. -# ClrVersion = '' - -# Processor architecture (None, X86, Amd64) required by this module -# ProcessorArchitecture = '' - -# Modules that must be imported into the global environment prior to importing this module -# RequiredModules = @() - -# Assemblies that must be loaded prior to importing this module -# RequiredAssemblies = @() - -# Script files (.ps1) that are run in the caller's environment prior to importing this module. -# ScriptsToProcess = @() - -# Type files (.ps1xml) to be loaded when importing this module -# TypesToProcess = @() - -# Format files (.ps1xml) to be loaded when importing this module -# FormatsToProcess = @() - -# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess -NestedModules = @('bin/PSParquet.dll') - -# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. -FunctionsToExport = 'Import-Parquet', 'Export-Parquet' - -# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. -CmdletsToExport = '*' - -# Variables to export from this module -VariablesToExport = '*' - -# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. -AliasesToExport = '*' - -# DSC resources to export from this module -# DscResourcesToExport = @() - -# List of all modules packaged with this module -# ModuleList = @() - -# List of all files packaged with this module -# FileList = @() - -# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. -PrivateData = @{ - - PSData = @{ - - # Tags applied to this module. These help with module discovery in online galleries. - # Tags = @() - - # A URL to the license for this module. - # LicenseUri = '' - - # A URL to the main website for this project. - # ProjectUri = '' - - # A URL to an icon representing this module. - # IconUri = '' - - # ReleaseNotes of this module - # ReleaseNotes = '' - - # Prerelease string of this module - # Prerelease = '' - - # Flag to indicate whether the module requires explicit user acceptance for install/update/save - # RequireLicenseAcceptance = $false - - # External dependent modules of this module - # ExternalModuleDependencies = @() - - } # End of PSData hashtable - -} # End of PrivateData hashtable - -# HelpInfo URI of this module -# HelpInfoURI = '' - -# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. -# DefaultCommandPrefix = '' - -} - diff --git a/output/PSParquet/PSParquet.psm1 b/output/PSParquet/PSParquet.psm1 deleted file mode 100644 index e69de29..0000000 diff --git a/output/PSParquet/README.md b/output/PSParquet/README.md deleted file mode 100644 index a2505eb..0000000 --- a/output/PSParquet/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Versions - -0.1.0: -* Parquet.Net updated to 4.16.4 -* Implemented low level API -* Export-Parquet InputObject takes values from pipeline diff --git a/output/PSParquet/bin/IronCompress.dll b/output/PSParquet/bin/IronCompress.dll deleted file mode 100644 index 330e1e1..0000000 Binary files a/output/PSParquet/bin/IronCompress.dll and /dev/null differ diff --git a/output/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll b/output/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll deleted file mode 100644 index 6d4515a..0000000 Binary files a/output/PSParquet/bin/Microsoft.IO.RecyclableMemoryStream.dll and /dev/null differ diff --git a/output/PSParquet/bin/PSParquet.dll b/output/PSParquet/bin/PSParquet.dll deleted file mode 100644 index 79cc3e1..0000000 Binary files a/output/PSParquet/bin/PSParquet.dll and /dev/null differ diff --git a/output/PSParquet/bin/Parquet.dll b/output/PSParquet/bin/Parquet.dll deleted file mode 100644 index 94225b9..0000000 Binary files a/output/PSParquet/bin/Parquet.dll and /dev/null differ diff --git a/output/PSParquet/changelog.txt b/output/PSParquet/changelog.txt deleted file mode 100644 index e69de29..0000000 diff --git a/output/PSParquet/en-US/PSParquet.dll-Help.xml b/output/PSParquet/en-US/PSParquet.dll-Help.xml deleted file mode 100644 index acd5b2b..0000000 --- a/output/PSParquet/en-US/PSParquet.dll-Help.xml +++ /dev/null @@ -1,253 +0,0 @@ - - - - - Export-Parquet - Export - Parquet - - Export objects to Parquet file - - - - Esports an array of objects to a parquet file - - - - Export-Parquet - - FilePath - - Path to the Parquet file. Existing file will be overwritten - - String - - String - - - None - - - InputObject - - An array of objects to export to Parquet - - Object - - Object - - - None - - - PassThru - - Passes the files to the pipeline - - - SwitchParameter - - - False - - - - - - FilePath - - Path to the Parquet file. Existing file will be overwritten - - String - - String - - - None - - - InputObject - - An array of objects to export to Parquet - - Object - - Object - - - None - - - PassThru - - Passes the files to the pipeline - - SwitchParameter - - SwitchParameter - - - False - - - - - - None - - - - - - - - - - System.Management.Automation.PSCustomObject - - - - - - - - - - - - - - -------------------------- Example 1 -------------------------- - PS C:\> $files = Get-ChildItem -File -Recurse -PS C:\> Export-Parquet -InputObject $files -FilePath C:\temp\files.parquet - - Get a list of files recursively from the current directory and export them tp a Parquet file - - - - - - - - Import-Parquet - Import - Parquet - - Import objects from Parquet file - - - - Imports a dataset from a Parquet file - - - - Import-Parquet - - FilePath - - Path to the Parquet file - - String - - String - - - None - - - - - - FilePath - - Path to the Parquet file - - String - - String - - - None - - - - - - System.String - - - - - - - - - - System.Management.Automation.PSCustomObject - - - - - - - - - - - - - - -------------------------- Example 1 -------------------------- - PS C:\> $FilesFromParquet = Import-Parquet -FilePath C:\temp\files.parquet -PS C:\> $FilesFromParquet[0] - - -PSPath : Microsoft.PowerShell.Core\FileSystem::C:\dev\PSParquet\output\PSParquet\changelog.txt -PSParentPath : Microsoft.PowerShell.Core\FileSystem::C:\dev\PSParquet\output\PSParquet -PSChildName : changelog.txt -PSDrive : -PSProvider : -PSIsContainer : False -Mode : -a--- -ModeWithoutHardLink : -a--- -VersionInfo : File: C:\dev\PSParquet\output\PSParquet\changelog.txt - InternalName: - OriginalFilename: - FileVersion: - FileDescription: - Product: - ProductVersion: - Debug: False - Patched: False - PreRelease: False - PrivateBuild: False - SpecialBuild: False - Language: - -BaseName : changelog -Target : -LinkType : -Length : 0 -DirectoryName : C:\dev\PSParquet\output\PSParquet -Directory : C:\dev\PSParquet\output\PSParquet -IsReadOnly : False -FullName : C:\dev\PSParquet\output\PSParquet\changelog.txt -Extension : .txt -Name : changelog.txt -Exists : True -CreationTime : 12-11-2021 14:37:46 +00:00 -CreationTimeUtc : 12-11-2021 13:37:46 +00:00 -LastAccessTime : 12-11-2021 18:12:59 +00:00 -LastAccessTimeUtc : 12-11-2021 17:12:59 +00:00 -LastWriteTime : 10-09-2021 07:40:41 +00:00 -LastWriteTimeUtc : 10-09-2021 05:40:41 +00:00 -LinkTarget : -Attributes : Archive - - Imports a Parquet file and stores the objects in the FilesFromParquet variable - - - - - - \ No newline at end of file diff --git a/output/PSParquet/license.txt b/output/PSParquet/license.txt deleted file mode 100644 index f3071c6..0000000 --- a/output/PSParquet/license.txt +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Axel B. Andersen - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/src/PSParquet/.vs/PSParquet/DesignTimeBuild/.dtbcache.v2 b/src/PSParquet/.vs/PSParquet/DesignTimeBuild/.dtbcache.v2 deleted file mode 100644 index 99a3b10..0000000 Binary files a/src/PSParquet/.vs/PSParquet/DesignTimeBuild/.dtbcache.v2 and /dev/null differ diff --git a/src/PSParquet/.vs/PSParquet/FileContentIndex/read.lock b/src/PSParquet/.vs/PSParquet/FileContentIndex/read.lock deleted file mode 100644 index e69de29..0000000 diff --git a/src/PSParquet/.vs/PSParquet/v17/.futdcache.v2 b/src/PSParquet/.vs/PSParquet/v17/.futdcache.v2 deleted file mode 100644 index 389897d..0000000 Binary files a/src/PSParquet/.vs/PSParquet/v17/.futdcache.v2 and /dev/null differ diff --git a/src/PSParquet/.vs/PSParquet/v17/.suo b/src/PSParquet/.vs/PSParquet/v17/.suo deleted file mode 100644 index 6ccad9c..0000000 Binary files a/src/PSParquet/.vs/PSParquet/v17/.suo and /dev/null differ diff --git a/src/PSParquet/.vs/PSParquet/v17/.wsuo b/src/PSParquet/.vs/PSParquet/v17/.wsuo deleted file mode 100644 index d32b4ac..0000000 Binary files a/src/PSParquet/.vs/PSParquet/v17/.wsuo and /dev/null differ diff --git a/src/PSParquet/.vs/PSParquet/v17/TestStore/0/000.testlog b/src/PSParquet/.vs/PSParquet/v17/TestStore/0/000.testlog deleted file mode 100644 index 6bbf10e..0000000 Binary files a/src/PSParquet/.vs/PSParquet/v17/TestStore/0/000.testlog and /dev/null differ diff --git a/src/PSParquet/.vs/PSParquet/v17/TestStore/0/testlog.manifest b/src/PSParquet/.vs/PSParquet/v17/TestStore/0/testlog.manifest deleted file mode 100644 index e92ede2..0000000 Binary files a/src/PSParquet/.vs/PSParquet/v17/TestStore/0/testlog.manifest and /dev/null differ diff --git a/src/PSParquet/.vs/ProjectEvaluation/psparquet.metadata.v5.2 b/src/PSParquet/.vs/ProjectEvaluation/psparquet.metadata.v5.2 deleted file mode 100644 index 9dc5e59..0000000 Binary files a/src/PSParquet/.vs/ProjectEvaluation/psparquet.metadata.v5.2 and /dev/null differ diff --git a/src/PSParquet/.vs/ProjectEvaluation/psparquet.projects.v5.2 b/src/PSParquet/.vs/ProjectEvaluation/psparquet.projects.v5.2 deleted file mode 100644 index 836db75..0000000 Binary files a/src/PSParquet/.vs/ProjectEvaluation/psparquet.projects.v5.2 and /dev/null differ diff --git a/src/PSParquet/ExportParquetCommand.cs b/src/PSParquet/ExportParquetCommand.cs index 5d750bd..2a0daf2 100644 --- a/src/PSParquet/ExportParquetCommand.cs +++ b/src/PSParquet/ExportParquetCommand.cs @@ -1,25 +1,10 @@ using System.Management.Automation; -using System.Management.Automation.Runspaces; using System.Collections.Generic; using System.IO; -using System; -using Parquet.Data; -using System.Linq; -using Parquet.Rows; -using Parquet.File; -using Parquet.Schema; -using Parquet.Serialization; -using Parquet; -using System.ComponentModel; -using System.Security.Cryptography.X509Certificates; -using System.Dynamic; -using PSParquet; -using System.ComponentModel.Design; -using System.Threading.Tasks; namespace PSParquet { - [Cmdlet("Export", "Parquet")] + [Cmdlet("Export", "Parquet", SupportsShouldProcess = true)] [OutputType(typeof(PSCustomObject))] public class ExportParquetCommand : PSCmdlet { @@ -41,29 +26,60 @@ public class ExportParquetCommand : PSCmdlet ValueFromPipeline = false, ValueFromPipelineByPropertyName = false)] public SwitchParameter PassThru { get; set; } + [Parameter( + Mandatory = false, + Position = 3, + ValueFromPipeline = false, + ValueFromPipelineByPropertyName = false)] + public SwitchParameter Force { get; set; } + + private readonly List inputObjects = new List(); + - // This method gets called once for each cmdlet in the pipeline when the pipeline starts executing protected override void BeginProcessing() - { - WriteVerbose("Begin block"); - } - // This method will be called for each input received from the pipeline to this cmdlet; if no input is received, this method is not called - protected override async void ProcessRecord() { WriteVerbose("Using: " + FilePath.FullName); - PSObject[] io = InputObject; - await PSParquet.WriteParquetFile(io, FilePath.FullName); + if (FilePath.Exists && !Force) + { + Force = ShouldContinue(FilePath.FullName, "Overwrite the existing file?"); + } + if (Force) + { + WriteVerbose($"Deleting: {FilePath}"); + FilePath.Delete(); + } + } - if (PassThru) + protected override void ProcessRecord() + { + if (!FilePath.Exists || Force) { - WriteObject(InputObject); + WriteDebug("Adding to List"); + inputObjects.AddRange(InputObject); } } - // This method will be called once at the end of pipeline execution; if no input is received, this method is not called protected override void EndProcessing() { - WriteVerbose("End!"); + if (!FilePath.Exists || Force) + { + var collectedObjects = inputObjects.ToArray(); + WriteVerbose($"Writing {collectedObjects.Length} objects to {FilePath.FullName}"); + bool result = PSParquet.WriteParquetFile(collectedObjects, FilePath.FullName).Result; + if (!result) + { + WriteWarning("InputObjects contains unsupported values. Transform the data prior to running Export-Parquet."); + } + else + { + WriteVerbose($"InputObject has been exported to {FilePath}"); + } + + if (PassThru) + { + WriteObject(collectedObjects); + } + } } } } diff --git a/src/PSParquet/ImportParquetCommand.cs b/src/PSParquet/ImportParquetCommand.cs index 04634f6..31ccdb9 100644 --- a/src/PSParquet/ImportParquetCommand.cs +++ b/src/PSParquet/ImportParquetCommand.cs @@ -1,16 +1,6 @@ using System.Management.Automation; -using System.Management.Automation.Runspaces; -using System.Collections.Generic; using System.IO; -using System.Linq; -using Parquet.Data; -using Parquet; -using IronCompress; -using Parquet.Rows; -//using static System.Runtime.InteropServices.JavaScript.JSType; -using Parquet.Schema; -using System.Dynamic; -using PSParquet; + namespace PSParquet { @@ -23,33 +13,26 @@ public class ImportParquetCommand : PSCmdlet Position = 0, ValueFromPipeline = true, ValueFromPipelineByPropertyName = true)] - public string FilePath { get; set; } + public FileInfo FilePath { get; set; } - // This method gets called once for each cmdlet in the pipeline when the pipeline starts executing protected override void BeginProcessing() { - WriteVerbose("Begin!"); - WriteVerbose("File: " + FilePath); - var fi = new FileInfo(FilePath); - WriteVerbose("File exists: " + fi.Exists); - if (!fi.Exists) + if (!FilePath.Exists) { - throw new FileNotFoundException("File not found: " + FilePath); + throw new FileNotFoundException($"File not found: {FilePath}"); } } - // This method will be called for each input received from the pipeline to this cmdlet; if no input is received, this method is not called + protected override void ProcessRecord() { - WriteVerbose("File: " + FilePath); - var fi = new FileInfo(FilePath); - var objs = PSParquet.GetParquetObjects(FilePath).GetAwaiter().GetResult(); - WriteObject(objs); + WriteVerbose($"File: {FilePath}"); } - // This method will be called once at the end of pipeline execution; if no input is received, this method is not called + protected override void EndProcessing() { - WriteVerbose("End!"); + var objs = PSParquet.GetParquetObjects(FilePath.FullName).GetAwaiter().GetResult(); + WriteObject(objs); } } } diff --git a/src/PSParquet/PSParquet.cs b/src/PSParquet/PSParquet.cs index 1a894fc..825e23c 100644 --- a/src/PSParquet/PSParquet.cs +++ b/src/PSParquet/PSParquet.cs @@ -4,11 +4,8 @@ using System; using System.Collections.Generic; using System.IO; -using System.ComponentModel; using System.Linq; using System.Management.Automation; -using System.Reflection.PortableExecutable; -using System.Text; using System.Threading.Tasks; using PSParquet.Classes; using System.Collections; @@ -19,9 +16,9 @@ public class PSParquet { string FilePath { get; set; } - public static async Task> GetParquetObjects(string FilePath) + public static async Task> GetParquetObjects(string FilePath) { - List objects = new List(); + List objects = new List(); Stream fileStream = File.OpenRead(FilePath); using (ParquetReader reader = await ParquetReader.CreateAsync(fileStream, leaveStreamOpen: false)) { @@ -45,38 +42,59 @@ public static async Task> GetParquetObjects(string FilePath) } return objects; } - public static object GetTypedNull(Type type) + + public static object GetTypedValue(Type type, dynamic value = null) { - dynamic val; - switch (type.ToString()) + dynamic valueResult = value; + if (valueResult is null) { - case ("System.String"): - val = ""; - break; - case ("System.DateTime"): - val = DateTime.MinValue; - break; - default: - val = 0; - break; - + switch (type.ToString()) + { + case ("System.String"): + valueResult = ""; + break; + case ("System.DateTime"): + valueResult = DateTime.MinValue; + break; + default: + valueResult = 0; + break; + } } - return Convert.ChangeType(val, type); + if (valueResult is PSObject) + { + valueResult = ((PSObject)value).BaseObject; + } + return Convert.ChangeType(valueResult, type); } - public static async Task WriteParquetFile(PSObject[] inputObject, string filePath) + + public static async Task WriteParquetFile(PSObject[] inputObject, string filePath) { var properties = inputObject[0].Members.Where(w => w.GetType() == typeof(PSNoteProperty)).ToList(); + bool dataIsValid = true; List parquetData = properties.Select(s => new ParquetData { Parameter = s.Name, - Type = Type.GetType(s.TypeNameOfValue), + // Making sure ps-typed int32s have sufficient size for all objects + Type = inputObject[0].Properties[s.Name].Value is PSObject ? Type.GetType("System.Double") : Type.GetType(s.TypeNameOfValue), Data = (from o in inputObject select o.Properties[s.Name].Value).ToArray() }).ToList(); - var schema = new ParquetSchema( - parquetData.Select(s => new DataField(s.Parameter, s.Type, false)) - ); + ParquetSchema schema = new(new DataField("Column1", typeof(string))); + + try + { + schema = new ParquetSchema( + parquetData.Select(s => new DataField(s.Parameter, s.Type, false)) + ); + } + catch (Exception ex) + { + Console.WriteLine($"Operation failed: {ex.Message}"); + dataIsValid = false; + return dataIsValid; + } using (Stream fileStream = File.OpenWrite(filePath)) @@ -88,12 +106,25 @@ public static async Task WriteParquetFile(PSObject[] inputObject, string filePat // create a new row group in the file using (ParquetRowGroupWriter groupWriter = parquetWriter.CreateRowGroup()) { - for (int i = 0; i < parquetData.Count; i++) + try + { + for (int i = 0; i < parquetData.Count; i++) + { + Type type = parquetData[i].Type; + Int64 count = parquetData[i].Data.Count(); + var rawData = parquetData[i].Data; + Array arr = Array.CreateInstance(type, count); + var data = rawData.Select(s => GetTypedValue(type, s)).ToArray(); + Array.Copy(data, arr, parquetData[i].Data.Count()); + await groupWriter.WriteColumnAsync(new DataColumn(schema.DataFields[i], arr)); + } + return true; + } + catch (Exception ex) { - Array arr = Array.CreateInstance(parquetData[i].Type, parquetData[i].Data.Count()); - var data = parquetData[i].Data.Select(s => s ?? GetTypedNull(parquetData[i].Type)).ToArray(); - Array.Copy(data, arr, parquetData[i].Data.Count()); - await groupWriter.WriteColumnAsync(new DataColumn(schema.DataFields[i], arr)); + Console.WriteLine($"Operation failed: {ex.Message}"); + dataIsValid = false; + return dataIsValid; } } } diff --git a/src/PSParquet/Properties/launchSettings.json b/src/PSParquet/Properties/launchSettings.json index ceab934..65abf6c 100644 --- a/src/PSParquet/Properties/launchSettings.json +++ b/src/PSParquet/Properties/launchSettings.json @@ -3,7 +3,7 @@ "PSParquet": { "commandName": "Executable", "executablePath": "C:\\Program Files\\PowerShell\\7\\pwsh.exe", - "commandLineArgs": "-NoExit -NoLogo -Command \"Import-Module 'C:\\repos\\PSParquet\\src\\PSParquet\\bin\\Debug\\net7.0\\PSParquet.dll';#Import-Parquet 'C:\\Temp\\filesRLE.parquet';\"" + "commandLineArgs": "-NoExit -NoLogo -Command \"Import-Module 'C:\\repos\\PSParquet\\src\\PSParquet\\bin\\Debug\\net7.0\\PSParquet.dll'\"" } } } \ No newline at end of file diff --git a/src/PSParquet/bin/Debug/net7.0/PSParquet.deps.json b/src/PSParquet/bin/Debug/net7.0/PSParquet.deps.json deleted file mode 100644 index a7d0404..0000000 --- a/src/PSParquet/bin/Debug/net7.0/PSParquet.deps.json +++ /dev/null @@ -1,255 +0,0 @@ -{ - "runtimeTarget": { - "name": ".NETCoreApp,Version=v7.0", - "signature": "" - }, - "compilationOptions": {}, - "targets": { - ".NETCoreApp,Version=v7.0": { - "PSParquet/1.0.0": { - "dependencies": { - "Parquet.Net": "4.16.4", - "PowerShellStandard.Library": "5.1.1" - }, - "runtime": { - "PSParquet.dll": {} - } - }, - "Apache.Arrow/2.0.0": { - "dependencies": { - "System.Buffers": "4.5.1", - "System.Memory": "4.5.3", - "System.Runtime.CompilerServices.Unsafe": "6.0.0", - "System.Threading.Tasks.Extensions": "4.5.2" - }, - "runtime": { - "lib/netcoreapp2.1/Apache.Arrow.dll": { - "assemblyVersion": "2.0.0.0", - "fileVersion": "2.0.0.0" - } - } - }, - "IronCompress/1.5.1": { - "dependencies": { - "Snappier": "1.1.1", - "ZstdSharp.Port": "0.7.2" - }, - "runtime": { - "lib/net7.0/IronCompress.dll": { - "assemblyVersion": "1.0.0.0", - "fileVersion": "1.5.1.0" - } - }, - "runtimeTargets": { - "runtimes/linux-arm64/native/libnironcompress.so": { - "rid": "linux-arm64", - "assetType": "native", - "fileVersion": "0.0.0.0" - }, - "runtimes/linux-x64/native/libnironcompress.so": { - "rid": "linux-x64", - "assetType": "native", - "fileVersion": "0.0.0.0" - }, - "runtimes/osx-arm64/native/libnironcompress.dylib": { - "rid": "osx-arm64", - "assetType": "native", - "fileVersion": "0.0.0.0" - }, - "runtimes/osx-x64/native/libnironcompress.dylib": { - "rid": "osx-x64", - "assetType": "native", - "fileVersion": "0.0.0.0" - }, - "runtimes/win-x64/native/nironcompress.dll": { - "rid": "win-x64", - "assetType": "native", - "fileVersion": "0.0.0.0" - } - } - }, - "Microsoft.Data.Analysis/0.20.1": { - "dependencies": { - "Apache.Arrow": "2.0.0", - "Microsoft.ML.DataView": "2.0.1", - "System.Buffers": "4.5.1", - "System.Memory": "4.5.3", - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - }, - "runtime": { - "lib/netstandard2.0/Microsoft.Data.Analysis.dll": { - "assemblyVersion": "1.0.0.0", - "fileVersion": "0.2000.123.8101" - } - } - }, - "Microsoft.IO.RecyclableMemoryStream/2.3.2": { - "runtime": { - "lib/net5.0/Microsoft.IO.RecyclableMemoryStream.dll": { - "assemblyVersion": "2.3.2.0", - "fileVersion": "2.3.2.0" - } - } - }, - "Microsoft.ML.DataView/2.0.1": { - "dependencies": { - "System.Collections.Immutable": "1.5.0", - "System.Memory": "4.5.3" - }, - "runtime": { - "lib/netstandard2.0/Microsoft.ML.DataView.dll": { - "assemblyVersion": "1.0.0.0", - "fileVersion": "2.0.123.8101" - } - } - }, - "Parquet.Net/4.16.4": { - "dependencies": { - "IronCompress": "1.5.1", - "Microsoft.Data.Analysis": "0.20.1", - "Microsoft.IO.RecyclableMemoryStream": "2.3.2" - }, - "runtime": { - "lib/net7.0/Parquet.dll": { - "assemblyVersion": "4.0.0.0", - "fileVersion": "4.16.4.0" - } - } - }, - "PowerShellStandard.Library/5.1.1": { - "runtime": { - "lib/netstandard2.0/System.Management.Automation.dll": { - "assemblyVersion": "3.0.0.0", - "fileVersion": "5.1.1.0" - } - } - }, - "Snappier/1.1.1": { - "runtime": { - "lib/net7.0/Snappier.dll": { - "assemblyVersion": "1.1.1.0", - "fileVersion": "1.1.1.0" - } - } - }, - "System.Buffers/4.5.1": {}, - "System.Collections.Immutable/1.5.0": {}, - "System.Memory/4.5.3": {}, - "System.Runtime.CompilerServices.Unsafe/6.0.0": {}, - "System.Threading.Tasks.Extensions/4.5.2": {}, - "ZstdSharp.Port/0.7.2": { - "runtime": { - "lib/net7.0/ZstdSharp.dll": { - "assemblyVersion": "0.7.2.0", - "fileVersion": "0.7.2.0" - } - } - } - } - }, - "libraries": { - "PSParquet/1.0.0": { - "type": "project", - "serviceable": false, - "sha512": "" - }, - "Apache.Arrow/2.0.0": { - "type": "package", - "serviceable": true, - "sha512": "sha512-VR+F7g41WMJhr7WoZwwp05OrbYgM5Kmj3FwFXv1g0GgAYhEoCJz3L3qpllUWq9+X/rFKkFRZ2B8XcmsbaqGhrw==", - "path": "apache.arrow/2.0.0", - "hashPath": "apache.arrow.2.0.0.nupkg.sha512" - }, - "IronCompress/1.5.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-gy/BbOvjQKw9oXd9E7ovDIB2guGJwROpUhK+8d4PQlZNLd4bQzFWYT5DiDhOFzHAowKuIpQx9XVI7heb0yEZow==", - "path": "ironcompress/1.5.1", - "hashPath": "ironcompress.1.5.1.nupkg.sha512" - }, - "Microsoft.Data.Analysis/0.20.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-Or8x/g9oM+ti2bGqUmgip9Wxwr5/bQyNP9sQpke92i/3Bi+syq31kwJ4z8Dk93FECYT8IDETfcyT21J+F4rcdw==", - "path": "microsoft.data.analysis/0.20.1", - "hashPath": "microsoft.data.analysis.0.20.1.nupkg.sha512" - }, - "Microsoft.IO.RecyclableMemoryStream/2.3.2": { - "type": "package", - "serviceable": true, - "sha512": "sha512-Oh1qXXFdJFcHozvb4H6XYLf2W0meZFuG0A+TfapFPj9z5fd4vxiARGEhAaLj/6XWQaMYIv4SH/9Q6H78Hw0E2Q==", - "path": "microsoft.io.recyclablememorystream/2.3.2", - "hashPath": "microsoft.io.recyclablememorystream.2.3.2.nupkg.sha512" - }, - "Microsoft.ML.DataView/2.0.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-w+GkAXlxaut65Lm+Fbp34YTfp0/9jGRn9uiVlL7Lls0/v+4IJM7SMTHfhvegPU48cyI6K2kzaK9j2Va/labhTA==", - "path": "microsoft.ml.dataview/2.0.1", - "hashPath": "microsoft.ml.dataview.2.0.1.nupkg.sha512" - }, - "Parquet.Net/4.16.4": { - "type": "package", - "serviceable": true, - "sha512": "sha512-xVifUzgZr7agqo9dclgsmrei2RtGOjvq/5OlbECDeoeBtIz0YTvj2r4d8/NN1FERvmwU3oZhVfxMiV4p7fCNiA==", - "path": "parquet.net/4.16.4", - "hashPath": "parquet.net.4.16.4.nupkg.sha512" - }, - "PowerShellStandard.Library/5.1.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-e31xJjG+Kjbv6YF3Yq6D4Dl3or8v7LrNF41k3CXrWozW6hR1zcOe5KYuZJaGSiAgLnwP8wcW+I3+IWEzMPZKXQ==", - "path": "powershellstandard.library/5.1.1", - "hashPath": "powershellstandard.library.5.1.1.nupkg.sha512" - }, - "Snappier/1.1.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-p0xEWUG8wYUfO24b6VTcFplBO+1ohgL9rK+1ogP8N4oGOKXlLuOLpzYtPBXNxTcYQxxmX3PJdCdqnv0tgcy5YQ==", - "path": "snappier/1.1.1", - "hashPath": "snappier.1.1.1.nupkg.sha512" - }, - "System.Buffers/4.5.1": { - "type": "package", - "serviceable": true, - "sha512": "sha512-Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==", - "path": "system.buffers/4.5.1", - "hashPath": "system.buffers.4.5.1.nupkg.sha512" - }, - "System.Collections.Immutable/1.5.0": { - "type": "package", - "serviceable": true, - "sha512": "sha512-EXKiDFsChZW0RjrZ4FYHu9aW6+P4MCgEDCklsVseRfhoO0F+dXeMSsMRAlVXIo06kGJ/zv+2w1a2uc2+kxxSaQ==", - "path": "system.collections.immutable/1.5.0", - "hashPath": "system.collections.immutable.1.5.0.nupkg.sha512" - }, - "System.Memory/4.5.3": { - "type": "package", - "serviceable": true, - "sha512": "sha512-3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", - "path": "system.memory/4.5.3", - "hashPath": "system.memory.4.5.3.nupkg.sha512" - }, - "System.Runtime.CompilerServices.Unsafe/6.0.0": { - "type": "package", - "serviceable": true, - "sha512": "sha512-/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==", - "path": "system.runtime.compilerservices.unsafe/6.0.0", - "hashPath": "system.runtime.compilerservices.unsafe.6.0.0.nupkg.sha512" - }, - "System.Threading.Tasks.Extensions/4.5.2": { - "type": "package", - "serviceable": true, - "sha512": "sha512-BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==", - "path": "system.threading.tasks.extensions/4.5.2", - "hashPath": "system.threading.tasks.extensions.4.5.2.nupkg.sha512" - }, - "ZstdSharp.Port/0.7.2": { - "type": "package", - "serviceable": true, - "sha512": "sha512-mT6Pb7qzDGapJ2ic4dAFqbzHpM+qgnasQIGR/Sofrk0fmKTQSTljG55OFXHbDJczLZWnUAFsg4bKR81uHiKzPg==", - "path": "zstdsharp.port/0.7.2", - "hashPath": "zstdsharp.port.0.7.2.nupkg.sha512" - } - } -} \ No newline at end of file diff --git a/src/PSParquet/bin/Debug/net7.0/PSParquet.dll b/src/PSParquet/bin/Debug/net7.0/PSParquet.dll deleted file mode 100644 index f42026a..0000000 Binary files a/src/PSParquet/bin/Debug/net7.0/PSParquet.dll and /dev/null differ diff --git a/src/PSParquet/bin/Debug/net7.0/PSParquet.pdb b/src/PSParquet/bin/Debug/net7.0/PSParquet.pdb deleted file mode 100644 index 21f063f..0000000 Binary files a/src/PSParquet/bin/Debug/net7.0/PSParquet.pdb and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/.NETCoreApp,Version=v7.0.AssemblyAttributes.cs b/src/PSParquet/obj/Debug/net7.0/.NETCoreApp,Version=v7.0.AssemblyAttributes.cs deleted file mode 100644 index a9058da..0000000 --- a/src/PSParquet/obj/Debug/net7.0/.NETCoreApp,Version=v7.0.AssemblyAttributes.cs +++ /dev/null @@ -1,4 +0,0 @@ -// -using System; -using System.Reflection; -[assembly: global::System.Runtime.Versioning.TargetFrameworkAttribute(".NETCoreApp,Version=v7.0", FrameworkDisplayName = ".NET 7.0")] diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfo.cs b/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfo.cs deleted file mode 100644 index 78666fe..0000000 --- a/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfo.cs +++ /dev/null @@ -1,23 +0,0 @@ -//------------------------------------------------------------------------------ -// -// This code was generated by a tool. -// Runtime Version:4.0.30319.42000 -// -// Changes to this file may cause incorrect behavior and will be lost if -// the code is regenerated. -// -//------------------------------------------------------------------------------ - -using System; -using System.Reflection; - -[assembly: System.Reflection.AssemblyCompanyAttribute("PSParquet")] -[assembly: System.Reflection.AssemblyConfigurationAttribute("Debug")] -[assembly: System.Reflection.AssemblyFileVersionAttribute("1.0.0.0")] -[assembly: System.Reflection.AssemblyInformationalVersionAttribute("1.0.0")] -[assembly: System.Reflection.AssemblyProductAttribute("PSParquet")] -[assembly: System.Reflection.AssemblyTitleAttribute("PSParquet")] -[assembly: System.Reflection.AssemblyVersionAttribute("1.0.0.0")] - -// Generated by the MSBuild WriteCodeFragment class. - diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfoInputs.cache b/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfoInputs.cache deleted file mode 100644 index ee45a90..0000000 --- a/src/PSParquet/obj/Debug/net7.0/PSParquet.AssemblyInfoInputs.cache +++ /dev/null @@ -1 +0,0 @@ -1ca522fb8bb99e57d48630a43860e14a9ef24120 diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.GeneratedMSBuildEditorConfig.editorconfig b/src/PSParquet/obj/Debug/net7.0/PSParquet.GeneratedMSBuildEditorConfig.editorconfig deleted file mode 100644 index ead75c4..0000000 --- a/src/PSParquet/obj/Debug/net7.0/PSParquet.GeneratedMSBuildEditorConfig.editorconfig +++ /dev/null @@ -1,11 +0,0 @@ -is_global = true -build_property.TargetFramework = net7.0 -build_property.TargetPlatformMinVersion = -build_property.UsingMicrosoftNETSdkWeb = -build_property.ProjectTypeGuids = -build_property.InvariantGlobalization = -build_property.PlatformNeutralAssembly = -build_property.EnforceExtendedAnalyzerRules = -build_property._SupportedPlatformList = Linux,macOS,Windows -build_property.RootNamespace = PSParquet -build_property.ProjectDir = C:\repos\PSParquet\src\PSParquet\ diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.assets.cache b/src/PSParquet/obj/Debug/net7.0/PSParquet.assets.cache deleted file mode 100644 index dffb6e2..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/PSParquet.assets.cache and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.AssemblyReference.cache b/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.AssemblyReference.cache deleted file mode 100644 index 79daa8e..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.AssemblyReference.cache and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.CoreCompileInputs.cache b/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.CoreCompileInputs.cache deleted file mode 100644 index 2da2ab9..0000000 --- a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.CoreCompileInputs.cache +++ /dev/null @@ -1 +0,0 @@ -990914655b81f2ac1d23483cf6a973be1d3c6be6 diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.FileListAbsolute.txt b/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.FileListAbsolute.txt deleted file mode 100644 index 6fca25b..0000000 --- a/src/PSParquet/obj/Debug/net7.0/PSParquet.csproj.FileListAbsolute.txt +++ /dev/null @@ -1,12 +0,0 @@ -C:\repos\PSParquet\src\PSParquet\bin\Debug\net7.0\PSParquet.deps.json -C:\repos\PSParquet\src\PSParquet\bin\Debug\net7.0\PSParquet.dll -C:\repos\PSParquet\src\PSParquet\bin\Debug\net7.0\PSParquet.pdb -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.csproj.AssemblyReference.cache -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.GeneratedMSBuildEditorConfig.editorconfig -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.AssemblyInfoInputs.cache -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.AssemblyInfo.cs -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.csproj.CoreCompileInputs.cache -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.dll -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\refint\PSParquet.dll -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\PSParquet.pdb -C:\repos\PSParquet\src\PSParquet\obj\Debug\net7.0\ref\PSParquet.dll diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.dll b/src/PSParquet/obj/Debug/net7.0/PSParquet.dll deleted file mode 100644 index f42026a..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/PSParquet.dll and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/PSParquet.pdb b/src/PSParquet/obj/Debug/net7.0/PSParquet.pdb deleted file mode 100644 index 21f063f..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/PSParquet.pdb and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/ref/PSParquet.dll b/src/PSParquet/obj/Debug/net7.0/ref/PSParquet.dll deleted file mode 100644 index f5add72..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/ref/PSParquet.dll and /dev/null differ diff --git a/src/PSParquet/obj/Debug/net7.0/refint/PSParquet.dll b/src/PSParquet/obj/Debug/net7.0/refint/PSParquet.dll deleted file mode 100644 index f5add72..0000000 Binary files a/src/PSParquet/obj/Debug/net7.0/refint/PSParquet.dll and /dev/null differ diff --git a/src/PSParquet/obj/PSParquet.csproj.nuget.dgspec.json b/src/PSParquet/obj/PSParquet.csproj.nuget.dgspec.json deleted file mode 100644 index b2dafa7..0000000 --- a/src/PSParquet/obj/PSParquet.csproj.nuget.dgspec.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "format": 1, - "restore": { - "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj": {} - }, - "projects": { - "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj": { - "version": "1.0.0", - "restore": { - "projectUniqueName": "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj", - "projectName": "PSParquet", - "projectPath": "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj", - "packagesPath": "C:\\Users\\Ax\\.nuget\\packages\\", - "outputPath": "C:\\repos\\PSParquet\\src\\PSParquet\\obj\\", - "projectStyle": "PackageReference", - "configFilePaths": [ - "C:\\Users\\Ax\\AppData\\Roaming\\NuGet\\NuGet.Config", - "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config" - ], - "originalTargetFrameworks": [ - "net7.0" - ], - "sources": { - "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {}, - "https://api.nuget.org/v3/index.json": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2/index.json": {} - }, - "frameworks": { - "net7.0": { - "targetAlias": "net7.0", - "projectReferences": {} - } - }, - "warningProperties": { - "warnAsError": [ - "NU1605" - ] - } - }, - "frameworks": { - "net7.0": { - "targetAlias": "net7.0", - "dependencies": { - "Parquet.Net": { - "target": "Package", - "version": "[4.16.4, )" - }, - "PowerShellStandard.Library": { - "target": "Package", - "version": "[5.1.1, )" - } - }, - "imports": [ - "net461", - "net462", - "net47", - "net471", - "net472", - "net48", - "net481" - ], - "assetTargetFallback": true, - "warn": true, - "frameworkReferences": { - "Microsoft.NETCore.App": { - "privateAssets": "all" - } - }, - "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\7.0.111\\RuntimeIdentifierGraph.json" - } - } - } - } -} \ No newline at end of file diff --git a/src/PSParquet/obj/PSParquet.csproj.nuget.g.props b/src/PSParquet/obj/PSParquet.csproj.nuget.g.props deleted file mode 100644 index 301126f..0000000 --- a/src/PSParquet/obj/PSParquet.csproj.nuget.g.props +++ /dev/null @@ -1,15 +0,0 @@ - - - - True - NuGet - $(MSBuildThisFileDirectory)project.assets.json - $(UserProfile)\.nuget\packages\ - C:\Users\Ax\.nuget\packages\ - PackageReference - 6.4.0 - - - - - \ No newline at end of file diff --git a/src/PSParquet/obj/PSParquet.csproj.nuget.g.targets b/src/PSParquet/obj/PSParquet.csproj.nuget.g.targets deleted file mode 100644 index 3dc06ef..0000000 --- a/src/PSParquet/obj/PSParquet.csproj.nuget.g.targets +++ /dev/null @@ -1,2 +0,0 @@ - - \ No newline at end of file diff --git a/src/PSParquet/obj/project.assets.json b/src/PSParquet/obj/project.assets.json deleted file mode 100644 index c925466..0000000 --- a/src/PSParquet/obj/project.assets.json +++ /dev/null @@ -1,596 +0,0 @@ -{ - "version": 3, - "targets": { - "net7.0": { - "Apache.Arrow/2.0.0": { - "type": "package", - "dependencies": { - "System.Buffers": "4.5.0", - "System.Memory": "4.5.2", - "System.Runtime.CompilerServices.Unsafe": "4.5.2", - "System.Threading.Tasks.Extensions": "4.5.2" - }, - "compile": { - "lib/netcoreapp2.1/Apache.Arrow.dll": {} - }, - "runtime": { - "lib/netcoreapp2.1/Apache.Arrow.dll": {} - } - }, - "IronCompress/1.5.1": { - "type": "package", - "dependencies": { - "Snappier": "1.1.1", - "ZstdSharp.Port": "0.7.2" - }, - "compile": { - "lib/net7.0/IronCompress.dll": {} - }, - "runtime": { - "lib/net7.0/IronCompress.dll": {} - }, - "runtimeTargets": { - "runtimes/linux-arm64/native/libnironcompress.so": { - "assetType": "native", - "rid": "linux-arm64" - }, - "runtimes/linux-x64/native/libnironcompress.so": { - "assetType": "native", - "rid": "linux-x64" - }, - "runtimes/osx-arm64/native/libnironcompress.dylib": { - "assetType": "native", - "rid": "osx-arm64" - }, - "runtimes/osx-x64/native/libnironcompress.dylib": { - "assetType": "native", - "rid": "osx-x64" - }, - "runtimes/win-x64/native/nironcompress.dll": { - "assetType": "native", - "rid": "win-x64" - } - } - }, - "Microsoft.Data.Analysis/0.20.1": { - "type": "package", - "dependencies": { - "Apache.Arrow": "2.0.0", - "Microsoft.ML.DataView": "2.0.1", - "System.Buffers": "4.5.1", - "System.Memory": "4.5.3", - "System.Runtime.CompilerServices.Unsafe": "6.0.0" - }, - "compile": { - "lib/netstandard2.0/Microsoft.Data.Analysis.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/netstandard2.0/Microsoft.Data.Analysis.dll": { - "related": ".xml" - } - } - }, - "Microsoft.IO.RecyclableMemoryStream/2.3.2": { - "type": "package", - "compile": { - "lib/net5.0/Microsoft.IO.RecyclableMemoryStream.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/net5.0/Microsoft.IO.RecyclableMemoryStream.dll": { - "related": ".xml" - } - } - }, - "Microsoft.ML.DataView/2.0.1": { - "type": "package", - "dependencies": { - "System.Collections.Immutable": "1.5.0", - "System.Memory": "4.5.3" - }, - "compile": { - "lib/netstandard2.0/Microsoft.ML.DataView.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/netstandard2.0/Microsoft.ML.DataView.dll": { - "related": ".xml" - } - } - }, - "Parquet.Net/4.16.4": { - "type": "package", - "dependencies": { - "IronCompress": "1.5.1", - "Microsoft.Data.Analysis": "0.20.1", - "Microsoft.IO.RecyclableMemoryStream": "2.3.2" - }, - "compile": { - "lib/net7.0/Parquet.dll": { - "related": ".pdb;.xml" - } - }, - "runtime": { - "lib/net7.0/Parquet.dll": { - "related": ".pdb;.xml" - } - } - }, - "PowerShellStandard.Library/5.1.1": { - "type": "package", - "compile": { - "lib/netstandard2.0/System.Management.Automation.dll": {} - }, - "runtime": { - "lib/netstandard2.0/System.Management.Automation.dll": {} - } - }, - "Snappier/1.1.1": { - "type": "package", - "compile": { - "lib/net7.0/Snappier.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/net7.0/Snappier.dll": { - "related": ".xml" - } - } - }, - "System.Buffers/4.5.1": { - "type": "package", - "compile": { - "ref/netcoreapp2.0/_._": {} - }, - "runtime": { - "lib/netcoreapp2.0/_._": {} - } - }, - "System.Collections.Immutable/1.5.0": { - "type": "package", - "compile": { - "lib/netstandard2.0/System.Collections.Immutable.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/netstandard2.0/System.Collections.Immutable.dll": { - "related": ".xml" - } - } - }, - "System.Memory/4.5.3": { - "type": "package", - "compile": { - "ref/netcoreapp2.1/_._": {} - }, - "runtime": { - "lib/netcoreapp2.1/_._": {} - } - }, - "System.Runtime.CompilerServices.Unsafe/6.0.0": { - "type": "package", - "compile": { - "lib/net6.0/System.Runtime.CompilerServices.Unsafe.dll": { - "related": ".xml" - } - }, - "runtime": { - "lib/net6.0/System.Runtime.CompilerServices.Unsafe.dll": { - "related": ".xml" - } - }, - "build": { - "buildTransitive/netcoreapp3.1/_._": {} - } - }, - "System.Threading.Tasks.Extensions/4.5.2": { - "type": "package", - "compile": { - "ref/netcoreapp2.1/_._": {} - }, - "runtime": { - "lib/netcoreapp2.1/_._": {} - } - }, - "ZstdSharp.Port/0.7.2": { - "type": "package", - "compile": { - "lib/net7.0/ZstdSharp.dll": {} - }, - "runtime": { - "lib/net7.0/ZstdSharp.dll": {} - } - } - } - }, - "libraries": { - "Apache.Arrow/2.0.0": { - "sha512": "VR+F7g41WMJhr7WoZwwp05OrbYgM5Kmj3FwFXv1g0GgAYhEoCJz3L3qpllUWq9+X/rFKkFRZ2B8XcmsbaqGhrw==", - "type": "package", - "path": "apache.arrow/2.0.0", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE.txt", - "apache.arrow.2.0.0.nupkg.sha512", - "apache.arrow.nuspec", - "lib/netcoreapp2.1/Apache.Arrow.dll", - "lib/netstandard1.3/Apache.Arrow.dll" - ] - }, - "IronCompress/1.5.1": { - "sha512": "gy/BbOvjQKw9oXd9E7ovDIB2guGJwROpUhK+8d4PQlZNLd4bQzFWYT5DiDhOFzHAowKuIpQx9XVI7heb0yEZow==", - "type": "package", - "path": "ironcompress/1.5.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "README.md", - "icon.png", - "ironcompress.1.5.1.nupkg.sha512", - "ironcompress.nuspec", - "lib/net6.0/IronCompress.dll", - "lib/net7.0/IronCompress.dll", - "lib/netcoreapp3.1/IronCompress.dll", - "lib/netstandard2.0/IronCompress.dll", - "lib/netstandard2.1/IronCompress.dll", - "runtimes/linux-arm64/native/libnironcompress.so", - "runtimes/linux-x64/native/libnironcompress.so", - "runtimes/osx-arm64/native/libnironcompress.dylib", - "runtimes/osx-x64/native/libnironcompress.dylib", - "runtimes/win-x64/native/nironcompress.dll" - ] - }, - "Microsoft.Data.Analysis/0.20.1": { - "sha512": "Or8x/g9oM+ti2bGqUmgip9Wxwr5/bQyNP9sQpke92i/3Bi+syq31kwJ4z8Dk93FECYT8IDETfcyT21J+F4rcdw==", - "type": "package", - "path": "microsoft.data.analysis/0.20.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE", - "THIRD-PARTY-NOTICES.TXT", - "interactive-extensions/dotnet/Microsoft.Data.Analysis.Interactive.dll", - "interactive-extensions/dotnet/Microsoft.Data.Analysis.Interactive.xml", - "lib/netstandard2.0/Microsoft.Data.Analysis.dll", - "lib/netstandard2.0/Microsoft.Data.Analysis.xml", - "microsoft.data.analysis.0.20.1.nupkg.sha512", - "microsoft.data.analysis.nuspec", - "mlnetlogo.png" - ] - }, - "Microsoft.IO.RecyclableMemoryStream/2.3.2": { - "sha512": "Oh1qXXFdJFcHozvb4H6XYLf2W0meZFuG0A+TfapFPj9z5fd4vxiARGEhAaLj/6XWQaMYIv4SH/9Q6H78Hw0E2Q==", - "type": "package", - "path": "microsoft.io.recyclablememorystream/2.3.2", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "README.md", - "lib/net462/Microsoft.IO.RecyclableMemoryStream.dll", - "lib/net462/Microsoft.IO.RecyclableMemoryStream.xml", - "lib/net5.0/Microsoft.IO.RecyclableMemoryStream.dll", - "lib/net5.0/Microsoft.IO.RecyclableMemoryStream.xml", - "lib/netcoreapp2.1/Microsoft.IO.RecyclableMemoryStream.dll", - "lib/netcoreapp2.1/Microsoft.IO.RecyclableMemoryStream.xml", - "lib/netstandard2.0/Microsoft.IO.RecyclableMemoryStream.dll", - "lib/netstandard2.0/Microsoft.IO.RecyclableMemoryStream.xml", - "lib/netstandard2.1/Microsoft.IO.RecyclableMemoryStream.dll", - "lib/netstandard2.1/Microsoft.IO.RecyclableMemoryStream.xml", - "microsoft.io.recyclablememorystream.2.3.2.nupkg.sha512", - "microsoft.io.recyclablememorystream.nuspec" - ] - }, - "Microsoft.ML.DataView/2.0.1": { - "sha512": "w+GkAXlxaut65Lm+Fbp34YTfp0/9jGRn9uiVlL7Lls0/v+4IJM7SMTHfhvegPU48cyI6K2kzaK9j2Va/labhTA==", - "type": "package", - "path": "microsoft.ml.dataview/2.0.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE", - "THIRD-PARTY-NOTICES.TXT", - "lib/netstandard2.0/Microsoft.ML.DataView.dll", - "lib/netstandard2.0/Microsoft.ML.DataView.xml", - "microsoft.ml.dataview.2.0.1.nupkg.sha512", - "microsoft.ml.dataview.nuspec", - "mlnetlogo.png" - ] - }, - "Parquet.Net/4.16.4": { - "sha512": "xVifUzgZr7agqo9dclgsmrei2RtGOjvq/5OlbECDeoeBtIz0YTvj2r4d8/NN1FERvmwU3oZhVfxMiV4p7fCNiA==", - "type": "package", - "path": "parquet.net/4.16.4", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "README.md", - "icon.png", - "lib/net6.0/Parquet.dll", - "lib/net6.0/Parquet.pdb", - "lib/net6.0/Parquet.xml", - "lib/net7.0/Parquet.dll", - "lib/net7.0/Parquet.pdb", - "lib/net7.0/Parquet.xml", - "lib/netcoreapp3.1/Parquet.dll", - "lib/netcoreapp3.1/Parquet.pdb", - "lib/netcoreapp3.1/Parquet.xml", - "lib/netstandard2.0/Parquet.dll", - "lib/netstandard2.0/Parquet.pdb", - "lib/netstandard2.0/Parquet.xml", - "lib/netstandard2.1/Parquet.dll", - "lib/netstandard2.1/Parquet.pdb", - "lib/netstandard2.1/Parquet.xml", - "parquet.net.4.16.4.nupkg.sha512", - "parquet.net.nuspec" - ] - }, - "PowerShellStandard.Library/5.1.1": { - "sha512": "e31xJjG+Kjbv6YF3Yq6D4Dl3or8v7LrNF41k3CXrWozW6hR1zcOe5KYuZJaGSiAgLnwP8wcW+I3+IWEzMPZKXQ==", - "type": "package", - "path": "powershellstandard.library/5.1.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "lib/net452/System.Management.Automation.dll", - "lib/netstandard2.0/System.Management.Automation.dll", - "powershellstandard.library.5.1.1.nupkg.sha512", - "powershellstandard.library.nuspec" - ] - }, - "Snappier/1.1.1": { - "sha512": "p0xEWUG8wYUfO24b6VTcFplBO+1ohgL9rK+1ogP8N4oGOKXlLuOLpzYtPBXNxTcYQxxmX3PJdCdqnv0tgcy5YQ==", - "type": "package", - "path": "snappier/1.1.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "COPYING.txt", - "lib/net6.0/Snappier.dll", - "lib/net6.0/Snappier.xml", - "lib/net7.0/Snappier.dll", - "lib/net7.0/Snappier.xml", - "lib/netstandard2.0/Snappier.dll", - "lib/netstandard2.0/Snappier.xml", - "snappier.1.1.1.nupkg.sha512", - "snappier.nuspec" - ] - }, - "System.Buffers/4.5.1": { - "sha512": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==", - "type": "package", - "path": "system.buffers/4.5.1", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE.TXT", - "THIRD-PARTY-NOTICES.TXT", - "lib/net461/System.Buffers.dll", - "lib/net461/System.Buffers.xml", - "lib/netcoreapp2.0/_._", - "lib/netstandard1.1/System.Buffers.dll", - "lib/netstandard1.1/System.Buffers.xml", - "lib/netstandard2.0/System.Buffers.dll", - "lib/netstandard2.0/System.Buffers.xml", - "lib/uap10.0.16299/_._", - "ref/net45/System.Buffers.dll", - "ref/net45/System.Buffers.xml", - "ref/netcoreapp2.0/_._", - "ref/netstandard1.1/System.Buffers.dll", - "ref/netstandard1.1/System.Buffers.xml", - "ref/netstandard2.0/System.Buffers.dll", - "ref/netstandard2.0/System.Buffers.xml", - "ref/uap10.0.16299/_._", - "system.buffers.4.5.1.nupkg.sha512", - "system.buffers.nuspec", - "useSharedDesignerContext.txt", - "version.txt" - ] - }, - "System.Collections.Immutable/1.5.0": { - "sha512": "EXKiDFsChZW0RjrZ4FYHu9aW6+P4MCgEDCklsVseRfhoO0F+dXeMSsMRAlVXIo06kGJ/zv+2w1a2uc2+kxxSaQ==", - "type": "package", - "path": "system.collections.immutable/1.5.0", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE.TXT", - "THIRD-PARTY-NOTICES.TXT", - "lib/netstandard1.0/System.Collections.Immutable.dll", - "lib/netstandard1.0/System.Collections.Immutable.xml", - "lib/netstandard1.3/System.Collections.Immutable.dll", - "lib/netstandard1.3/System.Collections.Immutable.xml", - "lib/netstandard2.0/System.Collections.Immutable.dll", - "lib/netstandard2.0/System.Collections.Immutable.xml", - "lib/portable-net45+win8+wp8+wpa81/System.Collections.Immutable.dll", - "lib/portable-net45+win8+wp8+wpa81/System.Collections.Immutable.xml", - "system.collections.immutable.1.5.0.nupkg.sha512", - "system.collections.immutable.nuspec", - "useSharedDesignerContext.txt", - "version.txt" - ] - }, - "System.Memory/4.5.3": { - "sha512": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==", - "type": "package", - "path": "system.memory/4.5.3", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE.TXT", - "THIRD-PARTY-NOTICES.TXT", - "lib/netcoreapp2.1/_._", - "lib/netstandard1.1/System.Memory.dll", - "lib/netstandard1.1/System.Memory.xml", - "lib/netstandard2.0/System.Memory.dll", - "lib/netstandard2.0/System.Memory.xml", - "ref/netcoreapp2.1/_._", - "system.memory.4.5.3.nupkg.sha512", - "system.memory.nuspec", - "useSharedDesignerContext.txt", - "version.txt" - ] - }, - "System.Runtime.CompilerServices.Unsafe/6.0.0": { - "sha512": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==", - "type": "package", - "path": "system.runtime.compilerservices.unsafe/6.0.0", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "Icon.png", - "LICENSE.TXT", - "THIRD-PARTY-NOTICES.TXT", - "buildTransitive/netcoreapp2.0/System.Runtime.CompilerServices.Unsafe.targets", - "buildTransitive/netcoreapp3.1/_._", - "lib/net461/System.Runtime.CompilerServices.Unsafe.dll", - "lib/net461/System.Runtime.CompilerServices.Unsafe.xml", - "lib/net6.0/System.Runtime.CompilerServices.Unsafe.dll", - "lib/net6.0/System.Runtime.CompilerServices.Unsafe.xml", - "lib/netcoreapp3.1/System.Runtime.CompilerServices.Unsafe.dll", - "lib/netcoreapp3.1/System.Runtime.CompilerServices.Unsafe.xml", - "lib/netstandard2.0/System.Runtime.CompilerServices.Unsafe.dll", - "lib/netstandard2.0/System.Runtime.CompilerServices.Unsafe.xml", - "system.runtime.compilerservices.unsafe.6.0.0.nupkg.sha512", - "system.runtime.compilerservices.unsafe.nuspec", - "useSharedDesignerContext.txt" - ] - }, - "System.Threading.Tasks.Extensions/4.5.2": { - "sha512": "BG/TNxDFv0svAzx8OiMXDlsHfGw623BZ8tCXw4YLhDFDvDhNUEV58jKYMGRnkbJNm7c3JNNJDiN7JBMzxRBR2w==", - "type": "package", - "path": "system.threading.tasks.extensions/4.5.2", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "LICENSE.TXT", - "THIRD-PARTY-NOTICES.TXT", - "lib/MonoAndroid10/_._", - "lib/MonoTouch10/_._", - "lib/netcoreapp2.1/_._", - "lib/netstandard1.0/System.Threading.Tasks.Extensions.dll", - "lib/netstandard1.0/System.Threading.Tasks.Extensions.xml", - "lib/netstandard2.0/System.Threading.Tasks.Extensions.dll", - "lib/netstandard2.0/System.Threading.Tasks.Extensions.xml", - "lib/portable-net45+win8+wp8+wpa81/System.Threading.Tasks.Extensions.dll", - "lib/portable-net45+win8+wp8+wpa81/System.Threading.Tasks.Extensions.xml", - "lib/xamarinios10/_._", - "lib/xamarinmac20/_._", - "lib/xamarintvos10/_._", - "lib/xamarinwatchos10/_._", - "ref/MonoAndroid10/_._", - "ref/MonoTouch10/_._", - "ref/netcoreapp2.1/_._", - "ref/xamarinios10/_._", - "ref/xamarinmac20/_._", - "ref/xamarintvos10/_._", - "ref/xamarinwatchos10/_._", - "system.threading.tasks.extensions.4.5.2.nupkg.sha512", - "system.threading.tasks.extensions.nuspec", - "useSharedDesignerContext.txt", - "version.txt" - ] - }, - "ZstdSharp.Port/0.7.2": { - "sha512": "mT6Pb7qzDGapJ2ic4dAFqbzHpM+qgnasQIGR/Sofrk0fmKTQSTljG55OFXHbDJczLZWnUAFsg4bKR81uHiKzPg==", - "type": "package", - "path": "zstdsharp.port/0.7.2", - "files": [ - ".nupkg.metadata", - ".signature.p7s", - "lib/net461/ZstdSharp.dll", - "lib/net5.0/ZstdSharp.dll", - "lib/net6.0/ZstdSharp.dll", - "lib/net7.0/ZstdSharp.dll", - "lib/netcoreapp3.1/ZstdSharp.dll", - "lib/netstandard2.0/ZstdSharp.dll", - "lib/netstandard2.1/ZstdSharp.dll", - "zstdsharp.port.0.7.2.nupkg.sha512", - "zstdsharp.port.nuspec" - ] - } - }, - "projectFileDependencyGroups": { - "net7.0": [ - "Parquet.Net >= 4.16.4", - "PowerShellStandard.Library >= 5.1.1" - ] - }, - "packageFolders": { - "C:\\Users\\Ax\\.nuget\\packages\\": {} - }, - "project": { - "version": "1.0.0", - "restore": { - "projectUniqueName": "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj", - "projectName": "PSParquet", - "projectPath": "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj", - "packagesPath": "C:\\Users\\Ax\\.nuget\\packages\\", - "outputPath": "C:\\repos\\PSParquet\\src\\PSParquet\\obj\\", - "projectStyle": "PackageReference", - "configFilePaths": [ - "C:\\Users\\Ax\\AppData\\Roaming\\NuGet\\NuGet.Config", - "C:\\Program Files (x86)\\NuGet\\Config\\Microsoft.VisualStudio.Offline.config" - ], - "originalTargetFrameworks": [ - "net7.0" - ], - "sources": { - "C:\\Program Files (x86)\\Microsoft SDKs\\NuGetPackages\\": {}, - "https://api.nuget.org/v3/index.json": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2": {}, - "https://pkgs.dev.azure.com/ateadbs/CSPX/_packaging/PSCSPX/nuget/v2/index.json": {} - }, - "frameworks": { - "net7.0": { - "targetAlias": "net7.0", - "projectReferences": {} - } - }, - "warningProperties": { - "warnAsError": [ - "NU1605" - ] - } - }, - "frameworks": { - "net7.0": { - "targetAlias": "net7.0", - "dependencies": { - "Parquet.Net": { - "target": "Package", - "version": "[4.16.4, )" - }, - "PowerShellStandard.Library": { - "target": "Package", - "version": "[5.1.1, )" - } - }, - "imports": [ - "net461", - "net462", - "net47", - "net471", - "net472", - "net48", - "net481" - ], - "assetTargetFallback": true, - "warn": true, - "frameworkReferences": { - "Microsoft.NETCore.App": { - "privateAssets": "all" - } - }, - "runtimeIdentifierGraphPath": "C:\\Program Files\\dotnet\\sdk\\7.0.111\\RuntimeIdentifierGraph.json" - } - } - } -} \ No newline at end of file diff --git a/src/PSParquet/obj/project.nuget.cache b/src/PSParquet/obj/project.nuget.cache deleted file mode 100644 index e79c47f..0000000 --- a/src/PSParquet/obj/project.nuget.cache +++ /dev/null @@ -1,23 +0,0 @@ -{ - "version": 2, - "dgSpecHash": "tPuwAKlFZnQF4IfWfTiAUmP28QgPJ3aYbWSjNhdaEYqwX0jphsp39jMBzcdvNNll5P5h0OaLSQ7dfb/WwbWegw==", - "success": true, - "projectFilePath": "C:\\repos\\PSParquet\\src\\PSParquet\\PSParquet.csproj", - "expectedPackageFiles": [ - "C:\\Users\\Ax\\.nuget\\packages\\apache.arrow\\2.0.0\\apache.arrow.2.0.0.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\ironcompress\\1.5.1\\ironcompress.1.5.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\microsoft.data.analysis\\0.20.1\\microsoft.data.analysis.0.20.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\microsoft.io.recyclablememorystream\\2.3.2\\microsoft.io.recyclablememorystream.2.3.2.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\microsoft.ml.dataview\\2.0.1\\microsoft.ml.dataview.2.0.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\parquet.net\\4.16.4\\parquet.net.4.16.4.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\powershellstandard.library\\5.1.1\\powershellstandard.library.5.1.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\snappier\\1.1.1\\snappier.1.1.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\system.buffers\\4.5.1\\system.buffers.4.5.1.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\system.collections.immutable\\1.5.0\\system.collections.immutable.1.5.0.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\system.memory\\4.5.3\\system.memory.4.5.3.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\system.runtime.compilerservices.unsafe\\6.0.0\\system.runtime.compilerservices.unsafe.6.0.0.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\system.threading.tasks.extensions\\4.5.2\\system.threading.tasks.extensions.4.5.2.nupkg.sha512", - "C:\\Users\\Ax\\.nuget\\packages\\zstdsharp.port\\0.7.2\\zstdsharp.port.0.7.2.nupkg.sha512" - ], - "logs": [] -} \ No newline at end of file