From 552552b93d092884208024fbcb54874d90893027 Mon Sep 17 00:00:00 2001 From: jhoneill Date: Thu, 4 Apr 2019 17:08:05 +0100 Subject: [PATCH] Perf improvments and direct table handling for Export Excel, --- Export-Excel.ps1 | 134 ++---------------- ImportExcel.psd1 | 75 +++++++++- Send-SqlDataToExcel.ps1 | 53 +------ __tests__/Copy-ExcelWorksheet.Tests.ps1 | 4 +- __tests__/First10Races.tests.ps1 | 2 +- __tests__/InputItemParameter.tests.ps1 | 79 +++++++++++ .../Set-Row_Set-Column-SetFormat.tests.ps1 | 2 +- 7 files changed, 171 insertions(+), 178 deletions(-) create mode 100644 __tests__/InputItemParameter.tests.ps1 diff --git a/Export-Excel.ps1 b/Export-Excel.ps1 index 475e1b9..d3a4927 100644 --- a/Export-Excel.ps1 +++ b/Export-Excel.ps1 @@ -523,111 +523,9 @@ begin { $numberRegex = [Regex]'\d' - function Add-CellValue { - <# - .SYNOPSIS - Save a value in an Excel cell. - - .DESCRIPTION - DateTime objects are always converted to a short DateTime format in Excel. When Excel loads the file, - it applies the local format for dates. And formulas are always saved as formulas. URIs are set as hyperlinks in the file. - - Numerical values will be converted to numbers as defined in the regional settings of the local - system. In case the parameter 'NoNumberConversion' is used, we don't convert to number and leave - the value 'as is'. In case of conversion failure, we also leave the value 'as is'. - #> - - Param ( - $TargetCell, - $CellValue - ) - #The write-verbose commands have been commented out below - even if verbose is silenced they cause a significiant performance impact and if it's on they will cause a flood of messages. - Switch ($CellValue) { - { $null -eq $_ } { - break - } - { $_ -is [DateTime]} { - # Save a date with one of Excel's built in formats format - $TargetCell.Value = $_ - $TargetCell.Style.Numberformat.Format = 'm/d/yy h:mm' # This is not a custom format, but a preset recognized as date and localized. - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$_' as date" - break - - } - { $_ -is [TimeSpan]} { - #Save a timespans with a built in format for elapsed hours, minutes and seconds - $TargetCell.Value = $_ - $TargetCell.Style.Numberformat.Format = '[h]:mm:ss' - break - } - { $_ -is [System.ValueType]} { - # Save numerics, setting format if need be. - $TargetCell.Value = $_ - if ($setNumformat) {$targetCell.Style.Numberformat.Format = $Numberformat } - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$_' as value" - break - } - { $_ -is [uri] } { - $targetCell.HyperLink = $_ - $TargetCell.Style.Font.Color.SetColor([System.Drawing.Color]::Blue) - $TargetCell.Style.Font.UnderLine = $true - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$($_.AbsoluteUri)' as Hyperlink" - break - } - { $_ -isnot [String]} { - $TargetCell.Value = $_.toString() - break - } - #All the remaining options are string as ... only strings get checked for formulas, URIs or being numbers - { $_[0] -eq '='} { - #region Save an Excel formula - we need = to spot the formula but the EPPLUS won't like it if we include it (Excel doesn't care if is there or not) - $TargetCell.Formula = ($_ -replace '^=','') - if ($setNumformat) {$targetCell.Style.Numberformat.Format = $Numberformat } - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$_' as formula" - break - } - { [System.Uri]::IsWellFormedUriString($_ , [System.UriKind]::Absolute) } { - if ($_ -match "^xl://internal/") { - $referenceAddress = $_ -replace "^xl://internal/" , "" - $display = $referenceAddress -replace "!A1$" , "" - $h = New-Object -TypeName OfficeOpenXml.ExcelHyperLink -ArgumentList $referenceAddress , $display - $TargetCell.HyperLink = $h - } - else {$TargetCell.HyperLink = $_ } #$TargetCell.Value = $_.AbsoluteUri - $TargetCell.Style.Font.Color.SetColor([System.Drawing.Color]::Blue) - $TargetCell.Style.Font.UnderLine = $true - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$_' as Hyperlink" - break - } - <# Logic for no-number conversion has been moved into default and only checked if value has digits. - {( $NoNumberConversion -and ( - ($NoNumberConversion -contains $Name) -or ($NoNumberConversion -eq '*'))) } { - #Save text without it to converting to number - $TargetCell.Value = $_ - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$($TargetCell.Value)' unconverted" - break - } #> - Default { - #Save a value as a number if possible - $number = $null - if ( $numberRegex.IsMatch($_) -and # if it contains digit(s) - this syntax is quicker than -match for many items and cuts out slow checks for non numbers - $NoNumberConversion -ne '*' -and # and NoNumberConversion isn't specified - $NoNumberConversion -notcontains $Name -and - [Double]::TryParse($_, [System.Globalization.NumberStyles]::Any, [System.Globalization.NumberFormatInfo]::CurrentInfo, [Ref]$number) - ) { - $TargetCell.Value = $number - if ($setNumformat) {$targetCell.Style.Numberformat.Format = $Numberformat } - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$($TargetCell.Value)' as number converted from '$_' with format '$Numberformat'" - } - else { - $TargetCell.Value = $_ - #Write-Verbose "Cell '$Row`:$ColumnIndex' header '$Name' add value '$($TargetCell.Value)' as string" - } - break - } - } - } - + $isDataTypeValueType = $false + if ($NoClobber) {Write-Warning -Message "-NoClobber parameter is no longer used" } + #Open the file, get the worksheet, and decide where in the sheet we are writing, and if there is a number format to apply. try { $script:Header = $null if ($Append -and $ClearSheet) {throw "You can't use -Append AND -ClearSheet."} @@ -639,7 +537,6 @@ $AutoFilter = $true } } - if ($ExcelPackage) { $pkg = $ExcelPackage $Path = $pkg.File @@ -647,7 +544,6 @@ Else { $pkg = Open-ExcelPackage -Path $Path -Create -KillExcel:$KillExcel -Password:$Password} } catch {throw "Could not open Excel Package $path"} - if ($NoClobber) {Write-Warning -Message "-NoClobber parameter is no longer used" } try { $params = @{WorksheetName=$WorksheetName} foreach ($p in @("ClearSheet", "MoveToStart", "MoveToEnd", "MoveBefore", "MoveAfter", "Activate")) {if ($PSBoundParameters[$p]) {$params[$p] = $PSBoundParameters[$p]}} @@ -722,19 +618,8 @@ $setNumformat = $false } else { $setNumformat = ($Numberformat -ne $ws.Cells.Style.Numberformat.Format) } - - $firstTimeThru = $true - $isDataTypeValueType = $false - } - catch { - if ($AlreadyExists) { - #Is this set anywhere ? - throw "Failed exporting worksheet '$WorksheetName' to '$Path': The worksheet '$WorksheetName' already exists." - } - else { - throw "Failed preparing to export to worksheet '$WorksheetName' to '$Path': $_" - } } + catch {throw "Failed preparing to export to worksheet '$WorksheetName' to '$Path': $_"} #region Special case -inputobject passed a dataTable object <# If inputObject was passed via the pipeline it won't be visible until the process block, we will only see it here if it was passed as a parameter if it was passed it is a data table don't do foreach on it (slow) put the whole table in and set dates on date columns, @@ -744,12 +629,17 @@ foreach ($c in $InputObject.Columns.where({$_.datatype -eq [datetime]})) { Set-ExcelColumn -Worksheet $ws -Column ($c.Ordinal + $StartColumn) -NumberFormat 'Date-Time' } - $row += $InputObject.Rows.Count - 1 - $ColumnIndex += $InputObject.Columns.Count - 1 + foreach ($c in $InputObject.Columns.where({$_.datatype -eq [timespan]})) { + Set-ExcelColumn -Worksheet $ws -Column ($c.Ordinal + $StartColumn) -NumberFormat '[h]:mm:ss' + } + $ColumnIndex += $InputObject.Columns.Count - 1 + if ($noHeader) {$row += $InputObject.Rows.Count -1 } + else {$row += $InputObject.Rows.Count } [void]$PSBoundParameters.Remove('InputObject') $firstTimeThru = $false } #endregion + else {$firstTimeThru = $true} } process { if ($PSBoundParameters.ContainsKey("InputObject")) { @@ -820,7 +710,7 @@ $ws.Cells[$Row, $ColumnIndex].Style.Font.Color.SetColor([System.Drawing.Color]::Blue) $ws.Cells[$Row, $ColumnIndex].Style.Font.UnderLine = $true } - elseif ($v -isnot [String] ) { + elseif ($v -isnot [String] ) { #Other objects or null. if ($null -ne $v) { $ws.Cells[$Row, $ColumnIndex].Value = $v.toString()} } elseif ($v[0] -eq '=') { diff --git a/ImportExcel.psd1 b/ImportExcel.psd1 index 35fbf8a..fedf1f2 100644 --- a/ImportExcel.psd1 +++ b/ImportExcel.psd1 @@ -61,16 +61,83 @@ Check out the How To Videos https://www.youtube.com/watch?v=U3Ne_yX4tYo&list=PL5 # NestedModules = @() # Functions to export from this module - FunctionsToExport = '*' + FunctionsToExport = @( + 'BarChart', + 'ColumnChart', + 'DoChart', + 'LineChart', + 'PieChart', + 'Pivot', + 'Get-XYRange', + 'Invoke-AllTests', + 'Test-Boolean', + 'Test-Date', + 'Test-Integer', + 'Test-Number', + 'Test-String', + 'New-PSItem', + 'Import-Html', + 'Import-UPS', + 'Import-USPS', + 'Add-ConditionalFormatting', + 'Add-ExcelChart', + 'Add-ExcelDataValidationRule', + 'Add-ExcelName', + 'Add-ExcelTable', + 'Add-PivotTable', + 'Add-WorkSheet', + 'Close-ExcelPackage', + 'Compare-WorkSheet', + 'ConvertFrom-ExcelData', + 'ConvertFrom-ExcelSheet', + 'ConvertFrom-ExcelToSQLInsert', + 'ConvertTo-ExcelXlsx', + 'Convert-XlRangeToImage', + 'Copy-ExcelWorkSheet', + 'Expand-NumberFormat', + 'Export-Excel', + 'Export-ExcelSheet', + 'Export-MultipleExcelSheets', + 'Get-ExcelColumnName', + 'Get-ExcelSheetInfo', + 'Get-ExcelWorkbookInfo', + 'Get-HtmlTable', + 'Get-Range', + 'Import-Excel', + 'Invoke-Sum', + 'Join-Worksheet', + 'Merge-MultipleSheets', + 'Merge-Worksheet', + 'New-ConditionalFormattingIconSet', + 'New-ConditionalText', + 'New-ExcelChartDefinition', + 'New-PivotTableDefinition', + 'New-Plot', + 'NumberFormatCompletion', + 'Open-ExcelPackage', + 'Remove-WorkSheet' + 'Select-Worksheet', + 'Send-SQLDataToExcel', + 'Set-CellStyle', + 'Set-ExcelColumn', + 'Set-ExcelRange', + 'Set-ExcelRow', + 'Update-FirstObjectProperties' + ) # Cmdlets to export from this module - CmdletsToExport = '*' + #CmdletsToExport = '*' # Variables to export from this module - VariablesToExport = '*' + #VariablesToExport = '*' # Aliases to export from this module - AliasesToExport = '*' + AliasesToExport = @('New-ExcelChart', + 'Set-Column', + 'Set-Format', + 'Set-Row', + 'Use-ExcelData' + ) # List of all modules packaged with this module # ModuleList = @() diff --git a/Send-SqlDataToExcel.ps1 b/Send-SqlDataToExcel.ps1 index 174ee0a..499f929 100644 --- a/Send-SqlDataToExcel.ps1 +++ b/Send-SqlDataToExcel.ps1 @@ -237,11 +237,6 @@ [Switch]$Passthru ) - if ($KillExcel) { - Get-Process excel -ErrorAction Ignore | Stop-Process - while (Get-Process excel -ErrorAction Ignore) {Start-Sleep -Milliseconds 250} - } - #We were either given a session object or a connection string (with, optionally a MSSQLServer parameter) # If we got -MSSQLServer, create a SQL connection, if we didn't but we got -Connection create an ODBC connection if ($MsSQLserver -and $Connection) { @@ -253,8 +248,7 @@ elseif ($Connection) { $Session = New-Object -TypeName System.Data.Odbc.OdbcConnection -ArgumentList $Connection ; $Session.ConnectionTimeout = 30 } - - If ($session) { + if ($Session) { #A session was either passed in or just created. If it's a SQL one make a SQL DataAdapter, otherwise make an ODBC one if ($Session.GetType().name -match "SqlConnection") { $dataAdapter = New-Object -TypeName System.Data.SqlClient.SqlDataAdapter -ArgumentList ( @@ -264,7 +258,7 @@ $dataAdapter = New-Object -TypeName System.Data.Odbc.OdbcDataAdapter -ArgumentList ( New-Object -TypeName System.Data.Odbc.OdbcCommand -ArgumentList $SQL, $Session ) } - if ($QueryTimeout) {$dataAdapter.SelectCommand.CommandTimeout = $ServerTimeout} + if ($QueryTimeout) {$dataAdapter.SelectCommand.CommandTimeout = $QueryTimeout} #Both adapter types output the same kind of table, create one and fill it from the adapter $DataTable = New-Object -TypeName System.Data.DataTable @@ -273,48 +267,11 @@ } if ($DataTable.Rows.Count) { #ExportExcel user a -NoHeader parameter so that's what we use here, but needs to be the other way around. - $printHeaders = -not $NoHeader - if ($Title) {$r = $StartRow +1 } - else {$r = $StartRow} - #Get our Excel sheet and fill it with the data - $excelPackage = Export-Excel -Path $Path -WorkSheetname $WorkSheetname -PassThru - $ws = $excelPackage.Workbook.Worksheets[$WorkSheetname] - $ws.Cells[$r,$StartColumn].LoadFromDataTable($dataTable, $printHeaders ) | Out-Null - - $LastRow = $StartRow + $DataTable.Rows.Count # if start row is 1, row 1 will be the header, row 2 will be data, so don't need to subtract 1 - $LastCol = $StartColumn + $DataTable.Columns.Count - 1 - $endAddress = [OfficeOpenXml.ExcelAddress]::GetAddress($LastRow , $LastCol) - $startAddress = [OfficeOpenXml.ExcelAddress]::GetAddress($StartRow, $StartColumn) - $dataRange = "{0}:{1}" -f $startAddress, $endAddress - - #Apply date format and range names - for ($c=0 ; $c -lt $DataTable.Columns.Count ; $c++) { - if ($DataTable.Columns[$c].DataType -eq [datetime]) { - Set-ExcelColumn -Worksheet $ws -Column ($c + $StartColumn) -NumberFormat 'Date-Time' - } - if ($AutoNameRange) { - Add-ExcelName -RangeName $DataTable.Columns[$c].ColumnName -Range $ws.Cells[($StartRow+1), ($StartColumn + $c ), $LastRow, ($StartColumn + $c )] - } - } - - #Apply range or table to whole - we can't leave this to Export-Excel if we are inserting onto a sheet where there is already data - if ($RangeName) { - Add-ExcelName -Range $ws.Cells[$dataRange] -RangeName $RangeName - $null = $PSBoundParameters.Remove('RangeName') - } - - if ($TableName) { - if ($PSBoundParameters.ContainsKey('TableStyle')) { - Add-ExcelTable -Range $ws.Cells[$dataRange] -TableName $TableName -TableStyle $TableStyle - $null = $PSBoundParameters.Remove('TableStyle') - } - else {Add-ExcelTable -Range $excelPackage.Workbook.Worksheets[$WorkSheetname].Cells[$dataRange] -TableName $TableName} - $null = $PSBoundParameters.Remove('TableName') - } #Call export-excel with any parameters which don't relate to the SQL query - "AutoNameRange", "Connection", "Database" , "Session", "MsSQLserver", "Destination" , "SQL" , "DataTable", "Path" | ForEach-Object {$null = $PSBoundParameters.Remove($_) } - Export-Excel -ExcelPackage $excelPackage @PSBoundParameters + + "Connection", "Database" , "Session", "MsSQLserver", "SQL" , "DataTable" | ForEach-Object {$null = $PSBoundParameters.Remove($_) } + Export-Excel @PSBoundParameters -InputObject $DataTable } else {Write-Warning -Message "No Data to insert."} #If we were passed a connection and opened a session, close that session. diff --git a/__tests__/Copy-ExcelWorksheet.Tests.ps1 b/__tests__/Copy-ExcelWorksheet.Tests.ps1 index 70785eb..a4c00f3 100644 --- a/__tests__/Copy-ExcelWorksheet.Tests.ps1 +++ b/__tests__/Copy-ExcelWorksheet.Tests.ps1 @@ -4,8 +4,8 @@ Remove-item -Path $path1, $path2 -ErrorAction SilentlyContinue $ProcRange = Get-Process | Export-Excel $path1 -DisplayPropertySet -WorkSheetname Processes -ReturnRange -if ((Get-Culture).NumberFormat.CurrencySymbol -eq "�") {$OtherCurrencySymbol = "$"} -else {$OtherCurrencySymbol = "�"} +if ((Get-Culture).NumberFormat.CurrencySymbol -eq "£") {$OtherCurrencySymbol = "$"} +else {$OtherCurrencySymbol = "£"} [PSCustOmobject][Ordered]@{ Date = Get-Date Formula1 = '=SUM(F2:G2)' diff --git a/__tests__/First10Races.tests.ps1 b/__tests__/First10Races.tests.ps1 index 482eeec..f0eb426 100644 --- a/__tests__/First10Races.tests.ps1 +++ b/__tests__/First10Races.tests.ps1 @@ -54,7 +54,7 @@ Describe "Creating small named ranges with hyperlinks" { $excel = Open-ExcelPackage $path $sheet = $excel.Workbook.Worksheets[1] - $m = $results | measure -sum -Property count + $m = $results | Measure-Object -sum -Property count $expectedRows = 1 + $m.count + $m.sum } Context "Creating hyperlinks" { diff --git a/__tests__/InputItemParameter.tests.ps1 b/__tests__/InputItemParameter.tests.ps1 new file mode 100644 index 0000000..57b0c94 --- /dev/null +++ b/__tests__/InputItemParameter.tests.ps1 @@ -0,0 +1,79 @@ +Describe "Exporting with -Inputobject" { + BeforeAll { + $path = "$env:TEMP\Results.xlsx" + Remove-Item -Path $path -ErrorAction SilentlyContinue + #Read race results, and group by race name : export 1 row to get headers, leaving enough rows aboce to put in a link for each race + $results = ((Get-Process) + (Get-Process -id $PID)) | Select-Object -last 10 -Property Name, cpu, pm, handles, StartTime + $DataTable = [System.Data.DataTable]::new('Test') + [void]$DataTable.Columns.Add('Name') + [void]$DataTable.Columns.Add('CPU', [double]) + [void]$DataTable.Columns.Add('PM', [Long]) + [void]$DataTable.Columns.Add('Handles', [Int]) + [void]$DataTable.Columns.Add('StartTime', [DateTime]) + foreach ($r in $results) { + [void]$DataTable.Rows.Add($r.name, $r.CPU, $R.PM, $r.Handles, $r.StartTime) + } + export-excel -Path $path -InputObject $results -WorksheetName Sheet1 -RangeName "Whole" + export-excel -Path $path -InputObject $DataTable -WorksheetName Sheet2 -AutoNameRange + Send-SQLDataToExcel -path $path -DataTable $DataTable -WorkSheetname Sheet3 -TableName "Data" + $excel = Open-ExcelPackage $path + $sheet = $excel.Sheet1 + } + Context "Array of processes" { + it "Put the correct rows and columns into the sheet " { + $sheet.Dimension.Rows | should be ($results.Count + 1) + $sheet.Dimension.Columns | should be 5 + $sheet.cells["A1"].Value | should be "Name" + $sheet.cells["E1"].Value | should be "StartTime" + $sheet.cells["A3"].Value | should be $results[1].Name + } + it "Created a range for the whole sheet " { + $sheet.Names[0].Name | should be "Whole" + $sheet.Names[0].Start.Address | should be "A1" + $sheet.Names[0].End.row | should be ($results.Count + 1) + $sheet.Names[0].End.Column | should be 5 + } + it "Formatted date fields with date type " { + $sheet.Cells["E11"].Style.Numberformat.NumFmtID | should be 22 + } + } + $sheet = $excel.Sheet2 + Context "Table of processes" { + it "Put the correct rows and columns into the sheet " { + $sheet.Dimension.Rows | should be ($results.Count + 1) + $sheet.Dimension.Columns | should be 5 + $sheet.cells["A1"].Value | should be "Name" + $sheet.cells["E1"].Value | should be "StartTime" + $sheet.cells["A3"].Value | should be $results[1].Name + } + it "Created named ranges for each column " { + $sheet.Names.count | should be 5 + $sheet.Names[0].Name | should be "Name" + $sheet.Names[1].Start.Address | should be "B2" + $sheet.Names[2].End.row | should be ($results.Count + 1) + $sheet.Names[3].End.Column | should be 4 + $sheet.Names[4].Start.Column | should be 5 + } + it "Formatted date fields with date type " { + $sheet.Cells["E11"].Style.Numberformat.NumFmtID | should be 22 + } + } + $sheet = $excel.Sheet3 + Context "Table of processes via Send-SQLDataToExcel" { + it "Put the correct rows and columns into the sheet " { + $sheet.Dimension.Rows | should be ($results.Count + 1) + $sheet.Dimension.Columns | should be 5 + $sheet.cells["A1"].Value | should be "Name" + $sheet.cells["E1"].Value | should be "StartTime" + $sheet.cells["A3"].Value | should be $results[1].Name + } + it "Created a table " { + $sheet.Tables.count | should be 1 + $sheet.Tables[0].Name | should be "Data" + $sheet.Tables[0].Columns[4].name | should be "StartTime" + } + it "Formatted date fields with date type " { + $sheet.Cells["E11"].Style.Numberformat.NumFmtID | should be 22 + } + } +} \ No newline at end of file diff --git a/__tests__/Set-Row_Set-Column-SetFormat.tests.ps1 b/__tests__/Set-Row_Set-Column-SetFormat.tests.ps1 index 7f0248f..c0d35ef 100644 --- a/__tests__/Set-Row_Set-Column-SetFormat.tests.ps1 +++ b/__tests__/Set-Row_Set-Column-SetFormat.tests.ps1 @@ -321,7 +321,7 @@ ID,Product,Quantity,Price,Total Describe "AutoNameRange data with a single property name" { BeforeEach { $xlfile = "$Env:TEMP\testNamedRange.xlsx" - rm $xlfile -ErrorAction SilentlyContinue + Remove-Item $xlfile -ErrorAction SilentlyContinue } it "Should have a single item as a named range" {