From d717aac5bd009f63670f1e164bacf2972931de1d Mon Sep 17 00:00:00 2001 From: JakobL-MSFT <110699333+JakobL-MSFT@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:36:56 -0800 Subject: Enable CodeQL to work thru reasonable workarounds - Cherry pick to main (#863) Enable CodeQL to work thru reasonable workarounds. (#857) --- .github/workflows/Code-Scanning.yml | 7 ++++--- .github/workflows/ci-pr.yml | 4 ++-- .github/workflows/ci.yml | 4 ++-- Build-AllSamples.ps1 | 20 +++++++++++--------- Build-Sample.ps1 | 6 ++++++ Build-SampleSet.ps1 | 32 ++++++++++++++++++-------------- Building-Locally.md | 9 +++------ 7 files changed, 46 insertions(+), 36 deletions(-) diff --git a/.github/workflows/Code-Scanning.yml b/.github/workflows/Code-Scanning.yml index 86be1ef0..1ba4c660 100644 --- a/.github/workflows/Code-Scanning.yml +++ b/.github/workflows/Code-Scanning.yml @@ -48,10 +48,11 @@ jobs: - name: Retrieve and build all available solutions id: build-all-samples run: | - .\Build-AllSamples.ps1 -Verbose + .\Build-AllSamples.ps1 -Verbose -ThrottleLimit 1 env: - Configuration: Debug - Platform: x64 + WDS_Configuration: Debug + WDS_Platform: x64 + WDS_WipeOutputs: ${{ true }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index c701bd23..fa115e25 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -36,5 +36,5 @@ jobs: $changedFiles = "${{ steps.get-changed-files.outputs.all_changed_files }}".Split(',') .\.github\scripts\Build-ChangedSamples.ps1 -ChangedFiles $changedFiles -Verbose env: - Configuration: ${{ matrix.configuration }} - Platform: ${{ matrix.platform }} + WDS_Configuration: ${{ matrix.configuration }} + WDS_Platform: ${{ matrix.platform }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8063d6e..6b97cfff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,5 +29,5 @@ jobs: run: | .\Build-AllSamples.ps1 -Verbose env: - Configuration: ${{ matrix.configuration }} - Platform: ${{ matrix.platform }} + WDS_Configuration: ${{ matrix.configuration }} + WDS_Platform: ${{ matrix.platform }} diff --git a/Build-AllSamples.ps1 b/Build-AllSamples.ps1 index fb876d99..2cf7f403 100644 --- a/Build-AllSamples.ps1 +++ b/Build-AllSamples.ps1 @@ -9,14 +9,17 @@ This script searches for all available Visual Studio Solutions (.sln files) and A regular expression matching the samples to be built. Default is '' that matches all samples. Examples include '^tools.' or '.dchu'. .PARAMETER Configurations -A list of configurations to build samples under. Values available are 'Debug' and 'Release'. By default, $env:Configuration will be used as the sole configuration to build for. If this environment variable is not set the default is 'Debug' and 'Release'. +A list of configurations to build samples under. Values available are 'Debug' and 'Release'. By default, $env:WDS_Configuration will be used as the sole configuration to build for. If this environment variable is not set the default is 'Debug' and 'Release'. .PARAMETER Platforms -A list of platforms to build samples under (e.g. 'x64', 'arm64'). By default, $env:Platform will be used as the sole platform to build for. If this environment variable is not set the default is 'x64' and'arm64'. +A list of platforms to build samples under (e.g. 'x64', 'arm64'). By default, $env:WDS_Platform will be used as the sole platform to build for. If this environment variable is not set the default is 'x64' and'arm64'. .PARAMETER LogFilesDirectory Path to a directory where the log files will be written to. If not provided, outputs will be logged to the '_logs' directory within the current working directory. +.PARAMETER ThrottleLimit +An integer indicating how many combinations to build in parallel. If 0 or not provided this defaults to 5 x number of logical processors. + .INPUTS None. @@ -27,16 +30,17 @@ None. .\Build-AllSamples .EXAMPLE -.\Build-AllSamples -Samples '^tools.' -Configurations 'Debug','Release' -Platforms 'x64','arm64' -LogFilesDirectory .\_logs +.\Build-AllSamples -Samples '^tools.' -Configurations 'Debug','Release' -Platforms 'x64','arm64' #> [CmdletBinding()] param( [string]$Samples = "", - [string[]]$Configurations = @([string]::IsNullOrEmpty($env:Configuration) ? ('Debug','Release') : $env:Configuration), - [string[]]$Platforms = @([string]::IsNullOrEmpty($env:Platform) ? ('x64','arm64') : $env:Platform), - [string]$LogFilesDirectory = (Join-Path (Get-Location) "_logs") + [string[]]$Configurations = @([string]::IsNullOrEmpty($env:WDS_Configuration) ? ('Debug','Release') : $env:WDS_Configuration), + [string[]]$Platforms = @([string]::IsNullOrEmpty($env:WDS_Platform) ? ('x64','arm64') : $env:WDS_Platform), + [string]$LogFilesDirectory = (Join-Path (Get-Location) "_logs"), + [int]$ThrottleLimit ) $Verbose = $false @@ -63,6 +67,4 @@ foreach ($file in $solutionFiles) { } } - - -.\Build-SampleSet -SampleSet $sampleSet -Configurations $Configurations -Platform $Platforms -LogFilesDirectory $LogFilesDirectory -Verbose:$Verbose +.\Build-SampleSet -SampleSet $sampleSet -Configurations $Configurations -Platform $Platforms -LogFilesDirectory $LogFilesDirectory -Verbose:$Verbose -ThrottleLimit $ThrottleLimit diff --git a/Build-Sample.ps1 b/Build-Sample.ps1 index 57f0ce7e..4f8f5b0b 100644 --- a/Build-Sample.ps1 +++ b/Build-Sample.ps1 @@ -140,6 +140,12 @@ $OutLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.out" Write-Verbose "Building Sample: $SampleName; Configuration: $Configuration; Platform: $Platform {" msbuild $solutionFile -clp:Verbosity=m -t:clean,build -property:Configuration=$Configuration -property:Platform=$Platform -p:TargetVersion=Windows10 -p:InfVerif_AdditionalOptions="/msft /sw1205 /sw1324 /sw1420 /sw1421" -p:SignToolWS=/fdws -p:DriverCFlagAddOn=/wd4996 -flp1:errorsonly`;logfile=$errorLogFilePath -flp2:WarningsOnly`;logfile=$warnLogFilePath -noLogo > $OutLogFilePath +if ($env:WDS_WipeOutputs -ne $null) +{ + Write-Verbose ("WipeOutputs: "+$Directory+" "+(((Get-Volume ($DriveLetter=(Get-Item ".").PSDrive.Name)).SizeRemaining/1GB))) + Get-ChildItem -path $Directory -Recurse -Include x64|Remove-Item -Recurse + Get-ChildItem -path $Directory -Recurse -Include arm64|Remove-Item -Recurse +} if ($LASTEXITCODE -ne 0) { diff --git a/Build-SampleSet.ps1 b/Build-SampleSet.ps1 index 3295c8b4..3ef00efe 100644 --- a/Build-SampleSet.ps1 +++ b/Build-SampleSet.ps1 @@ -1,11 +1,16 @@ [CmdletBinding()] param( [hashtable]$SampleSet, - [string[]]$Configurations = @([string]::IsNullOrEmpty($env:Configuration) ? "Debug" : $env:Configuration), - [string[]]$Platforms = @([string]::IsNullOrEmpty($env:Platform) ? "x64" : $env:Platform), - $LogFilesDirectory = (Get-Location) + [string[]]$Configurations = @([string]::IsNullOrEmpty($env:WDS_Configuration) ? "Debug" : $env:WDS_Configuration), + [string[]]$Platforms = @([string]::IsNullOrEmpty($env:WDS_Platform) ? "x64" : $env:WDS_Platform), + $LogFilesDirectory = (Get-Location), + [int]$ThrottleLimit ) +$ThrottleFactor = 5 +$LogicalProcessors = (Get-CIMInstance -Class 'CIM_Processor' -Verbose:$false).NumberOfLogicalProcessors +$ThrottleLimit = $ThrottleLimit -eq 0 ? ($ThrottleFactor * $LogicalProcessors) : $ThrottleLimit + $Verbose = $false if ($PSBoundParameters.ContainsKey('Verbose')) { $Verbose = $PsBoundParameters.Get_Item('Verbose') @@ -18,10 +23,6 @@ $sampleBuilderFilePath = "$LogFilesDirectory\overview.htm" Remove-Item -Recurse -Path $LogFilesDirectory 2>&1 | Out-Null New-Item -ItemType Directory -Force -Path $LogFilesDirectory | Out-Null -$NumberOfLogicalProcessors = (Get-CIMInstance -Class 'CIM_Processor' -Verbose:$false).NumberOfLogicalProcessors -$Throttlefactor = 5 -$SolutionsInParallel = $Throttlefactor * $NumberOfLogicalProcessors - $oldPreference = $ErrorActionPreference $ErrorActionPreference = "stop" try { @@ -58,9 +59,11 @@ Write-Output ("Samples: "+$sampleSet.Count) Write-Output ("Configurations: "+$Configurations.Count+" ("+$Configurations+")") Write-Output ("Platforms: "+$Platforms.Count+" ("+$Platforms+")") Write-Output "Combinations: $SolutionsTotal" -Write-Output "Logical Processors: $NumberOfLogicalProcessors" -Write-Output "Throttle factor: $Throttlefactor" -Write-Output "Throttle limit: $SolutionsInParallel" +Write-Output "LogicalProcessors: $LogicalProcessors" +Write-Output "ThrottleFactor: $ThrottleFactor" +Write-Output "ThrottleLimit: $ThrottleLimit" +Write-Output "WDS_WipeOutputs: $env:WDS_WipeOutputs" +Write-Output ("Disk Remaining (GB): "+(((Get-Volume ($DriveLetter=(Get-Item ".").PSDrive.Name)).SizeRemaining/1GB))) Write-Output "" Write-Output "T: Combinations" Write-Output "B: Built" @@ -78,7 +81,7 @@ $Results = @() $sw = [Diagnostics.Stopwatch]::StartNew() -$SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $SolutionsInParallel -Parallel { +$SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel { $LogFilesDirectory = $using:LogFilesDirectory $exclusionsSet = $using:exclusionsSet $Configurations = $using:Configurations @@ -133,11 +136,11 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $SolutionsInParallel ($using:jresult).SolutionsUnsupported += $thisunsupported ($using:jresult).SolutionsFailed += $thisfailed $SolutionsTotal = $using:SolutionsTotal - $SolutionsInParallel = $using:SolutionsInParallel + $ThrottleLimit = $using:ThrottleLimit $SolutionsBuilt = ($using:jresult).SolutionsBuilt $SolutionsRemaining = $SolutionsTotal - $SolutionsBuilt - $SolutionsRunning = $SolutionsRemaining -ge $SolutionsInParallel ? ($SolutionsInParallel) : ($SolutionsRemaining) - $SolutionsPending = $SolutionsRemaining -ge $SolutionsInParallel ? ($SolutionsRemaining - $SolutionsInParallel) : (0) + $SolutionsRunning = $SolutionsRemaining -ge $ThrottleLimit ? ($ThrottleLimit) : ($SolutionsRemaining) + $SolutionsPending = $SolutionsRemaining -ge $ThrottleLimit ? ($SolutionsRemaining - $ThrottleLimit) : (0) $SolutionsBuiltPercent = [Math]::Round(100 * ($SolutionsBuilt / $using:SolutionsTotal)) $TBRP = "T:" + ($SolutionsTotal) + "; B:" + (($using:jresult).SolutionsBuilt) + "; R:" + ($SolutionsRunning) + "; P:" + ($SolutionsPending) $rstr = "S:" + (($using:jresult).SolutionsSucceeded) + "; E:" + (($using:jresult).SolutionsExcluded) + "; U:" + (($using:jresult).SolutionsUnsupported) + "; F:" + (($using:jresult).SolutionsFailed) @@ -181,6 +184,7 @@ Write-Output "" Write-Output "Built all combinations." Write-Output "" Write-Output "Elapsed time: $min minutes, $seconds seconds." +Write-Output ("Disk Remaining (GB): "+(((Get-Volume ($DriveLetter=(Get-Item ".").PSDrive.Name)).SizeRemaining/1GB))) Write-Output ("Samples: "+$sampleSet.Count) Write-Output ("Configurations: "+$Configurations.Count+" ("+$Configurations+")") Write-Output ("Platforms: "+$Platforms.Count+" ("+$Platforms+")") diff --git a/Building-Locally.md b/Building-Locally.md index 294ecae8..8f62a609 100644 --- a/Building-Locally.md +++ b/Building-Locally.md @@ -15,10 +15,7 @@ You can also just download and mount the EWDK as well and in the following examp * Mount ISO image * Open a terminal * `.\LaunchBuildEnv` - * `set PLATFORM=` - -Note: The last command to clear the PLATFORM variable is added so as to allow the next part to iterate over all platforms automatically without having to add explicit arguments. - + ## Step 3: Clone Windows Driver Samples and checkout main branch ``` @@ -33,12 +30,12 @@ cd Windows-driver-samples pwsh .\Build-AllSamples ``` -Above builds all samples for all configurations and archictures. +Above builds all samples for all configurations and platforms. You can refine, for example as follows: ``` pwsh -.\Build-AllSamples -Samples 'tools.' -Configurations 'Debug','Release' -Platforms 'x64','arm64' -LogFilesDirectory .\_logs +.\Build-AllSamples -Samples '^tools.' -Configurations 'Debug','Release' -Platforms 'x64','arm64' ``` Expected output: -- cgit v1.3.1 From 080e9622071a08ccf89039520650958c09e1dcd4 Mon Sep 17 00:00:00 2001 From: Adonais Romero González Date: Tue, 14 Feb 2023 14:59:33 -0800 Subject: Fix logging when there are errors (#881) --- Build-SampleSet.ps1 | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Build-SampleSet.ps1 b/Build-SampleSet.ps1 index 3ef00efe..2c97f31a 100644 --- a/Build-SampleSet.ps1 +++ b/Build-SampleSet.ps1 @@ -40,7 +40,6 @@ finally { # TODO: Make exclusion more granular; allow for configuration|platform exclusions $exclusionsSet = @{} -$failSet = @() Import-Csv 'exclusions.csv' | ForEach-Object { $exclusionsSet[$_.Path.Replace($root, '').Trim('\').Replace('\', '.').ToLower()] = $_.Reason } @@ -50,6 +49,7 @@ $jresult = @{ SolutionsExcluded = 0 SolutionsFailed = 0 Results = @() + FailSet = @() lock = [System.Threading.Mutex]::new($false) } @@ -100,6 +100,7 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral $thisexcluded = 0 $thissucceeded = 0 $thisresult = "Not run" + $thisfailset = @() if ($exclusionsSet.ContainsKey($sampleName)) { # Verbose @@ -116,7 +117,7 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral $thisresult = "Succeeded" } elseif ($LASTEXITCODE -eq 1) { - $failSet += "$sampleName $configuration|$platform" + $thisfailset += "$sampleName $configuration|$platform" $thisfailed += 1 $thisresult = "Failed" } @@ -135,6 +136,7 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral ($using:jresult).SolutionsExcluded += $thisexcluded ($using:jresult).SolutionsUnsupported += $thisunsupported ($using:jresult).SolutionsFailed += $thisfailed + ($using:jresult).FailSet += $thisfailset $SolutionsTotal = $using:SolutionsTotal $ThrottleLimit = $using:ThrottleLimit $SolutionsBuilt = ($using:jresult).SolutionsBuilt @@ -162,10 +164,16 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral $sw.Stop() -if ($failSet.Count -gt 0) { +if ($jresult.FailSet.Count -gt 0) { Write-Output "Some combinations were built with errors:" - foreach ($failedSample in $failSet) { - Write-Output "$failedSample" + foreach ($failedSample in $jresult.FailSet) { + $failedSample -match "^(.*) (\w*)\|(\w*)$" | Out-Null + $failName = $Matches[1] + $failConfiguration = $Matches[2] + $failPlatform = $Matches[3] + Write-Output "Build errors in Sample $failName; Configuration: $failConfiguration; Platform: $failPlatform {" + Get-Content "$LogFilesDirectory\$failName.$failConfiguration.$failPlatform.err" | Write-Output + Write-Output "} $failedSample" } Write-Error "Some combinations were built with errors." } -- cgit v1.3.1 From 0b22a5da3df0d570a39681ffdd256cd876c30047 Mon Sep 17 00:00:00 2001 From: Adonais Romero González Date: Thu, 16 Feb 2023 16:19:48 -0800 Subject: Rename build instructions file to avoid conflicts with root README + small fix --- .github/Build-with-GitHub.md | 35 +++++++++++++++++++++++++++++++++++ .github/README.md | 35 ----------------------------------- 2 files changed, 35 insertions(+), 35 deletions(-) create mode 100644 .github/Build-with-GitHub.md delete mode 100644 .github/README.md diff --git a/.github/Build-with-GitHub.md b/.github/Build-with-GitHub.md new file mode 100644 index 00000000..8fe67c5f --- /dev/null +++ b/.github/Build-with-GitHub.md @@ -0,0 +1,35 @@ +# Using GitHub actions for building drivers + +If you use GitHub to host your code, you can leverage [GitHub Actions](https://docs.github.com/en/actions) to create automated workflows to build your driver projects. + +`windows-2022` runner (provided by `windows-latest`) is configured with Windows Driver Kit version 22H2 and Visual Studio 2022 off the box, so most solutions can be built directly running `msbuild` directly. + +```yaml +name: Build all driver samples +on: + push: + branches: + - main +jobs: + build: + strategy: + matrix: + configuration: [Debug, Release] + platform: [x64] + runs-on: windows-2022 + env: + Solution_Path: path\to\driver\solution.sln + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Add MSBuild to PATH + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Build solution + run: | + msbuild ${{ env.Solution_Path }} -p:Configuration:${{ env.Configuration }} -p:Platform:${{ env.Platform }} + env: + Configuration: ${{ matrix.configuration }} + Platform: ${{ matrix.platform }} +``` diff --git a/.github/README.md b/.github/README.md deleted file mode 100644 index b7569139..00000000 --- a/.github/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Using GitHub actions for building drivers - -If you use GitHub to host your code, you can leverage [GitHub Actions](https://docs.github.com/en/actions) to create automated workflows to build your driver projects. - -`windows-2022` runner (provided by `windows-latest`) is configured with Windows Driver Kit version 22H2 and Visual Studio 2022 off the box, so most solutions can be built directly running `msbuild` directly. - -```yaml -name: Build all driver samples -on: - push: - branches: - - main -jobs: - build: - strategy: - matrix: - configuration: [Debug, Release] - platform: [x64] - runs-on: windows-2022 - env: - Solution_Path: path\to\driver\solution.sln - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - - name: Add MSBuild to PATH - uses: microsoft/setup-msbuild@v1.0.2 - - - name: Build solution - run: | - msbuild $${{ env.Solution_Path }} -p:Configuration:${{ env.Configuration }} -p:Platform:${{ env.Platform }} - env: - Configuration: ${{ matrix.configuration }} - Platform: ${{ matrix.platform }} -``` -- cgit v1.3.1