Build System: Catch Sporadic Failures (#1123)

* Build System: Catch Sporadic Failures

* Build System: Catch Sporadic Failures

---------

Co-authored-by: Jakob Lichtenberg <[email protected]>

2 files changed, 93 insertions, 16 deletions

diff --git a/Build-Sample.ps1 b/Build-Sample.ps1
index 0ef9d051..8c614bd2 100644
--- a/Build-Sample.ps1
+++ b/Build-Sample.ps1
@@ -134,23 +134,60 @@ foreach ($line in Get-Content -Path $solutionFile)
 if (-not $configurationIsSupported)
 {
     Write-Verbose "[$SampleName] `u{23E9} Skipped. Configuration $Configuration|$Platform not supported."
-    exit 2
+    exit 3
 }
 
-$errorLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.err"
-$warnLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.wrn"
-$OutLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.out"
-
 Write-Verbose "Building Sample: $SampleName; Configuration: $Configuration; Platform: $Platform {"
-msbuild $solutionFile -clp:Verbosity=m -t:rebuild -property:Configuration=$Configuration -property:Platform=$Platform -p:TargetVersion=Windows10 -p:InfVerif_AdditionalOptions="$InfVerif_AdditionalOptions" -warnaserror -flp1:errorsonly`;logfile=$errorLogFilePath -flp2:WarningsOnly`;logfile=$warnLogFilePath -noLogo > $OutLogFilePath
-if ($env:WDS_WipeOutputs -ne $null)
-{
-    Write-Verbose ("WipeOutputs: "+$Directory+" "+(((Get-Volume ($DriveLetter=(Get-Item ".").PSDrive.Name)).SizeRemaining/1GB)))
-    Get-ChildItem -path $Directory -Recurse -Include x64|Remove-Item -Recurse
-    Get-ChildItem -path $Directory -Recurse -Include arm64|Remove-Item -Recurse
+
+$myexit=0
+
+#
+# Let us build up to three times (0th, 1st, and 2nd attempt).
+# If we succeed at first, then it is a success.
+# If we fail at first, but succeed at either of next two attempts, then it is a sporadic failure.
+# If we even at third attempt fail, then it is a true failure.
+#
+for ($i=0; $i -le 2; $i++) {
+    $errorLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.$i.err"
+    $warnLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.$i.wrn"
+    $OutLogFilePath = "$LogFilesDirectory\$SampleName.$Configuration.$Platform.$i.out"
+
+    msbuild $solutionFile -clp:Verbosity=m -t:rebuild -property:Configuration=$Configuration -property:Platform=$Platform -p:TargetVersion=Windows10 -p:InfVerif_AdditionalOptions="$InfVerif_AdditionalOptions" -warnaserror -flp1:errorsonly`;logfile=$errorLogFilePath -flp2:WarningsOnly`;logfile=$warnLogFilePath -noLogo > $OutLogFilePath
+    if ($env:WDS_WipeOutputs -ne $null)
+    {
+        Write-Verbose ("WipeOutputs: "+$Directory+" "+(((Get-Volume ($DriveLetter=(Get-Item ".").PSDrive.Name)).SizeRemaining/1GB)))
+        Get-ChildItem -path $Directory -Recurse -Include x64|Remove-Item -Recurse
+        Get-ChildItem -path $Directory -Recurse -Include arm64|Remove-Item -Recurse
+    }
+    if ($LASTEXITCODE -eq 0)
+    {
+        # We succeeded building. 
+        # If at first attempt, then $myexit=0
+        # If at later attempt, then $myexit=2
+        if ($i -eq 0)
+        {
+            $myexit=0
+        }
+        else
+        {
+            $myexit=2
+        }
+         break;
+    }
+    else
+    {
+        # We failed building. 
+        # Let us sleep for a bit.
+        # Then let the while loop do its thing and re-run.
+        sleep 1
+        if ($Verbose)
+        {
+            Write-Warning "`u{274C} Build failed. Retrying to see if sporadic..."
+        }
+    }
 }
 
-if ($LASTEXITCODE -ne 0)
+if ($myexit -eq 1)
 {
     if ($Verbose)
     {
@@ -159,4 +196,13 @@ if ($LASTEXITCODE -ne 0)
     exit 1
 }
 
+if ($myexit -eq 2)
+{
+    if ($Verbose)
+    {
+        Write-Warning "`u{274C} Build sporadically failed. Log available at $errorLogFilePath"
+    }
+    exit 2
+}
+
 Write-Verbose "Building Sample: $SampleName; Configuration: $Configuration; Platform: $Platform }"
diff --git a/Build-SampleSet.ps1 b/Build-SampleSet.ps1
index ffbb9b0b..a8876d60 100644
--- a/Build-SampleSet.ps1
+++ b/Build-SampleSet.ps1
@@ -145,6 +145,7 @@ $jresult = @{
     SolutionsExcluded    = 0
     SolutionsUnsupported = 0
     SolutionsFailed      = 0
+    SolutionsSporadic    = 0
     Results              = @()
     FailSet              = @()
     lock                 = [System.Threading.Mutex]::new($false)
@@ -174,6 +175,7 @@ Write-Output "S: Built and result was 'Succeeded'"
 Write-Output "E: Built and result was 'Excluded'"
 Write-Output "U: Built and result was 'Unsupported' (Platform and Configuration combination)"
 Write-Output "F: Built and result was 'Failed'"
+Write-Output "O: Built and result was 'Sporadic'"
 Write-Output ""
 Write-Output "Building all combinations..."
 
@@ -200,10 +202,12 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral
         foreach ($platform in $Platforms) {
             $thisunsupported = 0
             $thisfailed = 0
+            $thissporadic = 0
             $thisexcluded = 0
             $thissucceeded = 0
             $thisresult = "Not run"
             $thisfailset = @()
+            $thissporadicset = @()
 
             if ($exclusionConfigurations.ContainsKey($sampleName) -and ($exclusionConfigurations[$sampleName].Split(';') | Where-Object { "$configuration|$platform" -like $_ })) {
                 # Verbose
@@ -222,8 +226,13 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral
                     $thisfailed += 1
                     $thisresult = "Failed"
                 }
+                elseif ($LASTEXITCODE -eq 2) {
+                    $thissporadicset += "$sampleName $configuration|$platform"
+                    $thissporadic += 1
+                    $thisresult = "Sporadic"
+                }
                 else {
-                    # ($LASTEXITCODE -eq 2)
+                    # ($LASTEXITCODE -eq 3)
                     $thisunsupported += 1
                     $thisresult = "Unsupported"
                 }
@@ -237,7 +246,9 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral
                 ($using:jresult).SolutionsExcluded += $thisexcluded
                 ($using:jresult).SolutionsUnsupported += $thisunsupported
                 ($using:jresult).SolutionsFailed += $thisfailed
+                ($using:jresult).SolutionsSporadic += $thissporadic
                 ($using:jresult).FailSet += $thisfailset
+                ($using:jresult).SporadicSet += $thissporadicset
                 $SolutionsTotal = $using:SolutionsTotal
                 $ThrottleLimit = $using:ThrottleLimit
                 $SolutionsBuilt = ($using:jresult).SolutionsBuilt
@@ -246,7 +257,7 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral
                 $SolutionsPending = if ($SolutionsRemaining -ge $ThrottleLimit) { ($SolutionsRemaining - $ThrottleLimit) } else { 0 }
                 $SolutionsBuiltPercent = [Math]::Round(100 * ($SolutionsBuilt / $using:SolutionsTotal))
                 $TBRP = "T:" + ($SolutionsTotal) + "; B:" + (($using:jresult).SolutionsBuilt) + "; R:" + ($SolutionsRunning) + "; P:" + ($SolutionsPending)
-                $rstr = "S:" + (($using:jresult).SolutionsSucceeded) + "; E:" + (($using:jresult).SolutionsExcluded) + "; U:" + (($using:jresult).SolutionsUnsupported) + "; F:" + (($using:jresult).SolutionsFailed)
+                $rstr = "S:" + (($using:jresult).SolutionsSucceeded) + "; E:" + (($using:jresult).SolutionsExcluded) + "; U:" + (($using:jresult).SolutionsUnsupported) + "; F:" + (($using:jresult).SolutionsFailed) + "; O:" + (($using:jresult).SolutionsSporadic)
                 Write-Progress -Activity "Building combinations" -Status "$SolutionsBuilt of $using:SolutionsTotal combinations built ($SolutionsBuiltPercent%) | $TBRP | $rstr" -PercentComplete $SolutionsBuiltPercent
             }
             finally {
@@ -265,6 +276,8 @@ $SampleSet.GetEnumerator() | ForEach-Object -ThrottleLimit $ThrottleLimit -Paral
 
 $sw.Stop()
 
+Write-Output ""
+
 if ($jresult.FailSet.Count -gt 0) {
     Write-Output "Some combinations were built with errors:"
     $jresult.FailSet = $jresult.FailSet | Sort-Object
@@ -274,10 +287,27 @@ if ($jresult.FailSet.Count -gt 0) {
         $failConfiguration = $Matches[2]
         $failPlatform = $Matches[3]
         Write-Output "Build errors in Sample $failName; Configuration: $failConfiguration; Platform: $failPlatform {"
-        Get-Content "$LogFilesDirectory\$failName.$failConfiguration.$failPlatform.err" | Write-Output
+        Get-Content "$LogFilesDirectory\$failName.$failConfiguration.$failPlatform.0.err" | Write-Output
         Write-Output "} $failedSample"
     }
     Write-Error "Some combinations were built with errors."
+    Write-Output ""
+}
+
+if ($jresult.SporadicSet.Count -gt 0) {
+    Write-Output "Some combinations were built with sporadic error:"
+    $jresult.SporadicSet = $jresult.SporadicSet | Sort-Object
+    foreach ($sporadicSample in $jresult.SporadicSet) {
+        $sporadicSample -match "^(.*) (\w*)\|(\w*)$" | Out-Null
+        $sporadicName = $Matches[1]
+        $sporadicConfiguration = $Matches[2]
+        $sporadicPlatform = $Matches[3]
+        Write-Output "Build sporadic errors in Sample $sporadicName; Configuration: $sporadicConfiguration; Platform: $sporadicPlatform {"
+        Get-Content "$LogFilesDirectory\$sporadicName.$sporadicConfiguration.$sporadicPlatform.0.err" | Write-Output
+        Write-Output "} $sporadicSample"
+    }
+    Write-Error "Some combinations were built with sporadic errors."
+    Write-Output ""
 }
 
 # Display timer statistics to host
@@ -288,9 +318,9 @@ $SolutionsSucceeded = $jresult.SolutionsSucceeded
 $SolutionsExcluded = $jresult.SolutionsExcluded
 $SolutionsUnsupported = $jresult.SolutionsUnsupported
 $SolutionsFailed = $jresult.SolutionsFailed
+$SolutionsSporadic = $jresult.SolutionsSporadic
 $Results = $jresult.Results
 
-Write-Output ""
 Write-Output "Built all combinations."
 Write-Output ""
 Write-Output "Elapsed time:         $min minutes, $seconds seconds."
@@ -303,6 +333,7 @@ Write-Output "Succeeded:            $SolutionsSucceeded"
 Write-Output "Excluded:             $SolutionsExcluded"
 Write-Output "Unsupported:          $SolutionsUnsupported"
 Write-Output "Failed:               $SolutionsFailed"
+Write-Output "Sporadic:             $SolutionsSporadic"
 Write-Output "Log files directory:  $LogFilesDirectory"
 Write-Output "Overview report:      $reportFilePath"
 Write-Output ""