Port "ignore nonexistent queue" functionality to release/6.0 (#8156)

* In preparation of removing deadlettered queues, teach the SDK to optionally ignore when a queue does not exist.

Port of #8069 (bbb4187)

* put back property I shouldn't have removed

src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs

@@ -154,6 +154,17 @@ public async Task<ISentJob> SendAsync(Action<string> log, CancellationToken canc
 
             var (queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);
 
+            // Save time / resources by checking that the queue isn't missing before doing any potentially expensive storage operations
+            try
+            {
+                QueueInfo queueInfo = await HelixApi.Information.QueueInfoAsync(queueId, false, cancellationToken);
+            }
+            // 404 = this queue does not exist, or did and was removed.
+            catch (RestApiException ex) when (ex.Response?.Status == 404)
+            {
+                throw new ArgumentException($"Helix API does not contain an entry for {queueId}");
+            }
+
             IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId, cancellationToken);
             var jobList = new List<JobListEntry>();
 

src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs

@@ -21,6 +21,13 @@ public abstract class HelixTask : BaseTask, ICancelableTask
         /// </summary>
         public string AccessToken { get; set; }
 
+        /// <summary>
+        ///   If <see langword="true"/>, fail when posting jobs to non-existent queues; If <see langword="false"/> allow it and print a warning.
+        ///   Note if an MSBuild sequence starts and waits on jobs, and none are started, this will still fail.
+        ///   Defined on HelixTask so the catch block around Execute() can know about it.
+        /// </summary>
+        public bool FailOnMissingTargetQueue { get; set; } = true;
+
         protected IHelixApi HelixApi { get; private set; }
 
         protected IHelixApi AnonymousApi { get; private set; }
@@ -63,6 +70,17 @@ public sealed override bool Execute()
                 // Canceled
                 return false;
             }
+            catch (ArgumentException argEx) when (argEx.Message.StartsWith("Helix API does not contain an entry "))
+            {
+                if (FailOnMissingTargetQueue)
+                {
+                    Log.LogError(FailureCategory.Build, argEx.Message);
+                }
+                else
+                {
+                    Log.LogWarning($"{argEx.Message} (FailOnMissingTargetQueue is false, so this is just a warning.)");
+                }
+            }
             catch (Exception ex)
             {
                 Log.LogErrorFromException(FailureCategory.Helix, ex, true, true, null);

src/Microsoft.DotNet.Helix/Sdk/Readme.md

@@ -1,9 +1,9 @@
 # Microsoft.DotNet.Helix.Sdk
 
-This Package provides Helix Job sending functionality from an MSBuild project file.
+This Package provides Helix Job-sending functionality from an MSBuild project file.
 
 ## Examples
-Each of the following examples require dotnet-cli >= 2.1.300 and need the following files in a directory at or above the example project's directory.
+Each of the following examples require dotnet-cli >= 3.1.x, and need the following files in a directory at or above the example project's directory.
 #### NuGet.config
 ```xml
 <?xml version="1.0" encoding="utf-8"?>
@@ -104,13 +104,21 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <!-- The helix queue this job should run on. -->
     <HelixTargetQueue>Windows.10.Amd64.Open</HelixTargetQueue>
 
+    <!-- Whether to fail the build if any Helix queues supplied don't exist.
+         If set to false, sending to non-existent Helix Queues will only print a warning. Defaults to true. 
+         Only set this to false if losing this coverage when the target queue is deprecated is acceptable.
+         For any job waiting on runs, this will still cause failure if all queues do not exist as there must be
+         one or more runs started for waiting to not log errors.  Only set if you need it.
+    -->
+    <FailOnMissingTargetQueue>false</FailOnMissingTargetQueue>
+
     <!--
       The set of helix queues to send jobs to.
       This property is multiplexed over just like <TargetFrameworks> for C# projects.
       The project is built once per entry in this list with <HelixTargetQueue> set to the current list element value.
       Note that all payloads sent need to be able to run on all variations included.
     -->
-    <HelixTargetQueues>Ubuntu.1804.Amd64.Open;Ubuntu.1604.Amd64.Open;(Alpine.39.Amd64)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053</HelixTargetQueues>
+    <HelixTargetQueues>Ubuntu.1804.Amd64.Open;Ubuntu.1604.Amd64.Open;(Alpine.39.Amd64)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053</HelixTargetQueues>
 
     <!-- 'true' to download dotnet cli and add it to the path for every workitem. Default 'false' -->
     <IncludeDotNetCli>true</IncludeDotNetCli>
@@ -127,19 +135,7 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <FailOnTestFailure>true</FailOnTestFailure>
 
     <!--
-      'true' to enable the xunit reporter. Default 'false'
-      The xunit reporter will report test results from a test results
-      xml file found in the work item working directory.
-      The following file names are accepted:
-        testResults.xml
-        test-results.xml
-        test_results.xml
-    -->
     <EnableXUnitReporter>false</EnableXUnitReporter>
-    <!-- Instruct the sdk to wait for test result ingestion by MC, and fail if there are any failed work items or tests. -->
-    <FailOnMissionControlTestFailure>false</FailOnMissionControlTestFailure>
-
-    <!--
       Commands that are run before each workitem's command
       semicolon-separated; use ';;' to escape a single semicolon
     -->
@@ -188,7 +184,6 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <XUnitArguments></XUnitArguments>
   </PropertyGroup>
 
-
   <ItemGroup>
     <!--
       Another way to specify target queues

src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets

@@ -2,6 +2,10 @@
 <Project>
   <PropertyGroup>
     <EnableXUnitReporter Condition=" '$(EnableXUnitReporter)' != 'true' ">false</EnableXUnitReporter>
+    <!-- Helix Queues which do not exist (deprecation, typos, or purposful removal for use reduction) will error by default. 
+         For users that do not want this to break builds (such as in release branch testing) this property allows to downgrade 
+         this failure mode to just a warning, which hopefully still tells the user to remove usage when possible. -->
+    <FailOnMissingTargetQueue Condition=" '$(FailOnMissingTargetQueue)' != 'false' ">true</FailOnMissingTargetQueue>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(HelixTestConfigurationFilePath)' == '' ">
@@ -54,6 +58,7 @@
     </PropertyGroup>
     <SendHelixJob Type="$(HelixType)"
                   TargetQueue="$(HelixTargetQueue)"
+                  FailOnMissingTargetQueue="$(FailOnMissingTargetQueue)"
                   IsPosixShell="$(IsPosixShell)"
                   Creator="$(Creator)"
                   BaseUri="$(HelixBaseUri)"