From 29a04f0e12a4ad2091e2fcd8814de5c704671bde Mon Sep 17 00:00:00 2001
From: Matt Galbraith <mattgal@microsoft.com>
Date: Tue, 19 Oct 2021 13:10:15 -0700
Subject: [PATCH] In preparation of removing deadlettered queues, teach the SDK
 to optionally ignore when a queue does not exist.

---
 .../JobSender/JobDefinition.cs                | 11 ++++++++
 src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs   | 18 +++++++++++++
 src/Microsoft.DotNet.Helix/Sdk/Readme.md      | 27 ++++++++-----------
 ...crosoft.DotNet.Helix.Sdk.MonoQueue.targets | 10 ++++++-
 4 files changed, 49 insertions(+), 17 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs
index b5b782beec6..078cff49c71 100644
--- a/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs
+++ b/src/Microsoft.DotNet.Helix/JobSender/JobDefinition.cs
@@ -154,6 +154,17 @@ public async Task<ISentJob> SendAsync(Action<string> log, CancellationToken canc
 
             var (queueId, dockerTag, queueAlias) = ParseQueueId(TargetQueueId);
 
+            // Save time / resources by checking that the queue isn't missing before doing any potentially expensive storage operations
+            try
+            {
+                QueueInfo queueInfo = await HelixApi.Information.QueueInfoAsync(queueId, false, cancellationToken);
+            }
+            // 404 = this queue does not exist, or did and was removed.
+            catch (RestApiException ex) when (ex.Response?.Status == 404)
+            {
+                throw new ArgumentException($"Helix API does not contain an entry for {queueId}");
+            }
+
             IBlobContainer storageContainer = await storage.GetContainerAsync(TargetContainerName, queueId, cancellationToken);
             var jobList = new List<JobListEntry>();
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs b/src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs
index c411bdca5b3..caa5ae3aef1 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk/HelixTask.cs
@@ -21,6 +21,13 @@ public abstract class HelixTask : BaseTask, ICancelableTask
         /// </summary>
         public string AccessToken { get; set; }
 
+        /// <summary>
+        ///   If <see langword="true"/>, fail when posting jobs to non-existent queues; If <see langword="false"/> allow it and print a warning.
+        ///   Note if an MSBuild sequence starts and waits on jobs, and none are started, this will still fail.
+        ///   Defined on HelixTask so the catch block around Execute() can know about it.
+        /// </summary>
+        public bool FailOnMissingTargetQueue { get; set; } = true;
+
         protected IHelixApi HelixApi { get; private set; }
 
         protected IHelixApi AnonymousApi { get; private set; }
@@ -63,6 +70,17 @@ public sealed override bool Execute()
                 // Canceled
                 return false;
             }
+            catch (ArgumentException argEx) when (argEx.Message.StartsWith("Helix API does not contain an entry "))
+            {
+                if (FailOnMissingTargetQueue)
+                {
+                    Log.LogError(FailureCategory.Build, argEx.Message);
+                }
+                else
+                {
+                    Log.LogWarning($"{argEx.Message} (FailOnMissingTargetQueue is false, so this is just a warning.)");
+                }
+            }
             catch (Exception ex)
             {
                 Log.LogErrorFromException(FailureCategory.Helix, ex, true, true, null);
diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md
index eb8ca7fe33f..661cafdff4b 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md
+++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md
@@ -1,9 +1,9 @@
 # Microsoft.DotNet.Helix.Sdk
 
-This Package provides Helix Job sending functionality from an MSBuild project file.
+This Package provides Helix Job-sending functionality from an MSBuild project file.
 
 ## Examples
-Each of the following examples require dotnet-cli >= 2.1.300 and need the following files in a directory at or above the example project's directory.
+Each of the following examples require dotnet-cli >= 3.1.x, and need the following files in a directory at or above the example project's directory.
 #### NuGet.config
 ```xml
 <?xml version="1.0" encoding="utf-8"?>
@@ -149,13 +149,21 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <!-- The helix queue this job should run on. -->
     <HelixTargetQueue>Windows.10.Amd64.Open</HelixTargetQueue>
 
+    <!-- Whether to fail the build if any Helix queues supplied don't exist.
+         If set to false, sending to non-existent Helix Queues will only print a warning. Defaults to true. 
+         Only set this to false if losing this coverage when the target queue is deprecated is acceptable.
+         For any job waiting on runs, this will still cause failure if all queues do not exist as there must be
+         one or more runs started for waiting to not log errors.  Only set if you need it.
+    -->
+    <FailOnMissingTargetQueue>false</FailOnMissingTargetQueue>
+
     <!--
       The set of helix queues to send jobs to.
       This property is multiplexed over just like <TargetFrameworks> for C# projects.
       The project is built once per entry in this list with <HelixTargetQueue> set to the current list element value.
       Note that all payloads sent need to be able to run on all variations included.
     -->
-    <HelixTargetQueues>Ubuntu.1804.Amd64.Open;Ubuntu.1604.Amd64.Open;(Alpine.39.Amd64)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053</HelixTargetQueues>
+    <HelixTargetQueues>Ubuntu.1804.Amd64.Open;Ubuntu.1604.Amd64.Open;(Alpine.39.Amd64)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053</HelixTargetQueues>
 
     <!-- 'true' to download dotnet cli and add it to the path for every workitem. Default 'false' -->
     <IncludeDotNetCli>true</IncludeDotNetCli>
@@ -171,18 +179,6 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <!-- 'true' to produce a build error when tests fail. Default 'true' -->
     <FailOnTestFailure>true</FailOnTestFailure>
 
-    <!--
-      'true' to enable the xunit reporter. Default 'false'
-      The xunit reporter will report test results from a test results
-      xml file found in the work item working directory.
-      The following file names are accepted:
-        testResults.xml
-        test-results.xml
-        test_results.xml
-    -->
-    <!-- Instruct the sdk to wait for test result ingestion by MC, and fail if there are any failed work items or tests. -->
-    <FailOnMissionControlTestFailure>false</FailOnMissionControlTestFailure>
-
     <!--
       Commands that are run before each workitem's command
       semicolon-separated; use ';;' to escape a single semicolon
@@ -232,7 +228,6 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
     <XUnitArguments></XUnitArguments>
   </PropertyGroup>
 
-
   <ItemGroup>
     <!--
       Another way to specify target queues
diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets
index b3f79a5c026..20c875c5637 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets
+++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets
@@ -1,5 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project>  
+<Project>
+  <PropertyGroup>
+    <!-- Helix Queues which do not exist (deprecation, typos, or purposful removal for use reduction) will error by default. 
+         For users that do not want this to break builds (such as in release branch testing) this property allows to downgrade 
+         this failure mode to just a warning, which hopefully still tells the user to remove usage when possible. -->
+    <FailOnMissingTargetQueue Condition=" '$(FailOnMissingTargetQueue)' != 'false' ">true</FailOnMissingTargetQueue>
+  </PropertyGroup>
+  
   <PropertyGroup Condition=" '$(HelixTestConfigurationFilePath)' == '' ">
     <HelixTestConfigurationFilePath Condition=" '$(RepositoryEngineeringDir)' != '' ">$(RepositoryEngineeringDir)/test-configuration.json</HelixTestConfigurationFilePath>
   </PropertyGroup>
@@ -38,6 +45,7 @@
     </PropertyGroup>
     <SendHelixJob Type="$(HelixType)"
                   TargetQueue="$(HelixTargetQueue)"
+                  FailOnMissingTargetQueue="$(FailOnMissingTargetQueue)"
                   IsPosixShell="$(IsPosixShell)"
                   Creator="$(Creator)"
                   BaseUri="$(HelixBaseUri)"