Setting maxClients for parallelism in yaml (#7481)

* Set maxClients in yaml * test sdk * minor fix * test sdk * Revert image * revert test sdk * review comments fix * added tag to comment
dotnet · Jun 8, 2021 · 3edd05c · 3edd05c
1 parent cde3634
commit 3edd05c
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 6 deletions.
diff --git a/eng/publishing/v3/publish-assets.yml b/eng/publishing/v3/publish-assets.yml
@@ -102,6 +102,8 @@ jobs:
           /p:AzureDevOpsOrg='$(AzDOAccount)'
           /p:AzureProject='$(AzDOProjectName)'
           /p:UseStreamingPublishing='true'
+          /p:StreamingPublishingMaxClients=16
+          /p:NonStreamingPublishingMaxClients=12
     - template: /eng/common/templates/steps/publish-logs.yml
       parameters:
         StageLabel: '${{ parameters.stageName }}'

diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishArtifactsInManifest.proj
@@ -109,6 +109,8 @@
       <AllowFeedOverrides Condition="'$(AllowFeedOverrides)' == ''">false</AllowFeedOverrides>
       <UseStreamingPublishing Condition="'$(UseStreamingPublishing)' == ''">false</UseStreamingPublishing>
       <ArtifactsBasePath Condition="'$(ArtifactsBasePath)' == ''">$(BlobBasePath)</ArtifactsBasePath>
+      <NonStreamingPublishingMaxClients Condition="'$(NonStreamingPublishingMaxClients)' == ''">12</NonStreamingPublishingMaxClients>
+      <StreamingPublishingMaxClients Condition="'$(UseStreamingPublishing)' == 'true' and '$(StreamingPublishingMaxClients)' == ''">16</StreamingPublishingMaxClients>
     </PropertyGroup>
 
     <Error
@@ -160,8 +162,9 @@
       BuildId="$(BuildId)"
       AzureDevOpsOrg="$(AzureDevOpsOrg)"
       AzureProject="$(AzureProject)"
-      UseStreamingPublishing="$(UseStreamingPublishing)"/>
-
+      UseStreamingPublishing="$(UseStreamingPublishing)"
+      StreamingPublishingMaxClients="$(StreamingPublishingMaxClients)"
+      NonStreamingPublishingMaxClients="$(NonStreamingPublishingMaxClients)"/>
   </Target>
 
   <ItemGroup>

diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifest.cs
@@ -196,6 +196,10 @@ public string BuildQuality
         /// </summary>
         public bool UseStreamingPublishing { get; set; } = false;
 
+        public int StreamingPublishingMaxClients {get; set;}
+
+        public int NonStreamingPublishingMaxClients {get; set;}
+
         /// <summary>
         /// Just an internal flag to keep track whether we published assets via a V3 manifest or not.
         /// </summary>
@@ -389,7 +393,9 @@ internal PublishArtifactsInManifestBase ConstructPublishingV3Task(BuildModel bui
                 BuildId = this.BuildId,
                 AzureProject = this.AzureProject,
                 AzureDevOpsOrg = this.AzureDevOpsOrg,
-                UseStreamingPublishing = this.UseStreamingPublishing
+                UseStreamingPublishing = this.UseStreamingPublishing,
+                StreamingPublishingMaxClients = this.StreamingPublishingMaxClients,
+                NonStreamingPublishingMaxClients = this.NonStreamingPublishingMaxClients
             };
         }
     }

diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishArtifactsInManifestBase.cs
@@ -69,9 +69,13 @@ public abstract class PublishArtifactsInManifestBase : Microsoft.Build.Utilities
         /// </summary>
         [Required]
         public string NugetPath { get; set; }
-
-        private const int StreamingPublishingMaxClients = 16;
-        private const int NonStreamingPublishingMaxClients = 12;
+
+        /// <summary>
+        /// We are setting StreamingPublishingMaxClients=16 and NonStreamingPublishingMaxClients=12 through publish-asset.yml as we were hitting OOM issue 
+        /// https://github.com/dotnet/core-eng/issues/13098 for more details.
+        /// </summary>
+        public int StreamingPublishingMaxClients {get; set;}
+        public int NonStreamingPublishingMaxClients {get; set;}
 
         /// <summary>
         /// Maximum number of parallel uploads for the upload tasks.