Update dotnet6-agent branch from master (#4128)

* Catch updating exception (#4082)

* Node16 version updated to 16.17.1" (#4084)

* Bump ubuntu image version to 20.04 (#4079)

* Prevent creating host container network (#3788)

* Prevent creating host container network

* Remove redundant empty lines

* Fix invalid knob value fetching

* Update ContainerOperationProvider.cs

Co-authored-by: Denis Rumyantsev <mr.denis.rumyantsev@gmail.com>
Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com>
Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com>

* Allow env.sh to accept additional env var names (#3339)

The hard-coded list is not necessarily enough. This allows the caller
to specify additional environment variables to write to .env.

Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com>
Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com>

* Revert "Fix execution of scripts in variables (#4016)" (#4088)

This reverts commit af5328b.

* Fix CA1711 for ServiceBootFlag

* Fix CA2000 for JobRunner

* Resolve CA2000 in StepHost

* Resolve CA2000 in Windows service

* Resolve CA2000 for WorkerCommandManager

* Resove CA2000 for CodeCoverageCommands

* Add fixture for SecretMasker tests

* Add fixture for SecretMasker tests

* Fail the installation of agent for unsupported OS - RHEL6/CentOS6

* Change Fixtures -> Disposable

* git update-index --chmod=+x .\src\dev.sh

* Use dispose pattern to fix errors

Co-authored-by: Roman-Shchukin <111063382+Roman-Shchukin@users.noreply.github.com>
Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com>
Co-authored-by: Steve <hez2010@outlook.com>
Co-authored-by: Denis Rumyantsev <mr.denis.rumyantsev@gmail.com>
Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com>
Co-authored-by: Rick Brown <ricksbrown73@gmail.com>
Co-authored-by: Konstantin Tyukalov <v-ktyukalov@microsoft.com>

.azure-pipelines/pipeline.yml

@@ -112,7 +112,7 @@ stages:
         jobName: build_linux_x64
         displayName: Linux (x64)
         pool:
-          vmImage: ubuntu-18.04
+          vmImage: ubuntu-20.04
         os: linux
         arch: x64
         branch: ${{ parameters.branch }}
@@ -162,14 +162,14 @@ stages:
         publishArtifacts: ${{ parameters.publishArtifacts }}
         buildAlternatePackage: ${{ parameters.buildAlternatePackage }}
 
-  # RHEL6 (x64)
+  # RHEL7 (x64)
   - ${{ if parameters.rhel7_x64 }}:
     - template: build-jobs.yml
       parameters:
-        jobName: build_rhel6_x64
+        jobName: build_rhel7_x64
         displayName: RHEL7 (x64)
         pool:
-          vmImage: ubuntu-18.04
+          vmImage: ubuntu-20.04
         container: dotnetcore_centos7
         os: linux
         arch: x64
@@ -188,7 +188,7 @@ stages:
         jobName: build_osx
         displayName: macOS (x64)
         pool:
-          vmImage: macOS-10.15
+          vmImage: macOS-11
         os: osx
         arch: x64
         branch: ${{ parameters.branch }}

.vsts.release.yml

@@ -44,7 +44,7 @@ extends:
         ################################################################################
           displayName: Create Release Branch
           pool:
-            vmImage: ubuntu-18.04
+            vmImage: ubuntu-20.04
 
           steps:
           - checkout: self
@@ -177,7 +177,7 @@ extends:
           ################################################################################
             displayName: Create PRs in AzureDevOps and ConfigChange
             pool:
-              vmImage: ubuntu-18.04
+              vmImage: ubuntu-20.04
 
             steps:
             - checkout: self

src/Agent.Listener/Agent.cs

@@ -323,6 +323,7 @@ private async Task<int> RunAsync(AgentSettings settings, bool runOnce = false)
                     Task<bool> selfUpdateTask = null;
                     bool runOnceJobReceived = false;
                     jobDispatcher = HostContext.CreateService<IJobDispatcher>();
+                    TaskAgentMessage previuosMessage = null;
 
                     while (!HostContext.AgentShutdownToken.IsCancellationRequested)
                     {
@@ -338,7 +339,18 @@ private async Task<int> RunAsync(AgentSettings settings, bool runOnce = false)
                                 if (completeTask == selfUpdateTask)
                                 {
                                     autoUpdateInProgress = false;
-                                    if (await selfUpdateTask)
+
+                                    bool agentUpdated = false;
+                                    try
+                                    {
+                                        agentUpdated = await selfUpdateTask;
+                                    }
+                                    catch (Exception ex)
+                                    {
+                                        Trace.Info($"Ignore agent update exception. {ex}");
+                                    }
+
+                                    if (agentUpdated)
                                     {
                                         Trace.Info("Auto update task finished at backend, an agent update is ready to apply exit the current agent instance.");
                                         Trace.Info("Stop message queue looping.");
@@ -365,6 +377,9 @@ private async Task<int> RunAsync(AgentSettings settings, bool runOnce = false)
                                     {
                                         Trace.Info("Auto update task finished at backend, there is no available agent update needs to apply, continue message queue looping.");
                                     }
+
+                                    message = previuosMessage;// if agent wasn't updated it's needed to process the previous message
+                                    previuosMessage = null;
                                 }
                             }
 
@@ -390,7 +405,7 @@ private async Task<int> RunAsync(AgentSettings settings, bool runOnce = false)
                                 }
                             }
 
-                            message = await getNextMessage; //get next message
+                            message ??= await getNextMessage; //get next message
                             HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
                             if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
                             {
@@ -417,6 +432,11 @@ private async Task<int> RunAsync(AgentSettings settings, bool runOnce = false)
                             else if (string.Equals(message.MessageType, JobRequestMessageTypes.AgentJobRequest, StringComparison.OrdinalIgnoreCase) ||
                                     string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
                             {
+                                if (autoUpdateInProgress)
+                                {
+                                    previuosMessage = message;
+                                }
+
                                 if (autoUpdateInProgress || runOnceJobReceived)
                                 {
                                     skipMessageDeletion = true;

src/Agent.Listener/Configuration.Windows/NativeWindowsServiceHelper.cs

@@ -515,7 +515,7 @@ public void InstallService(string serviceName, string serviceDisplayName, string
                                         serviceDisplayName,
                                         ServiceRights.AllAccess,
                                         SERVICE_WIN32_OWN_PROCESS,
-                                        ServiceBootTag.AutoStart,
+                                        ServiceStartType.AutoStart,
                                         ServiceError.Normal,
                                         agentServiceExecutable,
                                         null,
@@ -911,7 +911,7 @@ public void GrantDirectoryPermissionForAccount(string accountName, IList<string>
             AddMemberToLocalGroup(accountName, groupName);
 
             // grant permssion for folders
-            foreach(var folder in folders)
+            foreach (var folder in folders)
             {
                 if (Directory.Exists(folder))
                 {
@@ -929,7 +929,7 @@ public void RevokeDirectoryPermissionForAccount(IList<string> folders)
             Trace.Info(StringUtil.Format("Calculated unique group name {0}", groupName));
 
             // remove the group from folders
-            foreach(var folder in folders)
+            foreach (var folder in folders)
             {
                 if (Directory.Exists(folder))
                 {
@@ -938,7 +938,7 @@ public void RevokeDirectoryPermissionForAccount(IList<string> folders)
                     {
                         RemoveGroupFromFolderSecuritySetting(folder, groupName);
                     }
-                    catch(Exception ex)
+                    catch (Exception ex)
                     {
                         Trace.Error(ex);
                     }
@@ -1314,9 +1314,9 @@ public enum ServiceError
             Critical = 0x00000003
         }
 
-        public enum ServiceBootTag
+        public enum ServiceStartType
         {
-            Start = 0x00000000,
+            BootStart = 0x00000000,
             SystemStart = 0x00000001,
             AutoStart = 0x00000002,
             DemandStart = 0x00000003,
@@ -1407,7 +1407,7 @@ private static extern IntPtr CreateService(
             string lpDisplayName,
             ServiceRights dwDesiredAccess,
             int dwServiceType,
-            ServiceBootTag dwStartType,
+            ServiceStartType dwStartType,
             ServiceError dwErrorControl,
             string lpBinaryPathName,
             string lpLoadOrderGroup,

src/Agent.Service/Windows/Program.cs

@@ -24,11 +24,13 @@ static int Main(String[] args)
                     return 1;
                 }
 
-                EventLog applicationLog = new EventLog("Application");
-                if (applicationLog.OverflowAction == OverflowAction.DoNotOverwrite)
+                using (EventLog applicationLog = new EventLog("Application"))
                 {
-                    Console.WriteLine("[WARNING] The retention policy for Application event log is set to \"Do not overwrite events\".");
-                    Console.WriteLine("[WARNING] Make sure manually clear logs as needed, otherwise AgentService will stop writing output to event log.");
+                    if (applicationLog.OverflowAction == OverflowAction.DoNotOverwrite)
+                    {
+                        Console.WriteLine("[WARNING] The retention policy for Application event log is set to \"Do not overwrite events\".");
+                        Console.WriteLine("[WARNING] Make sure manually clear logs as needed, otherwise AgentService will stop writing output to event log.");
+                    }
                 }
 
                 try

src/Agent.Worker/CodeCoverage/CodeCoverageCommands.cs

@@ -15,7 +15,7 @@
 
 namespace Microsoft.VisualStudio.Services.Agent.Worker.CodeCoverage
 {
-    public sealed class CodeCoverageCommandExtension: BaseWorkerCommandExtension
+    public sealed class CodeCoverageCommandExtension : BaseWorkerCommandExtension
     {
 
         public CodeCoverageCommandExtension()
@@ -27,7 +27,7 @@ public CodeCoverageCommandExtension()
     }
 
     #region publish code coverage helper methods
-    public sealed class PublishCodeCoverageCommand: IWorkerCommand
+    public sealed class PublishCodeCoverageCommand : IWorkerCommand
     {
         public string Name => "publish";
         public List<string> Aliases => null;
@@ -69,7 +69,7 @@ public void Execute(IExecutionContext context, Command command)
             context.Output(StringUtil.Loc("ReadingCodeCoverageSummary", _summaryFileLocation));
             var coverageData = reader.GetCodeCoverageSummary(context, _summaryFileLocation);
 
-            if (coverageData == null || coverageData.Count() == 0)
+            if (coverageData == null || !coverageData.Any())
             {
                 context.Warning(StringUtil.Loc("CodeCoverageDataIsNull"));
             }
@@ -84,7 +84,6 @@ public void Execute(IExecutionContext context, Command command)
             context.AsyncCommands.Add(commandContext);
         }
 
-        [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Maintainability", "CA2000:Dispose objects before losing scope")]
         private async Task PublishCodeCoverageAsync(
             IExecutionContext executionContext,
             IAsyncCommandContext commandContext,
@@ -96,7 +95,7 @@ private async Task PublishCodeCoverageAsync(
             CancellationToken cancellationToken)
         {
             //step 2: publish code coverage summary to TFS
-            if (coverageData != null && coverageData.Count() > 0)
+            if (coverageData != null && coverageData.Any())
             {
                 commandContext.Output(StringUtil.Loc("PublishingCodeCoverage"));
                 foreach (var coverage in coverageData)
@@ -150,7 +149,8 @@ private async Task PublishCodeCoverageAsync(
             }
             catch (SocketException ex)
             {
-                ExceptionsUtil.HandleSocketException(ex, WorkerUtilities.GetVssConnection(executionContext).Uri.ToString(), executionContext.Warning);
+                using var vssConnection = WorkerUtilities.GetVssConnection(executionContext);
+                ExceptionsUtil.HandleSocketException(ex, vssConnection.Uri.ToString(), executionContext.Warning);
             }
             catch (Exception ex)
             {

src/Agent.Worker/ContainerOperationProvider.cs

@@ -42,7 +42,13 @@ public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
             _dockerManger = HostContext.GetService<IDockerCommandManager>();
-            _containerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}";
+            _containerNetwork = $"vsts_network_{Guid.NewGuid():N}";
+        }
+
+        private string GetContainerNetwork(IExecutionContext executionContext)
+        {
+            var useHostNetwork = AgentKnobs.DockerNetworkCreateDriver.GetValue(executionContext).AsString() == "host";
+            return useHostNetwork ? "host" : _containerNetwork;
         }
 
         public async Task StartContainersAsync(IExecutionContext executionContext, object data)
@@ -110,8 +116,9 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec
 
             // Create local docker network for this job to avoid port conflict when multiple agents run on same machine.
             // All containers within a job join the same network
-            await CreateContainerNetworkAsync(executionContext, _containerNetwork);
-            containers.ForEach(container => container.ContainerNetwork = _containerNetwork);
+            var containerNetwork = GetContainerNetwork(executionContext);
+            await CreateContainerNetworkAsync(executionContext, containerNetwork);
+            containers.ForEach(container => container.ContainerNetwork = containerNetwork);
 
             foreach (var container in containers)
             {
@@ -146,7 +153,8 @@ public async Task StopContainersAsync(IExecutionContext executionContext, object
                 await StopContainerAsync(executionContext, container);
             }
             // Remove the container network
-            await RemoveContainerNetworkAsync(executionContext, _containerNetwork);
+            var containerNetwork = GetContainerNetwork(executionContext);
+            await RemoveContainerNetworkAsync(executionContext, containerNetwork);
         }
 
         private async Task<string> GetMSIAccessToken(IExecutionContext executionContext)
@@ -793,11 +801,20 @@ private async Task CreateContainerNetworkAsync(IExecutionContext executionContex
         {
             Trace.Entering();
             ArgUtil.NotNull(executionContext, nameof(executionContext));
-            int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, network);
-            if (networkExitCode != 0)
+
+            if (network != "host")
+            {
+                int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, network);
+                if (networkExitCode != 0)
+                {
+                    throw new InvalidOperationException($"Docker network create failed with exit code {networkExitCode}");
+                }
+            }
+            else
             {
-                throw new InvalidOperationException($"Docker network create failed with exit code {networkExitCode}");
+                Trace.Info("Skipping creation of a new docker network. Reusing the host network.");
             }
+
             // Expose docker network to env
             executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, network);
         }
@@ -808,13 +825,17 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex
             ArgUtil.NotNull(executionContext, nameof(executionContext));
             ArgUtil.NotNull(network, nameof(network));
 
-            executionContext.Output($"Remove container network: {network}");
-
-            int removeExitCode = await _dockerManger.DockerNetworkRemove(executionContext, network);
-            if (removeExitCode != 0)
+            if (network != "host")
             {
-                executionContext.Warning($"Docker network rm failed with exit code {removeExitCode}");
+                executionContext.Output($"Remove container network: {network}");
+
+                int removeExitCode = await _dockerManger.DockerNetworkRemove(executionContext, network);
+                if (removeExitCode != 0)
+                {
+                    executionContext.Warning($"Docker network rm failed with exit code {removeExitCode}");
+                }
             }
+
             // Remove docker network from env
             executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, null);
         }

src/Agent.Worker/Handlers/StepHost.cs

@@ -109,7 +109,8 @@ public string ResolvePathForStepHost(string path)
             StringComparison sc = (PlatformUtil.RunningOnWindows)
                                 ? StringComparison.OrdinalIgnoreCase
                                 : StringComparison.Ordinal;
-            if (Container.MountVolumes.Exists(x => {
+            if (Container.MountVolumes.Exists(x =>
+            {
                 if (!string.IsNullOrEmpty(x.SourceVolumePath))
                 {
                     return path.StartsWith(x.SourceVolumePath, sc);

src/Agent.Worker/JobRunner.cs

@@ -94,7 +94,9 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
 
             IExecutionContext jobContext = null;
             CancellationTokenRegistration? agentShutdownRegistration = null;
-            VssConnection connection = null;
+            VssConnection taskConnection = null;
+            VssConnection legacyTaskConnection = null;
+
             try
             {
                 // Create the job execution context.
@@ -190,8 +192,9 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
                 if (taskServerUri != null)
                 {
                     Trace.Info($"Creating task server with {taskServerUri}");
-                    connection = VssUtil.CreateConnection(taskServerUri, taskServerCredential, Trace);
-                    await taskServer.ConnectAsync(connection);
+
+                    taskConnection = VssUtil.CreateConnection(taskServerUri, taskServerCredential, Trace);
+                    await taskServer.ConnectAsync(taskConnection);
                 }
 
                 // for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url
@@ -202,9 +205,10 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
                         Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
                         var configStore = HostContext.GetService<IConfigurationStore>();
                         taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
+
                         Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
-                        connection = VssUtil.CreateConnection(taskServerUri, taskServerCredential, Trace);
-                        await taskServer.ConnectAsync(connection);
+                        legacyTaskConnection = VssUtil.CreateConnection(taskServerUri, taskServerCredential, trace: Trace);
+                        await taskServer.ConnectAsync(legacyTaskConnection);
                     }
                 }
 
@@ -224,7 +228,7 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
                     {
                         checkoutOptions = jobContext.Variables.ExpandValues(target: checkoutOptions);
                         checkoutOptions = VarUtil.ExpandEnvironmentVariables(HostContext, target: checkoutOptions);
-                        repository.Properties.Set<JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions); ;
+                        repository.Properties.Set<JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions);
                     }
 
                     // expand workspace mapping
@@ -346,8 +350,11 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
                     agentShutdownRegistration.Value.Dispose();
                     agentShutdownRegistration = null;
                 }
-                connection.Dispose();
-                jobConnection.Dispose();
+
+                legacyTaskConnection?.Dispose();
+                taskConnection?.Dispose();
+                jobConnection?.Dispose();
+
                 await ShutdownQueue(throwOnFailure: false);
             }
         }