From 463ceeeafbed7b577e3488f8b64da1c0690b36d4 Mon Sep 17 00:00:00 2001 From: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Date: Tue, 7 Feb 2023 17:56:12 +0400 Subject: [PATCH] Upd dotnet6 agent (#4150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Catch updating exception (#4082) * Node16 version updated to 16.17.1" (#4084) * Bump ubuntu image version to 20.04 (#4079) * Prevent creating host container network (#3788) * Prevent creating host container network * Remove redundant empty lines * Fix invalid knob value fetching * Update ContainerOperationProvider.cs Co-authored-by: Denis Rumyantsev Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Allow env.sh to accept additional env var names (#3339) The hard-coded list is not necessarily enough. This allows the caller to specify additional environment variables to write to .env. Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Revert "Fix execution of scripts in variables (#4016)" (#4088) This reverts commit af5328b2656c9bf5a70117044a707e011de77083. * Add Ubuntu 18.04 (#3943) Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Make agent update list of systems supporting .NET 6 periodically (#4081) * implemented logic to fetch net6.json from server every hour * added await keyword * Moved logic to set warning from ExecutionContext to JobRunner since InitializeJob method is not async * fixed issue with condition in Equals method of OperatingSystem class * small refactoring * net6.json must be read only if it was not fetched from server * refactored code of method "GetNet6SupportedSystems" * restored field net6SupportedSystems to reduce IO oprations and cover case when agent started but net6.json is not older than 1 hour Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Retries to lock Services database (#4085) * retries to lock database * Decrease timeout + Add retry message Co-authored-by: Konstantin Tyukalov Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Fixed name of DockerAdditionalNetworkOptions Knob (#3888) Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> * Format agent source code (#4101) * Run dotnet format * Remove extra semicolons * Fix SonarQube's "'Any()' should be used to test for emptiness" / Code Smell (#3404) * Add script to predict v2->v3 upgrade compatibility (#4092) * Add script to predict v2->v3 upgrade compatibility * Fix typo * Fixed another typo: frpom Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Added strict boolean knob values (#4103) * Add ConvertToBooleanStrict * Add AsBooleanStrict for agent knobs * update build-job (#4110) * update build-job - added UseDotNet task for linux os * Switch EsrpCodeSigning from V1 to V2 returned back UseDotNet task condition * Set UseDotNet version to 6 on osx * added dotnet 3.1.x before unit tests on win x86 * fix dotnet host search (#4124) * fix dotnet host search - added performMultiLevelLookup * Added runtime to unitTests step * Changed patch version in semver of runtime * remove sdk and runtime from unitTests script * Fixed problem with .NET Core 3.1 installation * Added UseDotNet task in functionalTests Co-authored-by: Roman Shchukin * Add option to skip rhel6 in pipeline and ci (#4121) * skipRhelRelease parameter added * skipRhelRelease parameter fixed * releaseRhelOnly parameter added * Logic of removing not filling lines from releaseNote * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Fix pipeline with test releaseNote job * Removed test logs * Added clear empty lines logic * Commented code for test. Test changes in pipeline * Removed test changes * skipRhelRelease parametr added to ci pipeline * Update dotnet to 3.1.32 (#4112) * DockerLogin retries (#4100) * DockerLogin retries - added changes from PR microsoft/azure-pipelines-agent#3840 - fixed incorrect variables * DockerLogin retries - added step UseDotNet@2 * DockerLogin retries - added linux to UseDotNet task * Docker Login retries Implement retry into DockerLogin method * Implement feature flag for docker login retries * revert build-job back * Added retry for docker start * fix delay * update knob description * Update agent Docker methods - reworked Docker methods with retries behavior - changed InvokeWithRetryIfNonZero on to incapsulated logic method * - renamed local variable as is in ADO * Added docker retry behavior to docker version method Co-authored-by: Your Name Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> Co-authored-by: Merlyn Oppenheim * Add the exact operation system version and environment to the telemetry (#4104) * Update TaskRunner.cs * Update build-job.yml * Added new variables to telemetry AgentName, MachineName, IsSelfHosted Co-authored-by: Kirill Ivlev <102740624+kirill-ivlev@users.noreply.github.com> * Mingit version upgraded to 2.39.1 (#4126) * Fix agent lint errors (#4117) * Fix CA2000 error with secrets masker * Fix CA1711 for ServiceBootFlag * Fix CA2000 for JobRunner * Resolve CA2000 in StepHost * Resolve CA2000 in Windows service * Resolve CA2000 for WorkerCommandManager * Resove CA2000 for CodeCoverageCommands * Remove unused namespaces * Add fixture for SecretMasker tests * Change Fixtures -> Disposable * Use dispose pattern to fix errors * Formatting * Revert "Allow env.sh to accept additional env var names (#3339)" (#4143) This reverts commit 27e3d4e177e83aac805bf93c38233fa1a9f06101. --------- Co-authored-by: Roman-Shchukin <111063382+Roman-Shchukin@users.noreply.github.com> Co-authored-by: Konstantin Tyukalov <52399739+KonstantinTyukalov@users.noreply.github.com> Co-authored-by: Steve Co-authored-by: Denis Rumyantsev Co-authored-by: Rick Brown Co-authored-by: Liliia Sabitova <107196662+LiliaSabitova@users.noreply.github.com> Co-authored-by: Eric van Wijk Co-authored-by: Sergey Koryshev Co-authored-by: Konstantin Tyukalov Co-authored-by: Rami Co-authored-by: Martin Co-authored-by: ivanduplenskikh <115665590+ivanduplenskikh@users.noreply.github.com> Co-authored-by: Roman Shchukin Co-authored-by: Your Name Co-authored-by: Merlyn Oppenheim Co-authored-by: İsmayıl İsmayılov <110806089+ismayilov-ismayil@users.noreply.github.com> --- .vsts.ci.yml | 2 +- src/Agent.Sdk/Util/ExceptionsUtil.cs | 3 -- src/Agent.Sdk/Util/ILoggedSecretMasker.cs | 3 -- src/Agent.Sdk/Util/LoggedSecretMasker.cs | 3 -- src/Agent.Sdk/Util/MaskingUtil.cs | 2 -- src/Agent.Sdk/Util/PlatformUtil.cs | 7 ---- .../ContainerOperationProvider.cs | 2 +- src/Agent.Worker/Handlers/StepHost.cs | 34 +++++++++---------- src/Agent.Worker/WorkerCommandManager.cs | 9 +++-- src/Misc/externals.sh | 2 +- .../SecretMaskerTests/LoggedSecretMaskerL0.cs | 2 +- 11 files changed, 24 insertions(+), 45 deletions(-) diff --git a/.vsts.ci.yml b/.vsts.ci.yml index 9917e704df..739583ae9b 100644 --- a/.vsts.ci.yml +++ b/.vsts.ci.yml @@ -29,7 +29,7 @@ parameters: default: true - name: macOS_arm64 type: boolean - displayName: macOS (arm64) + displayName: macOS (arm64)m default: true - name: skipRhelRelease type: boolean diff --git a/src/Agent.Sdk/Util/ExceptionsUtil.cs b/src/Agent.Sdk/Util/ExceptionsUtil.cs index 29da4be311..8412c64f47 100644 --- a/src/Agent.Sdk/Util/ExceptionsUtil.cs +++ b/src/Agent.Sdk/Util/ExceptionsUtil.cs @@ -2,10 +2,7 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.Net.Sockets; -using System.Text; -using Microsoft.VisualStudio.Services.Agent; using Microsoft.VisualStudio.Services.Agent.Util; namespace Agent.Sdk.Util diff --git a/src/Agent.Sdk/Util/ILoggedSecretMasker.cs b/src/Agent.Sdk/Util/ILoggedSecretMasker.cs index 5cb01c379c..acd832ae77 100644 --- a/src/Agent.Sdk/Util/ILoggedSecretMasker.cs +++ b/src/Agent.Sdk/Util/ILoggedSecretMasker.cs @@ -1,8 +1,5 @@ using Microsoft.TeamFoundation.DistributedTask.Logging; using System; -using System.Collections.Generic; -using System.Text; -using System.Text.RegularExpressions; namespace Agent.Sdk.Util { diff --git a/src/Agent.Sdk/Util/LoggedSecretMasker.cs b/src/Agent.Sdk/Util/LoggedSecretMasker.cs index 0ac9571c08..2eede77beb 100644 --- a/src/Agent.Sdk/Util/LoggedSecretMasker.cs +++ b/src/Agent.Sdk/Util/LoggedSecretMasker.cs @@ -1,8 +1,5 @@ using Microsoft.TeamFoundation.DistributedTask.Logging; using System; -using System.Collections.Generic; -using System.Text; -using System.Text.RegularExpressions; namespace Agent.Sdk.Util { diff --git a/src/Agent.Sdk/Util/MaskingUtil.cs b/src/Agent.Sdk/Util/MaskingUtil.cs index 15317d2d19..6ad41ffcee 100644 --- a/src/Agent.Sdk/Util/MaskingUtil.cs +++ b/src/Agent.Sdk/Util/MaskingUtil.cs @@ -1,7 +1,5 @@ using Microsoft.VisualStudio.Services.ServiceEndpoints.WebApi; using System; -using System.Collections.Generic; -using System.Text; namespace Microsoft.VisualStudio.Services.Agent.Util { diff --git a/src/Agent.Sdk/Util/PlatformUtil.cs b/src/Agent.Sdk/Util/PlatformUtil.cs index 53d5bfd108..29a5375e1b 100644 --- a/src/Agent.Sdk/Util/PlatformUtil.cs +++ b/src/Agent.Sdk/Util/PlatformUtil.cs @@ -2,12 +2,9 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; -using System.Net.Sockets; using System.Reflection; using System.Runtime.InteropServices; using System.Text; @@ -15,10 +12,6 @@ using System.Threading.Tasks; using System.Xml.Linq; using Agent.Sdk.Knob; -using Agent.Sdk.Util; -using BuildXL.Cache.MemoizationStore.Interfaces.Caches; -using BuildXL.Utilities; -using Microsoft.TeamFoundation.Build.WebApi; using Microsoft.VisualStudio.Services.Agent.Util; using Microsoft.Win32; using Newtonsoft.Json; diff --git a/src/Agent.Worker/ContainerOperationProvider.cs b/src/Agent.Worker/ContainerOperationProvider.cs index 64c8b40f1c..73c0f5c7dd 100644 --- a/src/Agent.Worker/ContainerOperationProvider.cs +++ b/src/Agent.Worker/ContainerOperationProvider.cs @@ -789,7 +789,7 @@ private async Task CreateContainerNetworkAsync(IExecutionContext executionContex { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); - + if (network != "host") { int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, network); diff --git a/src/Agent.Worker/Handlers/StepHost.cs b/src/Agent.Worker/Handlers/StepHost.cs index dbc97674de..aa0c2a07c7 100644 --- a/src/Agent.Worker/Handlers/StepHost.cs +++ b/src/Agent.Worker/Handlers/StepHost.cs @@ -210,24 +210,22 @@ public async Task ExecuteAsync(string workingDirectory, // It appears that node.exe outputs UTF8 when not in TTY mode. outputEncoding = Encoding.UTF8; } - using (var redirectStandardIn = new InputQueue()) - { - var payloadJson = JsonUtility.ToString(payload); - redirectStandardIn.Enqueue(payloadJson); - HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Payload: {payloadJson}"); - return await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work), - fileName: containerEnginePath, - arguments: containerExecutionArgs, - environment: null, - requireExitCodeZero: requireExitCodeZero, - outputEncoding: outputEncoding, - killProcessOnCancel: killProcessOnCancel, - redirectStandardIn: redirectStandardIn, - inheritConsoleHandler: inheritConsoleHandler, - continueAfterCancelProcessTreeKillAttempt: continueAfterCancelProcessTreeKillAttempt, - cancellationToken: cancellationToken); - } - + + using var redirectStandardIn = new InputQueue(); + var payloadJson = JsonUtility.ToString(payload); + redirectStandardIn.Enqueue(payloadJson); + HostContext.GetTrace(nameof(ContainerStepHost)).Info($"Payload: {payloadJson}"); + return await processInvoker.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work), + fileName: containerEnginePath, + arguments: containerExecutionArgs, + environment: null, + requireExitCodeZero: requireExitCodeZero, + outputEncoding: outputEncoding, + killProcessOnCancel: killProcessOnCancel, + redirectStandardIn: redirectStandardIn, + inheritConsoleHandler: inheritConsoleHandler, + continueAfterCancelProcessTreeKillAttempt: continueAfterCancelProcessTreeKillAttempt, + cancellationToken: cancellationToken); } } diff --git a/src/Agent.Worker/WorkerCommandManager.cs b/src/Agent.Worker/WorkerCommandManager.cs index f2e1c072b5..e88061a0ae 100644 --- a/src/Agent.Worker/WorkerCommandManager.cs +++ b/src/Agent.Worker/WorkerCommandManager.cs @@ -111,11 +111,10 @@ public bool TryProcessCommand(IExecutionContext context, string input) } catch (SocketException ex) { - using (var vssConnection = WorkerUtilities.GetVssConnection(context)) - { - ExceptionsUtil.HandleSocketException(ex, vssConnection.Uri.ToString(), context.Error); - context.CommandResult = TaskResult.Failed; - } + using var vssConnection = WorkerUtilities.GetVssConnection(context); + + ExceptionsUtil.HandleSocketException(ex, vssConnection.Uri.ToString(), context.Error); + context.CommandResult = TaskResult.Failed; } catch (Exception ex) { diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 23b4e9ddaf..beb2df95ec 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -11,7 +11,7 @@ NODE_URL=https://nodejs.org/dist NODE_VERSION="6.17.1" NODE10_VERSION="10.24.1" NODE16_VERSION="16.17.1" -MINGIT_VERSION="2.38.1" +MINGIT_VERSION="2.39.1" LFS_VERSION="2.13.3" get_abs_path() { diff --git a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs index 67373ddac6..982c93bba2 100644 --- a/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs +++ b/src/Test/L0/SecretMaskerTests/LoggedSecretMaskerL0.cs @@ -134,7 +134,7 @@ protected virtual void Dispose(bool disposing) } _secretMasker.Dispose(); - _secretMasker= null; + _secretMasker = null; disposedValue = true; }