From 979135d5ca4efaf6436ee13539cc3f1e039d570a Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Mon, 10 Jun 2024 18:31:30 +0000 Subject: [PATCH 1/8] Limit the size of OID supported by AsnDecoder/AsnReader --- .../Cryptography/DSAKeyFormatHelper.cs | 82 +++++-- .../DSA/DSAKeyFileTests.cs | 146 ++++++++++++ .../src/Resources/Strings.resx | 3 + .../src/System.Formats.Asn1.csproj | 6 + .../src/System/Formats/Asn1/AsnDecoder.Oid.cs | 53 +++++ .../Formats/Asn1/LocalAppContextSwitches.cs | 17 ++ .../tests/Reader/ReadObjectIdentifier.cs | 219 ++++++++++++++---- .../tests/System.Formats.Asn1.Tests.csproj | 1 + 8 files changed, 464 insertions(+), 63 deletions(-) create mode 100644 src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/LocalAppContextSwitches.cs diff --git a/src/libraries/Common/src/System/Security/Cryptography/DSAKeyFormatHelper.cs b/src/libraries/Common/src/System/Security/Cryptography/DSAKeyFormatHelper.cs index 63c7b451e142b..e63792f3ba1ac 100644 --- a/src/libraries/Common/src/System/Security/Cryptography/DSAKeyFormatHelper.cs +++ b/src/libraries/Common/src/System/Security/Cryptography/DSAKeyFormatHelper.cs @@ -25,16 +25,6 @@ internal static void ReadDsaPrivateKey( throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding); } - DssParms parms = DssParms.Decode(algId.Parameters.Value, AsnEncodingRules.BER); - - ret = new DSAParameters - { - P = parms.P.ToByteArray(isUnsigned: true, isBigEndian: true), - Q = parms.Q.ToByteArray(isUnsigned: true, isBigEndian: true), - }; - - ret.G = parms.G.ExportKeyParameter(ret.P.Length); - BigInteger x; try @@ -57,6 +47,34 @@ internal static void ReadDsaPrivateKey( throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding, e); } + DssParms parms = DssParms.Decode(algId.Parameters.Value, AsnEncodingRules.BER); + + // Sanity checks from FIPS 186-4 4.1/4.2. Since FIPS 186-5 withdrew DSA/DSS + // these will never change again. + // + // This technically allows a non-standard combination of 1024-bit P and 256-bit Q, + // but that will get filtered out by the underlying provider. + // These checks just prevent obviously bad data from wasting work on reinterpretation. + + if (parms.P.Sign < 0 || + parms.Q.Sign < 0 || + !IsValidPLength(parms.P.GetBitLength()) || + !IsValidQLength(parms.Q.GetBitLength()) || + parms.G <= 1 || + parms.G >= parms.P || + x <= 1 || + x >= parms.Q) + { + throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding); + } + + ret = new DSAParameters + { + P = parms.P.ToByteArray(isUnsigned: true, isBigEndian: true), + Q = parms.Q.ToByteArray(isUnsigned: true, isBigEndian: true), + }; + + ret.G = parms.G.ExportKeyParameter(ret.P.Length); ret.X = x.ExportKeyParameter(ret.Q.Length); // The public key is not contained within the format, calculate it. @@ -69,6 +87,11 @@ internal static void ReadDsaPublicKey( in AlgorithmIdentifierAsn algId, out DSAParameters ret) { + if (!algId.Parameters.HasValue) + { + throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding); + } + BigInteger y; try @@ -88,13 +111,27 @@ internal static void ReadDsaPublicKey( throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding, e); } - if (!algId.Parameters.HasValue) + DssParms parms = DssParms.Decode(algId.Parameters.Value, AsnEncodingRules.BER); + + // Sanity checks from FIPS 186-4 4.1/4.2. Since FIPS 186-5 withdrew DSA/DSS + // these will never change again. + // + // This technically allows a non-standard combination of 1024-bit P and 256-bit Q, + // but that will get filtered out by the underlying provider. + // These checks just prevent obviously bad data from wasting work on reinterpretation. + + if (parms.P.Sign < 0 || + parms.Q.Sign < 0 || + !IsValidPLength(parms.P.GetBitLength()) || + !IsValidQLength(parms.Q.GetBitLength()) || + parms.G <= 1 || + parms.G >= parms.P || + y <= 1 || + y >= parms.P) { throw new CryptographicException(SR.Cryptography_Der_Invalid_Encoding); } - DssParms parms = DssParms.Decode(algId.Parameters.Value, AsnEncodingRules.BER); - ret = new DSAParameters { P = parms.P.ToByteArray(isUnsigned: true, isBigEndian: true), @@ -105,6 +142,25 @@ internal static void ReadDsaPublicKey( ret.Y = y.ExportKeyParameter(ret.P.Length); } + private static bool IsValidPLength(long pBitLength) + { + return pBitLength switch + { + // FIPS 186-3/186-4 + 1024 or 2048 or 3072 => true, + // FIPS 186-1/186-2 + >= 512 and < 1024 => pBitLength % 64 == 0, + _ => false, + }; + } + + private static bool IsValidQLength(long qBitLength) + { + // FIPS 196-1/186-2 only allows 160 + // FIPS 186-3/186-4 allow 160/224/256 + return qBitLength is 160 or 224 or 256; + } + internal static void ReadSubjectPublicKeyInfo( ReadOnlySpan source, out int bytesRead, diff --git a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/DSA/DSAKeyFileTests.cs b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/DSA/DSAKeyFileTests.cs index 5f3a0f5b82a6d..27a94ab1907d1 100644 --- a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/DSA/DSAKeyFileTests.cs +++ b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/DSA/DSAKeyFileTests.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Formats.Asn1; +using System.Numerics; using System.Security.Cryptography.Encryption.RC2.Tests; using System.Text; using Test.Cryptography; @@ -302,6 +304,150 @@ public static void ReadWriteDsa2048SubjectPublicKeyInfo() DSATestData.GetDSA2048Params()); } + [Fact] + [SkipOnPlatform(TestPlatforms.OSX, "DSASecurityTransforms goes straight to OS, has different failure mode")] + public static void ImportNonsensePublicParameters() + { + AsnWriter writer = new AsnWriter(AsnEncodingRules.DER); + + DSAParameters validParameters = DSATestData.GetDSA2048Params(); + BigInteger p = new BigInteger(validParameters.P, true, true); + BigInteger q = new BigInteger(validParameters.Q, true, true); + BigInteger g = new BigInteger(validParameters.G, true, true); + BigInteger y = new BigInteger(validParameters.Y, true, true); + + using (DSA dsa = DSAFactory.Create()) + { + // 1 < y < p, 1 < g < p, q is 160/224/256 bits + // p is 512..1024 % 64, or 1024/2048/3072 bits + ImportSPKI(dsa, p, q, g, p, writer); + ImportSPKI(dsa, p, q, g, BigInteger.One, writer); + ImportSPKI(dsa, p, q, g, BigInteger.MinusOne, writer); + ImportSPKI(dsa, p, q, p, y, writer); + ImportSPKI(dsa, p, q, -g, y, writer); + ImportSPKI(dsa, p, q, BigInteger.One, y, writer); + ImportSPKI(dsa, p, q, BigInteger.MinusOne, y, writer); + ImportSPKI(dsa, p, q << 1, g, y, writer); + ImportSPKI(dsa, p, q >> 1, g, y, writer); + ImportSPKI(dsa, p, -q, g, y, writer); + ImportSPKI(dsa, p >> 1, q, g, y, writer); + ImportSPKI(dsa, p << 1, q, g, y, writer); + ImportSPKI(dsa, BigInteger.One << 4095, q, 2, 97, writer); + } + + static void ImportSPKI( + DSA key, + BigInteger p, + BigInteger q, + BigInteger g, + BigInteger y, + AsnWriter writer) + { + writer.Reset(); + writer.WriteInteger(y); + byte[] encodedPublicKey = writer.Encode(); + writer.Reset(); + + using (writer.PushSequence()) + { + using (writer.PushSequence()) + { + writer.WriteObjectIdentifier("1.2.840.10040.4.1"); + + using (writer.PushSequence()) + { + writer.WriteInteger(p); + writer.WriteInteger(q); + writer.WriteInteger(g); + } + } + + writer.WriteBitString(encodedPublicKey); + } + + byte[] spki = writer.Encode(); + writer.Reset(); + + AssertExtensions.ThrowsContains( + () => key.ImportSubjectPublicKeyInfo(spki, out _), + "corrupted"); + } + } + + [Fact] + public static void ImportNonsensePrivateParameters() + { + AsnWriter writer = new AsnWriter(AsnEncodingRules.DER); + + DSAParameters validParameters = DSATestData.GetDSA2048Params(); + BigInteger p = new BigInteger(validParameters.P, true, true); + BigInteger q = new BigInteger(validParameters.Q, true, true); + BigInteger g = new BigInteger(validParameters.G, true, true); + BigInteger x = new BigInteger(validParameters.X, true, true); + + using (DSA dsa = DSAFactory.Create()) + { + // 1 < x < q, 1 < g < p, q is 160/224/256 bits + // p is 512..1024 % 64, or 1024/2048/3072 bits + ImportPkcs8(dsa, p, q, g, q, writer); + ImportPkcs8(dsa, p, q, g, BigInteger.One, writer); + // x = -1 gets re-interpreted as x = 255 because of a CAPI compat issue. + //ImportPkcs8(dsa, p, q, g, BigInteger.MinusOne, writer); + ImportPkcs8(dsa, p, q, g, -x, writer); + ImportPkcs8(dsa, p, q, p, x, writer); + ImportPkcs8(dsa, p, q, -g, x, writer); + ImportPkcs8(dsa, p, q, BigInteger.One, x, writer); + ImportPkcs8(dsa, p, q, BigInteger.MinusOne, x, writer); + ImportPkcs8(dsa, p, q << 1, g, x, writer); + ImportPkcs8(dsa, p, q >> 1, g, x, writer); + ImportPkcs8(dsa, p >> 1, q, g, x, writer); + ImportPkcs8(dsa, p << 1, q, g, x, writer); + ImportPkcs8(dsa, -q, q, g, x, writer); + ImportPkcs8(dsa, BigInteger.One << 4095, q, 2, 97, writer); + ImportPkcs8(dsa, -p, q, g, x, writer); + } + + static void ImportPkcs8( + DSA key, + BigInteger p, + BigInteger q, + BigInteger g, + BigInteger x, + AsnWriter writer) + { + writer.Reset(); + + using (writer.PushSequence()) + { + writer.WriteInteger(0); + + using (writer.PushSequence()) + { + writer.WriteObjectIdentifier("1.2.840.10040.4.1"); + + using (writer.PushSequence()) + { + writer.WriteInteger(p); + writer.WriteInteger(q); + writer.WriteInteger(g); + } + } + + using (writer.PushOctetString()) + { + writer.WriteInteger(x); + } + } + + byte[] pkcs8 = writer.Encode(); + writer.Reset(); + + AssertExtensions.ThrowsContains( + () => key.ImportPkcs8PrivateKey(pkcs8, out _), + "corrupted"); + } + } + [Fact] public static void NoFuzzySubjectPublicKeyInfo() { diff --git a/src/libraries/System.Formats.Asn1/src/Resources/Strings.resx b/src/libraries/System.Formats.Asn1/src/Resources/Strings.resx index 10571924b9d2e..90918d9e4ef5c 100644 --- a/src/libraries/System.Formats.Asn1/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Asn1/src/Resources/Strings.resx @@ -141,6 +141,9 @@ The encoded named bit list value is larger than the value size of the '{0}' enum. + + The encoded object identifier (OID) exceeds the limits supported by this library. Supported OIDs are limited to 64 arcs and each subidentifier is limited to a 128-bit value. + The encoded value uses a constructed encoding, which is invalid for '{0}' values. diff --git a/src/libraries/System.Formats.Asn1/src/System.Formats.Asn1.csproj b/src/libraries/System.Formats.Asn1/src/System.Formats.Asn1.csproj index 6c8a6d9ed36c5..255d375562204 100644 --- a/src/libraries/System.Formats.Asn1/src/System.Formats.Asn1.csproj +++ b/src/libraries/System.Formats.Asn1/src/System.Formats.Asn1.csproj @@ -4,6 +4,8 @@ enable $(NetCoreAppCurrent);netstandard2.0;net461 true + true + 1 Provides classes that can read and write the ASN.1 BER, CER, and DER data formats. Commonly Used Types: @@ -11,6 +13,9 @@ System.Formats.Asn1.AsnReader System.Formats.Asn1.AsnWriter + + Common\System\LocalAppContextSwitches.Common.cs + Common\System\Security\Cryptography\CryptoPool.cs @@ -48,6 +53,7 @@ System.Formats.Asn1.AsnWriter + diff --git a/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/AsnDecoder.Oid.cs b/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/AsnDecoder.Oid.cs index 44106332bf015..67c04b7f1351e 100644 --- a/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/AsnDecoder.Oid.cs +++ b/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/AsnDecoder.Oid.cs @@ -71,15 +71,55 @@ private static void ReadSubIdentifier( throw new AsnContentException(); } + // Set semanticBits to a value such that on the first + // iteration of the loop it becomes the correct value. + // So each entry here is [real semantic bits for this value] - 7. + int semanticBits = source[0] switch + { + >= 0b1100_0000 => 0, + >= 0b1010_0000 => -1, + >= 0b1001_0000 => -2, + >= 0b1000_1000 => -3, + >= 0b1000_0100 => -4, + >= 0b1000_0010 => -5, + >= 0b1000_0001 => -6, + _ => 0, + }; + // First, see how long the segment is int end = -1; int idx; + // None of T-REC-X.660-201107, T-REC-X.680-201508, or T-REC-X.690-201508 + // have any recommendations for a minimum (or maximum) size of a + // sub-identifier. + // + // T-REC-X.667-201210 (and earlier versions) discuss the no-registration- + // required UUID space at 2.25.{UUID}, where UUIDs are defined as 128-bit + // values. This gives us a minimum lower bound of 128-bit. + // + // Windows Crypt32 has historically only supported 64-bit values, and + // the "size limitations" FAQ on oid-info.com says that the largest arc + // value is a 39-digit value that corresponds to a 2.25.UUID value. + // + // So, until something argues for a bigger number, our bit-limit is 128. + const int MaxAllowedBits = 128; + for (idx = 0; idx < source.Length; idx++) { // If the high bit isn't set this marks the end of the sub-identifier. bool endOfIdentifier = (source[idx] & 0x80) == 0; + if (!LocalAppContextSwitches.AllowAnySizeOid) + { + semanticBits += 7; + + if (semanticBits > MaxAllowedBits) + { + throw new AsnContentException(SR.ContentException_OidTooBig); + } + } + if (endOfIdentifier) { end = idx; @@ -258,8 +298,21 @@ private static string ReadObjectIdentifier( contents = contents.Slice(bytesRead); + const int MaxArcs = 64; + int remainingArcs = MaxArcs - 2; + while (!contents.IsEmpty) { + if (!LocalAppContextSwitches.AllowAnySizeOid) + { + if (remainingArcs <= 0) + { + throw new AsnContentException(SR.ContentException_OidTooBig); + } + + remainingArcs--; + } + ReadSubIdentifier(contents, out bytesRead, out smallValue, out largeValue); // Exactly one should be non-null. Debug.Assert((smallValue == null) != (largeValue == null)); diff --git a/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/LocalAppContextSwitches.cs b/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/LocalAppContextSwitches.cs new file mode 100644 index 0000000000000..62722d7f4d000 --- /dev/null +++ b/src/libraries/System.Formats.Asn1/src/System/Formats/Asn1/LocalAppContextSwitches.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System +{ + internal static partial class LocalAppContextSwitches + { + private static int s_allowAnySizeOid; + public static bool AllowAnySizeOid + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => GetCachedSwitchValue("System.Formats.Asn1.AllowAnySizeOid", ref s_allowAnySizeOid); + } + } +} diff --git a/src/libraries/System.Formats.Asn1/tests/Reader/ReadObjectIdentifier.cs b/src/libraries/System.Formats.Asn1/tests/Reader/ReadObjectIdentifier.cs index c1add074220a0..6584fd47b8f6b 100644 --- a/src/libraries/System.Formats.Asn1/tests/Reader/ReadObjectIdentifier.cs +++ b/src/libraries/System.Formats.Asn1/tests/Reader/ReadObjectIdentifier.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Text; +using Microsoft.DotNet.RemoteExecutor; using Test.Cryptography; using Xunit; @@ -198,70 +199,188 @@ public static void ExpectedTag_IgnoresConstructed( [InlineData(AsnEncodingRules.BER)] [InlineData(AsnEncodingRules.CER)] [InlineData(AsnEncodingRules.DER)] - public static void ReadVeryLongOid(AsnEncodingRules ruleSet) + public static void ReadMaximumArcOid(AsnEncodingRules ruleSet) { - byte[] inputData = new byte[100000]; - // 06 83 02 00 00 (OBJECT IDENTIFIER, 65536 bytes). - inputData[0] = 0x06; - inputData[1] = 0x83; - inputData[2] = 0x01; - inputData[3] = 0x00; - inputData[4] = 0x00; - // and the rest are all zero. - - // The first byte produces "0.0". Each of the remaining 65535 bytes produce - // another ".0". - const int ExpectedLength = 65536 * 2 + 1; - StringBuilder builder = new StringBuilder(ExpectedLength); - builder.Append('0'); - - for (int i = 0; i <= ushort.MaxValue; i++) + const int MaxArcs = 64; + // MaxArcs content bytes (all 0x7F) (which includes one for failure), plus one for the tag + // plus one for the encoded length. + byte[] input = new byte[MaxArcs + 2]; + input.AsSpan().Fill(0x7F); + input[0] = 0x06; + // The first two arcs are encoded in the first sub-identifier, so MaxArcs - 1. + input[1] = MaxArcs - 1; + + string decoded = AsnDecoder.ReadObjectIdentifier(input, ruleSet, out int consumed); + Assert.Equal(input.Length - 1, consumed); + + StringBuilder expected = new StringBuilder(4 * MaxArcs); + expected.Append("2.47"); + + for (int i = 2; i < MaxArcs; i++) { - builder.Append('.'); - builder.Append(0); + expected.Append(".127"); } - AsnReader reader = new AsnReader(inputData, ruleSet); - string oidString = reader.ReadObjectIdentifier(); + Assert.Equal(expected.ToString(), decoded); - Assert.Equal(ExpectedLength, oidString.Length); - Assert.Equal(builder.ToString(), oidString); + input[1] = MaxArcs; + AsnContentException ex = Assert.Throws( + () => AsnDecoder.ReadObjectIdentifier(input, ruleSet, out _)); + Assert.Contains("OID", ex.Message); } [Theory] [InlineData(AsnEncodingRules.BER)] [InlineData(AsnEncodingRules.CER)] [InlineData(AsnEncodingRules.DER)] - public static void ReadVeryLongOidArc(AsnEncodingRules ruleSet) + public static void ReadMaximumInitialSubIdentifier(AsnEncodingRules ruleSet) { - byte[] inputData = new byte[255]; - // 06 81 93 (OBJECT IDENTIFIER, 147 bytes). - inputData[0] = 0x06; - inputData[1] = 0x81; - inputData[2] = 0x93; - - // With 147 bytes we get 147*7 = 1029 value bits. - // The smallest legal number to encode would have a top byte of 0x81, - // leaving 1022 bits remaining. If they're all zero then we have 2^1022. - // - // Since it's our first sub-identifier it's really encoding "2.(2^1022 - 80)". - inputData[3] = 0x81; - // Leave the last byte as 0. - new Span(inputData, 4, 145).Fill(0x80); - - const string ExpectedOid = - "2." + - "449423283715578976932326297697256183404494244735576643183575" + - "202894331689513752407831771193306018840052800284699678483394" + - "146974422036041556232118576598685310944419733562163713190755" + - "549003115235298632707380212514422095376705856157203684782776" + - "352068092908376276711465745599868114846199290762088390824060" + - "56034224"; + // First sub-identifier is 2^128 - 1, second is 1 + byte[] valid = + { + 0x06, 0x14, 0x83, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0x01, + }; - AsnReader reader = new AsnReader(inputData, ruleSet); + // First sub-identifier is 2^128, second is 1 + byte[] invalid = + { + 0x06, 0x14, 0x84, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x00, 0x01, + }; + + string oid = AsnDecoder.ReadObjectIdentifier(valid, ruleSet, out int consumed); + Assert.Equal(valid.Length, consumed); + Assert.Equal("2.340282366920938463463374607431768211375.1", oid); + + AsnContentException ex = Assert.Throws( + () => AsnDecoder.ReadObjectIdentifier(invalid, ruleSet, out _)); + Assert.Contains("OID", ex.Message); + } - string oidString = reader.ReadObjectIdentifier(); - Assert.Equal(ExpectedOid, oidString); + [Theory] + [InlineData(AsnEncodingRules.BER)] + [InlineData(AsnEncodingRules.CER)] + [InlineData(AsnEncodingRules.DER)] + public static void ReadMaximumNonInitialSubIdentifier(AsnEncodingRules ruleSet) + { + // First sub-identifier is 1, second is 2^128 - 1 + byte[] valid = + { + 0x06, 0x14, 0x01, 0x83, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, + }; + + // First sub-identifier is 1, second is 2^128 + byte[] invalid = new byte[] + { + 0x06, 0x14, 0x01, 0x84, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, + }; + + string oid = AsnDecoder.ReadObjectIdentifier(valid, ruleSet, out int consumed); + Assert.Equal(valid.Length, consumed); + Assert.Equal("0.1.340282366920938463463374607431768211455", oid); + + AsnContentException ex = Assert.Throws( + () => AsnDecoder.ReadObjectIdentifier(invalid, ruleSet, out _)); + Assert.Contains("OID", ex.Message); + } + + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void ReadVeryLongOid_WithAppContext() + { + RemoteExecutor.Invoke( + static () => + { + AppContext.SetSwitch("System.Formats.Asn1.AllowAnySizeOid", true); + + byte[] inputData = new byte[100000]; + // 06 83 02 00 00 (OBJECT IDENTIFIER, 65536 bytes). + inputData[0] = 0x06; + inputData[1] = 0x83; + inputData[2] = 0x01; + inputData[3] = 0x00; + inputData[4] = 0x00; + // and the rest are all zero. + + // The first byte produces "0.0". Each of the remaining 65535 bytes produce + // another ".0". + const int ExpectedLength = 65536 * 2 + 1; + StringBuilder builder = new StringBuilder(ExpectedLength); + builder.Append('0'); + + for (int i = 0; i <= ushort.MaxValue; i++) + { + builder.Append('.'); + builder.Append(0); + } + + AsnEncodingRules[] ruleSets = + { + AsnEncodingRules.BER, + AsnEncodingRules.CER, + AsnEncodingRules.DER, + }; + + foreach (AsnEncodingRules ruleSet in ruleSets) + { + AsnReader reader = new AsnReader(inputData, ruleSet); + string oidString = reader.ReadObjectIdentifier(); + + Assert.Equal(ExpectedLength, oidString.Length); + Assert.Equal(builder.ToString(), oidString); + } + }).Dispose(); + } + + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void ReadVeryLongOidArc_WithAppContext() + { + RemoteExecutor.Invoke( + static () => + { + AppContext.SetSwitch("System.Formats.Asn1.AllowAnySizeOid", true); + + byte[] inputData = new byte[255]; + // 06 81 93 (OBJECT IDENTIFIER, 147 bytes). + inputData[0] = 0x06; + inputData[1] = 0x81; + inputData[2] = 0x93; + + // With 147 bytes we get 147*7 = 1029 value bits. + // The smallest legal number to encode would have a top byte of 0x81, + // leaving 1022 bits remaining. If they're all zero then we have 2^1022. + // + // Since it's our first sub-identifier it's really encoding "2.(2^1022 - 80)". + inputData[3] = 0x81; + // Leave the last byte as 0. + new Span(inputData, 4, 145).Fill(0x80); + + const string ExpectedOid = + "2." + + "449423283715578976932326297697256183404494244735576643183575" + + "202894331689513752407831771193306018840052800284699678483394" + + "146974422036041556232118576598685310944419733562163713190755" + + "549003115235298632707380212514422095376705856157203684782776" + + "352068092908376276711465745599868114846199290762088390824060" + + "56034224"; + + AsnEncodingRules[] ruleSets = + { + AsnEncodingRules.BER, + AsnEncodingRules.CER, + AsnEncodingRules.DER, + }; + + foreach (AsnEncodingRules ruleSet in ruleSets) + { + AsnReader reader = new AsnReader(inputData, ruleSet); + + string oidString = reader.ReadObjectIdentifier(); + Assert.Equal(ExpectedOid, oidString); + } + }).Dispose(); } } } diff --git a/src/libraries/System.Formats.Asn1/tests/System.Formats.Asn1.Tests.csproj b/src/libraries/System.Formats.Asn1/tests/System.Formats.Asn1.Tests.csproj index 1c4965e48777e..c4bfbf0a5ebc0 100644 --- a/src/libraries/System.Formats.Asn1/tests/System.Formats.Asn1.Tests.csproj +++ b/src/libraries/System.Formats.Asn1/tests/System.Formats.Asn1.Tests.csproj @@ -2,6 +2,7 @@ true $(NetCoreAppCurrent);net461 + true From de31d4653e01b601291facb28262b301c20554a9 Mon Sep 17 00:00:00 2001 From: Juan Sebastian Hoyos Ayala Date: Mon, 10 Jun 2024 19:46:16 +0000 Subject: [PATCH 2/8] Use GetTempPath2 on Windows if available ## Description Since Windows 10 Build 20348, there is a new API to get the temporary files path called [`GetTempPath2`](https://learn.microsoft.com/windows/win32/api/fileapi/nf-fileapi-gettemppath2w). This API returns a directory inaccessible to non-SYSTEM processes if the calling process runs as SYSTEM, and [it is recommended to call this function instead of `GetTempPath`](https://learn.microsoft.com/windows/win32/api/fileapi/nf-fileapi-gettemppathw#remarks). This PR tries to find `GetTempPath2A` / `GetTempPath2W` and uses that, otherwise it falls back to `GetTempPathA` / `GetTempPathW`. *Note:* this PR removes an unused function called `WszGetTempPath` that which referenced GetTempPathW ## Customer Impact - [ ] Customer reported - [X] Found internally This was found by code inspection. ## Regression - [ ] Yes - [X] No ## Testing PR validation ## Risk Low Cherry picked from !40221 Cherry-picked from commit `c845a53e`. --- src/coreclr/debug/createdump/createdump.h | 5 +++ .../debug/createdump/createdumpwindows.cpp | 35 ++++++++++++++++ src/coreclr/debug/createdump/main.cpp | 2 +- src/coreclr/inc/longfilepathwrappers.h | 4 -- src/coreclr/inc/winwrap.h | 1 - src/coreclr/utilcode/longfilepathwrappers.cpp | 40 ------------------- src/native/corehost/hostmisc/pal.windows.cpp | 34 +++++++++++++++- 7 files changed, 74 insertions(+), 47 deletions(-) diff --git a/src/coreclr/debug/createdump/createdump.h b/src/coreclr/debug/createdump/createdump.h index 930e48b916654..68012ffef7a3d 100644 --- a/src/coreclr/debug/createdump/createdump.h +++ b/src/coreclr/debug/createdump/createdump.h @@ -132,5 +132,10 @@ typedef struct extern bool CreateDump(const CreateDumpOptions& options); extern bool FormatDumpName(std::string& name, const char* pattern, const char* exename, int pid); +#ifdef HOST_WINDOWS +extern DWORD GetTempPathWrapper(IN DWORD nBufferLength, OUT LPSTR lpBuffer); +#else +#define GetTempPathWrapper GetTempPathA +#endif extern void printf_status(const char* format, ...); extern void printf_error(const char* format, ...); diff --git a/src/coreclr/debug/createdump/createdumpwindows.cpp b/src/coreclr/debug/createdump/createdumpwindows.cpp index b8384e4c8580f..d598ade93e81f 100644 --- a/src/coreclr/debug/createdump/createdumpwindows.cpp +++ b/src/coreclr/debug/createdump/createdumpwindows.cpp @@ -73,3 +73,38 @@ CreateDump(const CreateDumpOptions& options) return result; } + +typedef DWORD(WINAPI *pfnGetTempPathA)(DWORD nBufferLength, LPSTR lpBuffer); + +static volatile pfnGetTempPathA +g_pfnGetTempPathA = nullptr; + + +DWORD +GetTempPathWrapper( + IN DWORD nBufferLength, + OUT LPSTR lpBuffer) +{ + if (g_pfnGetTempPathA == nullptr) + { + HMODULE hKernel32 = LoadLibraryExW(L"kernel32.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + + pfnGetTempPathA pLocalGetTempPathA = NULL; + if (hKernel32 != NULL) + { + // store to thread local variable to prevent data race + pLocalGetTempPathA = (pfnGetTempPathA)::GetProcAddress(hKernel32, "GetTempPath2A"); + } + + if (pLocalGetTempPathA == NULL) // method is only available with Windows 10 Creators Update or later + { + g_pfnGetTempPathA = &GetTempPathA; + } + else + { + g_pfnGetTempPathA = pLocalGetTempPathA; + } + } + + return g_pfnGetTempPathA(nBufferLength, lpBuffer); +} \ No newline at end of file diff --git a/src/coreclr/debug/createdump/main.cpp b/src/coreclr/debug/createdump/main.cpp index 6dd116a7a3c33..89c8c7f404350 100644 --- a/src/coreclr/debug/createdump/main.cpp +++ b/src/coreclr/debug/createdump/main.cpp @@ -175,7 +175,7 @@ int __cdecl main(const int argc, const char* argv[]) if (options.DumpPathTemplate == nullptr) { - if (::GetTempPathA(MAX_LONGPATH, tmpPath) == 0) + if (::GetTempPathWrapper(MAX_LONGPATH, tmpPath) == 0) { printf_error("GetTempPath failed (0x%08x)\n", ::GetLastError()); return ::GetLastError(); diff --git a/src/coreclr/inc/longfilepathwrappers.h b/src/coreclr/inc/longfilepathwrappers.h index 149cdcf4443ac..e62f1ae6be3c7 100644 --- a/src/coreclr/inc/longfilepathwrappers.h +++ b/src/coreclr/inc/longfilepathwrappers.h @@ -87,10 +87,6 @@ UINT WINAPI GetTempFileNameWrapper( SString& lpTempFileName ); -DWORD WINAPI GetTempPathWrapper( - SString& lpBuffer - ); - DWORD WINAPI GetCurrentDirectoryWrapper( SString& lpBuffer ); diff --git a/src/coreclr/inc/winwrap.h b/src/coreclr/inc/winwrap.h index 4d97618984fd2..96fdb678ddbd7 100644 --- a/src/coreclr/inc/winwrap.h +++ b/src/coreclr/inc/winwrap.h @@ -220,7 +220,6 @@ //Long Files will not work on these till redstone #define WszGetCurrentDirectory GetCurrentDirectoryWrapper #define WszGetTempFileName GetTempFileNameWrapper -#define WszGetTempPath GetTempPathWrapper //APIS which have a buffer as an out parameter #define WszGetEnvironmentVariable GetEnvironmentVariableWrapper diff --git a/src/coreclr/utilcode/longfilepathwrappers.cpp b/src/coreclr/utilcode/longfilepathwrappers.cpp index 3afc611c3b218..a383090692595 100644 --- a/src/coreclr/utilcode/longfilepathwrappers.cpp +++ b/src/coreclr/utilcode/longfilepathwrappers.cpp @@ -514,46 +514,6 @@ UINT WINAPI GetTempFileNameWrapper( return ret; } -DWORD WINAPI GetTempPathWrapper( - SString& lpBuffer - ) -{ - CONTRACTL - { - NOTHROW; - } - CONTRACTL_END; - - HRESULT hr = S_OK; - DWORD ret = 0; - DWORD lastError; - - EX_TRY - { - //Change the behaviour in Redstone to retry - COUNT_T size = MAX_LONGPATH; - - ret = GetTempPathW( - size, - lpBuffer.OpenUnicodeBuffer(size - 1) - ); - - lastError = GetLastError(); - lpBuffer.CloseBuffer(ret); - } - EX_CATCH_HRESULT(hr); - - if (hr != S_OK) - { - SetLastError(hr); - } - else if (ret == 0) - { - SetLastError(lastError); - } - - return ret; -} DWORD WINAPI GetCurrentDirectoryWrapper( SString& lpBuffer diff --git a/src/native/corehost/hostmisc/pal.windows.cpp b/src/native/corehost/hostmisc/pal.windows.cpp index cd6701879389d..94c47541d2404 100644 --- a/src/native/corehost/hostmisc/pal.windows.cpp +++ b/src/native/corehost/hostmisc/pal.windows.cpp @@ -11,6 +11,38 @@ #include #include +namespace +{ + typedef DWORD(WINAPI *get_temp_path_func_ptr)(DWORD buffer_len, LPWSTR buffer); + static volatile get_temp_path_func_ptr s_get_temp_path_func = nullptr; + + DWORD get_temp_path(DWORD buffer_len, LPWSTR buffer) + { + if (s_get_temp_path_func == nullptr) + { + HMODULE kernel32 = ::LoadLibraryExW(L"kernel32.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); + + get_temp_path_func_ptr get_temp_path_func_local = NULL; + if (kernel32 != NULL) + { + // store to thread local variable to prevent data race + get_temp_path_func_local = (get_temp_path_func_ptr)::GetProcAddress(kernel32, "GetTempPath2W"); + } + + if (get_temp_path_func_local == NULL) // method is only available with Windows 10 Creators Update or later + { + s_get_temp_path_func = &GetTempPathW; + } + else + { + s_get_temp_path_func = get_temp_path_func_local; + } + } + + return s_get_temp_path_func(buffer_len, buffer); + } +} + bool GetModuleFileNameWrapper(HMODULE hModule, pal::string_t* recv) { pal::string_t path; @@ -569,7 +601,7 @@ bool get_extraction_base_parent_directory(pal::string_t& directory) const size_t max_len = MAX_PATH + 1; pal::char_t temp_path[max_len]; - size_t len = GetTempPathW(max_len, temp_path); + size_t len = get_temp_path(max_len, temp_path); if (len == 0) { return false; From d17e0d08ba877d6937e42cb34a8f25b6116b7883 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 11 Jun 2024 16:38:00 +0000 Subject: [PATCH 3/8] Bypassing Serialization Binders with BinaryFormatter Mutation --- .../Formatters/Binary/BinaryObjectWriter.cs | 19 ++++++++++++------- .../tests/BinaryFormatterTests.cs | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryObjectWriter.cs b/src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryObjectWriter.cs index cf57a01929bb1..deb7025593506 100644 --- a/src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryObjectWriter.cs +++ b/src/libraries/System.Runtime.Serialization.Formatters/src/System/Runtime/Serialization/Formatters/Binary/BinaryObjectWriter.cs @@ -375,7 +375,7 @@ private void WriteMembers(NameInfo memberNameInfo, return; } - if (!WriteKnownValueClass(memberNameInfo, memberTypeNameInfo, memberData!)) + if (!WriteKnownValueClass(memberNameInfo, memberTypeNameInfo, memberData!, ref assignUniqueIdToValueType)) { if (outType == null) { @@ -624,10 +624,10 @@ private void WriteArrayMember(WriteObjectInfo objectInfo, NameInfo arrayElemType actualTypeInfo._isArrayItem = true; } - if (!WriteKnownValueClass(arrayElemTypeNameInfo, actualTypeInfo, data!)) + bool assignUniqueIdForValueTypes = false; + if (!WriteKnownValueClass(arrayElemTypeNameInfo, actualTypeInfo, data!, ref assignUniqueIdForValueTypes)) { object obj = data!; - bool assignUniqueIdForValueTypes = false; if (ReferenceEquals(arrayElemTypeNameInfo._type, Converter.s_typeofObject)) { assignUniqueIdForValueTypes = true; @@ -807,11 +807,12 @@ private long Schedule(object obj, bool assignUniqueIdToValueType, Type? type, Wr } // Determines if a type is a primitive type, if it is it is written - private bool WriteKnownValueClass(NameInfo memberNameInfo, NameInfo typeNameInfo, object data) + private bool WriteKnownValueClass(NameInfo memberNameInfo, NameInfo typeNameInfo, object data, ref bool assignUniqueIdToValueType) { if (ReferenceEquals(typeNameInfo._type, Converter.s_typeofString)) { WriteString(memberNameInfo, typeNameInfo, data); + return true; } else { @@ -825,18 +826,22 @@ private bool WriteKnownValueClass(NameInfo memberNameInfo, NameInfo typeNameInfo if (typeNameInfo._isArray) // null if an array { _serWriter.WriteItem(memberNameInfo, typeNameInfo, data); + return true; } - else + else if (memberNameInfo._type == typeNameInfo._type + || memberNameInfo._type == typeof(object) + || (memberNameInfo._type != null && Nullable.GetUnderlyingType(memberNameInfo._type) != null)) { _serWriter.WriteMember(memberNameInfo, typeNameInfo, data); + return true; } } } - return true; + assignUniqueIdToValueType = true; + return false; } - // Writes an object reference to the stream. private void WriteObjectRef(NameInfo nameInfo, long objectId) => _serWriter!.WriteMemberObjectRef(nameInfo, (int)objectId); diff --git a/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs b/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs index 7bd5ede42c51b..5baa570e8621e 100644 --- a/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs +++ b/src/libraries/System.Runtime.Serialization.Formatters/tests/BinaryFormatterTests.cs @@ -535,6 +535,25 @@ public void Roundtrip_ArrayContainingArrayAtNonZeroLowerBound() BinaryFormatterHelpers.Clone(Array.CreateInstance(typeof(uint[]), new[] { 5 }, new[] { 1 })); } + [Fact] + [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, ".NET Framework has not been patched yet.")] + public void BypassingSerializationBinders() + { + Tuple tuple = new Tuple(42, new byte[] { 1, 2, 3, 4 }); + BinaryFormatter formatter = new BinaryFormatter(); + + using (MemoryStream stream = new MemoryStream()) + { + formatter.Serialize(stream, tuple); + + stream.Position = 0; + + Tuple deserialized = (Tuple)formatter.Deserialize(stream); + Assert.Equal(tuple.Item1, deserialized.Item1); + Assert.Equal(tuple.Item2, deserialized.Item2); + } + } + private static void ValidateEqualityComparer(object obj) { Type objType = obj.GetType(); From 0a0dd0e27560e692e11ee286ed9f45471b2131fa Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 11 Jun 2024 16:48:37 +0000 Subject: [PATCH 4/8] CHILD: .NET CORE | ElevationOfPrivilege backport of https://github.com/dotnet/runtime/pull/72452 --- .../Windows/Kernel32/Interop.GetTempPathW.cs | 13 ------ .../System.Private.CoreLib.Shared.projitems | 3 -- .../src/System/IO/Path.Windows.cs | 42 +++++++++++++++++-- 3 files changed, 39 insertions(+), 19 deletions(-) delete mode 100644 src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs deleted file mode 100644 index 5d555df58cd23..0000000000000 --- a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; - -internal static partial class Interop -{ - internal static partial class Kernel32 - { - [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, BestFitMapping = false, ExactSpelling = true)] - internal static extern uint GetTempPathW(int bufferLen, ref char buffer); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index ec495cc7817c9..21af116b5c8a0 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1567,9 +1567,6 @@ Common\Interop\Windows\Kernel32\Interop.GetTempFileNameW.cs - - Common\Interop\Windows\Kernel32\Interop.GetTempPathW.cs - Common\Interop\Windows\Kernel32\Interop.GetVolumeInformation.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs index e0ad5c0c26af2..0cd59e32d5189 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; using System.Text; #if MS_IO_REDIST @@ -14,8 +15,10 @@ namespace Microsoft.IO namespace System.IO #endif { - public static partial class Path + public static unsafe partial class Path { + private static volatile delegate* unmanaged s_GetTempPathWFunc; + public static char[] GetInvalidFileNameChars() => new char[] { '\"', '<', '>', '|', '\0', @@ -152,10 +155,22 @@ public static string GetTempPath() return path; } - private static void GetTempPath(ref ValueStringBuilder builder) + private static unsafe delegate* unmanaged GetGetTempPathWFunc() + { + IntPtr kernel32 = Interop.Kernel32.LoadLibraryEx(Interop.Libraries.Kernel32, IntPtr.Zero, Interop.Kernel32.LOAD_LIBRARY_SEARCH_SYSTEM32); + + if (!NativeLibrary.TryGetExport(kernel32, "GetTempPath2W", out IntPtr func)) + { + func = NativeLibrary.GetExport(kernel32, "GetTempPathW"); + } + + return (delegate* unmanaged)func; + } + + internal static void GetTempPath(ref ValueStringBuilder builder) { uint result; - while ((result = Interop.Kernel32.GetTempPathW(builder.Capacity, ref builder.GetPinnableReference())) > builder.Capacity) + while ((result = GetTempPathW(builder.Capacity, ref builder.GetPinnableReference())) > builder.Capacity) { // Reported size is greater than the buffer size. Increase the capacity. builder.EnsureCapacity(checked((int)result)); @@ -165,6 +180,27 @@ private static void GetTempPath(ref ValueStringBuilder builder) throw Win32Marshal.GetExceptionForLastWin32Error(); builder.Length = (int)result; + + static uint GetTempPathW(int bufferLen, ref char buffer) + { + delegate* unmanaged func = s_GetTempPathWFunc; + if (func == null) + { + func = s_GetTempPathWFunc = GetGetTempPathWFunc(); + } + + int lastError; + uint retVal; + fixed (char* ptr = &buffer) + { + Marshal.SetLastSystemError(0); + retVal = func(bufferLen, ptr); + lastError = Marshal.GetLastSystemError(); + } + + Marshal.SetLastPInvokeError(lastError); + return retVal; + } } // Returns a unique temporary file name, and creates a 0-byte file by that From 64331033cae32fd6166b321301baaf8f004e2336 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 12 Jun 2024 21:34:18 +0000 Subject: [PATCH 5/8] Microsoft.IO.Redist part (and a build fix) CHILD: .NET CORE | ElevationOfPrivilege: Microsoft.IO.Redist part (and a build fix) It turned out that https://dev.azure.com/dnceng/internal/_git/dotnet-runtime/pullrequest/39614 was incomplete and caused a build break. The code was compiling fine for the default config (.NET 6), but not Full Framework, which includes Microsoft.IO.Redist which compiles the affected files. I could not use exact same fix for Framework as for 6.0 as the code was using umanaged function pointers and causing error CS8889, so I've applied the same fix I did for Full Framework. --- .../Kernel32/Interop.GetProcAddress.cs | 2 ++ .../Windows/Kernel32/Interop.GetTempPathW.cs | 16 ++++++++++ .../src/Microsoft.IO.Redist.csproj | 6 ++++ .../src/System/IO/Path.Windows.cs | 30 +++++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetProcAddress.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetProcAddress.cs index 214050545ed15..08a2f9ffd33f0 100644 --- a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetProcAddress.cs +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetProcAddress.cs @@ -10,8 +10,10 @@ internal static partial class Interop { internal static partial class Kernel32 { +#if !MS_IO_REDIST // SafeLibraryHandle is inaccessible due to its protection level [DllImport(Libraries.Kernel32, CharSet = CharSet.Ansi, BestFitMapping = false)] public static extern IntPtr GetProcAddress(SafeLibraryHandle hModule, string lpProcName); +#endif [DllImport(Libraries.Kernel32, CharSet = CharSet.Ansi, BestFitMapping = false)] public static extern IntPtr GetProcAddress(IntPtr hModule, string lpProcName); diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs new file mode 100644 index 0000000000000..80c71479c8e51 --- /dev/null +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.GetTempPathW.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Kernel32 + { + [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, BestFitMapping = false, ExactSpelling = true)] + internal static extern uint GetTempPathW(int bufferLen, ref char buffer); + + [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, BestFitMapping = false, ExactSpelling = true)] + internal static extern uint GetTempPath2W(int bufferLen, ref char buffer); + } +} diff --git a/src/libraries/Microsoft.IO.Redist/src/Microsoft.IO.Redist.csproj b/src/libraries/Microsoft.IO.Redist/src/Microsoft.IO.Redist.csproj index 674d785cb7e8b..f86b4eda02cc0 100644 --- a/src/libraries/Microsoft.IO.Redist/src/Microsoft.IO.Redist.csproj +++ b/src/libraries/Microsoft.IO.Redist/src/Microsoft.IO.Redist.csproj @@ -1,6 +1,8 @@  true + true + 1 Downlevel support package for System.IO classes. $(DefineConstants);MS_IO_REDIST true @@ -108,6 +110,10 @@ Link="Interop\Windows\Interop.GetTempFileNameW.cs" /> + + s_GetTempPathWFunc; +#endif public static char[] GetInvalidFileNameChars() => new char[] { @@ -155,6 +159,20 @@ public static string GetTempPath() return path; } +#if MS_IO_REDIST + private static int GetGetTempPathVersion() + { + IntPtr kernel32 = Interop.Kernel32.GetModuleHandle(Interop.Libraries.Kernel32); + if (kernel32 != IntPtr.Zero) + { + if (Interop.Kernel32.GetProcAddress(kernel32, "GetTempPath2W") != IntPtr.Zero) + { + return 2; + } + } + return 1; + } +#else private static unsafe delegate* unmanaged GetGetTempPathWFunc() { IntPtr kernel32 = Interop.Kernel32.LoadLibraryEx(Interop.Libraries.Kernel32, IntPtr.Zero, Interop.Kernel32.LOAD_LIBRARY_SEARCH_SYSTEM32); @@ -166,6 +184,7 @@ public static string GetTempPath() return (delegate* unmanaged)func; } +#endif internal static void GetTempPath(ref ValueStringBuilder builder) { @@ -183,6 +202,16 @@ internal static void GetTempPath(ref ValueStringBuilder builder) static uint GetTempPathW(int bufferLen, ref char buffer) { +#if MS_IO_REDIST + int getTempPathVersion = s_GetTempPathVersion; + if (getTempPathVersion == 0) + { + s_GetTempPathVersion = getTempPathVersion = GetGetTempPathVersion(); + } + return getTempPathVersion == 2 + ? Interop.Kernel32.GetTempPath2W(bufferLen, ref buffer) + : Interop.Kernel32.GetTempPathW(bufferLen, ref buffer); +#else delegate* unmanaged func = s_GetTempPathWFunc; if (func == null) { @@ -200,6 +229,7 @@ static uint GetTempPathW(int bufferLen, ref char buffer) Marshal.SetLastPInvokeError(lastError); return retVal; +#endif } } From be26ec60dc363435afebb561a8ea5d602066060f Mon Sep 17 00:00:00 2001 From: Farhad Alizada <104755925+f-alizada@users.noreply.github.com> Date: Mon, 8 Jul 2024 20:01:58 +0200 Subject: [PATCH 6/8] [release/6.0] Onboard new inter-branch merge flow [Workflow] (#104417) Co-authored-by: Farhad Alizada --- .github/workflows/inter-branch-merge-flow.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/inter-branch-merge-flow.yml diff --git a/.github/workflows/inter-branch-merge-flow.yml b/.github/workflows/inter-branch-merge-flow.yml new file mode 100644 index 0000000000000..20246c14fc585 --- /dev/null +++ b/.github/workflows/inter-branch-merge-flow.yml @@ -0,0 +1,13 @@ +name: Inter-branch merge workflow +on: + push: + branches: + - release/** + +permissions: + contents: write + pull-requests: write + +jobs: + Merge: + uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main \ No newline at end of file From 89f7f624b0935d8dcf3ce010e28236b88e56b919 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:49:17 -0700 Subject: [PATCH 7/8] Update dependencies from https://github.com/dotnet/emsdk build 20240703.1 (#104631) Microsoft.NET.Workload.Emscripten.Manifest-6.0.100 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.300 , Microsoft.NET.Workload.Emscripten.Manifest-6.0.400 From Version 6.0.32 -> To Version 6.0.33 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NuGet.config b/NuGet.config index e233e479af6fc..86c28cf6f6d38 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6f3a59d99181d..8a9f25c245788 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -8,17 +8,17 @@ https://github.com/dotnet/msquic 7312355e44fd230b7aa26c7190f3870391751476 - + https://github.com/dotnet/emsdk - ab8bc4e2860e29d700542c68e57daa5e7663af62 + 849e1a63a8319f9578b9e5f02da592d24d4aa2df - + https://github.com/dotnet/emsdk - ab8bc4e2860e29d700542c68e57daa5e7663af62 + 849e1a63a8319f9578b9e5f02da592d24d4aa2df - + https://github.com/dotnet/emsdk - ab8bc4e2860e29d700542c68e57daa5e7663af62 + 849e1a63a8319f9578b9e5f02da592d24d4aa2df https://github.com/dotnet/wcf diff --git a/eng/Versions.props b/eng/Versions.props index 0da0f60f30b95..35844ef697864 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -175,9 +175,9 @@ 11.1.0-alpha.1.21416.1 11.1.0-alpha.1.21416.1 - 6.0.32 - 6.0.32 - 6.0.32 + 6.0.33 + 6.0.33 + 6.0.33 $(MicrosoftNETWorkloadEmscriptenManifest60100Version) 1.1.87-gba258badda From 67b32df67bc73fa9e3a28da2b770c4915e37106f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:49:45 -0700 Subject: [PATCH 8/8] Update dependencies from https://github.com/dotnet/arcade build 20240710.7 (#104747) Microsoft.DotNet.ApiCompat , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 6.0.0-beta.24266.4 -> To Version 6.0.0-beta.24360.7 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 76 +++++++++---------- eng/Versions.props | 30 ++++---- .../templates-official/job/source-build.yml | 8 ++ .../job/source-index-stage1.yml | 35 ++++++--- .../templates-official/jobs/source-build.yml | 8 ++ .../post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 +++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++ .../steps/get-federated-access-token.yml | 28 +++++++ eng/common/templates/job/source-build.yml | 8 ++ .../templates/job/source-index-stage1.yml | 35 ++++++--- eng/common/templates/jobs/source-build.yml | 8 ++ .../templates/post-build/common-variables.yml | 1 - .../steps/enable-internal-runtimes.yml | 28 +++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++ .../steps/get-federated-access-token.yml | 28 +++++++ global.json | 12 +-- 17 files changed, 339 insertions(+), 81 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8a9f25c245788..5fd1f2b339a3a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -26,77 +26,77 @@ - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 https://github.com/microsoft/vstest @@ -222,9 +222,9 @@ https://github.com/dotnet/xharness d1dd9c2ce3fc0b9358d2cda64c52d052d1a612c1 - + https://github.com/dotnet/arcade - 2eab07c3d7b78219d10099b19fafeef2ecae1779 + fbc993a9e8fb4926ce04c95ba2e48852c9d9df65 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 35844ef697864..0c5041e91c03e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -42,21 +42,21 @@ 1.1.0-preview.22164.17 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 2.5.1-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 - 6.0.0-beta.24266.4 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 2.5.1-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 + 6.0.0-beta.24360.7 6.0.0-preview.1.102 diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f5fa09f415117..a6f7efa776851 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -59,6 +65,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f04ad04c2b186..f6f0c15bfba42 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -17,14 +18,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -40,8 +41,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -53,7 +54,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index b9a1f67b9a9ac..9f60019cb01da 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index fae340f4d20d7..5404cc7ecbefc 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000000..93a8394a666b2 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 0000000000000..c0e8f91317f07 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 0000000000000..e3786cef6dfd7 --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index b6137f44ada13..8009ccb954acd 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -58,6 +64,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b710698eb4d4f..47f36e98bc16e 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240320.1 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -15,14 +16,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables pool: ${{ parameters.pool }} steps: @@ -38,8 +39,8 @@ jobs: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -51,7 +52,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index 7c240e6544724..e3fbd9f38e105 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index fae340f4d20d7..5404cc7ecbefc 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -2,7 +2,6 @@ variables: - group: AzureDevOps-Artifact-Feeds-Pats - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage - - group: DotNet-MSRC-Storage - group: Publish-Build-Assets # Whether the build is internal or not diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000000..54dc9416c5198 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 0000000000000..c0e8f91317f07 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 0000000000000..c8c49cc0e8f0f --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index ec967648df7ac..79600e9293346 100644 --- a/global.json +++ b/global.json @@ -1,21 +1,21 @@ { "sdk": { - "version": "6.0.130", + "version": "6.0.132", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "6.0.130" + "dotnet": "6.0.132" }, "native-tools": { "cmake": "3.16.4", "python3": "3.7.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24266.4", - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24266.4", - "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24266.4", - "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24266.4", + "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "6.0.0-beta.24360.7", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.24360.7", + "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.24360.7", + "Microsoft.DotNet.SharedFramework.Sdk": "6.0.0-beta.24360.7", "Microsoft.Build.NoTargets": "3.1.0", "Microsoft.Build.Traversal": "3.0.23", "Microsoft.NET.Sdk.IL": "6.0.0-rc.1.21415.6"