From dcf21c10e2b3490fedc12b20e1d4927dc1161f6b Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Fri, 10 Nov 2017 11:21:41 -0800 Subject: [PATCH 1/7] Initial commit of prototyping SignTool to both produce manifests with SHA256 info on all expected files and be able to unpack all specified files from such a manifest, assuming these files exist in any of the nested zip archives provided. --- src/SignTool/SignTool/BatchSignInput.cs | 136 ++++++++++++++++-- src/SignTool/SignTool/BatchSignUtil.cs | 100 +++++++++++-- src/SignTool/SignTool/ContentUtil.cs | 7 +- src/SignTool/SignTool/FileName.cs | 4 +- src/SignTool/SignTool/JsonTypes.cs | 61 ++++++-- .../SignTool/OrchestratedBatchSignInput.cs | 80 +++++++++++ src/SignTool/SignTool/PathUtil.cs | 4 +- src/SignTool/SignTool/Program.cs | 108 +++++++++++--- src/SignTool/SignTool/SignToolArgs.cs | 15 +- src/SignTool/SignTool/ZipData.cs | 5 - 10 files changed, 450 insertions(+), 70 deletions(-) create mode 100644 src/SignTool/SignTool/OrchestratedBatchSignInput.cs diff --git a/src/SignTool/SignTool/BatchSignInput.cs b/src/SignTool/SignTool/BatchSignInput.cs index 30c3dbf18c..d9b0b6e6b2 100644 --- a/src/SignTool/SignTool/BatchSignInput.cs +++ b/src/SignTool/SignTool/BatchSignInput.cs @@ -1,9 +1,13 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Linq; using System.Collections.Generic; using System.Collections.Immutable; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using SignTool.Json; namespace SignTool { @@ -18,41 +22,46 @@ internal sealed class BatchSignInput internal string OutputPath { get; } /// - /// The ordered names of the files to be signed. These are all relative paths off of the + /// Uri, to be consumed by later steps, which describes where these files get published to. + /// + internal string PublishUri { get; } + + /// + /// The ordered names of the files to be signed. These are all relative paths off of the /// property. /// internal ImmutableArray FileNames { get; } /// - /// These are binaries which are included in our zip containers but are already signed. This list is used for - /// validation purpsoes. These are all flat names and cannot be relative paths. + /// These are binaries which are included in our zip containers but are already signed. This list is used for + /// validation purposes. These are all flat names and cannot be relative paths. /// - internal ImmutableArray ExternalFileNames { get;} + internal ImmutableArray ExternalFileNames { get; } /// - /// Names of assemblies that need to be signed. This is a subset of + /// Names of assemblies that need to be signed. This is a subset of /// internal ImmutableArray AssemblyNames { get; } /// - /// Names of zip containers that need to be examined for signing. This is a subset of + /// Names of zip containers that need to be examined for signing. This is a subset of /// internal ImmutableArray ZipContainerNames { get; } /// - /// Names of other file types which aren't specifically handled by the tool. This is a subset of + /// Names of other file types which aren't specifically handled by the tool. This is a subset of /// internal ImmutableArray OtherNames { get; } /// - /// A map of all of the binaries that need to be signed to the actual signing data. + /// A map from all of the binaries that need to be signed to the actual signing data. /// internal ImmutableDictionary FileSignInfoMap { get; } - internal BatchSignInput(string outputPath, Dictionary fileSignDataMap, IEnumerable externalFileNames) + internal BatchSignInput(string outputPath, Dictionary fileSignDataMap, IEnumerable externalFileNames, string publishUri) { OutputPath = outputPath; - + PublishUri = publishUri; // Use order by to make the output of this tool as predictable as possible. var fileNames = fileSignDataMap.Keys; FileNames = fileNames.OrderBy(x => x).Select(x => new FileName(outputPath, x)).ToImmutableArray(); @@ -70,6 +79,111 @@ internal BatchSignInput(string outputPath, Dictionary fileSign } FileSignInfoMap = builder.ToImmutable(); } + + internal BatchSignInput(string outputPath, Dictionary fileSignDataMap, IEnumerable externalFileNames, string publishUri) + { + OutputPath = outputPath; + PublishUri = publishUri; + + List fileNames = fileSignDataMap.Keys.Select(x => new FileName(outputPath, x.FilePath, x.SHA256Hash)).ToList(); + ZipContainerNames = fileNames.Where(x => x.IsZipContainer).ToImmutableArray(); + // If there's any files we can't find, recursively unpack the zip archives we just made a list of above. + UnpackMissingContent(ref fileNames); + // After this point, if the files are available execution should be as before. + // Use OrderBy to make the output of this tool as predictable as possible. + FileNames = fileNames.OrderBy(x => x.RelativePath).ToImmutableArray(); + ExternalFileNames = externalFileNames.OrderBy(x => x).ToImmutableArray(); + AssemblyNames = FileNames.Where(x => x.IsAssembly).ToImmutableArray(); + OtherNames = FileNames.Where(x => !x.IsAssembly && !x.IsZipContainer).ToImmutableArray(); + + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var name in FileNames) + { + var data = fileSignDataMap.Keys.Where(k => k.SHA256Hash == name.SHA256Hash).Single(); + builder.Add(name, new FileSignInfo(name, fileSignDataMap[data])); + } + FileSignInfoMap = builder.ToImmutable(); + } + + private void UnpackMissingContent(ref List candidateFileNames) + { + bool success = true; + string unpackingDirectory = Path.Combine(OutputPath, "UnpackedZipArchives"); + StringBuilder missingFiles = new StringBuilder(); + Directory.CreateDirectory(unpackingDirectory); + + var unpackNeeded = (from file in candidateFileNames + where !File.Exists(file.FullPath) + select file).ToList(); + + // Nothing to do + if (unpackNeeded.Count() == 0) + { + return; + } + else + { + ContentUtil contentUtil = new ContentUtil(); + + // Get all Zip Archives in the manifest + // We'll use a non-immutable queue as we may need to add new zips to the list. + Queue allZipsWeKnowAbout = new Queue(ZipContainerNames); + + while (allZipsWeKnowAbout.Count > 0) + { + FileName zipFile = allZipsWeKnowAbout.Dequeue(); + string unpackFolder = Path.Combine(unpackingDirectory, zipFile.SHA256Hash); + + // Assumption: If a zip with a given hash is already unpacked, it's probably OK. + if (!Directory.Exists(unpackFolder)) + { + Directory.CreateDirectory(unpackFolder); + ZipFile.ExtractToDirectory(zipFile.FullPath, unpackFolder); + } + // Add any zips we just unzipped. + foreach (string file in Directory.GetFiles(unpackFolder, "*", SearchOption.AllDirectories)) + { + if (PathUtil.IsZipContainer(file)) + { + string relativePath = (string)(new Uri(unpackingDirectory).MakeRelativeUri(new Uri(file))).OriginalString; + allZipsWeKnowAbout.Enqueue(new FileName(unpackingDirectory, relativePath, contentUtil.GetChecksum(file))); + } + } + } + // Lazy : Disks are fast, just calculate ALL hashes. Could optimize by only files we intend to sign + Dictionary existingHashLookup = new Dictionary(); + foreach (string file in Directory.GetFiles(unpackingDirectory, "*", SearchOption.AllDirectories)) + { + existingHashLookup.Add(file, contentUtil.GetChecksum(file)); + } + + Dictionary fileNameUpdates = new Dictionary(); + // At this point, we've unpacked every Zip we can possibly pull out into folders named for the zip's hash into 'unpackingDirectory' + foreach (FileName missingFileWithHashToFind in unpackNeeded) + { + string matchFile = (from filePath in existingHashLookup.Keys + where Path.GetFileName(filePath).Equals(missingFileWithHashToFind.Name, StringComparison.OrdinalIgnoreCase) + where existingHashLookup[filePath] == missingFileWithHashToFind.SHA256Hash + select filePath).SingleOrDefault(); + if (matchFile == null) + { + success = false; + missingFiles.AppendLine($"Unable to find {missingFileWithHashToFind.Name} with SHA256 hash '{missingFileWithHashToFind.SHA256Hash}'"); + } + else + { + string relativePath = (string)(new Uri(OutputPath).MakeRelativeUri(new Uri(matchFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar); + FileName updatedFileName = new FileName(OutputPath, relativePath, missingFileWithHashToFind.SHA256Hash); + candidateFileNames.Remove(missingFileWithHashToFind); + candidateFileNames.Add(updatedFileName); + } + } + } + if (!success) + { + throw new Exception($"Failure attempting to find one or more files:\n{missingFiles.ToString()}"); + } + } } } diff --git a/src/SignTool/SignTool/BatchSignUtil.cs b/src/SignTool/SignTool/BatchSignUtil.cs index 6f44efc085..c85f085084 100644 --- a/src/SignTool/SignTool/BatchSignUtil.cs +++ b/src/SignTool/SignTool/BatchSignUtil.cs @@ -5,12 +5,11 @@ using System.Collections.Immutable; using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.IO.Packaging; using System.Linq; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; -using System.Text; -using System.Threading.Tasks; +using Newtonsoft.Json; +using SignTool.Json; using static SignTool.PathUtil; namespace SignTool @@ -22,11 +21,16 @@ internal sealed class BatchSignUtil private readonly BatchSignInput _batchData; private readonly ISignTool _signTool; private readonly ContentUtil _contentUtil = new ContentUtil(); + private readonly string _orchestrationManifestPath; + private readonly string _unpackingDirectory; - internal BatchSignUtil(ISignTool signTool, BatchSignInput batchData) + internal BatchSignUtil(ISignTool signTool, BatchSignInput batchData, string orchestrationManifestPath) { _signTool = signTool; _batchData = batchData; + _orchestrationManifestPath = orchestrationManifestPath; + // TODO: Better path; for now making sure the relative paths are all in the same "OutputDirectory" value should help things work. + _unpackingDirectory = Path.Combine(batchData.OutputPath, "ZipArchiveUnpackingDirectory"); } internal bool Go(TextWriter textWriter) @@ -36,12 +40,20 @@ internal bool Go(TextWriter textWriter) var allGood = VerifyCertificates(textWriter); var contentMap = BuildContentMap(textWriter, ref allGood); var zipDataMap = BuildAllZipData(contentMap, textWriter, ref allGood); + if (!allGood) { return false; } + // At this point we trust the content we're about to sign. We can now take the information from _batchData + // and put any content and zipDataMap hash values into the new manifest. + if (!string.IsNullOrEmpty(_orchestrationManifestPath)) + { + textWriter.WriteLine($"Writing updated SignTool Data Json information to '{_orchestrationManifestPath}'"); + return GenerateOrchestrationManifest(textWriter, _batchData, contentMap, _orchestrationManifestPath); + } - // Next remove public sign from all of the assemblies. It can interfere with the signing process. + // Next remove public signing from all of the assemblies; it can interfere with the signing process. RemovePublicSign(textWriter); // Next sign all of the files @@ -51,9 +63,56 @@ internal bool Go(TextWriter textWriter) return VerifyAfterSign(zipDataMap, textWriter); } + + + private bool GenerateOrchestrationManifest(TextWriter textWriter, BatchSignInput batchData, ContentMap contentMap, string outputPath) + { + try + { + textWriter.WriteLine($"Generating orchestration file manifest into {outputPath}"); + OrchestratedFileJson fileJsonWithInfo = new OrchestratedFileJson + { + ExcludeList = _batchData.ExternalFileNames.ToArray() ?? Array.Empty() + }; + + var distinctSigningCombos = batchData.FileSignInfoMap.Values.GroupBy(v => new { v.Certificate, v.StrongName }); + + List newList = new List(); + foreach (var combinationToSign in distinctSigningCombos) + { + var filesInThisGroup = combinationToSign.Select(c => new FileSignDataEntry() + { + FilePath = c.FileName.RelativePath, + SHA256Hash = contentMap.GetChecksum(c.FileName), + PublishToFeedUrl = batchData.PublishUri + }); + newList.Add(new OrchestratedFileSignData() + { + Certificate = combinationToSign.Key.Certificate, + StrongName = combinationToSign.Key.StrongName, + FileList = filesInThisGroup.ToArray() + }); + } + fileJsonWithInfo.SignList = newList.ToArray(); + + + using (StreamWriter file = File.CreateText(outputPath)) + { + file.Write(JsonConvert.SerializeObject(fileJsonWithInfo, Formatting.Indented)); + } + } + catch (Exception ex) + { + textWriter.WriteLine($"Unexpected exception: {ex.Message}"); + textWriter.WriteLine(ex.StackTrace); + return false; + } + return true; + } + private void RemovePublicSign(TextWriter textWriter) { - textWriter.WriteLine("Removing public sign"); + textWriter.WriteLine("Removing public signing"); foreach (var name in _batchData.AssemblyNames) { var fileSignInfo = _batchData.FileSignInfoMap[name]; @@ -68,7 +127,7 @@ private void RemovePublicSign(TextWriter textWriter) /// /// Actually sign all of the described files. /// - private void SignFiles(ContentMap contetMap, Dictionary zipDataMap, TextWriter textWriter) + private void SignFiles(ContentMap contentMap, Dictionary zipDataMap, TextWriter textWriter) { // Generate the list of signed files in a deterministic order. Makes it easier to track down // bugs if repeated runs use the same ordering. @@ -207,12 +266,27 @@ private List GetVsixPartRelativeNames(FileName vsixName) private ContentMap BuildContentMap(TextWriter textWriter, ref bool allGood) { var contentMap = new ContentMap(); + foreach (var fileName in _batchData.FileNames) { try { - var checksum = _contentUtil.GetChecksum(fileName.FullPath); - contentMap.Add(fileName, checksum); + if (string.IsNullOrEmpty(fileName.SHA256Hash)) + { + var checksum = _contentUtil.GetChecksum(fileName.FullPath); + contentMap.Add(fileName, checksum); + } + else + { + if (File.Exists(fileName.FullPath)) + { + contentMap.Add(fileName, fileName.SHA256Hash); + } + else + { + throw new FileNotFoundException(); + } + } } catch (Exception ex) { @@ -232,7 +306,7 @@ private ContentMap BuildContentMap(TextWriter textWriter, ref bool allGood) } /// - /// Sanity check the certificates that are attached to the various items. Ensure we aren't using say a VSIX + /// Sanity check the certificates that are attached to the various items. Ensure we aren't using, say, a VSIX /// certificate on a DLL for example. /// private bool VerifyCertificates(TextWriter textWriter) @@ -298,7 +372,7 @@ private Dictionary BuildAllZipData(ContentMap contentMap, Tex } /// - /// Build up the instance for a given zip container. This will also report any consintency + /// Build up the instance for a given zip container. This will also report any consistency /// errors found when examining the zip archive. /// private ZipData BuildZipData(FileName zipFileName, ContentMap contentMap, TextWriter textWriter, ref bool allGood) @@ -327,7 +401,7 @@ private ZipData BuildZipData(FileName zipFileName, ContentMap contentMap, TextWr if (!_batchData.FileNames.Any(x => FilePathComparer.Equals(x.Name, name))) { allGood = false; - textWriter.WriteLine($"VSIX {zipFileName} has part {name} which is not listed in the sign or external list"); + textWriter.WriteLine($"Zip Container '{zipFileName}' has part '{name}' which is not listed in the sign or external list"); continue; } diff --git a/src/SignTool/SignTool/ContentUtil.cs b/src/SignTool/SignTool/ContentUtil.cs index d94c1bbbac..82a3e3847e 100644 --- a/src/SignTool/SignTool/ContentUtil.cs +++ b/src/SignTool/SignTool/ContentUtil.cs @@ -3,21 +3,18 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Security.Cryptography; -using System.Text; -using System.Threading.Tasks; namespace SignTool { internal sealed class ContentUtil { private readonly Dictionary _filePathCache = new Dictionary(StringComparer.OrdinalIgnoreCase); - private readonly MD5 _md5 = MD5.Create(); + private readonly SHA256 _sha256 = SHA256.Create(); internal string GetChecksum(Stream stream) { - var hash = _md5.ComputeHash(stream); + var hash = _sha256.ComputeHash(stream); return HashBytesToString(hash); } diff --git a/src/SignTool/SignTool/FileName.cs b/src/SignTool/SignTool/FileName.cs index c1b5b66f4f..0c6ac3edc0 100644 --- a/src/SignTool/SignTool/FileName.cs +++ b/src/SignTool/SignTool/FileName.cs @@ -14,17 +14,19 @@ internal struct FileName : IEquatable internal string Name { get; } internal string FullPath { get; } internal string RelativePath { get; } + internal string SHA256Hash { get; } internal bool IsAssembly => PathUtil.IsAssembly(Name); internal bool IsVsix => PathUtil.IsVsix(Name); internal bool IsNupkg => PathUtil.IsVsix(Name); internal bool IsZipContainer => PathUtil.IsZipContainer(Name); - internal FileName(string rootBinaryPath, string relativePath) + internal FileName(string rootBinaryPath, string relativePath, string sha256Hash = null) { Name = Path.GetFileName(relativePath); FullPath = Path.Combine(rootBinaryPath, relativePath); RelativePath = relativePath; + SHA256Hash = sha256Hash; } public static bool operator ==(FileName left, FileName right) => left.FullPath == right.FullPath; diff --git a/src/SignTool/SignTool/JsonTypes.cs b/src/SignTool/SignTool/JsonTypes.cs index fc53a9ef22..0d9804ccfa 100644 --- a/src/SignTool/SignTool/JsonTypes.cs +++ b/src/SignTool/SignTool/JsonTypes.cs @@ -1,16 +1,14 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Newtonsoft.Json; namespace SignTool.Json { internal sealed class FileJson { + [JsonProperty(PropertyName = "publishUrl")] + public string PublishUrl { get; set; } + [JsonProperty(PropertyName = "sign")] public FileSignData[] SignList { get; set; } @@ -23,19 +21,62 @@ public FileJson() } } - internal sealed class FileSignData + internal sealed class OrchestratedFileJson + { + [JsonProperty(PropertyName = "sign")] + public OrchestratedFileSignData[] SignList { get; set; } + + [JsonProperty(PropertyName = "exclude")] + public string[] ExcludeList { get; set; } + + public OrchestratedFileJson() + { + } + } + + internal class FileSignDataBase { - [JsonProperty(PropertyName = "certificate")] + [JsonProperty(PropertyName = "certificate", Order = 1)] public string Certificate { get; set; } - [JsonProperty(PropertyName = "strongName")] + [JsonProperty(PropertyName = "strongName", Order = 2)] public string StrongName { get; set; } + } + - [JsonProperty(PropertyName = "values")] + internal sealed class FileSignData : FileSignDataBase + { + [JsonProperty(PropertyName = "values", Order = 3)] public string[] FileList { get; set; } public FileSignData() { } } + + internal sealed class OrchestratedFileSignData : FileSignDataBase + { + [JsonProperty(PropertyName = "values", Order = 3)] + public FileSignDataEntry[] FileList { get; set; } + + public OrchestratedFileSignData() + { + } + } + + internal sealed class FileSignDataEntry + { + [JsonProperty(PropertyName = "filePath")] + public string FilePath { get; set; } + + [JsonProperty(PropertyName = "sha256Hash")] + public string SHA256Hash { get; set; } + + [JsonProperty(PropertyName = "publishtofeedurl")] + public string PublishToFeedUrl { get; set; } + + public FileSignDataEntry() + { + } + } } diff --git a/src/SignTool/SignTool/OrchestratedBatchSignInput.cs b/src/SignTool/SignTool/OrchestratedBatchSignInput.cs new file mode 100644 index 0000000000..64e8f8a92b --- /dev/null +++ b/src/SignTool/SignTool/OrchestratedBatchSignInput.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using SignTool.Json; + +namespace SignTool +{ + /// + /// Represents a preprocessed version of all of the input to the batch signing process. + /// + internal sealed class OrchestratedBatchSignInput + { + /// + /// The path where the binaries are built to: e:\path\to\source\Binaries\Debug + /// + internal string OutputPath { get; } + + /// + /// The path where the binaries are built to: e:\path\to\source\Binaries\Debug + /// + internal string PublishUri { get; } + + /// + /// The ordered names of the files to be signed. These are all relative paths off of the + /// property. + /// + internal ImmutableArray FileNames { get; } + + /// + /// These are binaries which are included in our zip containers but are already signed. This list is used for + /// validation purposes. These are all flat names and cannot be relative paths. + /// + internal ImmutableArray ExternalFileNames { get;} + + /// + /// Names of assemblies that need to be signed. This is a subset of + /// + internal ImmutableArray AssemblyNames { get; } + + /// + /// Names of zip containers that need to be examined for signing. This is a subset of + /// + internal ImmutableArray ZipContainerNames { get; } + + /// + /// Names of other file types which aren't specifically handled by the tool. This is a subset of + /// + internal ImmutableArray OtherNames { get; } + + /// + /// A map from all of the binaries that need to be signed to the actual signing data. + /// + internal ImmutableDictionary FileSignInfoMap { get; } + + internal OrchestratedBatchSignInput(string outputPath, Dictionary fileSignDataMap, IEnumerable externalFileNames, string publishUri) + { + OutputPath = outputPath; + PublishUri = publishUri; + // Use order by to make the output of this tool as predictable as possible. + var fileNames = fileSignDataMap.Keys; + FileNames = fileNames.OrderBy(x => x).Select(x => new FileName(outputPath, x)).ToImmutableArray(); + ExternalFileNames = externalFileNames.OrderBy(x => x).ToImmutableArray(); + + AssemblyNames = FileNames.Where(x => x.IsAssembly).ToImmutableArray(); + ZipContainerNames = FileNames.Where(x => x.IsZipContainer).ToImmutableArray(); + OtherNames = FileNames.Where(x => !x.IsAssembly && !x.IsZipContainer).ToImmutableArray(); + + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var name in FileNames) + { + var data = fileSignDataMap[name.RelativePath]; + builder.Add(name, new FileSignInfo(name, data)); + } + FileSignInfoMap = builder.ToImmutable(); + } + } +} + diff --git a/src/SignTool/SignTool/PathUtil.cs b/src/SignTool/SignTool/PathUtil.cs index 26514027a2..ac158abeeb 100644 --- a/src/SignTool/SignTool/PathUtil.cs +++ b/src/SignTool/SignTool/PathUtil.cs @@ -9,8 +9,8 @@ namespace SignTool { internal static class PathUtil { - internal static bool IsVsix(string fileName) => Path.GetExtension(fileName) == ".vsix"; - internal static bool IsNupkg(string fileName) => Path.GetExtension(fileName) == ".nupkg"; + internal static bool IsVsix(string fileName) => Path.GetExtension(fileName).Equals(".vsix", StringComparison.OrdinalIgnoreCase); + internal static bool IsNupkg(string fileName) => Path.GetExtension(fileName).Equals(".nupkg", StringComparison.OrdinalIgnoreCase); internal static bool IsZipContainer(string fileName) => IsVsix(fileName) || diff --git a/src/SignTool/SignTool/Program.cs b/src/SignTool/SignTool/Program.cs index 93e93edcaf..4341f26ad3 100644 --- a/src/SignTool/SignTool/Program.cs +++ b/src/SignTool/SignTool/Program.cs @@ -2,14 +2,12 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Newtonsoft.Json; +using SignTool.Json; namespace SignTool { @@ -20,8 +18,7 @@ internal static class Program internal static int Main(string[] args) { - SignToolArgs signToolArgs; - if (!ParseCommandLineArguments(StandardHost.Instance, args, out signToolArgs)) + if (!ParseCommandLineArguments(StandardHost.Instance, args, out SignToolArgs signToolArgs)) { PrintUsage(); return ExitFailure; @@ -33,9 +30,17 @@ internal static int Main(string[] args) return ExitFailure; } + BatchSignInput batchData; var signTool = SignToolFactory.Create(signToolArgs); - var batchData = ReadConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); - var util = new BatchSignUtil(signTool, batchData); + if (signToolArgs.OrchestrationMode) + { + batchData = ReadOrchestrationConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); + } + else + { + batchData = ReadConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); + } + var util = new BatchSignUtil(signTool, batchData, signToolArgs.OrchestrationManifestPath); try { return util.Go(Console.Out) ? ExitSuccess : ExitFailure; @@ -52,12 +57,10 @@ internal static BatchSignInput ReadConfigFile(string outputPath, string configFi { using (var file = File.OpenText(configFile)) { - BatchSignInput batchData; - if (!TryReadConfigFile(Console.Out, file, outputPath, out batchData)) + if (!TryReadConfigFile(Console.Out, file, outputPath, out BatchSignInput batchData)) { Environment.Exit(ExitFailure); } - return batchData; } } @@ -91,13 +94,64 @@ internal static bool TryReadConfigFile(TextWriter output, TextReader configReade return false; } - batchData = new BatchSignInput(outputPath, map, fileJson.ExcludeList ?? Array.Empty()); + batchData = new BatchSignInput(outputPath, map, fileJson.ExcludeList ?? Array.Empty(), fileJson.PublishUrl ?? "unset"); + return true; + } + + internal static BatchSignInput ReadOrchestrationConfigFile(string outputPath, string configFile) + { + using (var file = File.OpenText(configFile)) + { + if (!TryReadOrchestrationConfigFile(Console.Out, file, outputPath, out BatchSignInput batchData)) + { + Environment.Exit(ExitFailure); + } + return batchData; + } + } + + internal static bool TryReadOrchestrationConfigFile(TextWriter output, TextReader configReader, string outputPath, out BatchSignInput batchData) + { + var serializer = new JsonSerializer(); + var fileJson = (Json.OrchestratedFileJson)serializer.Deserialize(configReader, typeof(Json.OrchestratedFileJson)); + var map = new Dictionary(); + // For now, a given json file will be assumed to serialize to one place and we'll throw otherwise + string publishUrl = (from OrchestratedFileSignData entry in fileJson.SignList + from FileSignDataEntry fileToSign in entry.FileList + select fileToSign.PublishToFeedUrl).Distinct().Single(); + var allGood = true; + foreach (var item in fileJson.SignList) + { + var data = new SignInfo(certificate: item.Certificate, strongName: item.StrongName); + + foreach (FileSignDataEntry entry in item.FileList) + { + if (map.ContainsKey(entry)) + { + Console.WriteLine($"Duplicate signing info entry for: {entry.FilePath}"); + allGood = false; + } + else + { + map.Add(entry, data); + } + } + } + + if (!allGood) + { + batchData = null; + return false; + } + + batchData = new BatchSignInput(outputPath, map, fileJson.ExcludeList ?? Array.Empty(), publishUrl ); return true; } + /// - /// The files to sign section supports globbing. The only caveat is that globs must expand to match at least a - /// single file else an error occurs. This function will expand those globas as necessary. + /// The 'files to sign' section supports globbing. The only caveat is that globs must expand to match at least a + /// single file else an error occurs. This function will expand those globs as necessary. /// private static List ExpandFileList(string outputPath, IEnumerable relativeFileNames, ref bool allGood) { @@ -143,11 +197,13 @@ internal static void PrintUsage() test: Run tool without actually modifying any state. testSign: The binaries will be test signed. The default is to real sign. +orchestrationmode: Support comma-separated list of one or more config files in the format outputted using the 'orchestrationConfigFile' argument. outputPath: Directory containing the binaries. intermediateOutputPath: Directory containing intermediate output. Default is (outputpath\..\Obj). nugetPackagesPath: Path containing downloaded NuGet packages. msbuildPath: Path to MSBuild.exe to use as signing mechanism. -config: Path to SignToolData.json. Default build\config\SignToolData.json. +config: Path to SignToolData.json. Default: build\config\SignToolData.json. +orchestrationConfigFile: Run tool to produce an orchestration json file. This will contain SHA256 hashes of files for verification to consume later. Cannot be used with -orchestrationmode "; Console.WriteLine(usage); } @@ -164,8 +220,10 @@ internal static bool ParseCommandLineArguments( string msbuildPath = null; string nugetPackagesPath = null; string configFile = null; + string orchestrationConfigFile = null; var test = false; var testSign = false; + var orchestrationMode = false; var i = 0; @@ -182,6 +240,10 @@ internal static bool ParseCommandLineArguments( testSign = true; i++; break; + case "-orchestrationmode": + orchestrationMode = true; + i++; + break; case "-intermediateoutputpath": if (!ParsePathOption(args, ref i, current, out intermediateOutputPath)) { @@ -206,12 +268,25 @@ internal static bool ParseCommandLineArguments( return false; } break; + case "-orchestrationconfigfile": + if (!ParsePathOption(args, ref i, current, out orchestrationConfigFile)) + { + return false; + } + orchestrationConfigFile = orchestrationConfigFile.TrimEnd('\"').TrimStart('\"'); + break; default: Console.Error.WriteLine($"Unrecognized option {current}"); return false; } } + if (orchestrationMode && !string.IsNullOrEmpty(orchestrationConfigFile)) + { + Console.WriteLine("Please specify either -orchestrationmode (for signing multiple json manifests w/ SHA256 entries) or a value for -orchestrationconfigfile to generate such a manifest, but not both."); + return false; + } + if (i + 1 != args.Length) { Console.Error.WriteLine("Need a value for outputPath"); @@ -256,7 +331,9 @@ internal static bool ParseCommandLineArguments( appPath: AppContext.BaseDirectory, configFile: configFile, test: test, - testSign: testSign); + testSign: testSign, + orchestrationManifestPath: orchestrationConfigFile, + orchestrationMode: orchestrationMode); return true; } @@ -287,7 +364,6 @@ private static string GetSourcesPath(IHost host, string outputPath) current = Path.GetDirectoryName(current); } - return null; } } diff --git a/src/SignTool/SignTool/SignToolArgs.cs b/src/SignTool/SignTool/SignToolArgs.cs index a7cec1859a..cada43e1fd 100644 --- a/src/SignTool/SignTool/SignToolArgs.cs +++ b/src/SignTool/SignTool/SignToolArgs.cs @@ -1,9 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace SignTool { internal struct SignToolArgs @@ -15,7 +9,9 @@ internal struct SignToolArgs internal string MSBuildPath { get; } internal string ConfigFile { get; } internal bool Test { get; } + internal bool OrchestrationMode { get; } internal bool TestSign { get; } + internal string OrchestrationManifestPath { get; } internal SignToolArgs( string outputPath, @@ -25,7 +21,10 @@ internal SignToolArgs( string nugetPackagesPath, string configFile, bool test, - bool testSign) + bool testSign, + bool orchestrationMode, + string orchestrationManifestPath + ) { OutputPath = outputPath; IntermediateOutputPath = intermediateOutputPath; @@ -35,6 +34,8 @@ internal SignToolArgs( ConfigFile = configFile; Test = test; TestSign = testSign; + OrchestrationManifestPath = orchestrationManifestPath; + OrchestrationMode = orchestrationMode; } } } diff --git a/src/SignTool/SignTool/ZipData.cs b/src/SignTool/SignTool/ZipData.cs index 5211ebc63c..4a86235476 100644 --- a/src/SignTool/SignTool/ZipData.cs +++ b/src/SignTool/SignTool/ZipData.cs @@ -1,11 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.Collections.Generic; using System.Collections.Immutable; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace SignTool { From 2fd64d5d299fe04a2fd4e414211f16ba4b985aab Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Wed, 22 Nov 2017 17:54:24 -0800 Subject: [PATCH 2/7] Plumb new arguments through Build.proj/Sign.proj --- sdks/RepoToolset/tools/Build.proj | 6 ++++-- sdks/RepoToolset/tools/Sign.proj | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sdks/RepoToolset/tools/Build.proj b/sdks/RepoToolset/tools/Build.proj index f2f8490035..5b83e8f8c5 100644 --- a/sdks/RepoToolset/tools/Build.proj +++ b/sdks/RepoToolset/tools/Build.proj @@ -14,9 +14,11 @@ Deploy "true" to deploy assets (e.g. VSIXes) Test "true" to run tests IntegrationTest "true" to run integration tests - Sign "true" to sign built binaries Pack "true" to build NuGet packages and VS insertion manifests + Sign "true" to sign built binaries SignType "real" to send binaries to signing service, "test" to only validate signing configuration. + OrchestrationConfigFile + Path to output Json for orchestrated-type signing (produce manifest that can be consumed later.) --> diff --git a/sdks/RepoToolset/tools/Sign.proj b/sdks/RepoToolset/tools/Sign.proj index ae25757c96..283d216a17 100644 --- a/sdks/RepoToolset/tools/Sign.proj +++ b/sdks/RepoToolset/tools/Sign.proj @@ -4,6 +4,12 @@ Required parameters: DirectoryBuildPropsPath Path to the Directory.Build.props file in the repo root. RealSign "true" to send binaries to the signing service, "false" to only validate signing configuration. + + Optional parameters: + OrchestrationConfigFile If specified, output a manifest similar to SignToolData.json but with checksums of all relevant files included + for reassembly later when only zip archives are available. + OrchestrationMode "true" if the input manifest is an orchestration manifest, false otherwise. Will attempt to unpack all zips + provided and match on checksums for any missing files. --> @@ -19,6 +25,8 @@ + + @@ -29,4 +37,5 @@ + From 5806f639db9e723028ff66c32ea25f7e5ed9a86f Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Wed, 22 Nov 2017 18:13:53 -0800 Subject: [PATCH 3/7] PR feedback: Get rid of OrchestrationMode, now just check for existence of a "kind" property. Some other light cleanup. --- sdks/RepoToolset/tools/Build.proj | 7 ++-- sdks/RepoToolset/tools/Sign.proj | 10 +++--- src/SignTool/SignTool/BatchSignInput.cs | 5 ++- src/SignTool/SignTool/BatchSignUtil.cs | 2 +- src/SignTool/SignTool/JsonTypes.cs | 6 ++++ src/SignTool/SignTool/Program.cs | 44 ++++++++++++++++--------- src/SignTool/SignTool/SignToolArgs.cs | 3 -- 7 files changed, 46 insertions(+), 31 deletions(-) diff --git a/sdks/RepoToolset/tools/Build.proj b/sdks/RepoToolset/tools/Build.proj index 5b83e8f8c5..41ff7fc37d 100644 --- a/sdks/RepoToolset/tools/Build.proj +++ b/sdks/RepoToolset/tools/Build.proj @@ -17,8 +17,6 @@ Pack "true" to build NuGet packages and VS insertion manifests Sign "true" to sign built binaries SignType "real" to send binaries to signing service, "test" to only validate signing configuration. - OrchestrationConfigFile - Path to output Json for orchestrated-type signing (produce manifest that can be consumed later.) --> false @@ -173,7 +174,7 @@ Sign artifacts. --> diff --git a/sdks/RepoToolset/tools/Sign.proj b/sdks/RepoToolset/tools/Sign.proj index 283d216a17..03f9dc6b31 100644 --- a/sdks/RepoToolset/tools/Sign.proj +++ b/sdks/RepoToolset/tools/Sign.proj @@ -6,10 +6,9 @@ RealSign "true" to send binaries to the signing service, "false" to only validate signing configuration. Optional parameters: - OrchestrationConfigFile If specified, output a manifest similar to SignToolData.json but with checksums of all relevant files included - for reassembly later when only zip archives are available. - OrchestrationMode "true" if the input manifest is an orchestration manifest, false otherwise. Will attempt to unpack all zips - provided and match on checksums for any missing files. + OutputConfigFile If specified, output a manifest similar to SignToolData.json but with checksums of all relevant files included + for reassembly later when only zip archives are available. If this file is later used as the '-config' file, + SignTool will attempt to unpack all zips provided and match on checksums for any missing files. --> @@ -25,8 +24,7 @@ - - + diff --git a/src/SignTool/SignTool/BatchSignInput.cs b/src/SignTool/SignTool/BatchSignInput.cs index d9b0b6e6b2..c3a4be33ac 100644 --- a/src/SignTool/SignTool/BatchSignInput.cs +++ b/src/SignTool/SignTool/BatchSignInput.cs @@ -125,8 +125,7 @@ private void UnpackMissingContent(ref List candidateFileNames) { ContentUtil contentUtil = new ContentUtil(); - // Get all Zip Archives in the manifest - // We'll use a non-immutable queue as we may need to add new zips to the list. + // Get all Zip Archives in the manifest recursively. Queue allZipsWeKnowAbout = new Queue(ZipContainerNames); while (allZipsWeKnowAbout.Count > 0) @@ -168,7 +167,7 @@ where Path.GetFileName(filePath).Equals(missingFileWithHashToFind.Name, StringCo if (matchFile == null) { success = false; - missingFiles.AppendLine($"Unable to find {missingFileWithHashToFind.Name} with SHA256 hash '{missingFileWithHashToFind.SHA256Hash}'"); + missingFiles.AppendLine($"File: {missingFileWithHashToFind.Name} Hash: '{missingFileWithHashToFind.SHA256Hash}'"); } else { diff --git a/src/SignTool/SignTool/BatchSignUtil.cs b/src/SignTool/SignTool/BatchSignUtil.cs index c85f085084..8353ff9101 100644 --- a/src/SignTool/SignTool/BatchSignUtil.cs +++ b/src/SignTool/SignTool/BatchSignUtil.cs @@ -94,7 +94,7 @@ private bool GenerateOrchestrationManifest(TextWriter textWriter, BatchSignInput }); } fileJsonWithInfo.SignList = newList.ToArray(); - + fileJsonWithInfo.Kind = "orchestration"; using (StreamWriter file = File.CreateText(outputPath)) { diff --git a/src/SignTool/SignTool/JsonTypes.cs b/src/SignTool/SignTool/JsonTypes.cs index 0d9804ccfa..bda64b8a95 100644 --- a/src/SignTool/SignTool/JsonTypes.cs +++ b/src/SignTool/SignTool/JsonTypes.cs @@ -6,6 +6,9 @@ namespace SignTool.Json { internal sealed class FileJson { + [JsonProperty(PropertyName = "kind")] + public string Kind { get; set; } + [JsonProperty(PropertyName = "publishUrl")] public string PublishUrl { get; set; } @@ -23,6 +26,9 @@ public FileJson() internal sealed class OrchestratedFileJson { + [JsonProperty(PropertyName = "kind")] + public string Kind { get; set; } + [JsonProperty(PropertyName = "sign")] public OrchestratedFileSignData[] SignList { get; set; } diff --git a/src/SignTool/SignTool/Program.cs b/src/SignTool/SignTool/Program.cs index 4341f26ad3..ab28f4036a 100644 --- a/src/SignTool/SignTool/Program.cs +++ b/src/SignTool/SignTool/Program.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using SignTool.Json; namespace SignTool @@ -32,14 +33,21 @@ internal static int Main(string[] args) BatchSignInput batchData; var signTool = SignToolFactory.Create(signToolArgs); - if (signToolArgs.OrchestrationMode) - { - batchData = ReadOrchestrationConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); - } - else + string configFileKind = GetConfigFileKind(signToolArgs.ConfigFile); + + switch (configFileKind.ToLower()) { - batchData = ReadConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); + case "default": + batchData = ReadConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); + break; + case "orchestration": + batchData = ReadOrchestrationConfigFile(signToolArgs.OutputPath, signToolArgs.ConfigFile); + break; + default: + Console.WriteLine($"Don't know how to deal with manifest kind '{configFileKind}'"); + return 1; } + var util = new BatchSignUtil(signTool, batchData, signToolArgs.OrchestrationManifestPath); try { @@ -213,14 +221,14 @@ internal static bool ParseCommandLineArguments( string[] args, out SignToolArgs signToolArgs) { - signToolArgs = default(SignToolArgs); + signToolArgs = default; string intermediateOutputPath = null; string outputPath = null; string msbuildPath = null; string nugetPackagesPath = null; string configFile = null; - string orchestrationConfigFile = null; + string outputConfigFile = null; var test = false; var testSign = false; var orchestrationMode = false; @@ -268,12 +276,12 @@ internal static bool ParseCommandLineArguments( return false; } break; - case "-orchestrationconfigfile": - if (!ParsePathOption(args, ref i, current, out orchestrationConfigFile)) + case "-outputconfig": + if (!ParsePathOption(args, ref i, current, out outputConfigFile)) { return false; } - orchestrationConfigFile = orchestrationConfigFile.TrimEnd('\"').TrimStart('\"'); + outputConfigFile = outputConfigFile.TrimEnd('\"').TrimStart('\"'); break; default: Console.Error.WriteLine($"Unrecognized option {current}"); @@ -281,9 +289,9 @@ internal static bool ParseCommandLineArguments( } } - if (orchestrationMode && !string.IsNullOrEmpty(orchestrationConfigFile)) + if (orchestrationMode && !string.IsNullOrEmpty(outputConfigFile)) { - Console.WriteLine("Please specify either -orchestrationmode (for signing multiple json manifests w/ SHA256 entries) or a value for -orchestrationconfigfile to generate such a manifest, but not both."); + Console.WriteLine("Please specify either -orchestrationmode (signing multiple json manifests with SHA256 entries) or a value for -outputconfig to generate such a manifest, but not both."); return false; } @@ -332,8 +340,7 @@ internal static bool ParseCommandLineArguments( configFile: configFile, test: test, testSign: testSign, - orchestrationManifestPath: orchestrationConfigFile, - orchestrationMode: orchestrationMode); + orchestrationManifestPath: outputConfigFile); return true; } @@ -366,5 +373,12 @@ private static string GetSourcesPath(IHost host, string outputPath) } return null; } + + private static string GetConfigFileKind(string path) + { + JObject configFile = JObject.Parse(File.ReadAllText(path)); + var kind = configFile["kind"]?.Value(); + return string.IsNullOrEmpty(kind) ? "default" : kind; + } } } diff --git a/src/SignTool/SignTool/SignToolArgs.cs b/src/SignTool/SignTool/SignToolArgs.cs index cada43e1fd..f76bee6709 100644 --- a/src/SignTool/SignTool/SignToolArgs.cs +++ b/src/SignTool/SignTool/SignToolArgs.cs @@ -9,7 +9,6 @@ internal struct SignToolArgs internal string MSBuildPath { get; } internal string ConfigFile { get; } internal bool Test { get; } - internal bool OrchestrationMode { get; } internal bool TestSign { get; } internal string OrchestrationManifestPath { get; } @@ -22,7 +21,6 @@ internal SignToolArgs( string configFile, bool test, bool testSign, - bool orchestrationMode, string orchestrationManifestPath ) { @@ -35,7 +33,6 @@ string orchestrationManifestPath Test = test; TestSign = testSign; OrchestrationManifestPath = orchestrationManifestPath; - OrchestrationMode = orchestrationMode; } } } From a9ce6c8493953487747bbf072e67e69b98014913 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Mon, 27 Nov 2017 14:08:25 -0800 Subject: [PATCH 4/7] Remove leftover PB_SigningOrchestrationMode variable mentions --- sdks/RepoToolset/tools/Build.proj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/RepoToolset/tools/Build.proj b/sdks/RepoToolset/tools/Build.proj index 41ff7fc37d..accf98c130 100644 --- a/sdks/RepoToolset/tools/Build.proj +++ b/sdks/RepoToolset/tools/Build.proj @@ -30,7 +30,6 @@ PB_PublishBlobFeedKey {''|string} Account key. PB_SigningOrchestrationConfig {''|string} Path to output Json for orchestrated-type signing (produce manifest that can be consumed later.) - PB_SigningOrchestrationMode {''|'true'|'false'} Set 'true' to have SignTool use manifests as provided above. --> false @@ -174,7 +173,7 @@ Sign artifacts. --> From a45db94fd1583f733f6ec0e5b2072fa6ddca1312 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Mon, 27 Nov 2017 14:35:27 -0800 Subject: [PATCH 5/7] Address PR feedback --- src/SignTool/SignTool/BatchSignInput.cs | 93 ++++++++++++------------- src/SignTool/SignTool/BatchSignUtil.cs | 58 +++++++-------- src/SignTool/SignTool/Program.cs | 11 --- 3 files changed, 69 insertions(+), 93 deletions(-) diff --git a/src/SignTool/SignTool/BatchSignInput.cs b/src/SignTool/SignTool/BatchSignInput.cs index c3a4be33ac..7d4b531ec7 100644 --- a/src/SignTool/SignTool/BatchSignInput.cs +++ b/src/SignTool/SignTool/BatchSignInput.cs @@ -118,66 +118,63 @@ private void UnpackMissingContent(ref List candidateFileNames) // Nothing to do if (unpackNeeded.Count() == 0) - { return; - } - else - { - ContentUtil contentUtil = new ContentUtil(); - // Get all Zip Archives in the manifest recursively. - Queue allZipsWeKnowAbout = new Queue(ZipContainerNames); + ContentUtil contentUtil = new ContentUtil(); - while (allZipsWeKnowAbout.Count > 0) - { - FileName zipFile = allZipsWeKnowAbout.Dequeue(); - string unpackFolder = Path.Combine(unpackingDirectory, zipFile.SHA256Hash); + // Get all Zip Archives in the manifest recursively. + Queue allZipsWeKnowAbout = new Queue(ZipContainerNames); - // Assumption: If a zip with a given hash is already unpacked, it's probably OK. - if (!Directory.Exists(unpackFolder)) - { - Directory.CreateDirectory(unpackFolder); - ZipFile.ExtractToDirectory(zipFile.FullPath, unpackFolder); - } - // Add any zips we just unzipped. - foreach (string file in Directory.GetFiles(unpackFolder, "*", SearchOption.AllDirectories)) + while (allZipsWeKnowAbout.Count > 0) + { + FileName zipFile = allZipsWeKnowAbout.Dequeue(); + string unpackFolder = Path.Combine(unpackingDirectory, zipFile.SHA256Hash); + + // Assumption: If a zip with a given hash is already unpacked, it's probably OK. + if (!Directory.Exists(unpackFolder)) + { + Directory.CreateDirectory(unpackFolder); + ZipFile.ExtractToDirectory(zipFile.FullPath, unpackFolder); + } + // Add any zips we just unzipped. + foreach (string file in Directory.GetFiles(unpackFolder, "*", SearchOption.AllDirectories)) + { + if (PathUtil.IsZipContainer(file)) { - if (PathUtil.IsZipContainer(file)) - { - string relativePath = (string)(new Uri(unpackingDirectory).MakeRelativeUri(new Uri(file))).OriginalString; - allZipsWeKnowAbout.Enqueue(new FileName(unpackingDirectory, relativePath, contentUtil.GetChecksum(file))); - } + string relativePath = (new Uri(unpackingDirectory).MakeRelativeUri(new Uri(file))).OriginalString; + allZipsWeKnowAbout.Enqueue(new FileName(unpackingDirectory, relativePath, contentUtil.GetChecksum(file))); } } - // Lazy : Disks are fast, just calculate ALL hashes. Could optimize by only files we intend to sign - Dictionary existingHashLookup = new Dictionary(); - foreach (string file in Directory.GetFiles(unpackingDirectory, "*", SearchOption.AllDirectories)) + } + // Lazy : Disks are fast, just calculate ALL hashes. Could optimize by only files we intend to sign + Dictionary existingHashLookup = new Dictionary(); + foreach (string file in Directory.GetFiles(unpackingDirectory, "*", SearchOption.AllDirectories)) + { + existingHashLookup.Add(file, contentUtil.GetChecksum(file)); + } + + Dictionary fileNameUpdates = new Dictionary(); + // At this point, we've unpacked every Zip we can possibly pull out into folders named for the zip's hash into 'unpackingDirectory' + foreach (FileName missingFileWithHashToFind in unpackNeeded) + { + string matchFile = (from filePath in existingHashLookup.Keys + where Path.GetFileName(filePath).Equals(missingFileWithHashToFind.Name, StringComparison.OrdinalIgnoreCase) + where existingHashLookup[filePath] == missingFileWithHashToFind.SHA256Hash + select filePath).SingleOrDefault(); + if (matchFile == null) { - existingHashLookup.Add(file, contentUtil.GetChecksum(file)); + success = false; + missingFiles.AppendLine($"File: {missingFileWithHashToFind.Name} Hash: '{missingFileWithHashToFind.SHA256Hash}'"); } - - Dictionary fileNameUpdates = new Dictionary(); - // At this point, we've unpacked every Zip we can possibly pull out into folders named for the zip's hash into 'unpackingDirectory' - foreach (FileName missingFileWithHashToFind in unpackNeeded) + else { - string matchFile = (from filePath in existingHashLookup.Keys - where Path.GetFileName(filePath).Equals(missingFileWithHashToFind.Name, StringComparison.OrdinalIgnoreCase) - where existingHashLookup[filePath] == missingFileWithHashToFind.SHA256Hash - select filePath).SingleOrDefault(); - if (matchFile == null) - { - success = false; - missingFiles.AppendLine($"File: {missingFileWithHashToFind.Name} Hash: '{missingFileWithHashToFind.SHA256Hash}'"); - } - else - { - string relativePath = (string)(new Uri(OutputPath).MakeRelativeUri(new Uri(matchFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar); - FileName updatedFileName = new FileName(OutputPath, relativePath, missingFileWithHashToFind.SHA256Hash); - candidateFileNames.Remove(missingFileWithHashToFind); - candidateFileNames.Add(updatedFileName); - } + string relativePath = (new Uri(OutputPath).MakeRelativeUri(new Uri(matchFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar); + FileName updatedFileName = new FileName(OutputPath, relativePath, missingFileWithHashToFind.SHA256Hash); + candidateFileNames.Remove(missingFileWithHashToFind); + candidateFileNames.Add(updatedFileName); } } + if (!success) { throw new Exception($"Failure attempting to find one or more files:\n{missingFiles.ToString()}"); diff --git a/src/SignTool/SignTool/BatchSignUtil.cs b/src/SignTool/SignTool/BatchSignUtil.cs index 8353ff9101..fba46f8c0a 100644 --- a/src/SignTool/SignTool/BatchSignUtil.cs +++ b/src/SignTool/SignTool/BatchSignUtil.cs @@ -63,50 +63,40 @@ internal bool Go(TextWriter textWriter) return VerifyAfterSign(zipDataMap, textWriter); } - - private bool GenerateOrchestrationManifest(TextWriter textWriter, BatchSignInput batchData, ContentMap contentMap, string outputPath) { - try + textWriter.WriteLine($"Generating orchestration file manifest into {outputPath}"); + OrchestratedFileJson fileJsonWithInfo = new OrchestratedFileJson { - textWriter.WriteLine($"Generating orchestration file manifest into {outputPath}"); - OrchestratedFileJson fileJsonWithInfo = new OrchestratedFileJson - { - ExcludeList = _batchData.ExternalFileNames.ToArray() ?? Array.Empty() - }; + ExcludeList = _batchData.ExternalFileNames.ToArray() ?? Array.Empty() + }; - var distinctSigningCombos = batchData.FileSignInfoMap.Values.GroupBy(v => new { v.Certificate, v.StrongName }); + var distinctSigningCombos = batchData.FileSignInfoMap.Values.GroupBy(v => new { v.Certificate, v.StrongName }); - List newList = new List(); - foreach (var combinationToSign in distinctSigningCombos) + List newList = new List(); + foreach (var combinationToSign in distinctSigningCombos) + { + var filesInThisGroup = combinationToSign.Select(c => new FileSignDataEntry() { - var filesInThisGroup = combinationToSign.Select(c => new FileSignDataEntry() - { - FilePath = c.FileName.RelativePath, - SHA256Hash = contentMap.GetChecksum(c.FileName), - PublishToFeedUrl = batchData.PublishUri - }); - newList.Add(new OrchestratedFileSignData() - { - Certificate = combinationToSign.Key.Certificate, - StrongName = combinationToSign.Key.StrongName, - FileList = filesInThisGroup.ToArray() - }); - } - fileJsonWithInfo.SignList = newList.ToArray(); - fileJsonWithInfo.Kind = "orchestration"; - - using (StreamWriter file = File.CreateText(outputPath)) + FilePath = c.FileName.RelativePath, + SHA256Hash = contentMap.GetChecksum(c.FileName), + PublishToFeedUrl = batchData.PublishUri + }); + newList.Add(new OrchestratedFileSignData() { - file.Write(JsonConvert.SerializeObject(fileJsonWithInfo, Formatting.Indented)); - } + Certificate = combinationToSign.Key.Certificate, + StrongName = combinationToSign.Key.StrongName, + FileList = filesInThisGroup.ToArray() + }); } - catch (Exception ex) + fileJsonWithInfo.SignList = newList.ToArray(); + fileJsonWithInfo.Kind = "orchestration"; + + using (StreamWriter file = File.CreateText(outputPath)) { - textWriter.WriteLine($"Unexpected exception: {ex.Message}"); - textWriter.WriteLine(ex.StackTrace); - return false; + file.Write(JsonConvert.SerializeObject(fileJsonWithInfo, Formatting.Indented)); } + return true; } diff --git a/src/SignTool/SignTool/Program.cs b/src/SignTool/SignTool/Program.cs index ab28f4036a..68a2344668 100644 --- a/src/SignTool/SignTool/Program.cs +++ b/src/SignTool/SignTool/Program.cs @@ -231,7 +231,6 @@ internal static bool ParseCommandLineArguments( string outputConfigFile = null; var test = false; var testSign = false; - var orchestrationMode = false; var i = 0; @@ -248,10 +247,6 @@ internal static bool ParseCommandLineArguments( testSign = true; i++; break; - case "-orchestrationmode": - orchestrationMode = true; - i++; - break; case "-intermediateoutputpath": if (!ParsePathOption(args, ref i, current, out intermediateOutputPath)) { @@ -289,12 +284,6 @@ internal static bool ParseCommandLineArguments( } } - if (orchestrationMode && !string.IsNullOrEmpty(outputConfigFile)) - { - Console.WriteLine("Please specify either -orchestrationmode (signing multiple json manifests with SHA256 entries) or a value for -outputconfig to generate such a manifest, but not both."); - return false; - } - if (i + 1 != args.Length) { Console.Error.WriteLine("Need a value for outputPath"); From 481993292c9152dd9c8b081efb1a975fe56f180a Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Mon, 27 Nov 2017 14:46:54 -0800 Subject: [PATCH 6/7] PR feedback: Fix Usage --- src/SignTool/SignTool/Program.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/SignTool/SignTool/Program.cs b/src/SignTool/SignTool/Program.cs index 68a2344668..568135b817 100644 --- a/src/SignTool/SignTool/Program.cs +++ b/src/SignTool/SignTool/Program.cs @@ -205,13 +205,12 @@ internal static void PrintUsage() test: Run tool without actually modifying any state. testSign: The binaries will be test signed. The default is to real sign. -orchestrationmode: Support comma-separated list of one or more config files in the format outputted using the 'orchestrationConfigFile' argument. outputPath: Directory containing the binaries. intermediateOutputPath: Directory containing intermediate output. Default is (outputpath\..\Obj). nugetPackagesPath: Path containing downloaded NuGet packages. msbuildPath: Path to MSBuild.exe to use as signing mechanism. config: Path to SignToolData.json. Default: build\config\SignToolData.json. -orchestrationConfigFile: Run tool to produce an orchestration json file. This will contain SHA256 hashes of files for verification to consume later. Cannot be used with -orchestrationmode +outputConfig: Run tool to produce an orchestration json file with specified name. This will contain SHA256 hashes of files for verification to consume later. "; Console.WriteLine(usage); } From 6247730aa0cc2aa75e48b0c68262ac85c1ad3f69 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Mon, 27 Nov 2017 16:48:52 -0800 Subject: [PATCH 7/7] Add ResolveMissingZipArchives --- src/SignTool/SignTool/BatchSignInput.cs | 38 +++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/SignTool/SignTool/BatchSignInput.cs b/src/SignTool/SignTool/BatchSignInput.cs index 7d4b531ec7..5e210df397 100644 --- a/src/SignTool/SignTool/BatchSignInput.cs +++ b/src/SignTool/SignTool/BatchSignInput.cs @@ -58,6 +58,8 @@ internal sealed class BatchSignInput /// internal ImmutableDictionary FileSignInfoMap { get; } + private ContentUtil contentUtil = new ContentUtil(); + internal BatchSignInput(string outputPath, Dictionary fileSignDataMap, IEnumerable externalFileNames, string publishUri) { OutputPath = outputPath; @@ -86,6 +88,11 @@ internal BatchSignInput(string outputPath, Dictionary fileNames = fileSignDataMap.Keys.Select(x => new FileName(outputPath, x.FilePath, x.SHA256Hash)).ToList(); + + // We need to tolerate not being able to find zips in the same path as they were in on the original machine. + // As long as we can find a FileName with the same hash for each one, we should be OK. + ResolveMissingZipArchives(ref fileNames); + ZipContainerNames = fileNames.Where(x => x.IsZipContainer).ToImmutableArray(); // If there's any files we can't find, recursively unpack the zip archives we just made a list of above. UnpackMissingContent(ref fileNames); @@ -105,6 +112,35 @@ internal BatchSignInput(string outputPath, Dictionary fileNames) + { + StringBuilder missingFiles = new StringBuilder(); + List missingZips = fileNames.Where(x => x.IsZipContainer && !File.Exists(x.FullPath)).ToList(); + foreach (FileName missingZip in missingZips) + { + // Throw if somehow the same missing zip is present more than once. May be OK to take FirstOrDefault. + var matchingFile = (from path in Directory.GetFiles(OutputPath, missingZip.Name, SearchOption.AllDirectories) + where contentUtil.GetChecksum(path).Equals(missingZip.SHA256Hash, StringComparison.OrdinalIgnoreCase) + select path).SingleOrDefault(); + + if (!string.IsNullOrEmpty(matchingFile)) + { + fileNames.Remove(missingZip); + string relativePath = (new Uri(OutputPath).MakeRelativeUri(new Uri(matchingFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar); + FileName updatedFileName = new FileName(OutputPath, relativePath, missingZip.SHA256Hash); + fileNames.Add(updatedFileName); + } + else + { + missingFiles.AppendLine($"File: {missingZip.Name} Hash: {missingZip.SHA256Hash}"); + } + } + if (!(missingFiles.Length == 0)) + { + throw new FileNotFoundException($"Could not find one or more Zip-archive files referenced:\n{missingFiles}"); + } + } + private void UnpackMissingContent(ref List candidateFileNames) { bool success = true; @@ -120,8 +156,6 @@ private void UnpackMissingContent(ref List candidateFileNames) if (unpackNeeded.Count() == 0) return; - ContentUtil contentUtil = new ContentUtil(); - // Get all Zip Archives in the manifest recursively. Queue allZipsWeKnowAbout = new Queue(ZipContainerNames);