Skip to content

Commit

Permalink
Merge pull request #100 from MattGal/SignTool_Orchestration_Manifests
Browse files Browse the repository at this point in the history 
POC for Orchestratable SignTool
  • Loading branch information
@tmat
tmat committed Nov 28, 2017
2 parents f773b66 + 6247730 commit b2bdaef
Show file tree
Hide file tree
Showing 12 changed files with 487 additions and 73 deletions.
6 changes: 4 additions & 2 deletions sdks/RepoToolset/tools/Build.proj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
Deploy "true" to deploy assets (e.g. VSIXes)
Test "true" to run tests
IntegrationTest "true" to run integration tests
Sign "true" to sign built binaries
Pack "true" to build NuGet packages and VS insertion manifests
Sign "true" to sign built binaries
SignType "real" to send binaries to signing service, "test" to only validate signing configuration.
-->

Expand All @@ -28,6 +28,8 @@
PB_PublishType {''|store1-store2-...-storeN} List of stores where to publish assets to.
PB_PublishBlobFeedUrl {''|URL} Target feed URL.
PB_PublishBlobFeedKey {''|string} Account key.
PB_SigningOrchestrationConfig {''|string} Path to output Json for orchestrated-type signing (produce manifest that can be consumed later.)
-->
<PropertyGroup>
<RealSign>false</RealSign>
Expand Down Expand Up @@ -171,7 +173,7 @@
Sign artifacts.
-->
<MSBuild Projects="Sign.proj"
Properties="$(_CommonProps);RealSign=$(RealSign);DirectoryBuildPropsPath=$(DirectoryBuildPropsPath)"
Properties="$(_CommonProps);RealSign=$(RealSign);DirectoryBuildPropsPath=$(DirectoryBuildPropsPath);OutputConfigFile=$(PB_SigningOrchestrationConfig)"
Targets="Sign"
Condition="'$(Sign)' == 'true'"/>

Expand Down
7 changes: 7 additions & 0 deletions sdks/RepoToolset/tools/Sign.proj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
Required parameters:
DirectoryBuildPropsPath Path to the Directory.Build.props file in the repo root.
RealSign "true" to send binaries to the signing service, "false" to only validate signing configuration.
Optional parameters:
OutputConfigFile If specified, output a manifest similar to SignToolData.json but with checksums of all relevant files included
for reassembly later when only zip archives are available. If this file is later used as the '-config' file,
SignTool will attempt to unpack all zips provided and match on checksums for any missing files.
-->

<Import Project="$(DirectoryBuildPropsPath)" Condition="Exists('$(DirectoryBuildPropsPath)')"/>
Expand All @@ -19,6 +24,7 @@
</Exec>

<ItemGroup>
<SignToolArgs Include='-outputconfig "$(OutputConfigFile)"' Condition="'$(OutputConfigFile)' != ''" />
<SignToolArgs Include='-nugetPackagesPath "$(NuGetPackageRoot)\"' />
<SignToolArgs Include='-intermediateOutputPath "$(ArtifactsObjDir)\"' />
<SignToolArgs Include='-config "$(SignToolDataPath)"' />
Expand All @@ -29,4 +35,5 @@

<Exec Command="$(NuGetPackageRoot)roslyntools.microsoft.signtool\$(RoslynToolsMicrosoftSignToolVersion)\tools\SignTool.exe @(SignToolArgs, ' ')" />
</Target>

</Project>
166 changes: 155 additions & 11 deletions src/SignTool/SignTool/BatchSignInput.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Linq;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Text;
using SignTool.Json;

namespace SignTool
{
Expand All @@ -18,41 +22,48 @@ internal sealed class BatchSignInput
internal string OutputPath { get; }

/// <summary>
/// The ordered names of the files to be signed. These are all relative paths off of the <see cref="OutputPath"/>
/// Uri, to be consumed by later steps, which describes where these files get published to.
/// </summary>
internal string PublishUri { get; }

/// <summary>
/// The ordered names of the files to be signed. These are all relative paths off of the <see cref="OutputPath"/>
/// property.
/// </summary>
internal ImmutableArray<FileName> FileNames { get; }

/// <summary>
/// These are binaries which are included in our zip containers but are already signed. This list is used for
/// validation purpsoes. These are all flat names and cannot be relative paths.
/// These are binaries which are included in our zip containers but are already signed. This list is used for
/// validation purposes. These are all flat names and cannot be relative paths.
/// </summary>
internal ImmutableArray<string> ExternalFileNames { get;}
internal ImmutableArray<string> ExternalFileNames { get; }

/// <summary>
/// Names of assemblies that need to be signed. This is a subset of <see cref="FileNames"/>
/// Names of assemblies that need to be signed. This is a subset of <see cref="FileNames"/>
/// </summary>
internal ImmutableArray<FileName> AssemblyNames { get; }

/// <summary>
/// Names of zip containers that need to be examined for signing. This is a subset of <see cref="FileNames"/>
/// Names of zip containers that need to be examined for signing. This is a subset of <see cref="FileNames"/>
/// </summary>
internal ImmutableArray<FileName> ZipContainerNames { get; }

/// <summary>
/// Names of other file types which aren't specifically handled by the tool. This is a subset of <see cref="FileNames"/>
/// Names of other file types which aren't specifically handled by the tool. This is a subset of <see cref="FileNames"/>
/// </summary>
internal ImmutableArray<FileName> OtherNames { get; }

/// <summary>
/// A map of all of the binaries that need to be signed to the actual signing data.
/// A map from all of the binaries that need to be signed to the actual signing data.
/// </summary>
internal ImmutableDictionary<FileName, FileSignInfo> FileSignInfoMap { get; }

internal BatchSignInput(string outputPath, Dictionary<string, SignInfo> fileSignDataMap, IEnumerable<string> externalFileNames)
private ContentUtil contentUtil = new ContentUtil();

internal BatchSignInput(string outputPath, Dictionary<string, SignInfo> fileSignDataMap, IEnumerable<string> externalFileNames, string publishUri)
{
OutputPath = outputPath;

PublishUri = publishUri;
// Use order by to make the output of this tool as predictable as possible.
var fileNames = fileSignDataMap.Keys;
FileNames = fileNames.OrderBy(x => x).Select(x => new FileName(outputPath, x)).ToImmutableArray();
Expand All @@ -70,6 +81,139 @@ internal BatchSignInput(string outputPath, Dictionary<string, SignInfo> fileSign
}
FileSignInfoMap = builder.ToImmutable();
}

internal BatchSignInput(string outputPath, Dictionary<FileSignDataEntry, SignInfo> fileSignDataMap, IEnumerable<string> externalFileNames, string publishUri)
{
OutputPath = outputPath;
PublishUri = publishUri;

List<FileName> fileNames = fileSignDataMap.Keys.Select(x => new FileName(outputPath, x.FilePath, x.SHA256Hash)).ToList();

// We need to tolerate not being able to find zips in the same path as they were in on the original machine.
// As long as we can find a FileName with the same hash for each one, we should be OK.
ResolveMissingZipArchives(ref fileNames);

ZipContainerNames = fileNames.Where(x => x.IsZipContainer).ToImmutableArray();
// If there's any files we can't find, recursively unpack the zip archives we just made a list of above.
UnpackMissingContent(ref fileNames);
// After this point, if the files are available execution should be as before.
// Use OrderBy to make the output of this tool as predictable as possible.
FileNames = fileNames.OrderBy(x => x.RelativePath).ToImmutableArray();
ExternalFileNames = externalFileNames.OrderBy(x => x).ToImmutableArray();
AssemblyNames = FileNames.Where(x => x.IsAssembly).ToImmutableArray();
OtherNames = FileNames.Where(x => !x.IsAssembly && !x.IsZipContainer).ToImmutableArray();

var builder = ImmutableDictionary.CreateBuilder<FileName, FileSignInfo>();
foreach (var name in FileNames)
{
var data = fileSignDataMap.Keys.Where(k => k.SHA256Hash == name.SHA256Hash).Single();
builder.Add(name, new FileSignInfo(name, fileSignDataMap[data]));
}
FileSignInfoMap = builder.ToImmutable();
}

private void ResolveMissingZipArchives(ref List<FileName> fileNames)
{
StringBuilder missingFiles = new StringBuilder();
List<FileName> missingZips = fileNames.Where(x => x.IsZipContainer && !File.Exists(x.FullPath)).ToList();
foreach (FileName missingZip in missingZips)
{
// Throw if somehow the same missing zip is present more than once. May be OK to take FirstOrDefault.
var matchingFile = (from path in Directory.GetFiles(OutputPath, missingZip.Name, SearchOption.AllDirectories)
where contentUtil.GetChecksum(path).Equals(missingZip.SHA256Hash, StringComparison.OrdinalIgnoreCase)
select path).SingleOrDefault();

if (!string.IsNullOrEmpty(matchingFile))
{
fileNames.Remove(missingZip);
string relativePath = (new Uri(OutputPath).MakeRelativeUri(new Uri(matchingFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar);
FileName updatedFileName = new FileName(OutputPath, relativePath, missingZip.SHA256Hash);
fileNames.Add(updatedFileName);
}
else
{
missingFiles.AppendLine($"File: {missingZip.Name} Hash: {missingZip.SHA256Hash}");
}
}
if (!(missingFiles.Length == 0))
{
throw new FileNotFoundException($"Could not find one or more Zip-archive files referenced:\n{missingFiles}");
}
}

private void UnpackMissingContent(ref List<FileName> candidateFileNames)
{
bool success = true;
string unpackingDirectory = Path.Combine(OutputPath, "UnpackedZipArchives");
StringBuilder missingFiles = new StringBuilder();
Directory.CreateDirectory(unpackingDirectory);

var unpackNeeded = (from file in candidateFileNames
where !File.Exists(file.FullPath)
select file).ToList();

// Nothing to do
if (unpackNeeded.Count() == 0)
return;

// Get all Zip Archives in the manifest recursively.
Queue<FileName> allZipsWeKnowAbout = new Queue<FileName>(ZipContainerNames);

while (allZipsWeKnowAbout.Count > 0)
{
FileName zipFile = allZipsWeKnowAbout.Dequeue();
string unpackFolder = Path.Combine(unpackingDirectory, zipFile.SHA256Hash);

// Assumption: If a zip with a given hash is already unpacked, it's probably OK.
if (!Directory.Exists(unpackFolder))
{
Directory.CreateDirectory(unpackFolder);
ZipFile.ExtractToDirectory(zipFile.FullPath, unpackFolder);
}
// Add any zips we just unzipped.
foreach (string file in Directory.GetFiles(unpackFolder, "*", SearchOption.AllDirectories))
{
if (PathUtil.IsZipContainer(file))
{
string relativePath = (new Uri(unpackingDirectory).MakeRelativeUri(new Uri(file))).OriginalString;
allZipsWeKnowAbout.Enqueue(new FileName(unpackingDirectory, relativePath, contentUtil.GetChecksum(file)));
}
}
}
// Lazy : Disks are fast, just calculate ALL hashes. Could optimize by only files we intend to sign
Dictionary<string, string> existingHashLookup = new Dictionary<string, string>();
foreach (string file in Directory.GetFiles(unpackingDirectory, "*", SearchOption.AllDirectories))
{
existingHashLookup.Add(file, contentUtil.GetChecksum(file));
}

Dictionary<FileName, FileName> fileNameUpdates = new Dictionary<FileName, FileName>();
// At this point, we've unpacked every Zip we can possibly pull out into folders named for the zip's hash into 'unpackingDirectory'
foreach (FileName missingFileWithHashToFind in unpackNeeded)
{
string matchFile = (from filePath in existingHashLookup.Keys
where Path.GetFileName(filePath).Equals(missingFileWithHashToFind.Name, StringComparison.OrdinalIgnoreCase)
where existingHashLookup[filePath] == missingFileWithHashToFind.SHA256Hash
select filePath).SingleOrDefault();
if (matchFile == null)
{
success = false;
missingFiles.AppendLine($"File: {missingFileWithHashToFind.Name} Hash: '{missingFileWithHashToFind.SHA256Hash}'");
}
else
{
string relativePath = (new Uri(OutputPath).MakeRelativeUri(new Uri(matchFile))).OriginalString.Replace('/', Path.DirectorySeparatorChar);
FileName updatedFileName = new FileName(OutputPath, relativePath, missingFileWithHashToFind.SHA256Hash);
candidateFileNames.Remove(missingFileWithHashToFind);
candidateFileNames.Add(updatedFileName);
}
}

if (!success)
{
throw new Exception($"Failure attempting to find one or more files:\n{missingFiles.ToString()}");
}
}
}
}

Loading

0 comments on commit b2bdaef

Please sign in to comment.