Add support for indexing .binlog files.

Similar to .sln files, Csc invocations are extracted from the .binlog.
KirillOsenkov · Apr 18, 2020 · b79ff9c · b79ff9c
1 parent 3e03563
commit b79ff9c
Show file tree

Hide file tree

Showing 9 changed files with 313 additions and 18 deletions.
diff --git a/RunTestSite.cmd b/RunTestSite.cmd
@@ -1,2 +1,3 @@
-cd src\HtmlGenerator\bin\Debug\net472\Web\Index
-dotnet Microsoft.SourceBrowser.SourceIndexServer.dll
+pushd src\HtmlGenerator\bin\Debug\net472\Web\Index
+dotnet Microsoft.SourceBrowser.SourceIndexServer.dll
+popd
diff --git a/src/Common/AssemblyNameExtractor.cs b/src/Common/AssemblyNameExtractor.cs
@@ -22,10 +22,18 @@ public static IEnumerable<string> GetAssemblyNames(string projectOrSolutionFileP
                 return null;
             }
 
-            if (projectOrSolutionFilePath.EndsWith(".sln"))
+            if (projectOrSolutionFilePath.EndsWith(".sln", System.StringComparison.OrdinalIgnoreCase))
             {
                 return GetAssemblyNamesFromSolution(projectOrSolutionFilePath);
             }
+            else if (projectOrSolutionFilePath.EndsWith(".dll", System.StringComparison.OrdinalIgnoreCase) ||
+                     projectOrSolutionFilePath.EndsWith(".exe", System.StringComparison.OrdinalIgnoreCase))
+            {
+                return new[]
+                {
+                    Path.GetFileNameWithoutExtension(projectOrSolutionFilePath)
+                };
+            }
             else
             {
                 return new[] { GetAssemblyNameFromProject(projectOrSolutionFilePath) };

diff --git a/src/HtmlGenerator/HtmlGenerator.csproj b/src/HtmlGenerator/HtmlGenerator.csproj
@@ -53,6 +53,7 @@
     <PackageReference Include="Microsoft.CodeAnalysis.VisualBasic.Features" Version="$(NuGetVersionRoslyn)" />
     <PackageReference Include="Microsoft.CodeAnalysis.VisualBasic.Workspaces" Version="$(NuGetVersionRoslyn)" />
     <PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="$(NuGetVersionRoslyn)" />
+    <PackageReference Include="MSBuild.StructuredLogger" Version="2.0.174" />
     <PackageReference Include="GuiLabs.Language.Xml" Version="1.2.46" />
     <PackageReference Include="Microsoft.VisualStudio.Language.Intellisense" Version="16.4.280" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />

diff --git a/src/HtmlGenerator/Pass1-Generation/GenerateFromBuildLog.cs b/src/HtmlGenerator/Pass1-Generation/GenerateFromBuildLog.cs
@@ -2,6 +2,10 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Runtime.InteropServices;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.VisualBasic;
 using Microsoft.SourceBrowser.BuildLogParser;
 using Microsoft.SourceBrowser.Common;
 
@@ -40,7 +44,11 @@ public static void IndexLogFile(string logFile, string serverPath = null, string
             }
         }
 
-        public static void GenerateInvocation(CompilerInvocation invocation)
+        public static void GenerateInvocation(CompilerInvocation invocation,
+            IReadOnlyDictionary<string, string> serverPathMappings = null,
+            HashSet<string> processedAssemblyList = null,
+            HashSet<string> assemblyNames = null,
+            Folder<ProjectSkeleton> solutionExplorerRoot = null)
         {
             try
             {
@@ -61,7 +69,9 @@ public static void GenerateInvocation(CompilerInvocation invocation)
                         Paths.SolutionDestinationFolder,
                         invocation.ServerPath,
                         invocation.NetworkShare);
-                    solutionGenerator.Generate();
+                    solutionGenerator.ServerPathMappings = serverPathMappings;
+                    solutionGenerator.GlobalAssemblyList = assemblyNames;
+                    solutionGenerator.Generate(processedAssemblyList, solutionExplorerRoot);
                 }
                 else
                 {
@@ -118,6 +128,7 @@ private static IEnumerable<CompilerInvocation> GetInvocationsToProcess()
         public class CompilerInvocation
         {
             public string ProjectFilePath { get; set; }
+            public string ProjectDirectory => ProjectFilePath == null ? "" : Path.GetDirectoryName(ProjectFilePath);
             public string OutputAssemblyPath { get; set; }
             public string CommandLineArguments { get; set; }
             public string ServerPath { get; set; }
@@ -133,22 +144,64 @@ public string AssemblyName
                 }
             }
 
+            private string language;
             public string Language
             {
                 get
                 {
-                    if (ProjectFilePath == null && TypeScriptFiles != null)
+                    if (language == null)
                     {
-                        return "TypeScript";
+                        if (ProjectFilePath == null && TypeScriptFiles != null)
+                        {
+                            language = "TypeScript";
+                        }
+                        else if (".vbproj".Equals(Path.GetExtension(ProjectFilePath), StringComparison.OrdinalIgnoreCase))
+                        {
+                            language = "Visual Basic";
+                        }
+                        else
+                        {
+                            language = "C#";
+                        }
                     }
-                    else if (".vbproj".Equals(Path.GetExtension(ProjectFilePath), StringComparison.OrdinalIgnoreCase))
+
+                    return language;
+                }
+
+                set
+                {
+                    language = value;
+                }
+            }
+
+            private string[] GetCommandLineArguments()
+            {
+                return CommandLineParser.SplitCommandLineIntoArguments(CommandLineArguments, removeHashComments: false).ToArray();
+            }
+
+            private CommandLineArguments parsed;
+            public CommandLineArguments Parsed
+            {
+                get
+                {
+                    if (parsed != null)
                     {
-                        return "Visual Basic";
+                        return parsed;
+                    }
+
+                    var sdkDirectory = RuntimeEnvironment.GetRuntimeDirectory();
+                    var args = GetCommandLineArguments();
+
+                    if (Language == LanguageNames.CSharp)
+                    {
+                        parsed = CSharpCommandLineParser.Default.Parse(args, ProjectDirectory, sdkDirectory);
                     }
                     else
                     {
-                        return "C#";
+                        parsed = VisualBasicCommandLineParser.Default.Parse(args, ProjectDirectory, sdkDirectory);
                     }
+
+                    return parsed;
                 }
             }
         }

diff --git a/src/HtmlGenerator/Pass1-Generation/ProjectGenerator.cs b/src/HtmlGenerator/Pass1-Generation/ProjectGenerator.cs
@@ -99,7 +99,14 @@ public async Task Generate()
 
                 Log.Write(ProjectDestinationFolder, ConsoleColor.DarkCyan);
 
-                ProjectSourcePath = Paths.MakeRelativeToFolder(ProjectFilePath, SolutionGenerator.SolutionSourceFolder);
+                if (SolutionGenerator.SolutionSourceFolder is string solutionFolder)
+                {
+                    ProjectSourcePath = Paths.MakeRelativeToFolder(ProjectFilePath, solutionFolder);
+                }
+                else
+                {
+                    ProjectSourcePath = ProjectFilePath;
+                }
 
                 if (File.Exists(Path.Combine(ProjectDestinationFolder, Constants.DeclaredSymbolsFileName + ".txt")))
                 {

diff --git a/src/HtmlGenerator/Pass1-Generation/SolutionGenerator.cs b/src/HtmlGenerator/Pass1-Generation/SolutionGenerator.cs
@@ -17,7 +17,7 @@ public partial class SolutionGenerator : IDisposable
         public string SolutionDestinationFolder { get; private set; }
         public string ProjectFilePath { get; private set; }
         public string ServerPath { get; set; }
-        public IReadOnlyDictionary<string, string> ServerPathMappings { get; }
+        public IReadOnlyDictionary<string, string> ServerPathMappings { get; set; }
         public string NetworkShare { get; private set; }
         private Federation Federation { get; set; }
         public IEnumerable<string> PluginBlacklist { get; private set; }

diff --git a/src/HtmlGenerator/Program.cs b/src/HtmlGenerator/Program.cs
@@ -3,6 +3,7 @@
 using System.Collections.Immutable;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Reflection;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -233,8 +234,12 @@ private static bool IsSupportedProject(string filePath)
             }
 
             return filePath.EndsWith(".sln", StringComparison.OrdinalIgnoreCase) ||
+                   filePath.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ||
+                   filePath.EndsWith(".buildlog", StringComparison.OrdinalIgnoreCase) ||
                    filePath.EndsWith(".csproj", StringComparison.OrdinalIgnoreCase) ||
-                   filePath.EndsWith(".vbproj", StringComparison.OrdinalIgnoreCase);
+                   filePath.EndsWith(".vbproj", StringComparison.OrdinalIgnoreCase) ||
+                   filePath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) ||
+                   filePath.EndsWith(".exe", StringComparison.OrdinalIgnoreCase);
         }
 
         private static void PrintUsage()
@@ -244,7 +249,7 @@ private static void PrintUsage()
                 + "[/force] "
                 + "[/noplugins] "
                 + "[/noplugin:Git] "
-                + "<pathtosolution1.csproj|vbproj|sln> [more solutions/projects..] "
+                + "<pathtosolution1.csproj|vbproj|sln|binlog|buildlog|dll|exe> [more solutions/projects..] "
                 + "[/in:<filecontaingprojectlist>] "
                 + "[/nobuiltinfederations] "
                 + "[/offlinefederation:server=assemblyListFile] "
@@ -253,6 +258,18 @@ private static void PrintUsage()
 
         private static readonly Folder<ProjectSkeleton> mergedSolutionExplorerRoot = new Folder<ProjectSkeleton>();
 
+        private static IEnumerable<string> GetAssemblyNames(string filePath)
+        {
+            if (filePath.EndsWith(".binlog", System.StringComparison.OrdinalIgnoreCase) ||
+                filePath.EndsWith(".buildlog", System.StringComparison.OrdinalIgnoreCase))
+            {
+                var invocations = BinLogCompilerInvocationsReader.ExtractInvocations(filePath);
+                return invocations.Select(i => Path.GetFileNameWithoutExtension(i.Parsed.OutputFileName)).ToArray();
+            }
+
+            return AssemblyNameExtractor.GetAssemblyNames(filePath);
+        }
+
         private static void IndexSolutions(
             IEnumerable<string> solutionFilePaths,
             Dictionary<string, string> properties,
@@ -267,7 +284,7 @@ private static void IndexSolutions(
             {
                 using (Disposable.Timing("Reading assembly names from " + path))
                 {
-                    foreach (var assemblyName in AssemblyNameExtractor.GetAssemblyNames(path))
+                    foreach (var assemblyName in GetAssemblyNames(path))
                     {
                         assemblyNames.Add(assemblyName);
                     }
@@ -280,6 +297,23 @@ private static void IndexSolutions(
             {
                 using (Disposable.Timing("Generating " + path))
                 {
+                    if (path.EndsWith(".binlog", StringComparison.OrdinalIgnoreCase) ||
+                        path.EndsWith(".buildlog", StringComparison.OrdinalIgnoreCase))
+                    {
+                        var invocations = BinLogCompilerInvocationsReader.ExtractInvocations(path);
+                        foreach (var invocation in invocations)
+                        {
+                            GenerateFromBuildLog.GenerateInvocation(
+                                invocation,
+                                serverPathMappings,
+                                processedAssemblyList,
+                                assemblyNames,
+                                mergedSolutionExplorerRoot);
+                        }
+
+                        continue;
+                    }
+
                     using (var solutionGenerator = new SolutionGenerator(
                         path,
                         Paths.SolutionDestinationFolder,