Support Scala 3's Best Effort compilation

This commit allows bloop to support best effort compilation for Metals,
with it being enabled with the other Metals options.

Best Effort is meant to be a set of Scala 3 compiler options that allow
the errored, but typed programs, to be able to be serialized into a
TASTy aligned format (Best Effort TASTy), and later to reuse those
typed program trees in both the dependent projects, and in the
presentation compiler.

Those best effort tasty files are always serialized to a seperate
directory, which in this PR is managed by bloop. Best effort compilation
may fail, similarly to the regular compilation.

The best effort directory is kept outside of the regular build
directory, in `.bloop/project-id/scala-3/best-effort`. There, two
directories are managed, `build` and `dep`. Whenever metals compiles
a project the `build` directory is populated with best effort artifacts.
Then, if the best effort compilation is successful, they are copied to
the 'dep' directory, which is used for dependencies, metals presentation
compiler etc. This way if a compilation fails, the artifacts from the
previous compilation can still be kept. This is not completely
dissimilar to how the regular compilation works here, with the
difference being the more straigthforward implementation, without any
counters that monitor how many tasks are being executed with the `dep`
directory concurrently.

backend/src/main/scala/bloop/CompileProducts.scala

@@ -32,6 +32,7 @@ import xsbti.compile.PreviousResult
 case class CompileProducts(
     readOnlyClassesDir: Path,
     newClassesDir: Path,
+    bestEffortDepDir: Option[Path],
     resultForDependentCompilationsInSameRun: PreviousResult,
     resultForFutureCompilationRuns: PreviousResult,
     invalidatedCompileProducts: Set[File],

backend/src/main/scala/bloop/Compiler.scala

@@ -22,6 +22,7 @@ import bloop.reporter.ZincReporter
 import bloop.task.Task
 import bloop.tracing.BraveTracer
 import bloop.util.AnalysisUtils
+import bloop.util.BestEffortDirs
 import bloop.util.CacheHashCode
 import bloop.util.UUIDUtil
 
@@ -70,6 +71,7 @@ case class CompileOutPaths(
     analysisOut: AbsolutePath,
     genericClassesDir: AbsolutePath,
     externalClassesDir: AbsolutePath,
+    bestEffortDirs: Option[BestEffortDirs],
     internalReadOnlyClassesDir: AbsolutePath
 ) {
   // Don't change the internals of this method without updating how they are cleaned up
@@ -94,22 +96,6 @@ case class CompileOutPaths(
     createInternalNewDir(classesName => s"${classesName}-${UUIDUtil.randomUUID}")
   }
 
-  /**
-   * Creates an internal directory where symbol pickles are stored when build
-   * pipelining is enabled. This directory is removed whenever compilation
-   * process has finished because pickles are useless when class files are
-   * present. This might change when we expose pipelining to clients.
-   *
-   * Watch out: for the moment this pickles dir is not used anywhere.
-   */
-  lazy val internalNewPicklesDir: AbsolutePath = {
-    createInternalNewDir { classesName =>
-      val newName = s"${classesName.replace("classes", "pickles")}-${UUIDUtil.randomUUID}"
-      // If original classes name didn't contain `classes`, add pickles at the beginning
-      if (newName.contains("pickles")) newName
-      else "pickles-" + newName
-    }
-  }
 }
 
 object CompileOutPaths {
@@ -211,7 +197,8 @@ object Compiler {
         problems: List[ProblemPerPhase],
         t: Option[Throwable],
         elapsed: Long,
-        backgroundTasks: CompileBackgroundTasks
+        backgroundTasks: CompileBackgroundTasks,
+        products: Option[CompileProducts]
     ) extends Result
         with CacheHashCode
 
@@ -231,7 +218,7 @@ object Compiler {
 
     object NotOk {
       def unapply(result: Result): Option[Result] = result match {
-        case f @ (Failed(_, _, _, _) | Cancelled(_, _, _) | Blocked(_) | GlobalError(_, _)) =>
+        case f @ (_: Failed | _: Cancelled | _: Blocked | _: GlobalError) =>
           Some(f)
         case _ => None
       }
@@ -249,6 +236,9 @@ object Compiler {
     val readOnlyClassesDirPath = readOnlyClassesDir.toString
     val newClassesDir = compileOut.internalNewClassesDir.underlying
     val newClassesDirPath = newClassesDir.toString
+    val bestEffortBuildDir = compileOut.bestEffortDirs.map(_.buildDir.underlying)
+    val bestEffortDepDir = compileOut.bestEffortDirs.map(_.depDir.underlying)
+    bestEffortBuildDir.foreach(dir => if (!Files.exists(dir)) Files.createDirectories(dir))
 
     logger.debug(s"External classes directory ${externalClassesDirPath}")
     logger.debug(s"Read-only classes directory ${readOnlyClassesDirPath}")
@@ -465,6 +455,7 @@ object Compiler {
             val products = CompileProducts(
               readOnlyClassesDir,
               readOnlyClassesDir,
+              bestEffortDepDir,
               noOpPreviousResult,
               noOpPreviousResult,
               Set(),
@@ -493,7 +484,16 @@ object Compiler {
                 }
 
                 val deleteNewClassesDir = Task(BloopPaths.delete(AbsolutePath(newClassesDir)))
-                val allTasks = List(deleteNewClassesDir, updateClientState, writeAnalysisIfMissing)
+                val deleteBestEffortDirsTasks = compileOut.bestEffortDirs match {
+                  case Some(BestEffortDirs(buildDir, depDir)) =>
+                    List(Task(BloopPaths.delete(buildDir)), Task(BloopPaths.delete(depDir)))
+                  case None => List(Task.unit)
+                }
+                val allTasks = List(
+                  deleteNewClassesDir,
+                  updateClientState,
+                  writeAnalysisIfMissing
+                ) ++ deleteBestEffortDirsTasks
                 Task
                   .gatherUnordered(allTasks)
                   .map(_ => ())
@@ -572,7 +572,15 @@ object Compiler {
                       }
                     }
                   }
-                  Task.gatherUnordered(List(firstTask, secondTask)).map(_ => ())
+                  val deleteBestEffortDirsTasks =
+                    compileOut.bestEffortDirs match { // TODO duplicate
+                      case Some(BestEffortDirs(buildDir, depDir)) =>
+                        List(Task(BloopPaths.delete(buildDir)), Task(BloopPaths.delete(depDir)))
+                      case None => List(Task.unit)
+                    }
+                  Task
+                    .gatherUnordered(List(firstTask, secondTask) ++ deleteBestEffortDirsTasks)
+                    .map(_ => ())
                 }
 
                 allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation()))
@@ -582,6 +590,7 @@ object Compiler {
             val products = CompileProducts(
               readOnlyClassesDir,
               newClassesDir,
+              bestEffortDepDir,
               resultForDependentCompilationsInSameRun,
               resultForFutureCompilationRuns,
               allInvalidated.toSet,
@@ -616,12 +625,58 @@ object Compiler {
               val failedProblems = reportedProblems ++ newProblems.toList
               val backgroundTasks =
                 toBackgroundTasks(backgroundTasksForFailedCompilation.toList)
-              Result.Failed(failedProblems, None, elapsed, backgroundTasks)
+
+              val products = compileOut.bestEffortDirs.map {
+                case BestEffortDirs(buildDir, depDir) =>
+                  // For Best Effort, delete previous dep directory contents and update with new ones
+                  if (Files.exists(depDir.underlying)) BloopPaths.delete(depDir)
+                  if (!Files.exists(depDir.underlying)) Files.createDirectories(depDir.underlying)
+                  java.nio.file.Files
+                    .walk(buildDir.underlying)
+                    .forEach { srcFile =>
+                      val targetFile =
+                        depDir.underlying.resolve(buildDir.underlying.relativize(srcFile))
+                      if (Files.isDirectory(srcFile)) {
+                        Files.createDirectories(targetFile)
+                      } else if (Files.isRegularFile(srcFile)) {
+                        Files.copy(
+                          srcFile,
+                          targetFile,
+                          java.nio.file.StandardCopyOption.REPLACE_EXISTING
+                        )
+                      }
+                    }
+                  BloopPaths.delete(buildDir)
+
+                  val noOpPreviousResult = {
+                    updatePreviousResultWithRecentClasspathHashes(
+                      compileInputs.previousResult,
+                      uniqueInputs
+                    )
+                  }
+                  CompileProducts(
+                    readOnlyClassesDir,
+                    newClassesDir,
+                    bestEffortDepDir,
+                    noOpPreviousResult,
+                    noOpPreviousResult,
+                    Set.empty,
+                    Map.empty
+                  )
+              }
+              Result.Failed(failedProblems, None, elapsed, backgroundTasks, products)
             case t: Throwable =>
+              compileOut.bestEffortDirs.foreach {
+                case BestEffortDirs(buildDir, _) =>
+                  logger.info(
+                    "Unsuccessful best effort compilation. No best effort artifacts were created."
+                  )
+                  BloopPaths.delete(buildDir)
+              }
               t.printStackTrace()
               val backgroundTasks =
                 toBackgroundTasks(backgroundTasksForFailedCompilation.toList)
-              Result.Failed(Nil, Some(t), elapsed, backgroundTasks)
+              Result.Failed(Nil, Some(t), elapsed, backgroundTasks, None)
           }
       }
   }

backend/src/main/scala/bloop/util/BestEffortDirs.scala

@@ -0,0 +1,8 @@
+package bloop.util
+
+import bloop.io.AbsolutePath
+
+case class BestEffortDirs(
+    buildDir: AbsolutePath,
+    depDir: AbsolutePath
+)

frontend/src/it/scala/bloop/CommunityBuild.scala

@@ -132,6 +132,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) {
         resources = Nil,
         compileSetup = Config.CompileSetup.empty,
         genericClassesDir = dummyClassesDir,
+        bestEffortDirs = None,
         scalacOptions = Nil,
         javacOptions = Nil,
         sources = Nil,

frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala

@@ -12,7 +12,8 @@ object BloopBspDefinitions {
       clientClassesRootDir: Option[Uri],
       semanticdbVersion: Option[String],
       supportedScalaVersions: Option[List[String]],
-      javaSemanticdbVersion: Option[String]
+      javaSemanticdbVersion: Option[String],
+      enableBestEffortMode: Option[Boolean]
   )
 
   object BloopExtraBuildParams {
@@ -21,7 +22,8 @@ object BloopBspDefinitions {
       clientClassesRootDir = None,
       semanticdbVersion = None,
       supportedScalaVersions = None,
-      javaSemanticdbVersion = None
+      javaSemanticdbVersion = None,
+      enableBestEffortMode = None
     )
 
     implicit val codec: JsonValueCodec[BloopExtraBuildParams] =

frontend/src/main/scala/bloop/bsp/BloopBspServices.scala

@@ -274,6 +274,7 @@ final class BloopBspServices(
       } else {
         val javaSemanticDBVersion = extraBuildParams.flatMap(_.javaSemanticdbVersion)
         val scalaSemanticDBVersion = extraBuildParams.flatMap(_.semanticdbVersion)
+        val enableBestEffortMode = extraBuildParams.flatMap(_.enableBestEffortMode)
         val supportedScalaVersions =
           if (scalaSemanticDBVersion.nonEmpty)
             extraBuildParams.map(_.supportedScalaVersions.toList.flatten)
@@ -285,7 +286,8 @@ final class BloopBspServices(
               scalaSemanticDBVersion,
               supportedScalaVersions,
               currentRefreshProjectsCommand,
-              currentTraceSettings
+              currentTraceSettings,
+              enableBestEffortMode
             )
           )
         else None
@@ -430,6 +432,7 @@ final class BloopBspServices(
     }
 
     val isPipeline = compileArgs.exists(_ == "--pipeline")
+    val isBestEffort = compileArgs.exists(_ == "--best-effort")
     def compile(projects: List[Project]): Task[State] = {
       val config = ReporterConfig.defaultFormat.copy(reverseOrder = false)
 
@@ -479,6 +482,7 @@ final class BloopBspServices(
         dag,
         createReporter,
         isPipeline,
+        isBestEffort,
         cancelCompilation,
         store,
         logger
@@ -504,7 +508,7 @@ final class BloopBspServices(
             case Compiler.Result.GlobalError(problem, _) => List(problem)
             case Compiler.Result.Cancelled(problems, elapsed, _) =>
               List(reportError(p, problems, elapsed))
-            case f @ Compiler.Result.Failed(problems, t, elapsed, _) =>
+            case f @ Compiler.Result.Failed(problems, t, elapsed, _, _) =>
               previouslyFailedCompilations.put(p, f)
               val acc = List(reportError(p, problems, elapsed))
               t match {
@@ -789,7 +793,8 @@ final class BloopBspServices(
               fullClasspath.toList,
               javaOptions,
               workingDirectory,
-              environmentVariables
+              environmentVariables,
+              None
             )
           }).toList
           Task.now((state, Right(environmentEntries)))
@@ -1256,7 +1261,10 @@ final class BloopBspServices(
               target = target,
               options = project.scalacOptions.toList,
               classpath = classpath,
-              classDirectory = classesDir
+              classDirectory = classesDir,
+              bestEffortDirectory = project.bestEffortDirs.map(bestEffortDirs =>
+                bsp.Uri(bestEffortDirs.depDir.toBspUri)
+              )
             )
         }.toList
       )

frontend/src/main/scala/bloop/cli/Commands.scala

@@ -153,6 +153,10 @@ object Commands {
       incremental: Boolean = true,
       @HelpMessage("Pipeline the compilation of modules in your build. By default, false.")
       pipeline: Boolean = false,
+      @HelpMessage(
+        "Activates the best effort mode, used for specific compiler versions. By default, false."
+      )
+      bestEffort: Boolean = false,
       @HelpMessage("Pick reporter to show compilation messages. By default, bloop's used.")
       reporter: ReporterKind = BloopReporter,
       @ExtraName("w")