All parallel optimizations enabled (#18998)

Author: majocha

docs/release-notes/.FSharp.Compiler.Service/11.0.0.md

@@ -11,4 +11,6 @@
 
 ### Changed
 
+* Parallel compilation stabilised and enabled by default ([PR #18998](https://github.com/dotnet/fsharp/pull/18998))
+
 ### Breaking Changes

src/Compiler/Driver/CompilerConfig.fs

@@ -805,18 +805,11 @@ type TcConfigBuilder =
             doTLR = false
             doFinalSimplify = false
             optsOn = false
-            optSettings =
-                { OptimizationSettings.Defaults with
-                    processingMode =
-                        if FSharpExperimentalFeaturesEnabledAutomatically then
-                            OptimizationProcessingMode.Parallel
-                        else
-                            OptimizationProcessingMode.Sequential
-                }
+            optSettings = OptimizationSettings.Defaults
             emitTailcalls = true
             deterministic = false
             parallelParsing = true
-            parallelIlxGen = FSharpExperimentalFeaturesEnabledAutomatically
+            parallelIlxGen = true
             emitMetadataAssembly = MetadataAssemblyGeneration.None
             preferredUiLang = None
             lcid = None
@@ -858,15 +851,11 @@ type TcConfigBuilder =
             sdkDirOverride = sdkDirOverride
             xmlDocInfoLoader = None
             exiter = QuitProcessExiter
-            parallelReferenceResolution = ParallelReferenceResolution.Off
+            parallelReferenceResolution = ParallelReferenceResolution.On
             captureIdentifiersWhenParsing = false
             typeCheckingConfig =
                 {
-                    TypeCheckingConfig.Mode =
-                        if FSharpExperimentalFeaturesEnabledAutomatically then
-                            TypeCheckingMode.Graph
-                        else
-                            TypeCheckingMode.Sequential
+                    TypeCheckingConfig.Mode = TypeCheckingMode.Graph
                     DumpGraph = false
                 }
             dumpSignatureData = false

src/Compiler/Driver/CompilerImports.fs

@@ -1586,8 +1586,10 @@ and [<Sealed>] TcImports
 
 #if !NO_TYPEPROVIDERS
     member private tcImports.AttachDisposeTypeProviderAction action =
-        CheckDisposed()
-        disposeTypeProviderActions.Add action
+        tciLock.AcquireLock(fun tcitok ->
+            CheckDisposed()
+            RequireTcImportsLock(tcitok, disposeTypeProviderActions)
+            disposeTypeProviderActions.Add action)
 #endif
 
     // Note: the returned binary reader is associated with the tcImports, i.e. when the tcImports are closed
@@ -2247,110 +2249,106 @@ and [<Sealed>] TcImports
         phase2
 
     // NOTE: When used in the Language Service this can cause the transitive checking of projects. Hence it must be cancellable.
-    member tcImports.TryRegisterAndPrepareToImportReferencedDll
-        (ctok, r: AssemblyResolution)
-        : Async<(_ * (unit -> AvailableImportedAssembly list)) option> =
-        async {
-            CheckDisposed()
-            let m = r.originalReference.Range
-            let fileName = r.resolvedPath
+    member tcImports.RegisterAndImportReferencedAssemblies(ctok, nms: AssemblyResolution list) =
+        let tryGetAssemblyData (r: AssemblyResolution) =
+            async {
+                CheckDisposed()
+                let m = r.originalReference.Range
+                let fileName = r.resolvedPath
 
-            let! contentsOpt =
-                async {
-                    match r.ProjectReference with
-                    | Some ilb -> return! ilb.EvaluateRawContents()
-                    | None -> return ProjectAssemblyDataResult.Unavailable true
-                }
+                try
 
-            // If we have a project reference but did not get any valid contents,
-            //     just return None and do not attempt to read elsewhere.
-            match contentsOpt with
-            | ProjectAssemblyDataResult.Unavailable false -> return None
-            | _ ->
+                    let! contentsOpt =
+                        async {
+                            match r.ProjectReference with
+                            | Some ilb -> return! ilb.EvaluateRawContents()
+                            | None -> return ProjectAssemblyDataResult.Unavailable true
+                        }
 
-                let assemblyData =
+                    // If we have a project reference but did not get any valid contents,
+                    //     just return None and do not attempt to read elsewhere.
                     match contentsOpt with
-                    | ProjectAssemblyDataResult.Available ilb -> ilb
-                    | ProjectAssemblyDataResult.Unavailable _ ->
-                        let ilModule, ilAssemblyRefs = tcImports.OpenILBinaryModule(ctok, fileName, m)
-                        RawFSharpAssemblyDataBackedByFileOnDisk(ilModule, ilAssemblyRefs) :> IRawFSharpAssemblyData
+                    | ProjectAssemblyDataResult.Unavailable false -> return None
+                    | _ ->
+
+                        match contentsOpt with
+                        | ProjectAssemblyDataResult.Available ilb -> return Some(r, ilb)
+                        | ProjectAssemblyDataResult.Unavailable _ ->
+                            let ilModule, ilAssemblyRefs = tcImports.OpenILBinaryModule(ctok, fileName, m)
+                            return Some(r, RawFSharpAssemblyDataBackedByFileOnDisk(ilModule, ilAssemblyRefs))
+
+                with e ->
+                    errorR (Error(FSComp.SR.buildProblemReadingAssembly (fileName, e.Message), m))
+                    return None
+            }
 
-                let ilShortAssemName = assemblyData.ShortAssemblyName
-                let ilScopeRef = assemblyData.ILScopeRef
+        let registerDll (r: AssemblyResolution, assemblyData: IRawFSharpAssemblyData) =
+            let m = r.originalReference.Range
+            let fileName = r.resolvedPath
+            let ilShortAssemName = assemblyData.ShortAssemblyName
+            let ilScopeRef = assemblyData.ILScopeRef
 
-                if tcImports.IsAlreadyRegistered ilShortAssemName then
-                    let dllinfo = tcImports.FindDllInfo(ctok, m, ilShortAssemName)
+            if tcImports.IsAlreadyRegistered ilShortAssemName then
 
-                    let phase2 () =
-                        [ tcImports.FindCcuInfo(ctok, m, ilShortAssemName, lookupOnly = true) ]
+                let phase2 () =
+                    [ tcImports.FindCcuInfo(ctok, m, ilShortAssemName, lookupOnly = true) ]
 
-                    return Some(dllinfo, phase2)
-                else
-                    let dllinfo =
-                        {
-                            RawMetadata = assemblyData
-                            FileName = fileName
+                async { return phase2 () }
+            else
+                let dllinfo =
+                    {
+                        RawMetadata = assemblyData
+                        FileName = fileName
 #if !NO_TYPEPROVIDERS
-                            ProviderGeneratedAssembly = None
-                            IsProviderGenerated = false
-                            ProviderGeneratedStaticLinkMap = None
+                        ProviderGeneratedAssembly = None
+                        IsProviderGenerated = false
+                        ProviderGeneratedStaticLinkMap = None
 #endif
-                            ILScopeRef = ilScopeRef
-                            ILAssemblyRefs = assemblyData.ILAssemblyRefs
-                        }
+                        ILScopeRef = ilScopeRef
+                        ILAssemblyRefs = assemblyData.ILAssemblyRefs
+                    }
 
-                    tcImports.RegisterDll dllinfo
+                tcImports.RegisterDll dllinfo
 
-                    let phase2 =
-                        if assemblyData.HasAnyFSharpSignatureDataAttribute then
-                            if not assemblyData.HasMatchingFSharpSignatureDataAttribute then
-                                errorR (Error(FSComp.SR.buildDifferentVersionMustRecompile fileName, m))
-                                tcImports.PrepareToImportReferencedILAssembly(ctok, m, fileName, dllinfo)
-                            else
-                                try
-                                    tcImports.PrepareToImportReferencedFSharpAssembly(ctok, m, fileName, dllinfo)
-                                with e ->
-                                    error (Error(FSComp.SR.buildErrorOpeningBinaryFile (fileName, e.Message), m))
-                        else
+                let phase2 =
+                    if assemblyData.HasAnyFSharpSignatureDataAttribute then
+                        if not assemblyData.HasMatchingFSharpSignatureDataAttribute then
+                            errorR (Error(FSComp.SR.buildDifferentVersionMustRecompile fileName, m))
                             tcImports.PrepareToImportReferencedILAssembly(ctok, m, fileName, dllinfo)
+                        else
+                            try
+                                tcImports.PrepareToImportReferencedFSharpAssembly(ctok, m, fileName, dllinfo)
+                            with e ->
+                                error (Error(FSComp.SR.buildErrorOpeningBinaryFile (fileName, e.Message), m))
+                    else
+                        tcImports.PrepareToImportReferencedILAssembly(ctok, m, fileName, dllinfo)
 
-                    return Some(dllinfo, phase2)
-        }
+                async { return phase2 () }
 
-    // NOTE: When used in the Language Service this can cause the transitive checking of projects. Hence it must be cancellable.
-    member tcImports.RegisterAndImportReferencedAssemblies(ctok, nms: AssemblyResolution list) =
         async {
             CheckDisposed()
 
             let tcConfig = tcConfigP.Get ctok
 
-            let runMethod =
+            let runMethod computations =
                 match tcConfig.parallelReferenceResolution with
-                | ParallelReferenceResolution.On -> MultipleDiagnosticsLoggers.Parallel
-                | ParallelReferenceResolution.Off -> MultipleDiagnosticsLoggers.Sequential
+                | ParallelReferenceResolution.On -> MultipleDiagnosticsLoggers.Parallel computations
+                | ParallelReferenceResolution.Off -> MultipleDiagnosticsLoggers.Sequential computations
+
+            let! assemblyData = nms |> List.map tryGetAssemblyData |> runMethod
+
+            // Preserve determinicstic order of references, because types from later assemblies may shadow earlier ones.
+            let phase2s = assemblyData |> Seq.choose id |> Seq.map registerDll |> List.ofSeq
 
-            let! results =
-                nms
-                |> List.map (fun nm ->
-                    async {
-                        try
-                            use _ = new CompilationGlobalsScope()
-                            return! tcImports.TryRegisterAndPrepareToImportReferencedDll(ctok, nm)
-                        with e ->
-                            errorR (Error(FSComp.SR.buildProblemReadingAssembly (nm.resolvedPath, e.Message), nm.originalReference.Range))
-                            return None
-                    })
-                |> runMethod
-
-            let _dllinfos, phase2s = results |> Array.choose id |> List.ofArray |> List.unzip
             fixupOrphanCcus ()
-            let ccuinfos = List.collect (fun phase2 -> phase2 ()) phase2s
+
+            let! ccuinfos = phase2s |> runMethod
 
             if importsBase.IsSome then
                 importsBase.Value.CcuTable.Values |> Seq.iter addConstraintSources
                 ccuTable.Values |> Seq.iter addConstraintSources
 
-            return ccuinfos
+            return ccuinfos |> List.concat
         }
 
     /// Note that implicit loading is not used for compilations from MSBuild, which passes ``--noframework``
@@ -2376,7 +2374,7 @@ and [<Sealed>] TcImports
                     ReportWarnings warns
 
                     tcImports.RegisterAndImportReferencedAssemblies(ctok, res)
-                    |> Async.RunImmediate
+                    |> Async.RunSynchronously
                     |> ignore
 
                     true
@@ -2684,7 +2682,7 @@ let RequireReferences (ctok, tcImports: TcImports, tcEnv, thisAssemblyName, reso
 
     let ccuinfos =
         tcImports.RegisterAndImportReferencedAssemblies(ctok, resolutions)
-        |> Async.RunImmediate
+        |> Async.RunSynchronously
 
     let asms =
         ccuinfos

src/Compiler/Driver/CompilerOptions.fs

@@ -625,10 +625,12 @@ let callVirtSwitch (tcConfigB: TcConfigBuilder) switch =
 let callParallelCompilationSwitch (tcConfigB: TcConfigBuilder) switch =
     tcConfigB.parallelIlxGen <- switch = OptionSwitch.On
 
-    let (graphCheckingMode, optMode) =
+    let (graphCheckingMode, optMode, parallelReferenceResolution) =
         match switch with
-        | OptionSwitch.On -> TypeCheckingMode.Graph, OptimizationProcessingMode.Parallel
-        | OptionSwitch.Off -> TypeCheckingMode.Sequential, OptimizationProcessingMode.Sequential
+        | OptionSwitch.On -> TypeCheckingMode.Graph, OptimizationProcessingMode.Parallel, ParallelReferenceResolution.On
+        | OptionSwitch.Off -> TypeCheckingMode.Sequential, OptimizationProcessingMode.Sequential, ParallelReferenceResolution.Off
+
+    tcConfigB.parallelReferenceResolution <- parallelReferenceResolution
 
     if tcConfigB.typeCheckingConfig.Mode <> graphCheckingMode then
         tcConfigB.typeCheckingConfig <-

src/Compiler/Driver/GraphChecking/DependencyResolution.fs

@@ -206,6 +206,7 @@ let mkGraph (filePairs: FilePairMap) (files: FileInProject array) : Graph<FileIn
         if file.Idx = 0 then
             // First file cannot have any dependencies.
             Array.empty
+
         else
             let fileContent = fileContents[file.Idx]
 
@@ -235,20 +236,47 @@ let mkGraph (filePairs: FilePairMap) (files: FileInProject array) : Graph<FileIn
                 | None -> Array.empty
                 | Some sigIdx -> Array.singleton sigIdx
 
+            let wrongOrderSignature =
+                match filePairs.TryGetWrongOrderSignatureToImplementationIndex file.Idx with
+                | None -> Array.empty
+                | Some idx -> Array.singleton idx
+
             let allDependencies =
                 [|
                     yield! depsResult.FoundDependencies
                     yield! ghostDependencies
                     yield! signatureDependency
+                    yield! wrongOrderSignature
                 |]
                 |> Array.distinct
 
             allDependencies
 
-    let graph =
+    // If there is a script in the project, we just process sequentially all the files that may have been added as part of the script closure.
+    // That means all files up to the last script file.
+    let scriptCompilationLength =
         files
+        |> Array.tryFindIndexBack (fun f -> f.IsScript)
+        |> Option.map (fun idx -> idx + 1)
+        |> Option.defaultValue 0
+
+    let sequentialPartForScriptCompilation =
+        files
+        |> Array.take scriptCompilationLength
+        |> Array.map (fun file ->
+            file.Idx,
+            [|
+                if file.Idx > 0 then
+                    file.Idx - 1
+            |])
+
+    let normalPart =
+        files
+        |> Array.skip scriptCompilationLength
         |> Array.Parallel.map (fun file -> file.Idx, findDependencies file)
-        |> readOnlyDict
+
+    let graph =
+        Array.append sequentialPartForScriptCompilation normalPart |> readOnlyDict
 
     let trie = trie |> Array.last |> snd