account for extensions indirectly depending on eachother in parallel package precompilation

Author: KristofferC

base/precompilation.jl

@@ -402,7 +402,6 @@ function precompilepkgs(pkgs::Vector{String}=String[];
         depsmap[pkg] = filter!(!Base.in_sysimage, deps)
         # add any extensions
         pkg_exts = Dict{Base.PkgId, Vector{Base.PkgId}}()
-        prev_ext = nothing
         for (ext_name, extdep_uuids) in env.extensions[dep]
             ext_deps = Base.PkgId[]
             push!(ext_deps, pkg) # depends on parent package
@@ -417,13 +416,8 @@ function precompilepkgs(pkgs::Vector{String}=String[];
                 end
             end
             all_extdeps_available || continue
-            if prev_ext isa Base.PkgId
-                # also make the exts depend on eachother sequentially to avoid race
-                push!(ext_deps, prev_ext)
-            end
             ext_uuid = Base.uuid5(pkg.uuid, ext_name)
             ext = Base.PkgId(ext_uuid, ext_name)
-            prev_ext = ext
             filter!(!Base.in_sysimage, ext_deps)
             depsmap[ext] = ext_deps
             exts[ext] = pkg.name
@@ -434,6 +428,51 @@ function precompilepkgs(pkgs::Vector{String}=String[];
         end
     end
 
+    # An extension effectively depends on another extension if it has all the the
+    # dependencies of that other extension
+    function expand_dependencies(depsmap)
+        function visit!(visited, node, all_deps)
+            if node in visited
+                return
+            end
+            push!(visited, node)
+            for dep in get(Set{Base.PkgId}, depsmap, node)
+                if !(dep in all_deps)
+                    push!(all_deps, dep)
+                    visit!(visited, dep, all_deps)
+                end
+            end
+        end
+
+        depsmap_transitive = Dict{Base.PkgId, Set{Base.PkgId}}()
+        for package in keys(depsmap)
+            # Initialize a set to keep track of all dependencies for 'package'
+            all_deps = Set{Base.PkgId}()
+            visited = Set{Base.PkgId}()
+            visit!(visited, package, all_deps)
+            # Update depsmap with the complete set of dependencies for 'package'
+            depsmap_transitive[package] = all_deps
+        end
+        return depsmap_transitive
+    end
+
+    depsmap_transitive = expand_dependencies(depsmap)
+
+    for (_, extensions_1) in pkg_exts_map
+        for extension_1 in extensions_1
+            deps_ext_1 = depsmap_transitive[extension_1]
+            for (_, extensions_2) in pkg_exts_map
+                for extension_2 in extensions_2
+                    extension_1 == extension_2 && continue
+                    deps_ext_2 = depsmap_transitive[extension_2]
+                    if issubset(deps_ext_2, deps_ext_1)
+                        push!(depsmap[extension_1], extension_2)
+                    end
+                end
+            end
+        end
+    end
+
     @debug "precompile: deps collected"
     # this loop must be run after the full depsmap has been populated
     for (pkg, pkg_exts) in pkg_exts_map
@@ -748,6 +787,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
     end
     @debug "precompile: starting precompilation loop" depsmap direct_deps
     ## precompilation loop
+
     for (pkg, deps) in depsmap
         cachepaths = Base.find_all_in_cache_path(pkg)
         sourcepath = Base.locate_package(pkg)
@@ -820,6 +860,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
                             end
                             loaded && (n_loaded += 1)
                         catch err
+                            @show err
                             close(std_pipe.in) # close pipe to end the std output monitor
                             wait(t_monitor)
                             if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file"))
@@ -843,7 +884,8 @@ function precompilepkgs(pkgs::Vector{String}=String[];
                     notify(was_processed[pkg_config])
                 catch err_outer
                     # For debugging:
-                    # println("Task failed $err_outer") # logging doesn't show here
+                    println("Task failed $err_outer")
+                    Base.display_error(ErrorException(""), Base.catch_backtrace())# logging doesn't show here
                     handle_interrupt(err_outer) || rethrow()
                     notify(was_processed[pkg_config])
                 finally