fixes for cloning and fetching

Author: KristofferC

src/API.jl

@@ -7,7 +7,7 @@ import Dates
 import LibGit2
 
 import ..depots, ..logdir, ..devdir, ..print_first_command_header
-import ..Operations, ..Display
+import ..Operations, ..Display, ..GitTools
 using ..Types, ..TOML
 
 
@@ -75,7 +75,6 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
         for reg in registries()
             if isdir(joinpath(reg, ".git"))
                 regpath = pathrepr(reg)
-                printpkgstyle(ctx, :Updating, "registry at ", regpath)
                 LibGit2.with(LibGit2.GitRepo, reg) do repo
                     if LibGit2.isdirty(repo)
                         push!(errors, (regpath, "registry dirty"))
@@ -86,7 +85,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
                         return
                     end
                     branch = LibGit2.headname(repo)
-                    LibGit2.fetch(repo)
+                    GitTools.fetch(repo)
                     ff_succeeded = try
                         LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
                     catch e

src/GitTools.jl

@@ -1,5 +1,6 @@
 module GitTools
 
+using ..Pkg3
 import LibGit2
 using Printf
 
@@ -16,6 +17,8 @@ end
 function showprogress(io::IO, p::MiniProgressBar)
     perc = p.current / p.max * 100
     prev_perc = p.prev / p.max * 100
+    # Bail early if we are not updating the progress bar,
+    # Saves printing to the terminal
     if p.has_shown && !(perc - prev_perc > 0.1)
         return
     end
@@ -32,43 +35,45 @@ function showprogress(io::IO, p::MiniProgressBar)
     print(io, "\r")
 end
 
-function transfer_progress(progress::Ptr{LibGit2.GitTransferProgress}, p::Any)
+Base.@kwdef struct GitTransferProgress
+    total_objects::Cuint
+    indexed_objects::Cuint
+    received_objects::Cuint
+    local_objects::Cuint
+    total_deltas::Cuint
+    indexed_deltas::Cuint
+received_bytes::Csize_t
+end
+
+function transfer_progress(progress::Ptr{GitTransferProgress}, p::Any)
     progress = unsafe_load(progress)
+    bar = unsafe_pointer_to_objref(p.transfer_progress)
+    @assert typeof(bar) == MiniProgressBar
     if progress.total_deltas != 0
-        p.header = "Resolving Deltas:"
-        p.max = progress.total_deltas
-        p.current = progress.indexed_deltas
+        bar.header = "Resolving Deltas:"
+        bar.max = progress.total_deltas
+        bar.current = progress.indexed_deltas
     else
-        p.max = progress.total_objects
-        p.current = progress.received_objects
+        bar.max = progress.total_objects
+        bar.current = progress.received_objects
     end
-    showprogress(stdout, p)
+    showprogress(stdout, bar)
     return Cint(0)
 end
 
 function clone(url, source_path; isbare::Bool=false, header = nothing, branch = nothing, credentials = nothing)
-    isdir(source_path) && error("$source_path already exists")
-    printstyled(stdout, "Cloning "; color = :green, bold=true)
-    if header == nothing
-        println(stdout, "from git repo: ", url, ".")
-    else
-        println(stdout, header)
-    end
+    Pkg3.Types.printpkgstyle(stdout, :Cloning, header == nothing ? string("git-repo `", url, "`") : header)
     transfer_payload = MiniProgressBar(header = "Fetching:", color = Base.info_color())
     cred_payload = LibGit2.CredentialPayload(credentials)
-    print(stdout, "\e[?25l")
+    print(stdout, "\e[?25l") # disable cursor
     try
-        GC.@preserve p branch begin
+        GC.@preserve transfer_payload branch begin
             callbacks = LibGit2.RemoteCallbacks(
-                credentials=(LibGit2.credential_cb(), pointer_from_objref(cred_payload)),
-                transfer_progress=(cfunction(transfer_progress, Cint, Tuple{Ptr{LibGit2.GitTransferProgress}, Any}), pointer_from_objref(transfer_payload)),
+                credentials=(LibGit2.credentials_cb(), pointer_from_objref(cred_payload)),
+                transfer_progress=(cfunction(transfer_progress, Cint, Tuple{Ptr{GitTransferProgress}, Any}), pointer_from_objref(transfer_payload)),
             )
             fetch_opts = LibGit2.FetchOptions(callbacks = callbacks)
-            if branch == nothing
-                clone_opts = LibGit2.CloneOptions(fetch_opts=fetch_opts, bare=isbare)
-            else
-                clone_opts = LibGit2.CloneOptions(fetch_opts=fetch_opts, bare=isbare, checkout_branch=Cstring(pointer(branch)))
-            end
+            clone_opts = LibGit2.CloneOptions(fetch_opts=fetch_opts, bare=isbare, checkout_branch= branch == nothing ? C_NULL : Cstring(pointer(branch)))
             return LibGit2.clone(url, source_path, clone_opts)
         end
     catch e
@@ -79,10 +84,42 @@ function clone(url, source_path; isbare::Bool=false, header = nothing, branch =
         rm(source_path; force=true, recursive=true)
         rethrow(e)
     finally
-        print(stdout, "\e[?25h")
-        println(stdout)
+        print(stdout, "\033[2K") # clear line
+        print(stdout, "\e[?25h") # put back cursor
+    end
+    LibGit2.approve(cred_payload)
+end
+
+function fetch(repo::LibGit2.GitRepo, remoteurl=nothing; header = nothing, refspecs=String[], credentials=nothing)
+    remote = if remoteurl == nothing
+        LibGit2.get(LibGit2.GitRemote, repo, "origin")
+    else
+        LibGit2.GitRemoteAnon(repo, remoteurl)
+    end
+    Pkg3.Types.printpkgstyle(stdout, :Updating, header == nothing ? string("git-repo `", LibGit2.url(remote), "`") : header)
+    transfer_payload = MiniProgressBar(header = "Fetching:", color = Base.info_color())
+    cred_payload = LibGit2.CredentialPayload(credentials)
+    print(stdout, "\e[?25l") # disable cursor
+    try
+        GC.@preserve transfer_payload begin
+            callbacks = LibGit2.RemoteCallbacks(
+                credentials=(LibGit2.credentials_cb(), pointer_from_objref(cred_payload)),
+                transfer_progress=(cfunction(transfer_progress, Cint, Tuple{Ptr{GitTransferProgress}, Any}), pointer_from_objref(transfer_payload)),
+            )
+            fetch_opts = LibGit2.FetchOptions(callbacks = callbacks)
+            return LibGit2.fetch(remote, refspecs; options=fetch_opts)
+        end
+    catch e
+        if isa(e, LibGit2.GitError) && e.code == LibGit2.Error.EAUTH
+            LibGit2.reject(cred_payload)
+        end
+        rethrow(e)
+    finally
+        close(remote)
+        print(stdout, "\033[2K") # clear line
+        print(stdout, "\e[?25h") # put back cursor
     end
     LibGit2.approve(cred_payload)
 end
 
-end # module
+end # module

src/Operations.jl

@@ -391,7 +391,7 @@ function install_git(
             err isa LibGit2.GitError && err.code == LibGit2.Error.ENOTFOUND || rethrow(err)
         end
         url = urls[i]
-        LibGit2.fetch(repo, remoteurl=url, refspecs=refspecs, credentials=creds)
+        GitTools.fetch(repo, url, refspecs=refspecs, credentials=creds)
     end
     tree = try
         with(LibGit2.GitObject, repo, git_hash) do g

src/Types.jl

@@ -622,14 +622,12 @@ function handle_repos_develop!(ctx::Context, pkgs::AbstractVector{PackageSpec})
             mkpath(clone_path)
             repo_path = joinpath(clone_path, string(hash(pkg.repo.url), "_full"))
             repo, just_cloned = ispath(repo_path) ? (LibGit2.GitRepo(repo_path), false) : begin
-                printpkgstyle(ctx, :Cloning, "package from $(pkg.repo.url)")
-                r = LibGit2.clone(pkg.repo.url, repo_path)
-                GitTools.fetch(r, remoteurl=pkg.repo.url, refspecs=refspecs, credentials=creds)
+                r = GitTools.clone(pkg.repo.url, repo_path)
+                GitTools.fetch(r, pkg.repo.url; refspecs=refspecs, credentials=creds)
                 r, true
             end
             if !just_cloned
-                printpkgstyle(ctx, :Updating, "repo from $(pkg.repo.url)")
-                GitTools.fetch(repo, remoteurl=pkg.repo.url, refspecs=refspecs, credentials=creds)
+                GitTools.fetch(repo, pkg.repo.url; refspecs=refspecs, credentials=creds)
             end
             close(repo)
 
@@ -672,14 +670,14 @@ function handle_repos_add!(ctx::Context, pkgs::AbstractVector{PackageSpec}; upgr
         repo_path = joinpath(clones_dir, string(hash(pkg.repo.url)))
         repo, just_cloned = ispath(repo_path) ? (LibGit2.GitRepo(repo_path), false) : begin
             r = GitTools.clone(pkg.repo.url, repo_path, isbare=true, credentials=creds)
-            GitTools.fetch(r, remoteurl=pkg.repo.url, refspecs=refspecs, credentials=creds)
+            GitTools.fetch(r, pkg.repo.url; refspecs=refspecs, credentials=creds)
             r, true
         end
         info = manifest_info(env, pkg.uuid)
         pinned = (info != nothing && get(info, "pinned", false))
-        if upgrade_or_add  && !pinned && !just_cloned
-            printpkgstyle(ctx, :Updating, "repo from $(pkg.repo.url)")
-            GitTools.fetch(repo, remoteurl=pkg.repo.url, refspecs=refspecs, credentials=creds)
+        if upgrade_or_add && !pinned && !just_cloned
+            rev = pkg.repo.rev
+            GitTools.fetch(repo, pkg.repo.url; refspecs=refspecs, credentials=creds)
         end
         if upgrade_or_add && !pinned
             rev = pkg.repo.rev
@@ -934,7 +932,8 @@ function registries()::Vector{String}
         printpkgstyle(stdout, :Cloning, "default registries into $user_regs")
         for (reg, url) in DEFAULT_REGISTRIES
             path = joinpath(user_regs, reg)
-            GitTools.clone(url, path; header = "registry $reg from $(repr(url))", credentials = creds)
+            repo = Pkg3.GitTools.clone(url, path; header = "registry $reg from $(repr(url))", credentials = creds)
+            close(repo)
         end
     end
     return [r for d in depots() for r in registries(d)]
@@ -1151,14 +1150,13 @@ pathrepr(path::String, base::String=pwd()) = pathrepr(nothing, path, base)
 function pathrepr(env::Union{Nothing, EnvCache}, path::String, base::String=pwd())
     path = abspath(base, path)
     if env isa EnvCache && env.git != nothing
-        LibGit2.with(LibGit2.GitRepo, LibGit2.path(env.git)) do repo
-            if startswith(base, repo)
-                # we're in the repo => path relative to pwd()
-                path = relpath(path, base)
-            elseif startswith(path, repo)
-                # we're not in repo but path is => path relative to repo
-                path = relpath(path, repo)
-            end
+        repo = LibGit2.path(env.git)
+        if startswith(base, repo)
+            # we're in the repo => path relative to pwd()
+            path = relpath(path, base)
+        elseif startswith(path, repo)
+            # we're not in repo but path is => path relative to repo
+            path = relpath(path, repo)
         end
     end
     if !Sys.iswindows() && isabspath(path)