Skip to content

Commit

Permalink
Parallelize artifact downloads. Update progress bar styling. (#3952)
Browse files Browse the repository at this point in the history 
* parallelize artifact downloads

* show largest first

* non fancyprint mode &. formatting

* update progress bar styling

* fix alignment

* fix color

* fix non-fancy print formatting

* fix scope

* rpad artifact names on CI

* improve error handling

* error handling improvements

* comment

* fix test

* comments & tidies

---------

Co-authored-by: KristofferC <[email protected]>
  • Loading branch information
@IanButterworth
IanButterworth and KristofferC authored Aug 16, 2024
1 parent a717900 commit d1d2fc9
Show file tree
Hide file tree
Showing 6 changed files with 283 additions and 96 deletions.
2 changes: 1 addition & 1 deletion src/API.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1268,7 +1268,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
# Install all packages
new_apply = Operations.download_source(ctx)
# Install all artifacts
Operations.download_artifacts(ctx.env; platform, verbose, io=ctx.io)
Operations.download_artifacts(ctx; platform, verbose)
# Run build scripts
allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose)

Expand Down
128 changes: 74 additions & 54 deletions src/Artifacts.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ function download_artifact(
verbose::Bool = false,
quiet_download::Bool = false,
io::IO=stderr_f(),
progress::Union{Function, Nothing} = nothing,
)
if artifact_exists(tree_hash)
return true
Expand All @@ -323,8 +324,8 @@ function download_artifact(
temp_dir = mktempdir(artifacts_dir)

try
download_verify_unpack(tarball_url, tarball_hash, temp_dir, ignore_existence=true, verbose=verbose,
quiet_download=quiet_download, io=io)
download_verify_unpack(tarball_url, tarball_hash, temp_dir;
ignore_existence=true, verbose, quiet_download, io, progress)
calc_hash = SHA1(GitTools.tree_hash(temp_dir))

# Did we get what we expected? If not, freak out.
Expand Down Expand Up @@ -394,82 +395,101 @@ function ensure_artifact_installed(name::String, artifacts_toml::String;
pkg_uuid::Union{Base.UUID,Nothing}=nothing,
verbose::Bool = false,
quiet_download::Bool = false,
progress::Union{Function,Nothing} = nothing,
io::IO=stderr_f())
meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform)
if meta === nothing
error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'")
end

return ensure_artifact_installed(name, meta, artifacts_toml; platform=platform,
verbose=verbose, quiet_download=quiet_download, io=io)
return ensure_artifact_installed(name, meta, artifacts_toml;
platform, verbose, quiet_download, progress, io)
end

function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String;
platform::AbstractPlatform = HostPlatform(),
verbose::Bool = false,
quiet_download::Bool = false,
progress::Union{Function,Nothing} = nothing,
io::IO=stderr_f())
hash = SHA1(meta["git-tree-sha1"])

hash = SHA1(meta["git-tree-sha1"])
if !artifact_exists(hash)
errors = Any[]
# first try downloading from Pkg server
# TODO: only do this if Pkg server knows about this package
if (server = pkg_server()) !== nothing
url = "$server/artifact/$hash"
download_success = let url=url
@debug "Downloading artifact from Pkg server" name artifacts_toml platform url
with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url; verbose=verbose, quiet_download=quiet_download, io=io)
end
end
# download_success is either `true` or an error object
if download_success === true
return artifact_path(hash)
else
@debug "Failed to download artifact from Pkg server" download_success
push!(errors, (url, download_success))
end
if isnothing(progress) || verbose == true
return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io)
else
# if a custom progress handler is given it is taken to mean the caller wants to handle the download scheduling
return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, quiet_download=true, io, progress)
end
else
return artifact_path(hash)
end
end

# If this artifact does not exist on-disk already, ensure it has download
# information, then download it!
if !haskey(meta, "download")
error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
function try_artifact_download_sources(
name::String, hash::SHA1, meta::Dict, artifacts_toml::String;
platform::AbstractPlatform=HostPlatform(),
verbose::Bool=false,
quiet_download::Bool=false,
io::IO=stderr_f(),
progress::Union{Function,Nothing}=nothing)

errors = Any[]
# first try downloading from Pkg server
# TODO: only do this if Pkg server knows about this package
if (server = pkg_server()) !== nothing
url = "$server/artifact/$hash"
download_success = let url = url
@debug "Downloading artifact from Pkg server" name artifacts_toml platform url
with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url; verbose, quiet_download, io, progress)
end
end
# download_success is either `true` or an error object
if download_success === true
return artifact_path(hash)
else
@debug "Failed to download artifact from Pkg server" download_success
push!(errors, (url, download_success))
end
end

# Attempt to download from all sources
for entry in meta["download"]
url = entry["url"]
tarball_hash = entry["sha256"]
download_success = let url=url
@debug "Downloading artifact" name artifacts_toml platform url
with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url, tarball_hash; verbose=verbose, quiet_download=quiet_download, io=io)
end
end
# download_success is either `true` or an error object
if download_success === true
return artifact_path(hash)
else
@debug "Failed to download artifact" download_success
push!(errors, (url, download_success))
# If this artifact does not exist on-disk already, ensure it has download
# information, then download it!
if !haskey(meta, "download")
error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
end

# Attempt to download from all sources
for entry in meta["download"]
url = entry["url"]
tarball_hash = entry["sha256"]
download_success = let url = url
@debug "Downloading artifact" name artifacts_toml platform url
with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url, tarball_hash; verbose, quiet_download, io, progress)
end
end
errmsg = """
Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
Sources attempted:
"""
for (url, err) in errors
errmsg *= "- $(url)\n"
errmsg *= " Error: $(sprint(showerror, err))\n"
# download_success is either `true` or an error object
if download_success === true
return artifact_path(hash)
else
@debug "Failed to download artifact" download_success
push!(errors, (url, download_success))
end
error(errmsg)
else
return artifact_path(hash)
end
errmsg = """
Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
Sources attempted:
"""
for (url, err) in errors
errmsg *= "- $(url)\n"
errmsg *= " Error: $(sprint(showerror, err))\n"
end
error(errmsg)
end


function with_show_download_info(f, io, name, quiet_download)
fancyprint = can_fancyprint(io)
if !quiet_download
Expand All @@ -485,7 +505,7 @@ function with_show_download_info(f, io, name, quiet_download)
if !quiet_download
fancyprint && print(io, "\033[1A") # move cursor up one line
fancyprint && print(io, "\033[2K") # clear line
if success
if success
fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name")
else
printpkgstyle(io, :Failure, "artifact: $name", color = :red)
Expand Down
46 changes: 35 additions & 11 deletions src/MiniProgressBars.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,31 @@ export MiniProgressBar, start_progress, end_progress, show_progress, print_progr

using Printf

# Until Base.format_bytes supports sigdigits
function pkg_format_bytes(bytes; binary=true, sigdigits::Integer=3)
units = binary ? Base._mem_units : Base._cnt_units
factor = binary ? 1024 : 1000
bytes, mb = Base.prettyprint_getunits(bytes, length(units), Int64(factor))
if mb == 1
return string(Int(bytes), " ", Base._mem_units[mb], bytes==1 ? "" : "s")
else
return string(Base.Ryu.writefixed(Float64(bytes), sigdigits), binary ? " $(units[mb])" : "$(units[mb])B")
end
end

Base.@kwdef mutable struct MiniProgressBar
max::Int = 1.0
header::String = ""
color::Symbol = :nothing
width::Int = 40
current::Int = 0.0
prev::Int = 0.0
current::Int = 0
prev::Int = 0
has_shown::Bool = false
time_shown::Float64 = 0.0
percentage::Bool = true
mode::Symbol = :percentage # :percentage :int :data
always_reprint::Bool = false
indent::Int = 4
main::Bool = true
end

const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps
Expand Down Expand Up @@ -47,21 +60,32 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere
p.prev = p.current
p.has_shown = true

progress_text = if p.percentage
progress_text = if p.mode == :percentage
@sprintf "%2.1f %%" perc
else
elseif p.mode == :int
string(p.current, "/", p.max)
elseif p.mode == :data
lpad(string(pkg_format_bytes(p.current; sigdigits=1), "/", pkg_format_bytes(p.max; sigdigits=1)), 20)
else
error("Unknown mode $(p.mode)")
end
termwidth = @something termwidth displaysize(io)[2]
max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width))
n_filled = ceil(Int, max_progress_width * perc / 100)
n_left = max_progress_width - n_filled
headers = split(p.header)
to_print = sprint(; context=io) do io
print(io, " "^p.indent)
printstyled(io, p.header, color=p.color, bold=true)
print(io, " [")
print(io, "="^n_filled, ">")
print(io, " "^n_left, "] ", )
if p.main
printstyled(io, headers[1], " "; color=:green, bold=true)
printstyled(io, join(headers[2:end], ' '))
else
print(io, p.header)
end
print(io, " ")
printstyled(io, "━"^n_filled; color=p.color)
printstyled(io, perc >= 95 ? "━" : "╸"; color=p.color)
printstyled(io, "━"^n_left, " "; color=:light_black)
print(io, progress_text)
carriagereturn && print(io, "\r")
end
Expand All @@ -80,10 +104,10 @@ end
# prog = MiniProgressBar(...)
# prog.end = n
# for progress in 1:n
# print_progree_bottom(io)
# print_progress_bottom(io)
# println("stuff")
# prog.current = progress
# showproress(io, prog)
# showprogress(io, prog)
# end
#
function print_progress_bottom(io::IO)
Expand Down
Loading

0 comments on commit d1d2fc9

Please sign in to comment.