Merge pull request #4036 from JuliaLang/backports-release-1.10

docs/src/getting-started.md

@@ -154,7 +154,7 @@ We can see that the `tutorial` environment now contains `Example` and `JSON`.
     If you have the same
     package (at the same version) installed in multiple environments, the package
     will only be downloaded and stored on the hard drive once. This makes environments
-    very lightweight and effectively free to create. Only using the default
+    very lightweight and effectively free to create. Using only the default
     environment with a huge number of packages in it is a common beginners mistake in
     Julia. Learning how to use environments effectively will improve your experience with
     Julia packages.

src/Artifacts.jl

@@ -65,20 +65,42 @@ Either rename the directory at `temp_dir` to `new_path` and set it to read-only
 or if `new_path` artifact already exists try to do nothing.
 """
 function _mv_temp_artifact_dir(temp_dir::String, new_path::String)::Nothing
-    if !isdir(new_path)
+    # Sometimes a rename can fail because the temp_dir is locked by
+    # anti-virus software scanning the new files.
+    # In this case we want to sleep and try again.
+    # I am using the list of error codes to retry from:
+    # https://github.com/isaacs/node-graceful-fs/blob/234379906b7d2f4c9cfeb412d2516f42b0fb4953/polyfills.js#L87
+    # Retry for up to about 60 seconds by retrying 20 times with exponential backoff.
+    retry = 0
+    max_num_retries = 20 # maybe this should be configurable?
+    sleep_amount = 0.01 # seconds
+    max_sleep_amount = 5.0 # seconds
+    while true
+        isdir(new_path) && return
         # This next step is like
         # `mv(temp_dir, new_path)`.
         # However, `mv` defaults to `cp` if `rename` returns an error.
         # `cp` is not atomic, so avoid the potential of calling it.
         err = ccall(:jl_fs_rename, Int32, (Cstring, Cstring), temp_dir, new_path)
-        # Ignore rename error, but ensure `new_path` exists.
-        if !isdir(new_path)
-            error("$(repr(new_path)) could not be made")
+        if err ≥ 0
+            # rename worked
+            chmod(new_path, filemode(dirname(new_path)))
+            set_readonly(new_path)
+            return
+        else
+            # Ignore rename error if `new_path` exists.
+            isdir(new_path) && return
+            if retry < max_num_retries && err ∈ (Base.UV_EACCES, Base.UV_EPERM, Base.UV_EBUSY)
+                sleep(sleep_amount)
+                sleep_amount = min(sleep_amount*2.0, max_sleep_amount)
+                retry += 1
+            else
+                Base.uv_error("rename of $(repr(temp_dir)) to $(repr(new_path))", err)
+            end
         end
         chmod(new_path, filemode(dirname(new_path)))
         set_readonly(new_path)
     end
-    nothing
 end
 
 """
@@ -485,7 +507,7 @@ function with_show_download_info(f, io, name, quiet_download)
         if !quiet_download
             fancyprint && print(io, "\033[1A") # move cursor up one line
             fancyprint && print(io, "\033[2K") # clear line
-            if success 
+            if success
                 fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name")
             else
                 printpkgstyle(io, :Failure, "artifact: $name", color = :red)

src/MiniProgressBars.jl

@@ -40,7 +40,7 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere
         return
     end
     t = time()
-    if p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[]
+    if !p.always_reprint && p.has_shown && (t - p.time_shown) < PROGRESS_BAR_TIME_GRANULARITY[]
         return
     end
     p.time_shown = t

src/Operations.jl

@@ -2210,6 +2210,9 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache)
             # Check if deps are loaded
             extdeps_info= Tuple{String, Bool}[]
             for extdep in extdeps
+                haskey(weakdepses, extdep) ||
+                    pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a m",
+                             "alformed Project.toml, the extension package $extdep is not listed in [weakdeps]")
                 uuid = weakdepses[extdep]
                 loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, extdep))
                 push!(extdeps_info, (extdep, loaded))

src/REPLMode/REPLMode.jl

@@ -10,7 +10,7 @@ import REPL: TerminalMenus
 
 import ..casesensitive_isdir, ..OFFLINE_MODE, ..linewrap, ..pathrepr
 using ..Types, ..Operations, ..API, ..Registry, ..Resolve
-import ..stdout_f, ..stderr_f
+import ..stdout_f, ..stderr_f, ..safe_realpath
 
 const TEST_MODE = Ref{Bool}(false)
 const PRINTED_REPL_WARNING = Ref{Bool}(false)
@@ -167,7 +167,7 @@ Base.@kwdef mutable struct Statement
 end
 
 function lex(cmd::String)::Vector{QString}
-    replace_comma = (nothing!=match(r"^(add|rm|remove|status|precompile)+\s", cmd))
+    replace_comma = (nothing!=match(r"^(add|dev|develop|rm|remove|status|precompile)+\s", cmd))
     in_doublequote = false
     in_singlequote = false
     qstrings = QString[]
@@ -497,7 +497,7 @@ function projname(project_file::String)
     end
     for depot in Base.DEPOT_PATH
         envdir = joinpath(depot, "environments")
-        if startswith(abspath(project_file), abspath(envdir))
+        if startswith(safe_realpath(project_file), safe_realpath(envdir))
             return "@" * name
         end
     end

src/REPLMode/command_declarations.jl

@@ -364,6 +364,7 @@ Create a minimal project called `pkgname` in the current folder. For more featur
 ],
 PSA[:name => "precompile",
     :api => API.precompile,
+    :should_splat => false,
     :arg_count => 0 => Inf,
     :completions => complete_installed_packages,
     :description => "precompile all the project dependencies",

src/Registry/Registry.jl

@@ -429,6 +429,15 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(),
                                     registry_update_log[string(reg.uuid)] = now()
                                     @label done_tarball_read
                                 else
+                                    if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true)
+                                        @info """
+                                            The General registry is installed via unpacked tarball.
+                                            Consider reinstalling it via the newer faster direct from
+                                            tarball format by running:
+                                              pkg> registry rm General; registry add General
+
+                                            """ maxlog=1
+                                    end
                                     mktempdir() do tmp
                                         try
                                             download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
@@ -447,6 +456,14 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(),
                         end
                     elseif isdir(joinpath(reg.path, ".git"))
                         printpkgstyle(io, :Updating, "registry at " * regpath)
+                        if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true)
+                            @info """
+                                The General registry is installed via git. Consider reinstalling it via
+                                the newer faster direct from tarball format by running:
+                                  pkg> registry rm General; registry add General
+
+                                """ maxlog=1
+                        end
                         LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
                             if LibGit2.isdirty(repo)
                                 push!(errors, (regpath, "registry dirty"))

src/Resolve/Resolve.jl

@@ -341,6 +341,8 @@ function greedysolver(graph::Graph)
                     return (false, Int[])
                 elseif old_v1 == spp[p1]
                     sol[p1] = v1
+                    fill!(gconstr[p1], false)
+                    gconstr[p1][v1] = true
                     push!(staged_next, p1)
                 end
             end

test/new.jl

@@ -800,7 +800,7 @@ end
         @test Pkg.dependencies()[exuuid].version == v"0.3.0"
         @test Pkg.dependencies()[pngjll_uuid].version == v"1.6.37+4"
         Pkg.add(Pkg.PackageSpec(;name="JSON", version="0.18.0"); preserve=Pkg.PRESERVE_NONE)
-        @test Pkg.dependencies()[exuuid].version == v"0.5.3"
+        @test Pkg.dependencies()[exuuid].version > v"0.3.0"
         @test Pkg.dependencies()[json_uuid].version == v"0.18.0"
         @test Pkg.dependencies()[pngjll_uuid].version > v"1.6.37+4"
     end
@@ -2185,19 +2185,17 @@ end
 
         api, arg, opts = first(Pkg.pkg"precompile Foo")
         @test api == Pkg.precompile
-        @test arg == "Foo"
+        @test arg == ["Foo"]
         @test isempty(opts)
 
-        api, arg1, arg2, opts = first(Pkg.pkg"precompile Foo Bar")
+        api, arg, opts = first(Pkg.pkg"precompile Foo Bar")
         @test api == Pkg.precompile
-        @test arg1 == "Foo"
-        @test arg2 == "Bar"
+        @test arg == ["Foo", "Bar"]
         @test isempty(opts)
 
-        api, arg1, arg2, opts = first(Pkg.pkg"precompile Foo, Bar")
+        api, arg, opts = first(Pkg.pkg"precompile Foo, Bar")
         @test api == Pkg.precompile
-        @test arg1 == "Foo"
-        @test arg2 == "Bar"
+        @test arg == ["Foo", "Bar"]
         @test isempty(opts)
     end
 end

test/pkg.jl

@@ -268,8 +268,9 @@ temp_pkg_dir() do project_path
     end
 
     @testset "develop / freeing" begin
-        Pkg.add(TEST_PKG.name)
+        Pkg.add(name=TEST_PKG.name, version=v"0.5.3")
         old_v = Pkg.dependencies()[TEST_PKG.uuid].version
+        @test old_v == v"0.5.3"
         Pkg.rm(TEST_PKG.name)
         mktempdir() do devdir
             withenv("JULIA_PKG_DEVDIR" => devdir) do
@@ -298,11 +299,27 @@ temp_pkg_dir() do project_path
                     touch("deps.jl")
                     """
                 )
+                exa_proj = joinpath(devdir, TEST_PKG.name, "Project.toml")
+                proj_str = read(exa_proj, String)
+                compat_onwards = split(proj_str, "[compat]")[2]
+                open(exa_proj, "w") do io
+                    println(io, """
+                    name = "Example"
+                    uuid = "$(TEST_PKG.uuid)"
+                    version = "100.0.0"
+
+                    [compat]
+                    $compat_onwards
+                    """)
+                end
+                Pkg.resolve()
+                @test Pkg.dependencies()[TEST_PKG.uuid].version == v"100.0.0"
                 Pkg.build(TEST_PKG.name)
                 @test isfile(joinpath(devdir, TEST_PKG.name, "deps", "deps.jl"))
                 Pkg.test(TEST_PKG.name)
                 Pkg.free(TEST_PKG.name)
-                @test Pkg.dependencies()[TEST_PKG.uuid].version == old_v
+                @test Pkg.dependencies()[TEST_PKG.uuid].version < v"100.0.0"
+                @test Pkg.dependencies()[TEST_PKG.uuid].version >= old_v
             end
         end
     end

test/repl.jl

@@ -55,9 +55,10 @@ temp_pkg_dir(;rm=false) do project_path; cd(project_path) do;
     tmp_pkg_path = mktempdir()
 
     pkg"activate ."
-    pkg"add [email protected]"
+    pkg"add [email protected].3"
     @test isinstalled(TEST_PKG)
     v = Pkg.dependencies()[TEST_PKG.uuid].version
+    @test v == v"0.5.3"
     pkg"rm Example"
     pkg"add Example, Random"
     pkg"rm Example Random"
@@ -120,6 +121,8 @@ temp_pkg_dir(;rm=false) do project_path; cd(project_path) do;
     mktempdir() do tmp_dev_dir
     withenv("JULIA_PKG_DEVDIR" => tmp_dev_dir) do
         pkg"develop Example"
+        pkg"develop Example,PackageCompiler"
+        pkg"develop Example PackageCompiler"
 
         # Copy the manifest + project and see that we can resolve it in a new environment
         # and get all the packages installed
@@ -723,4 +726,15 @@ end
     end
 end
 
+@testset "JuliaLang/julia #55850" begin
+    tmp_55850 = mktempdir()
+    tmp_sym_link = joinpath(tmp_55850, "sym")
+    symlink(tmp_55850, tmp_sym_link; dir_target=true)
+    # DEPOT_PATH must stay only the temp directory otherwise the bug is hidden
+    withenv("JULIA_DEPOT_PATH" => tmp_sym_link, "JULIA_LOAD_PATH" => nothing) do
+        prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg: Pkg, REPLMode; Pkg.activate(io=devnull); print(REPLMode.promptf())"`)
+        @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> "
+    end
+end
+
 end # module

test/resolve.jl

@@ -611,7 +611,6 @@ end
     )
     @test resolve_tst(deps_data, reqs_data, want_data)
 
-
     # require A, D, and lower version of Y
     reqs_data = Any[
         ["A", "*"],
@@ -632,6 +631,39 @@ end
     )
     @test resolve_tst(deps_data, reqs_data, want_data)
 
+
+    VERBOSE && @info("SCHEME 15")
+    ## DEPENDENCY SCHEME 15: A GRAPH WITH A WEAK DEPENDENCE
+    ## (REDUCED VERSION OF A REALISTIC SCHEME, ref Pkg.jl issue #4030)
+    deps_data = Any[
+        ["A", v"1"],
+        ["A", v"2", "C", "*"],
+        ["B", v"1", "D", "1", :weak],
+        ["C", v"1", "E", "*"],
+        ["C", v"2", "E", "*"],
+        ["C", v"2", "B", "1"],
+        ["E", v"1", "D", "1"],
+        ["E", v"2", "F", "1"],
+        ["F", v"1", "D", "*"],
+        ["D", v"1"],
+        ["D", v"2"],
+    ]
+
+    @test sanity_tst(deps_data)
+
+    reqs_data = Any[
+        ["A", "*"],
+    ]
+    want_data = Dict(
+        "A" => v"2",
+        "B" => v"1",
+        "C" => v"2",
+        "D" => v"1",
+        "E" => v"2",
+        "F" => v"1",
+    )
+    @test resolve_tst(deps_data, reqs_data, want_data)
+
 end
 
 @testset "realistic" begin