From 5c2dba19d21796d756b80517cac752ea183d7371 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Tue, 23 Sep 2025 14:12:07 +0200 Subject: [PATCH 01/10] Change delayed delete mechanism to prevent cross-drive mv failure for in-use DLL --- base/file.jl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/base/file.jl b/base/file.jl index 24a21396721d3..6ba5f30082d97 100644 --- a/base/file.jl +++ b/base/file.jl @@ -255,7 +255,7 @@ end # Files that were requested to be deleted but can't be by the current process # i.e. loaded DLLs on Windows -delayed_delete_dir() = joinpath(tempdir(), "julia_delayed_deletes") +delayed_delete_list() = joinpath(tempdir(), "julia_delayed_deletes.txt") """ rm(path::AbstractString; force::Bool=false, recursive::Bool=false) @@ -278,8 +278,7 @@ Stacktrace: [...] ``` """ -function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allow_delayed_delete::Bool=true) - # allow_delayed_delete is used by Pkg.gc() but is otherwise not part of the public API +function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) if islink(path) || !isdir(path) try unlink(path) @@ -287,14 +286,16 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allo if isa(err, IOError) force && err.code==Base.UV_ENOENT && return @static if Sys.iswindows() - if allow_delayed_delete && err.code==Base.UV_EACCES && endswith(path, ".dll") + if err.code==Base.UV_EACCES && endswith(path, ".dll") # Loaded DLLs cannot be deleted on Windows, even with posix delete mode - # but they can be moved. So move out to allow the dir to be deleted. - # Pkg.gc() cleans up this dir when possible - dir = mkpath(delayed_delete_dir()) - temp_path = tempname(dir, cleanup = false, suffix = string("_", basename(path))) - @debug "Could not delete DLL most likely because it is loaded, moving to tempdir" path temp_path - mv(path, temp_path) + # but they can be renamed. Do so temporarily, until later cleanup by Pkg.gc() + temp_path = tempname(dirname(path), cleanup = false, suffix = string("_", basename(path))) + # ensure that temp_path is on the same drive as path to avoid issue #59589 + @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path + Base.open(delayed_delete_list(), "a") do io + println(io, temp_path) # record the temporary path for Pkg.gc() + end + rename(path, temp_path) # do not call mv which could recursively call rm(path) return end end From a849be7ca6b6b4fcb6d7dd8d22d67c8abbe31f43 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Tue, 23 Sep 2025 15:04:55 +0200 Subject: [PATCH 02/10] Record the absolute path --- base/file.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/file.jl b/base/file.jl index 6ba5f30082d97..b1c5a1a54fa7b 100644 --- a/base/file.jl +++ b/base/file.jl @@ -293,7 +293,7 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) # ensure that temp_path is on the same drive as path to avoid issue #59589 @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path Base.open(delayed_delete_list(), "a") do io - println(io, temp_path) # record the temporary path for Pkg.gc() + println(io, abspath(temp_path)) # record the temporary path for Pkg.gc() end rename(path, temp_path) # do not call mv which could recursively call rm(path) return From 8e14282ab403b6cdf35d0821b275d00035482b89 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Wed, 24 Sep 2025 13:55:18 +0200 Subject: [PATCH 03/10] Include review comments and move obsolete DLLs out of depot --- base/file.jl | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/base/file.jl b/base/file.jl index b1c5a1a54fa7b..6a60a1d07d039 100644 --- a/base/file.jl +++ b/base/file.jl @@ -253,9 +253,9 @@ function mkpath(path::AbstractString; mode::Integer = 0o777) return path end -# Files that were requested to be deleted but can't be by the current process -# i.e. loaded DLLs on Windows -delayed_delete_list() = joinpath(tempdir(), "julia_delayed_deletes.txt") +# Files that were requested to be deleted but can't be by the current process, +# i.e. loaded DLLs on Windows, are listed in the directory below +delayed_delete_ref() = joinpath(tempdir(), "julia_delayed_deletes_ref") """ rm(path::AbstractString; force::Bool=false, recursive::Bool=false) @@ -278,7 +278,8 @@ Stacktrace: [...] ``` """ -function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) +function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allow_delayed_delete::Bool=true) + # allow_delayed_delete is used by Pkg.gc() but is otherwise not part of the public API if islink(path) || !isdir(path) try unlink(path) @@ -286,16 +287,8 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) if isa(err, IOError) force && err.code==Base.UV_ENOENT && return @static if Sys.iswindows() - if err.code==Base.UV_EACCES && endswith(path, ".dll") - # Loaded DLLs cannot be deleted on Windows, even with posix delete mode - # but they can be renamed. Do so temporarily, until later cleanup by Pkg.gc() - temp_path = tempname(dirname(path), cleanup = false, suffix = string("_", basename(path))) - # ensure that temp_path is on the same drive as path to avoid issue #59589 - @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path - Base.open(delayed_delete_list(), "a") do io - println(io, abspath(temp_path)) # record the temporary path for Pkg.gc() - end - rename(path, temp_path) # do not call mv which could recursively call rm(path) + if allow_delayed_delete && err.code==Base.UV_EACCES && endswith(path, ".dll") + delayed_delete_dll(path) return end end @@ -331,6 +324,27 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) end +# Loaded DLLs cannot be deleted on Windows, even with posix delete mode but they can be renamed. +# delayed_delete_dll(path) does so temporarily, until later cleanup by Pkg.gc(). +function delayed_delete_dll(path) + drive = first(splitdrive(path)) + tmpdrive = first(splitdrive(tempdir())) + # in-use DLL must be kept on the same drive + deletedir = if drive == tmpdrive + joinpath(tempdir(), "julia_delayed_deletes") + else + joinpath(drive, "julia_delayed_deletes") + end + mkpath(deletedir) + temp_path = tempname(deletedir; cleanup=false, suffix=string("_", basename(path))) + @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path + mkpath(delayed_delete_ref()) + io = last(mktemp(delayed_delete_ref(); cleanup=false)) + println(io, abspath(temp_path)) # record the temporary path for Pkg.gc() + close(io) + rename(path, temp_path) # do not call mv which could recursively call rm(path) +end + # The following use Unix command line facilities function checkfor_mv_cp_cptree(src::AbstractString, dst::AbstractString, txt::AbstractString; force::Bool=false) From 7f39af570f8678a682319eea07152ea1e23fd18a Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Wed, 24 Sep 2025 22:25:19 +0200 Subject: [PATCH 04/10] Review comment Co-authored-by: Jameson Nash --- base/file.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/file.jl b/base/file.jl index 6a60a1d07d039..ef53de36b5f43 100644 --- a/base/file.jl +++ b/base/file.jl @@ -340,7 +340,7 @@ function delayed_delete_dll(path) @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path mkpath(delayed_delete_ref()) io = last(mktemp(delayed_delete_ref(); cleanup=false)) - println(io, abspath(temp_path)) # record the temporary path for Pkg.gc() + print(io, temp_path) # record the temporary path for Pkg.gc() close(io) rename(path, temp_path) # do not call mv which could recursively call rm(path) end From 4390aaf1cb66d8235002b72c4b700d40ed655bfb Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Fri, 26 Sep 2025 16:11:53 +0200 Subject: [PATCH 05/10] Keep DLLs in the same folder --- base/file.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/base/file.jl b/base/file.jl index ef53de36b5f43..da49364f498c0 100644 --- a/base/file.jl +++ b/base/file.jl @@ -330,13 +330,7 @@ function delayed_delete_dll(path) drive = first(splitdrive(path)) tmpdrive = first(splitdrive(tempdir())) # in-use DLL must be kept on the same drive - deletedir = if drive == tmpdrive - joinpath(tempdir(), "julia_delayed_deletes") - else - joinpath(drive, "julia_delayed_deletes") - end - mkpath(deletedir) - temp_path = tempname(deletedir; cleanup=false, suffix=string("_", basename(path))) + temp_path = tempname(abspath(dirname(path)); cleanup=false, suffix=string("_", basename(path))) @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path mkpath(delayed_delete_ref()) io = last(mktemp(delayed_delete_ref(); cleanup=false)) From 253588b558cf7438941263792eb5dc532967ff3b Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Fri, 26 Sep 2025 16:50:00 +0200 Subject: [PATCH 06/10] Adapt tests to prevent removal of depots with in-use DLLs --- Compiler/test/special_loading.jl | 3 +- test/compileall.jl | 4 ++- test/core.jl | 9 ++--- test/loading.jl | 51 +++++++++++----------------- test/precompile.jl | 10 ++---- test/precompile_utils.jl | 18 +++------- test/relocatedepot.jl | 13 +++---- test/tempdepot.jl | 58 ++++++++++++++++++++++++++++++++ 8 files changed, 99 insertions(+), 67 deletions(-) create mode 100755 test/tempdepot.jl diff --git a/Compiler/test/special_loading.jl b/Compiler/test/special_loading.jl index ba8cbc635eae8..ca29618a44d17 100644 --- a/Compiler/test/special_loading.jl +++ b/Compiler/test/special_loading.jl @@ -2,7 +2,8 @@ # Only run when testing Base compiler if Base.identify_package("Compiler") === nothing - mktempdir() do dir + include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "test", "tempdepot.jl")) + mkdepottempdir() do dir withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do cd(joinpath(@__DIR__, "CompilerLoadingTest")) do @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) diff --git a/test/compileall.jl b/test/compileall.jl index 1987bda7f04df..726cadbba68c8 100644 --- a/test/compileall.jl +++ b/test/compileall.jl @@ -2,8 +2,10 @@ # We make it a separate test target here, so that it can run in parallel # with the rest of the tests. +include("tempdepot.jl") + function precompile_test_harness(@nospecialize(f)) - load_path = mktempdir() + load_path = mkdepottempdir() try pushfirst!(LOAD_PATH, load_path) pushfirst!(DEPOT_PATH, load_path) diff --git a/test/core.jl b/test/core.jl index b4774b6dc7608..5e8acf5740f34 100644 --- a/test/core.jl +++ b/test/core.jl @@ -9,6 +9,8 @@ const Bottom = Union{} # For curmod_* include("testenv.jl") +include("tempdepot.jl") + ## tests that `const` field declarations # sanity tests that our built-in types are marked correctly for const fields @@ -8443,7 +8445,7 @@ end # precompilation let load_path = mktempdir() - depot_path = mktempdir() + depot_path = mkdepottempdir() try pushfirst!(LOAD_PATH, load_path) pushfirst!(DEPOT_PATH, depot_path) @@ -8485,11 +8487,6 @@ let load_path = mktempdir() filter!((≠)(load_path), LOAD_PATH) filter!((≠)(depot_path), DEPOT_PATH) rm(load_path, recursive=true, force=true) - try - rm(depot_path, force=true, recursive=true) - catch err - @show err - end end end diff --git a/test/loading.jl b/test/loading.jl index 738aa7d1419d8..8ccf24bcf1fe1 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -34,6 +34,7 @@ end @test @nested_LINE_expansion2() == ((@__LINE__() - 5, @__LINE__() - 9), @__LINE__()) original_depot_path = copy(Base.DEPOT_PATH) +include("tempdepot.jl") include("precompile_utils.jl") loaded_files = String[] @@ -226,7 +227,6 @@ end end end - ## functional testing of package identification, location & loading ## saved_load_path = copy(LOAD_PATH) @@ -236,8 +236,9 @@ watcher_counter = Ref(0) push!(Base.active_project_callbacks, () -> watcher_counter[] += 1) push!(Base.active_project_callbacks, () -> error("broken")) +const testdefaultdepot = mkdepottempdir() push!(empty!(LOAD_PATH), joinpath(@__DIR__, "project")) -append!(empty!(DEPOT_PATH), [mktempdir(), joinpath(@__DIR__, "depot")]) +append!(empty!(DEPOT_PATH), [testdefaultdepot, joinpath(@__DIR__, "depot")]) @test watcher_counter[] == 0 @test_logs (:error, r"active project callback .* failed") Base.set_active_project(nothing) @test watcher_counter[] == 1 @@ -461,7 +462,7 @@ function make_env(flat, root, roots, graph, paths, dummies) ) end -const depots = [mktempdir() for _ = 1:3] +const depots = [mkdepottempdir() for _ = 1:3] const envs = Dict{String,Any}() append!(empty!(DEPOT_PATH), depots) @@ -755,13 +756,6 @@ end for env in keys(envs) rm(env, force=true, recursive=true) end -for depot in depots - try - rm(depot, force=true, recursive=true) - catch err - @show err - end -end append!(empty!(LOAD_PATH), saved_load_path) append!(empty!(DEPOT_PATH), saved_depot_path) @@ -1043,9 +1037,10 @@ end _pkgversion == pkgversion(parent) || error("unexpected extension \$ext version: \$_pkgversion") end """ - depot_path = mktempdir() - try - proj = joinpath(@__DIR__, "project", "Extensions", "HasDepWithExtensions.jl") + depot_path = mkdepottempdir() + proj = joinpath(@__DIR__, "project", "Extensions", "HasDepWithExtensions.jl") + + begin function gen_extension_cmd(compile, distr=false) load_distr = distr ? "using Distributed; addprocs(1)" : "" @@ -1155,7 +1150,7 @@ end # Extension-to-extension dependencies - mktempdir() do depot # Parallel pre-compilation + mkdepottempdir() do depot # Parallel pre-compilation code = """ Base.disable_parallel_precompile = false using ExtToExtDependency @@ -1171,7 +1166,7 @@ end ) @test occursin("Hello ext-to-ext!", String(read(cmd))) end - mktempdir() do depot # Serial pre-compilation + mkdepottempdir() do depot # Serial pre-compilation code = """ Base.disable_parallel_precompile = true using ExtToExtDependency @@ -1188,7 +1183,7 @@ end @test occursin("Hello ext-to-ext!", String(read(cmd))) end - mktempdir() do depot # Parallel pre-compilation + mkdepottempdir() do depot # Parallel pre-compilation code = """ Base.disable_parallel_precompile = false using CrossPackageExtToExtDependency @@ -1204,7 +1199,7 @@ end ) @test occursin("Hello x-package ext-to-ext!", String(read(cmd))) end - mktempdir() do depot # Serial pre-compilation + mkdepottempdir() do depot # Serial pre-compilation code = """ Base.disable_parallel_precompile = true using CrossPackageExtToExtDependency @@ -1224,7 +1219,7 @@ end # Extensions for "parent" dependencies # (i.e. an `ExtAB` where A depends on / loads B, but B provides the extension) - mktempdir() do depot # Parallel pre-compilation + mkdepottempdir() do depot # Parallel pre-compilation code = """ Base.disable_parallel_precompile = false using Parent @@ -1239,7 +1234,7 @@ end ) @test occursin("Hello parent!", String(read(cmd))) end - mktempdir() do depot # Serial pre-compilation + mkdepottempdir() do depot # Serial pre-compilation code = """ Base.disable_parallel_precompile = true using Parent @@ -1254,13 +1249,6 @@ end ) @test occursin("Hello parent!", String(read(cmd))) end - - finally - try - rm(depot_path, force=true, recursive=true) - catch err - @show err - end end end @@ -1364,7 +1352,7 @@ end end @testset "relocatable upgrades #51989" begin - mktempdir() do depot + mkdepottempdir() do depot # realpath is needed because Pkg is used for one of the precompile paths below, and Pkg calls realpath on the # project path so the cache file slug will be different if the tempdir is given as a symlink # (which it often is on MacOS) which would break the test. @@ -1438,7 +1426,7 @@ end end @testset "code coverage disabled during precompilation" begin - mktempdir() do depot + mkdepottempdir() do depot cov_test_dir = joinpath(@__DIR__, "project", "deps", "CovTest.jl") cov_cache_dir = joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "CovTest") function rm_cov_files() @@ -1482,7 +1470,7 @@ end end @testset "command-line flags" begin - mktempdir() do depot_path mktempdir() do dir + mkdepottempdir() do depot_path mktempdir() do dir # generate a Parent.jl and Child.jl package, with Parent depending on Child open(joinpath(dir, "Child.jl"), "w") do io println(io, """ @@ -1565,7 +1553,7 @@ end end @testset "including non-existent file throws proper error #52462" begin - mktempdir() do depot + mkdepottempdir() do depot project_path = joinpath(depot, "project") mkpath(project_path) @@ -1707,7 +1695,7 @@ end end @testset "require_stdlib loading duplication" begin - depot_path = mktempdir() + depot_path = mkdepottempdir() oldBase64 = nothing try push!(empty!(DEPOT_PATH), depot_path) @@ -1731,7 +1719,6 @@ end finally oldBase64 === nothing || Base.register_root_module(oldBase64) copy!(DEPOT_PATH, original_depot_path) - rm(depot_path, force=true, recursive=true) end end diff --git a/test/precompile.jl b/test/precompile.jl index 12e35210449ed..8e0b067b66afa 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -4,6 +4,7 @@ using Test, Distributed, Random, Logging, Libdl using REPL # testing the doc lookup function should be outside of the scope of this file, but is currently tested here include("precompile_utils.jl") +include("tempdepot.jl") Foo_module = :Foo4b3a94a1a081a8cb foo_incl_dep = :foo4b3a94a1a081a8cb @@ -1524,11 +1525,11 @@ end test_workers = addprocs(1) push!(test_workers, myid()) save_cwd = pwd() - temp_path = mktempdir() + temp_path = mkdepottempdir() try cd(temp_path) load_path = mktempdir(temp_path) - load_cache_path = mktempdir(temp_path) + load_cache_path = mkdepottempdir(temp_path) ModuleA = :Issue19960A ModuleB = :Issue19960B @@ -1576,11 +1577,6 @@ end end finally cd(save_cwd) - try - rm(temp_path, recursive=true) - catch err - @show err - end pop!(test_workers) # remove myid rmprocs(test_workers) end diff --git a/test/precompile_utils.jl b/test/precompile_utils.jl index 55eba353f2ada..c9a7c98d262e0 100644 --- a/test/precompile_utils.jl +++ b/test/precompile_utils.jl @@ -1,28 +1,18 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +include("tempdepot.jl") + function precompile_test_harness(@nospecialize(f), testset::String) @testset "$testset" precompile_test_harness(f, true) end function precompile_test_harness(@nospecialize(f), separate::Bool=true) - load_path = mktempdir() - load_cache_path = separate ? mktempdir() : load_path + load_path = mkdepottempdir() + load_cache_path = separate ? mkdepottempdir() : load_path try pushfirst!(LOAD_PATH, load_path) pushfirst!(DEPOT_PATH, load_cache_path) f(load_path) finally - try - rm(load_path, force=true, recursive=true) - catch err - @show err - end - if separate - try - rm(load_cache_path, force=true, recursive=true) - catch err - @show err - end - end filter!((≠)(load_path), LOAD_PATH) separate && filter!((≠)(load_cache_path), DEPOT_PATH) end diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index 2ef6dec90dbc1..e8758365e3ff4 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -4,6 +4,7 @@ using Test include("testenv.jl") +include("tempdepot.jl") function test_harness(@nospecialize(fn); empty_load_path=true, empty_depot_path=true) @@ -32,7 +33,7 @@ if !test_relocated_depot # insert @depot only once for first match test_harness() do - mktempdir() do dir + mkdepottempdir() do dir pushfirst!(DEPOT_PATH, dir) if Sys.iswindows() # dirs start with a drive letter instead of a path separator @@ -46,7 +47,7 @@ if !test_relocated_depot # 55340 empty!(DEPOT_PATH) - mktempdir() do dir + mkdepottempdir() do dir jlrc = joinpath(dir, "julia-rc2") jl = joinpath(dir, "julia") mkdir(jl) @@ -61,7 +62,7 @@ if !test_relocated_depot # deal with and without trailing path separators test_harness() do - mktempdir() do dir + mkdepottempdir() do dir pushfirst!(DEPOT_PATH, dir) path = joinpath(dir, "foo") if isdirpath(DEPOT_PATH[1]) @@ -176,7 +177,7 @@ if !test_relocated_depot # add them as include_dependency()s to a new pkg Foo, which will be precompiled into depot3. # After loading the include_dependency()s of Foo should refer to depot1 depot2 each. test_harness() do - mktempdir() do depot1 + mkdepottempdir() do depot1 # precompile Example in depot1 example1_root = joinpath(depot1, "Example1") mkpath(joinpath(example1_root, "src")) @@ -196,7 +197,7 @@ if !test_relocated_depot end pushfirst!(LOAD_PATH, depot1); pushfirst!(DEPOT_PATH, depot1) pkg = Base.identify_package("Example1"); Base.require(pkg) - mktempdir() do depot2 + mkdepottempdir() do depot2 # precompile Example in depot2 example2_root = joinpath(depot2, "Example2") mkpath(joinpath(example2_root, "src")) @@ -216,7 +217,7 @@ if !test_relocated_depot end pushfirst!(LOAD_PATH, depot2); pushfirst!(DEPOT_PATH, depot2) pkg = Base.identify_package("Example2"); Base.require(pkg) - mktempdir() do depot3 + mkdepottempdir() do depot3 # precompile Foo in depot3 open(joinpath(depot3, "Module52161.jl"), write=true) do io println(io, """ diff --git a/test/tempdepot.jl b/test/tempdepot.jl new file mode 100755 index 0000000000000..fc9eaaa0d2450 --- /dev/null +++ b/test/tempdepot.jl @@ -0,0 +1,58 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# This includes the `mkdepottempdir` and `rmdepot` functions, used to +# respectively create and remove temporary depots to use in tests. +# `mktempdir` and `rm` cannot be used because, on Windows, the DLLs generated by +# precompilation in the depots cannot be removed by the program that uses them. +# This file can be included multiple times in the same module if necessary, +# which can happen with unisolated test runs. + +if Sys.iswindows() && !@isdefined(DEPOTS_TOREMOVE) + const DEPOTS_TOREMOVE = String[] + atexit() do # launch a process that will rm the depots + rmcmd = """ + for _ in 1:3600 # wait up to 1h from atexit() until the julia testing process dies + sleep(1) + ccall(:uv_kill, Cint, (Cuint, Cint), $(getpid()), 0) == Base.UV_ESRCH && break + end + for path in $DEPOTS_TOREMOVE + try + rm(path, force=true, recursive=true) + catch + end + end + """ + cmd = Cmd(`$(Base.julia_cmd()) --startup-file=no -e $rmcmd`; ignorestatus=true, detach=true) + run(cmd; wait=false) + end +end + +function rmdepot(depot) + try + @static if Sys.iswindows() # on Windows, delay the rm + push!(DEPOTS_TOREMOVE, depot) + else # on the other systems, do it immediately + rm(depot, force=true, recursive=true) + end + catch err + @show err + end +end + +function mkdepottempdir(f::Function, parent=tempdir(); prefix="jltestdepot_") + tmpdir = mktempdir(parent; prefix, cleanup=false) + try + f(tmpdir) + finally + rmdepot(tmpdir) + end +end +function mkdepottempdir(parent=tempdir(); prefix="jltestdepot_", cleanup=true) + @static if Sys.iswindows() + tmpdir = mktempdir(parent; prefix, cleanup=false) + cleanup && push!(DEPOTS_TOREMOVE, tmpdir) + tmpdir + else + mktempdir(parent; prefix, cleanup) + end +end From ec6b7abf1447508ed0b12a9f8a522b82fde2838b Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Sun, 28 Sep 2025 23:23:24 +0200 Subject: [PATCH 07/10] More review comments, add temp_cleanup_postprocess Co-authored-by: Jameson Nash --- base/file.jl | 37 ++++++++++++++++++++++++++++++++++--- test/tempdepot.jl | 17 +---------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/base/file.jl b/base/file.jl index da49364f498c0..46677dc0c7ed4 100644 --- a/base/file.jl +++ b/base/file.jl @@ -327,8 +327,6 @@ end # Loaded DLLs cannot be deleted on Windows, even with posix delete mode but they can be renamed. # delayed_delete_dll(path) does so temporarily, until later cleanup by Pkg.gc(). function delayed_delete_dll(path) - drive = first(splitdrive(path)) - tmpdrive = first(splitdrive(tempdir())) # in-use DLL must be kept on the same drive temp_path = tempname(abspath(dirname(path)); cleanup=false, suffix=string("_", basename(path))) @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path @@ -687,8 +685,41 @@ end # deprecated internal function used by some packages temp_cleanup_purge(; force=false) = force ? temp_cleanup_purge_all() : @lock TEMP_CLEANUP_LOCK temp_cleanup_purge_prelocked(false) +function temp_cleanup_postprocess(cleanup_dirs) + if !isempty(cleanup_dirs) + rmcmd = """ + eof(stdin) + for path in eachline(stdin) + try + rm(path, force=true, recursive=true) + catch ex + @warn "Failed to clean up temporary path \$(repr(path))\n\$ex" _group=:file + end + end + """ + cmd = Cmd(`$(Base.julia_cmd()) --startup-file=no -e $rmcmd`; ignorestatus=true, detach=true) + pw = Base.PipeEndpoint() + rd, wr = Base.link_pipe(true, true) + try + Base.open_pipe!(pw, wr) + catch + Base.close_pipe_sync(wr) + rethrow() + end + run(cmd, rd, devnull, stderr; wait=false) + join(pw, cleanup_dirs, "\n") + Base.dup(wr) # intentionally leak a reference, until the process exits + close(pw) + end +end + +function temp_cleanup_atexit() + temp_cleanup_purge_all() + temp_cleanup_postprocess(keys(TEMP_CLEANUP)) +end + function __postinit__() - Base.atexit(temp_cleanup_purge_all) + Base.atexit(temp_cleanup_atexit) end const temp_prefix = "jl_" diff --git a/test/tempdepot.jl b/test/tempdepot.jl index fc9eaaa0d2450..18fad0cf15346 100755 --- a/test/tempdepot.jl +++ b/test/tempdepot.jl @@ -9,22 +9,7 @@ if Sys.iswindows() && !@isdefined(DEPOTS_TOREMOVE) const DEPOTS_TOREMOVE = String[] - atexit() do # launch a process that will rm the depots - rmcmd = """ - for _ in 1:3600 # wait up to 1h from atexit() until the julia testing process dies - sleep(1) - ccall(:uv_kill, Cint, (Cuint, Cint), $(getpid()), 0) == Base.UV_ESRCH && break - end - for path in $DEPOTS_TOREMOVE - try - rm(path, force=true, recursive=true) - catch - end - end - """ - cmd = Cmd(`$(Base.julia_cmd()) --startup-file=no -e $rmcmd`; ignorestatus=true, detach=true) - run(cmd; wait=false) - end + atexit(() -> Base.Filesystem.temp_cleanup_postprocess(DEPOTS_TOREMOVE)) end function rmdepot(depot) From 13e0b92153ce410444ae937888ce81c9c10ca7e3 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Sun, 28 Sep 2025 23:31:12 +0200 Subject: [PATCH 08/10] No macro form for bootstrap? --- base/file.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/file.jl b/base/file.jl index 46677dc0c7ed4..b6c05f3847b06 100644 --- a/base/file.jl +++ b/base/file.jl @@ -697,7 +697,7 @@ function temp_cleanup_postprocess(cleanup_dirs) end end """ - cmd = Cmd(`$(Base.julia_cmd()) --startup-file=no -e $rmcmd`; ignorestatus=true, detach=true) + cmd = Cmd(Base.cmd_gen(((Base.julia_cmd(),), ("--startup-file=no",), ("-e",), (rmcmd,))); ignorestatus = true, detach = true) pw = Base.PipeEndpoint() rd, wr = Base.link_pipe(true, true) try From b9018d7e81e31554d4945ff14c858f3ae28d974a Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Mon, 29 Sep 2025 22:23:08 +0200 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Jameson Nash --- base/file.jl | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/base/file.jl b/base/file.jl index b6c05f3847b06..b567646e57d66 100644 --- a/base/file.jl +++ b/base/file.jl @@ -688,8 +688,9 @@ temp_cleanup_purge(; force=false) = force ? temp_cleanup_purge_all() : @lock TEM function temp_cleanup_postprocess(cleanup_dirs) if !isempty(cleanup_dirs) rmcmd = """ - eof(stdin) - for path in eachline(stdin) + cleanuplist = readlines(stdin) # This loop won't start running until stdin is closed, which is supposed to be sequenced after the process exits + sleep(1) # Wait for the operating system to hopefully be ready, since the OS implementation is probably incorrect, given the history of buggy work-arounds like this that have existed for ages in dotNet and libuv + for path in cleanuplist try rm(path, force=true, recursive=true) catch ex @@ -699,23 +700,16 @@ function temp_cleanup_postprocess(cleanup_dirs) """ cmd = Cmd(Base.cmd_gen(((Base.julia_cmd(),), ("--startup-file=no",), ("-e",), (rmcmd,))); ignorestatus = true, detach = true) pw = Base.PipeEndpoint() - rd, wr = Base.link_pipe(true, true) - try - Base.open_pipe!(pw, wr) - catch - Base.close_pipe_sync(wr) - rethrow() - end - run(cmd, rd, devnull, stderr; wait=false) + run(cmd, pw, devnull, stderr; wait=false) join(pw, cleanup_dirs, "\n") - Base.dup(wr) # intentionally leak a reference, until the process exits + Base.dup(Base._fd(pw)) # intentionally leak a reference, until the process exits close(pw) end end function temp_cleanup_atexit() temp_cleanup_purge_all() - temp_cleanup_postprocess(keys(TEMP_CLEANUP)) + @lock TEMP_CLEANUP_LOCK temp_cleanup_postprocess(keys(TEMP_CLEANUP)) end function __postinit__() From 542e4651febe2a4d3e8c1a7bc6e5b69e5fbd1e00 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Tue, 30 Sep 2025 10:13:52 +0200 Subject: [PATCH 10/10] Prevent leaking file handles in case of error Co-authored-by: Elliot Saba --- base/file.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/base/file.jl b/base/file.jl index b567646e57d66..efc642b099b6a 100644 --- a/base/file.jl +++ b/base/file.jl @@ -332,8 +332,11 @@ function delayed_delete_dll(path) @debug "Could not delete DLL most likely because it is loaded, moving to a temporary path" path temp_path mkpath(delayed_delete_ref()) io = last(mktemp(delayed_delete_ref(); cleanup=false)) - print(io, temp_path) # record the temporary path for Pkg.gc() - close(io) + try + print(io, temp_path) # record the temporary path for Pkg.gc() + finally + close(io) + end rename(path, temp_path) # do not call mv which could recursively call rm(path) end