https://github.com/JuliaLang/julia
Tip revision: 2bc58db83ea4b78c2f03644a0408f0521a46f7f0 authored by Jameson Nash on 26 August 2015, 22:13:21 UTC
define summarysize for a tuple to be same as any other iterable and only include the summarysize of env in the count when at the toplevel
define summarysize for a tuple to be same as any other iterable and only include the summarysize of env in the count when at the toplevel
Tip revision: 2bc58db
loading.jl
# This file is a part of Julia. License is MIT: http://julialang.org/license
# Base.require is the implementation for the `import` statement
# `wd` is a working directory to search. defaults to current working directory.
# if `wd === nothing`, no extra path is searched.
function find_in_path(name::AbstractString, wd = pwd())
isabspath(name) && return name
base = name
if endswith(name,".jl")
base = name[1:end-3]
else
name = string(base,".jl")
end
if wd !== nothing
isfile(joinpath(wd,name)) && return joinpath(wd,name)
end
for prefix in [Pkg.dir(); LOAD_PATH]
path = joinpath(prefix, name)
isfile(path) && return abspath(path)
path = joinpath(prefix, base, "src", name)
isfile(path) && return abspath(path)
path = joinpath(prefix, name, "src", name)
isfile(path) && return abspath(path)
end
return nothing
end
function find_in_node_path(name, srcpath, node::Int=1)
if myid() == node
find_in_path(name, srcpath)
else
remotecall_fetch(node, find_in_path, name, srcpath)
end
end
function find_source_file(file)
(isabspath(file) || isfile(file)) && return file
file2 = find_in_path(file)
file2 !== nothing && return file2
file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
isfile(file2) ? file2 : nothing
end
function find_all_in_cache_path(mod::Symbol)
name = string(mod)
paths = AbstractString[]
for prefix in LOAD_CACHE_PATH
path = joinpath(prefix, name*".ji")
if isfile(path)
push!(paths, path)
end
end
paths
end
function _include_from_serialized(content::Vector{UInt8})
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content))
end
# returns an array of modules loaded, or nothing if failed
function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteString, toplevel_load::Bool)
restored = nothing
if toplevel_load && myid() == 1 && nprocs() > 1
recompile_stale(mod, path_to_try)
# broadcast top-level import/using from node 1 (only)
if node == myid()
content = open(readbytes, path_to_try)
else
content = remotecall_fetch(node, open, readbytes, path_to_try)
end
restored = _include_from_serialized(content)
if restored !== nothing
others = filter(x -> x != myid(), procs())
refs = Any[ @spawnat p (nothing !== _include_from_serialized(content)) for p in others]
for (id, ref) in zip(others, refs)
if !fetch(ref)
warn("node state is inconsistent: node $id failed to load cache from $path_to_try")
end
end
end
elseif node == myid()
myid() == 1 && recompile_stale(mod, path_to_try)
restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try)
else
content = remotecall_fetch(node, open, readbytes, path_to_try)
restored = _include_from_serialized(content)
end
# otherwise, continue search
if restored !== nothing
for M in restored
if isdefined(M, :__META__)
push!(Base.Docs.modules, M)
end
end
end
return restored
end
function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
if node == myid()
paths = find_all_in_cache_path(mod)
else
paths = @fetchfrom node find_all_in_cache_path(mod)
end
sort!(paths, by=mtime, rev=true) # try newest cachefiles first
for path_to_try in paths
restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
if restored === nothing
warn("deserialization checks failed while attempting to load cache from $path_to_try")
else
return restored
end
end
return nothing
end
# to synchronize multiple tasks trying to import/using something
const package_locks = Dict{Symbol,Condition}()
const package_loaded = Set{Symbol}()
# used to optionally track dependencies when requiring a module:
const _require_dependencies = Tuple{ByteString,Float64}[]
const _track_dependencies = [false]
function _include_dependency(_path::AbstractString)
prev = source_path(nothing)
path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path)
if myid() == 1 && _track_dependencies[1]
apath = abspath(path)
push!(_require_dependencies, (apath, mtime(apath)))
end
return path, prev
end
function include_dependency(path::AbstractString)
_include_dependency(path)
return nothing
end
# We throw PrecompilableError(true) when a module wants to be precompiled but isn't,
# and PrecompilableError(false) when a module doesn't want to be precompiled but is
immutable PrecompilableError <: Exception
isprecompilable::Bool
end
function show(io::IO, ex::PrecompilableError)
if ex.isprecompilable
print(io, "__precompile__(true) is only allowed in module files being imported")
else
print(io, "__precompile__(false) is not allowed in files that are being precompiled")
end
end
precompilableerror(ex::PrecompilableError, c) = ex.isprecompilable == c
precompilableerror(ex::WrappedException, c) = precompilableerror(ex.error, c)
precompilableerror(ex, c) = false
# Call __precompile__ at the top of a file to force it to be precompiled (true), or
# to be prevent it from being precompiled (false). __precompile__(true) is
# ignored except within "require" call.
function __precompile__(isprecompilable::Bool=true)
if myid() == 1 && isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) &&
!(isprecompilable && toplevel_load::Bool)
throw(PrecompilableError(isprecompilable))
end
end
# require always works in Main scope and loads files from node 1
toplevel_load = true
function require(mod::Symbol)
# dependency-tracking is only used for one top-level include(path),
# and is not applied recursively to imported modules:
old_track_dependencies = _track_dependencies[1]
_track_dependencies[1] = false
global toplevel_load
loading = get(package_locks, mod, false)
if loading !== false
# load already in progress for this module
wait(loading)
return
end
package_locks[mod] = Condition()
last = toplevel_load::Bool
try
toplevel_load = false
if nothing !== _require_from_serialized(1, mod, last)
return
end
if JLOptions().incremental != 0
# spawn off a new incremental precompile task from node 1 for recursive `require` calls
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
warn("require failed to create a precompiled cache file")
end
return
end
name = string(mod)
path = find_in_node_path(name, nothing, 1)
path === nothing && throw(ArgumentError("$name not found in path"))
try
if last && myid() == 1 && nprocs() > 1
# include on node 1 first to check for PrecompilableErrors
eval(Main, :(Base.include_from_node1($path)))
# broadcast top-level import/using from node 1 (only)
refs = Any[ @spawnat p eval(Main, :(Base.include_from_node1($path))) for p in filter(x -> x != 1, procs()) ]
for r in refs; wait(r); end
else
eval(Main, :(Base.include_from_node1($path)))
end
catch ex
if !precompilableerror(ex, true)
rethrow() # rethrow non-precompilable=true errors
end
isinteractive() && info("Precompiling module $mod...")
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
error("__precompile__(true) but require failed to create a precompiled cache file")
end
end
finally
toplevel_load = last
loading = pop!(package_locks, mod)
notify(loading, all=true)
_track_dependencies[1] = old_track_dependencies
end
nothing
end
# remote/parallel load
include_string(txt::ByteString, fname::ByteString) =
ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t),
txt, sizeof(txt), fname, sizeof(fname))
include_string(txt::AbstractString, fname::AbstractString) = include_string(bytestring(txt), bytestring(fname))
include_string(txt::AbstractString) = include_string(txt, "string")
function source_path(default::Union{AbstractString,Void}="")
t = current_task()
while true
s = t.storage
if !is(s, nothing) && haskey(s, :SOURCE_PATH)
return s[:SOURCE_PATH]
end
if is(t, t.parent)
return default
end
t = t.parent
end
end
function source_dir()
p = source_path(nothing)
p === nothing ? p : dirname(p)
end
macro __FILE__() source_path() end
function include_from_node1(_path::AbstractString)
path, prev = _include_dependency(_path)
tls = task_local_storage()
tls[:SOURCE_PATH] = path
local result
try
if myid()==1
# sleep a bit to process file requests from other nodes
nprocs()>1 && sleep(0.005)
result = Core.include(path)
nprocs()>1 && sleep(0.005)
else
result = include_string(remotecall_fetch(1, readall, path), path)
end
finally
if prev === nothing
delete!(tls, :SOURCE_PATH)
else
tls[:SOURCE_PATH] = prev
end
end
result
end
function evalfile(path::AbstractString, args::Vector{UTF8String}=UTF8String[])
return eval(Module(:__anon__),
Expr(:toplevel,
:(const ARGS = $args),
:(eval(x) = Main.Core.eval(__anon__,x)),
:(eval(m,x) = Main.Core.eval(m,x)),
:(Main.Base.include($path))))
end
evalfile(path::AbstractString, args::Vector) = evalfile(path, UTF8String[args...])
function create_expr_cache(input::AbstractString, output::AbstractString)
isfile(output) && rm(output)
code_object = """
while !eof(STDIN)
eval(Main, deserialize(STDIN))
end
"""
io, pobj = open(detach(`$(julia_cmd())
--output-ji $output --output-incremental=yes
--startup-file=no --history-file=no
--eval $code_object`), "w", STDOUT)
serialize(io, quote
empty!(Base.LOAD_PATH)
append!(Base.LOAD_PATH, $LOAD_PATH)
empty!(Base.LOAD_CACHE_PATH)
append!(Base.LOAD_CACHE_PATH, $LOAD_CACHE_PATH)
empty!(Base.DL_LOAD_PATH)
append!(Base.DL_LOAD_PATH, $DL_LOAD_PATH)
end)
source = source_path(nothing)
if source !== nothing
serialize(io, quote
task_local_storage()[:SOURCE_PATH] = $(source)
end)
end
serialize(io, :(Base._track_dependencies[1] = true))
serialize(io, :(Base.include($(abspath(input)))))
if source !== nothing
serialize(io, quote
delete!(task_local_storage(), :SOURCE_PATH)
end)
end
close(io)
wait(pobj)
return pobj
end
compilecache(mod::Symbol) = compilecache(string(mod))
function compilecache(name::ByteString)
myid() == 1 || error("can only precompile from node 1")
path = find_in_path(name, nothing)
path === nothing && throw(ArgumentError("$name not found in path"))
cachepath = LOAD_CACHE_PATH[1]
if !isdir(cachepath)
mkpath(cachepath)
end
cachefile = abspath(cachepath, name*".ji")
if !success(create_expr_cache(path, cachefile))
error("Failed to precompile $name to $cachefile")
end
return cachefile
end
module_uuid(m::Module) = ccall(:jl_module_uuid, UInt64, (Any,), m)
isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_deserialize_verify_header, Cint, (Ptr{Void},), f.ios)
function cache_dependencies(f::IO)
modules = Tuple{Symbol,UInt64}[]
files = Tuple{ByteString,Float64}[]
while true
n = ntoh(read(f, Int32))
n == 0 && break
push!(modules,
(symbol(readbytes(f, n)), # module symbol
ntoh(read(f, UInt64)))) # module UUID (timestamp)
end
read(f, Int64) # total bytes for file dependencies
while true
n = ntoh(read(f, Int32))
n == 0 && break
push!(files, (bytestring(readbytes(f, n)), ntoh(read(f, Float64))))
end
return modules, files
end
function cache_dependencies(cachefile::AbstractString)
io = open(cachefile, "r")
try
!isvalid_cache_header(io) && throw(ArgumentError("invalid cache file $cachefile"))
return cache_dependencies(io)
finally
close(io)
end
end
function stale_cachefile(modpath, cachefile)
io = open(cachefile, "r")
try
if !isvalid_cache_header(io)
return true # invalid cache file
end
modules, files = cache_dependencies(io)
if files[1][1] != modpath
return true # cache file was compiled from a different path
end
for (f,ftime) in files
if mtime(f) != ftime
return true
end
end
# files are not stale, so module list is valid and needs checking
for (M,uuid) in modules
if !isdefined(Main, M)
require(M) # should recursively recompile module M if stale
end
if module_uuid(Main.(M)) != uuid
return true
end
end
return false # fresh cachefile
finally
close(io)
end
end
function recompile_stale(mod, cachefile)
path = find_in_path(string(mod), nothing)
if path === nothing
rm(cachefile)
error("module $mod not found in current path; removed orphaned cache file $cachefile")
end
if stale_cachefile(path, cachefile)
info("Recompiling stale cache file $cachefile for module $mod.")
if !success(create_expr_cache(path, cachefile))
error("Failed to precompile $mod to $cachefile")
end
end
end