https://github.com/JuliaLang/julia
Revision 6b41538bef339dba09d80568c0065fa236c89b5a authored by Jeff Bezanson on 29 July 2016, 20:22:57 UTC, committed by GitHub on 29 July 2016, 20:22:57 UTC
ensure known_object_data is assigned before deserialize is called again
Tip revision: 6b41538bef339dba09d80568c0065fa236c89b5a authored by Jeff Bezanson on 29 July 2016, 20:22:57 UTC
Merge pull request #17619 from JuliaLang/jn/ser-typename
Merge pull request #17619 from JuliaLang/jn/ser-typename
Tip revision: 6b41538
util.jl
# This file is a part of Julia. License is MIT: http://julialang.org/license
# timing
# time() in libc.jl
# high-resolution relative time, in nanoseconds
time_ns() = ccall(:jl_hrtime, UInt64, ())
# This type must be kept in sync with the C struct in src/gc.h
immutable GC_Num
allocd ::Int64 # GC internal
deferred_alloc::Int64
freed ::Int64 # GC internal
malloc ::UInt64
realloc ::UInt64
poolalloc ::UInt64
bigalloc ::UInt64
freecall ::UInt64
total_time ::UInt64
total_allocd::UInt64 # GC internal
since_sweep ::UInt64 # GC internal
collect ::Csize_t # GC internal
pause ::Cint
full_sweep ::Cint
end
gc_num() = ccall(:jl_gc_num, GC_Num, ())
# This type is to represent differences in the counters, so fields may be negative
immutable GC_Diff
allocd ::Int64 # Bytes allocated
malloc ::Int64 # Number of GC aware malloc()
realloc ::Int64 # Number of GC aware realloc()
poolalloc ::Int64 # Number of pool allocation
bigalloc ::Int64 # Number of big (non-pool) allocation
freecall ::Int64 # Number of GC aware free()
total_time ::Int64 # Time spent in garbage collection
pause ::Int64 # Number of GC pauses
full_sweep ::Int64 # Number of GC full collection
end
function GC_Diff(new::GC_Num, old::GC_Num)
# logic from `src/gc.c:jl_gc_total_bytes`
old_allocd = old.allocd + Int64(old.collect) + Int64(old.total_allocd)
new_allocd = new.allocd + Int64(new.collect) + Int64(new.total_allocd)
return GC_Diff(new_allocd - old_allocd,
Int64(new.malloc - old.malloc),
Int64(new.realloc - old.realloc),
Int64(new.poolalloc - old.poolalloc),
Int64(new.bigalloc - old.bigalloc),
Int64(new.freecall - old.freecall),
Int64(new.total_time - old.total_time),
new.pause - old.pause,
new.full_sweep - old.full_sweep)
end
function gc_alloc_count(diff::GC_Diff)
diff.malloc + diff.realloc + diff.poolalloc + diff.bigalloc
end
# total time spend in garbage collection, in nanoseconds
gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ())
# total number of bytes allocated so far
gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ())
function tic()
t0 = time_ns()
task_local_storage(:TIMERS, (t0, get(task_local_storage(), :TIMERS, ())))
return t0
end
function toq()
t1 = time_ns()
timers = get(task_local_storage(), :TIMERS, ())
if is(timers,())
error("toc() without tic()")
end
t0 = timers[1]::UInt64
task_local_storage(:TIMERS, timers[2])
(t1-t0)/1e9
end
function toc()
t = toq()
println("elapsed time: ", t, " seconds")
return t
end
# print elapsed time, return expression value
const _mem_units = ["byte", "KB", "MB", "GB", "TB", "PB"]
const _cnt_units = ["", " k", " M", " G", " T", " P"]
function prettyprint_getunits(value, numunits, factor)
if value == 0 || value == 1
return (value, 1)
end
unit = ceil(Int, log(value) / log(factor))
unit = min(numunits, unit)
number = value/factor^(unit-1)
return number, unit
end
function padded_nonzero_print(value,str)
if value != 0
blanks = " "[1:18-length(str)]
println("$str:$blanks$value")
end
end
function time_print(elapsedtime, bytes, gctime, allocs)
@printf("%10.6f seconds", elapsedtime/1e9)
if bytes != 0 || allocs != 0
bytes, mb = prettyprint_getunits(bytes, length(_mem_units), Int64(1024))
allocs, ma = prettyprint_getunits(allocs, length(_cnt_units), Int64(1000))
if ma == 1
@printf(" (%d%s allocation%s: ", allocs, _cnt_units[ma], allocs==1 ? "" : "s")
else
@printf(" (%.2f%s allocations: ", allocs, _cnt_units[ma])
end
if mb == 1
@printf("%d %s%s", bytes, _mem_units[mb], bytes==1 ? "" : "s")
else
@printf("%.3f %s", bytes, _mem_units[mb])
end
if gctime > 0
@printf(", %.2f%% gc time", 100*gctime/elapsedtime)
end
print(")")
elseif gctime > 0
@printf(", %.2f%% gc time", 100*gctime/elapsedtime)
end
println()
end
function timev_print(elapsedtime, diff::GC_Diff)
allocs = gc_alloc_count(diff)
time_print(elapsedtime, diff.allocd, diff.total_time, allocs)
print("elapsed time (ns): $elapsedtime\n")
padded_nonzero_print(diff.total_time, "gc time (ns)")
padded_nonzero_print(diff.allocd, "bytes allocated")
padded_nonzero_print(diff.poolalloc, "pool allocs")
padded_nonzero_print(diff.bigalloc, "non-pool GC allocs")
padded_nonzero_print(diff.malloc, "malloc() calls")
padded_nonzero_print(diff.realloc, "realloc() calls")
padded_nonzero_print(diff.freecall, "free() calls")
padded_nonzero_print(diff.pause, "GC pauses")
padded_nonzero_print(diff.full_sweep, "full collections")
end
macro time(ex)
quote
local stats = gc_num()
local elapsedtime = time_ns()
local val = $(esc(ex))
elapsedtime = time_ns() - elapsedtime
local diff = GC_Diff(gc_num(), stats)
time_print(elapsedtime, diff.allocd, diff.total_time,
gc_alloc_count(diff))
val
end
end
macro timev(ex)
quote
local stats = gc_num()
local elapsedtime = time_ns()
local val = $(esc(ex))
elapsedtime = time_ns() - elapsedtime
timev_print(elapsedtime, GC_Diff(gc_num(), stats))
val
end
end
# print nothing, return elapsed time
macro elapsed(ex)
quote
local t0 = time_ns()
local val = $(esc(ex))
(time_ns()-t0)/1e9
end
end
# measure bytes allocated without *most* contamination from compilation
# Note: This reports a different value from the @time macros, because
# it wraps the call in a function, however, this means that things
# like: @allocated y = foo()
# will not work correctly, because it will set y in the context of
# the local function made by the macro, not the current function
macro allocated(ex)
quote
let
local f
function f()
b0 = gc_bytes()
$(esc(ex))
gc_bytes() - b0
end
f()
end
end
end
# print nothing, return value, elapsed time, bytes allocated & gc time
macro timed(ex)
quote
local stats = gc_num()
local elapsedtime = time_ns()
local val = $(esc(ex))
elapsedtime = time_ns() - elapsedtime
local diff = GC_Diff(gc_num(), stats)
val, elapsedtime/1e9, diff.allocd, diff.total_time/1e9, diff
end
end
function fftw_vendor()
if Base.libfftw_name == "libmkl_rt"
return :mkl
else
return :fftw
end
end
## printing with color ##
function with_output_color(f::Function, color::Symbol, io::IO, args...)
buf = IOBuffer()
have_color && print(buf, get(text_colors, color, color_normal))
try f(buf, args...)
finally
have_color && print(buf, color_normal)
print(io, takebuf_string(buf))
end
end
print_with_color(color::Symbol, io::IO, msg::AbstractString...) =
with_output_color(print, color, io, msg...)
print_with_color(color::Symbol, msg::AbstractString...) =
print_with_color(color, STDOUT, msg...)
println_with_color(color::Symbol, io::IO, msg::AbstractString...) =
with_output_color(println, color, io, msg...)
println_with_color(color::Symbol, msg::AbstractString...) =
println_with_color(color, STDOUT, msg...)
## warnings and messages ##
function info(io::IO, msg...; prefix="INFO: ")
println_with_color(info_color(), io, prefix, chomp(string(msg...)))
end
info(msg...; prefix="INFO: ") = info(STDERR, msg..., prefix=prefix)
# print a warning only once
const have_warned = Set()
warn_once(io::IO, msg...) = warn(io, msg..., once=true)
warn_once(msg...) = warn(STDERR, msg..., once=true)
function warn(io::IO, msg...;
prefix="WARNING: ", once=false, key=nothing, bt=nothing,
filename=nothing, lineno::Int=0)
str = chomp(string(msg...))
if once
if key === nothing
key = str
end
(key in have_warned) && return
push!(have_warned, key)
end
print_with_color(warn_color(), io, prefix, str)
if bt !== nothing
show_backtrace(io, bt)
end
if filename !== nothing
print(io, "\nwhile loading $filename, in expression starting on line $lineno")
end
println(io)
return
end
warn(msg...; kw...) = warn(STDERR, msg...; kw...)
warn(io::IO, err::Exception; prefix="ERROR: ", kw...) =
warn(io, sprint(buf->showerror(buf, err)), prefix=prefix; kw...)
warn(err::Exception; prefix="ERROR: ", kw...) =
warn(STDERR, err, prefix=prefix; kw...)
function julia_cmd(julia=joinpath(JULIA_HOME, julia_exename()))
opts = JLOptions()
cpu_target = unsafe_string(opts.cpu_target)
image_file = unsafe_string(opts.image_file)
compile = if opts.compile_enabled == 0
"no"
elseif opts.compile_enabled == 2
"all"
elseif opts.compile_enabled == 3
"min"
else
"yes"
end
depwarn = if opts.depwarn == 0
"no"
elseif opts.depwarn == 2
"error"
else
"yes"
end
`$julia -C$cpu_target -J$image_file --compile=$compile --depwarn=$depwarn`
end
julia_exename() = ccall(:jl_is_debugbuild,Cint,())==0 ? "julia" : "julia-debug"
"""
securezero!(o)
`securezero!` fills the memory associated with an object `o` with zeros.
Unlike `fill!(o,0)` and similar code, which might be optimized away by
the compiler for objects about to be discarded, the `securezero!` function
will always be called.
"""
function securezero! end
@noinline securezero!{T<:Number}(a::AbstractArray{T}) = fill!(a, 0)
securezero!(s::String) = securezero!(s.data)
@noinline unsafe_securezero!{T}(p::Ptr{T}, len::Integer=1) =
ccall(:memset, Ptr{T}, (Ptr{T}, Cint, Csize_t), p, 0, len*sizeof(T))
unsafe_securezero!(p::Ptr{Void}, len::Integer=1) = Ptr{Void}(unsafe_securezero!(Ptr{UInt8}(p), len))
if is_windows()
function getpass(prompt::AbstractString)
print(prompt)
flush(STDOUT)
p = Array{UInt8}(128) # mimic Unix getpass in ignoring more than 128-char passwords
# (also avoids any potential memory copies arising from push!)
try
plen = 0
while true
c = ccall(:_getch, UInt8, ())
if c == 0xff || c == UInt8('\n') || c == UInt8('\r')
break # EOF or return
elseif c == 0x00 || c == 0xe0
ccall(:_getch, UInt8, ()) # ignore function/arrow keys
elseif c == UInt8('\b') && plen > 0
plen -= 1 # delete last character on backspace
elseif !iscntrl(Char(c)) && plen < 128
p[plen += 1] = c
end
end
return unsafe_string(pointer(p), plen) # use unsafe_string rather than String(p[1:plen])
# to be absolutely certain we never make an extra copy
finally
securezero!(p)
end
return ""
end
else
getpass(prompt::AbstractString) = unsafe_string(ccall(:getpass, Cstring, (Cstring,), prompt))
end
# Windows authentication prompt
if is_windows()
immutable CREDUI_INFO
cbSize::UInt32
parent::Ptr{Void}
pszMessageText::Ptr{UInt16}
pszCaptionText::Ptr{UInt16}
banner::Ptr{Void}
end
const CREDUIWIN_GENERIC = 0x0001
const CREDUIWIN_IN_CRED_ONLY = 0x0020
const CREDUIWIN_ENUMERATE_CURRENT_USER = 0x0200
const CRED_PACK_GENERIC_CREDENTIALS = 0x0004
const ERROR_SUCCESS = 0x0000
const ERROR_CANCELLED = 0x04c7
function winprompt(message, caption, default_username; prompt_username = true)
# Step 1: Create an encrypted username/password bundle that will be used to set
# the default username (in theory could also provide a default password)
credbuf = Array{UInt8,1}(1024)
credbufsize = Ref{UInt32}(sizeof(credbuf))
succeeded = ccall((:CredPackAuthenticationBufferW, "credui.dll"), stdcall, Bool,
(UInt32, Cwstring, Cwstring, Ptr{UInt8}, Ptr{UInt32}),
CRED_PACK_GENERIC_CREDENTIALS, default_username, "", credbuf, credbufsize)
@assert succeeded
# Step 2: Create the actual dialog
# 2.1: Set up the window
messageArr = Base.cwstring(message)
captionArr = Base.cwstring(caption)
pfSave = Ref{Bool}(false)
cred = Ref{CREDUI_INFO}(CREDUI_INFO(sizeof(CREDUI_INFO), C_NULL, pointer(messageArr), pointer(captionArr), C_NULL))
dwflags = CREDUIWIN_GENERIC | CREDUIWIN_ENUMERATE_CURRENT_USER
if !prompt_username
# Disable setting anything other than default_username
dwflags |= CREDUIWIN_IN_CRED_ONLY
end
authPackage = Ref{Culong}(0)
outbuf_data = Ref{Ptr{Void}}(C_NULL)
outbuf_size = Ref{Culong}(0)
# 2.2: Do the actual request
code = ccall((:CredUIPromptForWindowsCredentialsW, "credui.dll"), stdcall, UInt32, (Ptr{CREDUI_INFO}, UInt32, Ptr{Culong},
Ptr{Void}, Culong, Ptr{Ptr{Void}}, Ptr{Culong}, Ptr{Bool}, UInt32), cred, 0, authPackage, credbuf, credbufsize[],
outbuf_data, outbuf_size, pfSave, dwflags)
# 2.3: If that failed for any reason other than the user canceling, error out.
# If the user canceled, just return a nullable
if code == ERROR_CANCELLED
return Nullable{Tuple{String,String}}()
elseif code != ERROR_SUCCESS
error(Base.Libc.FormatMessage(code))
end
# Step 3: Convert encrypted credentials back to plain text
passbuf = Array{UInt16,1}(1024)
passlen = Ref{UInt32}(length(passbuf))
usernamebuf = Array{UInt16,1}(1024)
usernamelen = Ref{UInt32}(length(usernamebuf))
# Need valid buffers for domain, even though we don't care
dummybuf = Array{UInt16,1}(1024)
succeeded = ccall((:CredUnPackAuthenticationBufferW, "credui.dll"), Bool,
(UInt32, Ptr{Void}, UInt32, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}, Ptr{UInt16}, Ptr{UInt32}),
0, outbuf_data[], outbuf_size[], usernamebuf, usernamelen, dummybuf, Ref{UInt32}(1024), passbuf, passlen)
if !succeeded
error(Base.Libc.FormatMessage())
end
# Step 4: Free the encrypted buffer
# ccall(:SecureZeroMemory, Ptr{Void}, (Ptr{Void}, Csize_t), outbuf_data[], outbuf_size[]) - not an actual function
unsafe_securezero!(outbuf_data[], outbuf_size[])
ccall((:CoTaskMemFree, "ole32.dll"), Void, (Ptr{Void},), outbuf_data[])
# Done.
passbuf_ = passbuf[1:passlen[]-1]
result = Nullable((String(transcode(UInt8, usernamebuf[1:usernamelen[]-1])),
String(transcode(UInt8, passbuf_))))
securezero!(passbuf_)
securezero!(passbuf)
return result
end
end
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...