https://github.com/JuliaLang/julia
Tip revision: 3b4e7b46ccf602e004905273ebebf85c47e220f7 authored by Keno Fischer on 10 October 2018, 00:26:12 UTC
HACK: Option to ignore all inlining heuristics
HACK: Option to ignore all inlining heuristics
Tip revision: 3b4e7b4
staticdata.c
// This file is a part of Julia. License is MIT: https://julialang.org/license
/*
saving and restoring system images
*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h> // printf
#include "julia.h"
#include "julia_internal.h"
#include "builtin_proto.h"
#include "processor.h"
#ifndef _OS_WINDOWS_
#include <dlfcn.h>
#endif
#ifndef _COMPILER_MICROSOFT_
#include "valgrind.h"
#else
#define RUNNING_ON_VALGRIND 0
#endif
#include "julia_assert.h"
#ifdef __cplusplus
extern "C" {
#endif
// TODO: put WeakRefs on the weak_refs list during deserialization
// TODO: handle finalizers
// hash of definitions for predefined tagged object
static htable_t sertag_table;
static htable_t symbol_table;
static uintptr_t nsym_tag;
// array of definitions for the predefined tagged object types
// (reverse of sertag_table and symbol_table)
static arraylist_t deser_tag;
static arraylist_t deser_sym;
// table of all objects that are serialized
static htable_t backref_table;
static int backref_table_numel;
static arraylist_t layout_table;
// list of (size_t pos, (void *f)(jl_value_t*)) entries
// for the serializer to mark values in need of rework by function f
// during deserialization later
static arraylist_t reinit_list;
// list of modules being deserialized with __init__ methods
// (not used in MODE_AST)
jl_array_t *jl_module_init_order;
// hash of definitions for predefined function pointers
static htable_t fptr_to_id;
// array of definitions for the predefined function pointers
// (reverse of fptr_to_id)
static const jl_fptr_args_t id_to_fptrs[] = {
jl_f_throw, jl_f_is, jl_f_typeof, jl_f_issubtype, jl_f_isa,
jl_f_typeassert, jl_f__apply, jl_f__apply_pure, jl_f__apply_latest, jl_f_isdefined,
jl_f_tuple, jl_f_svec, jl_f_intrinsic_call, jl_f_invoke_kwsorter,
jl_f_getfield, jl_f_setfield, jl_f_fieldtype, jl_f_nfields,
jl_f_arrayref, jl_f_arrayset, jl_f_arraysize, jl_f_apply_type,
jl_f_applicable, jl_f_invoke, jl_f_sizeof, jl_f__expr, jl_f__typevar,
jl_f_ifelse,
NULL };
typedef enum _DUMP_MODES {
// not in the serializer at all, or
// something is seriously wrong
MODE_INVALID = 0,
// jl_restore_system_image
// restoring an entire system image from disk
MODE_SYSTEM_IMAGE,
} DUMP_MODES;
typedef struct {
ios_t *s;
ios_t *const_data;
ios_t *symbols;
ios_t *relocs;
ios_t *gvar_record;
ios_t *fptr_record;
arraylist_t relocs_list;
arraylist_t gctags_list;
DUMP_MODES mode;
jl_ptls_t ptls;
} jl_serializer_state;
static jl_value_t *jl_idtable_type = NULL;
static jl_typename_t *jl_idtable_typename = NULL;
static jl_value_t *jl_bigint_type = NULL;
static int gmp_limb_size = 0;
static arraylist_t builtin_typenames;
enum RefTags {
DataRef,
ConstDataRef,
TagRef,
SymbolRef,
BindingRef,
FunctionRef,
BuiltinFunctionRef
};
// calling conventions for internal entry points.
// this is used to set the method-instance->invoke field
typedef enum {
JL_API_TRAMPOLINE,
JL_API_BOXED,
JL_API_CONST,
JL_API_WITH_PARAMETERS,
JL_API_INTERPRETED,
JL_API_BUILTIN,
JL_API_MAX
} jl_callingconv_t;
// this supports up to 1 GB images and 16 RefTags
// if a larger size is required, will need to add support for writing larger relocations in many cases below
#define RELOC_TAG_OFFSET 28
/* read and write in host byte order */
#define write_uint8(s, n) ios_putc((n), (s))
#define read_uint8(s) ((uint8_t)ios_getc((s)))
static void write_uint32(ios_t *s, uint32_t i) JL_NOTSAFEPOINT
{
ios_write(s, (char*)&i, 4);
}
static uint32_t read_uint32(ios_t *s) JL_NOTSAFEPOINT
{
uint32_t x = 0;
ios_read(s, (char*)&x, 4);
return x;
}
// --- Static Compile ---
extern int globalUnique;
static void *jl_sysimg_handle = NULL;
static uint64_t sysimage_base = 0;
static uintptr_t *sysimg_gvars_base = NULL;
static const int32_t *sysimg_gvars_offsets = NULL;
static jl_sysimg_fptrs_t sysimg_fptrs;
static inline uintptr_t *sysimg_gvars(uintptr_t *base, size_t idx)
{
return base + sysimg_gvars_offsets[idx] / sizeof(base[0]);
}
JL_DLLEXPORT int jl_running_on_valgrind(void)
{
return RUNNING_ON_VALGRIND;
}
static void jl_load_sysimg_so(void)
{
int imaging_mode = jl_generating_output() && !jl_options.incremental;
// in --build mode only use sysimg data, not precompiled native code
if (!imaging_mode && jl_options.use_sysimage_native_code==JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES) {
jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars_base", (void **)&sysimg_gvars_base, 1);
jl_dlsym(jl_sysimg_handle, "jl_sysimg_gvars_offsets", (void **)&sysimg_gvars_offsets, 1);
sysimg_gvars_offsets += 1;
assert(sysimg_fptrs.base);
jl_dlsym(jl_sysimg_handle, "jl_globalUnique", (void **)&globalUnique, 1);
#ifdef JULIA_ENABLE_THREADING
uintptr_t *tls_getter_slot;
jl_dlsym(jl_sysimg_handle, "jl_get_ptls_states_slot", (void **)&tls_getter_slot, 1);
*tls_getter_slot = (uintptr_t)jl_get_ptls_states_getter();
size_t *tls_offset_idx;
jl_dlsym(jl_sysimg_handle, "jl_tls_offset", (void **)&tls_offset_idx, 1);
*tls_offset_idx = (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset);
#endif
#ifdef _OS_WINDOWS_
sysimage_base = (intptr_t)jl_sysimg_handle;
#else
Dl_info dlinfo;
if (dladdr((void*)sysimg_gvars_base, &dlinfo) != 0) {
sysimage_base = (intptr_t)dlinfo.dli_fbase;
}
else {
sysimage_base = 0;
}
#endif
}
else {
memset(&sysimg_fptrs, 0, sizeof(sysimg_fptrs));
}
const char *sysimg_data;
jl_dlsym(jl_sysimg_handle, "jl_system_image_data", (void **)&sysimg_data, 1);
size_t len;
jl_dlsym(jl_sysimg_handle, "jl_system_image_size", (void **)&len, 1);
jl_restore_system_image_data(sysimg_data, len);
}
// --- serializer ---
static uintptr_t jl_fptr_id(void *fptr)
{
void **pbp = ptrhash_bp(&fptr_to_id, fptr);
if (*pbp == HT_NOTFOUND || fptr == NULL)
return 0;
else
return *(uintptr_t*)pbp;
}
#define jl_serialize_value(s, v) jl_serialize_value_(s,(jl_value_t*)(v))
static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v);
static void jl_serialize_module(jl_serializer_state *s, jl_module_t *m)
{
jl_serialize_value(s, m->name);
jl_serialize_value(s, m->parent);
size_t i;
void **table = m->bindings.table;
for (i = 1; i < m->bindings.size; i += 2) {
if (table[i] != HT_NOTFOUND) {
jl_binding_t *b = (jl_binding_t*)table[i];
if (b->owner == m || m != jl_main_module) {
jl_serialize_value(s, b->name);
jl_serialize_value(s, b->value);
jl_serialize_value(s, b->globalref);
jl_serialize_value(s, b->owner);
}
}
}
if (m != jl_main_module) {
for (i = 0; i < m->usings.len; i++) {
jl_serialize_value(s, (jl_value_t*)m->usings.items[i]);
}
}
}
static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
{
if (v == NULL || jl_is_symbol(v)) {
return;
}
void *builtin = ptrhash_get(&sertag_table, v);
if (builtin != HT_NOTFOUND) {
return;
}
void **bp = ptrhash_bp(&backref_table, v);
if (*bp != HT_NOTFOUND) {
return;
}
size_t item = ++backref_table_numel;
assert(item < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many items to serialize");
char *pos = (char*)HT_NOTFOUND + item;
*bp = (void*)pos;
// some values have special representations
jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
jl_serialize_value(s, t);
if (t->layout->npointers == 0) {
// skip it
}
else if (jl_is_svec(v)) {
size_t i, l = jl_svec_len(v);
jl_value_t **data = jl_svec_data(v);
for (i = 0; i < l; i++) {
jl_serialize_value(s, data[i]);
}
}
else if (jl_is_array(v)) {
jl_array_t *ar = (jl_array_t*)v;
jl_serialize_value(s, jl_typeof(ar));
if (ar->flags.ptrarray) {
size_t i, l = jl_array_len(ar);
for (i = 0; i < l; i++) {
jl_serialize_value(s, jl_array_ptr_ref(ar, i));
}
}
}
else if (jl_typeis(v, jl_module_type)) {
jl_serialize_module(s, (jl_module_t*)v);
}
else if (jl_typeis(v, jl_task_type)) {
jl_error("Task cannot be serialized");
}
else {
char *data = (char*)jl_data_ptr(v);
size_t i, nf = jl_datatype_nfields(t);
for (i = 0; i < nf; i++) {
if (jl_field_isptr(t, i)) {
char *slot = data + jl_field_offset(t, i);
jl_value_t *fld = *(jl_value_t**)slot;
jl_serialize_value(s, fld);
}
}
}
}
static void ios_ensureroom(ios_t *s, size_t newsize) JL_NOTSAFEPOINT
{
size_t prevsize = s->size;
if (prevsize < newsize) {
ios_trunc(s, newsize);
assert(s->size == newsize);
memset(&s->buf[prevsize], 0, newsize - prevsize);
}
}
static void record_gvar(jl_serializer_state *s, int gid, uintptr_t reloc_id) JL_NOTSAFEPOINT
{
if (gid == 0)
return;
ios_ensureroom(s->gvar_record, gid * sizeof(uint32_t));
ios_seek(s->gvar_record, (gid - 1) * sizeof(uint32_t));
assert(reloc_id < UINT32_MAX);
write_uint32(s->gvar_record, reloc_id);
}
static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT
{
static const char zeros[16] = {0};
while (nb > 16) {
ios_write(s, zeros, 16);
nb -= 16;
}
if (nb != 0)
ios_write(s, zeros, nb);
}
static void write_pointer(ios_t *s) JL_NOTSAFEPOINT
{
assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value");
write_padding(s, sizeof(void*));
}
#define backref_id(s, v) _backref_id(s, (jl_value_t*)(v))
static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v) JL_NOTSAFEPOINT
{
assert(v != NULL && "cannot get backref to NULL object");
void *idx = HT_NOTFOUND;
if (jl_is_symbol(v)) {
void **pidx = ptrhash_bp(&symbol_table, v);
idx = *pidx;
if (idx == HT_NOTFOUND) {
size_t l = strlen(jl_symbol_name((jl_sym_t*)v));
write_uint32(s->symbols, l);
ios_write(s->symbols, jl_symbol_name((jl_sym_t*)v), l + 1);
size_t offset = ++nsym_tag;
assert(offset < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many symbols");
idx = (void*)((char*)HT_NOTFOUND + ((uintptr_t)SymbolRef << RELOC_TAG_OFFSET) + offset);
*pidx = idx;
}
}
else {
idx = ptrhash_get(&sertag_table, v);
}
if (idx == HT_NOTFOUND) {
idx = ptrhash_get(&backref_table, v);
assert(idx != HT_NOTFOUND && "object missed during jl_serialize_value pass");
}
return (char*)idx - 1 - (char*)HT_NOTFOUND;
}
static void write_pointerfield(jl_serializer_state *s, jl_value_t *fld) JL_NOTSAFEPOINT
{
if (fld != NULL) {
arraylist_push(&s->relocs_list, (void*)(uintptr_t)ios_pos(s->s));
arraylist_push(&s->relocs_list, (void*)backref_id(s, fld));
}
write_pointer(s->s);
}
static void write_gctaggedfield(jl_serializer_state *s, uintptr_t ref) JL_NOTSAFEPOINT
{
arraylist_push(&s->gctags_list, (void*)(uintptr_t)ios_pos(s->s));
arraylist_push(&s->gctags_list, (void*)ref);
write_pointer(s->s);
}
static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t *m)
{
size_t reloc_offset = ios_pos(s->s);
size_t tot = sizeof(jl_module_t);
ios_write(s->s, (char*)m, tot);
jl_module_t *newm = (jl_module_t*)&s->s->buf[reloc_offset];
newm->name = NULL;
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, name)));
arraylist_push(&s->relocs_list, (void*)backref_id(s, m->name));
newm->parent = NULL;
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, parent)));
arraylist_push(&s->relocs_list, (void*)backref_id(s, m->parent));
newm->primary_world = jl_world_counter;
// write out the bindings table as a list
// immediately after jl_module_t
// (the ptrhash will need to be recreated on load)
size_t count = 0;
size_t i;
void **table = m->bindings.table;
for (i = 1; i < m->bindings.size; i += 2) {
if (table[i] != HT_NOTFOUND) {
jl_binding_t *b = (jl_binding_t*)table[i];
if (b->owner == m || m != jl_main_module) {
write_gctaggedfield(s, (uintptr_t)BindingRef << RELOC_TAG_OFFSET);
tot += sizeof(void*);
size_t binding_reloc_offset = ios_pos(s->s);
record_gvar(s, jl_get_llvm_gv((jl_value_t*)b), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + binding_reloc_offset);
write_pointerfield(s, (jl_value_t*)b->name);
write_pointerfield(s, b->value);
write_pointerfield(s, b->globalref);
write_pointerfield(s, (jl_value_t*)b->owner);
size_t flag_offset = offsetof(jl_binding_t, owner) + sizeof(b->owner);
ios_write(s->s, (char*)b + flag_offset, sizeof(*b) - flag_offset);
tot += sizeof(jl_binding_t);
count += 1;
}
}
}
assert(ios_pos(s->s) - reloc_offset == tot);
newm = (jl_module_t*)&s->s->buf[reloc_offset];
newm->bindings.size = count; // stash the count in newm->size
newm->bindings.table = NULL;
memset(&newm->bindings._space, 0, sizeof(newm->bindings._space));
// write out the usings list
memset(&newm->usings._space, 0, sizeof(newm->usings._space));
if (m == jl_main_module) {
newm->usings.len = 1;
newm->usings.max = AL_N_INLINE;
newm->usings.items = (void**)offsetof(jl_module_t, usings._space);
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items)));
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item));
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings._space[0])));
arraylist_push(&s->relocs_list, (void*)backref_id(s, jl_core_module));
}
else {
if (newm->usings.items == &newm->usings._space[0]) {
m->usings.max = AL_N_INLINE;
newm->usings.items = (void**)offsetof(jl_module_t, usings._space);
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items)));
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item));
size_t i;
for (i = 0; i < m->usings.len; i++) {
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings._space[i])));
arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings._space[i]));
}
}
else {
newm->usings.items = (void**)tot;
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings.items)));
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item));
size_t i;
for (i = 0; i < m->usings.len; i++) {
write_pointerfield(s, (jl_value_t*)m->usings.items[i]);
tot += sizeof(void*);
}
for (; i < m->usings.max; i++) {
write_pointer(s->s);
tot += sizeof(void*);
}
// newm = (jl_module_t*)&s->s->buf[reloc_offset];
}
}
}
#if 0
static size_t jl_sort_size(jl_datatype_t *dt)
{
if (dt == jl_simplevector_type)
return SIZE_MAX - 5;
if (dt == jl_string_type)
return SIZE_MAX - 4;
if (dt->name == jl_array_typename)
return SIZE_MAX - 3;
if (dt == jl_datatype_type)
return SIZE_MAX - 2;
if (dt == jl_module_type)
return SIZE_MAX - 1;
return jl_datatype_size(dt);
}
#endif
static int sysimg_sort_order(const void *pa, const void *pb)
{
uintptr_t sa = ((uintptr_t*)pa)[1];
uintptr_t sb = ((uintptr_t*)pb)[1];
return (sa > sb ? 1 : (sa < sb ? -1 : 0));
#if 0
jl_value_t *a = *(jl_value_t**)pa;
jl_datatype_t *tya = (jl_datatype_t*)jl_typeof(a);
size_t sa = jl_sort_size(tya);
jl_value_t *b = *(jl_value_t**)pb;
jl_datatype_t *tyb = (jl_datatype_t*)jl_typeof(b);
size_t sb = jl_sort_size(tyb);
if (sa == sb) {
sa = tya->uid;
sb = tyb->uid;
}
return (sa > sb ? 1 : (sa < sb ? -1 : 0));
#endif
}
jl_value_t *jl_find_ptr = NULL;
static void jl_write_values(jl_serializer_state *s)
{
arraylist_t objects_list;
arraylist_new(&objects_list, backref_table_numel * 2);
arraylist_new(&layout_table, 0);
arraylist_grow(&layout_table, backref_table_numel);
memset(layout_table.items, 0, backref_table_numel * sizeof(void*));
size_t i, len = backref_table.size;
void **p = backref_table.table;
for (i = 0; i < len; i += 2) {
char *reloc_id = (char*)p[i + 1];
if (reloc_id != HT_NOTFOUND) {
jl_value_t *v = (jl_value_t*)p[i];
uintptr_t item = reloc_id - 1 - (char*)HT_NOTFOUND;
objects_list.items[objects_list.len++] = (void*)v;
objects_list.items[objects_list.len++] = (void*)item;
}
}
assert(backref_table_numel * 2 == objects_list.len);
qsort(objects_list.items, backref_table_numel, sizeof(void*) * 2, sysimg_sort_order);
for (i = 0, len = backref_table_numel * 2; i < len; i += 2) {
jl_value_t *v = (jl_value_t*)objects_list.items[i];
uintptr_t item = (uintptr_t)objects_list.items[i + 1];
jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v);
assert((t->instance == NULL || t->instance == v) && "detected singleton construction corruption");
// realign stream to expected gc alignment (16 bytes)
uintptr_t skip_header_pos = ios_pos(s->s) + sizeof(jl_taggedvalue_t);
write_padding(s->s, LLT_ALIGN(skip_header_pos, 16) - skip_header_pos);
// write header
write_gctaggedfield(s, backref_id(s, t));
size_t reloc_offset = ios_pos(s->s);
assert(item < layout_table.len && layout_table.items[item] == NULL);
layout_table.items[item] = (void*)reloc_offset;
record_gvar(s, jl_get_llvm_gv(v), ((uintptr_t)DataRef << RELOC_TAG_OFFSET) + reloc_offset);
// write data
if (jl_is_cpointer(v)) {
write_pointer(s->s);
}
else if (jl_is_array(v)) {
#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes)
jl_array_t *ar = (jl_array_t*)v;
int ndimwords = jl_array_ndimwords(ar->flags.ndims);
size_t tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords * sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT);
// copy header
ios_write(s->s, (char*)v, tsz);
// make some header modifications in-place
jl_array_t *newa = (jl_array_t*)&s->s->buf[reloc_offset];
size_t alen = jl_array_len(ar);
size_t tot = alen * ar->elsize;
if (newa->flags.ndims == 1)
newa->maxsize = alen;
newa->offset = 0;
newa->flags.how = 0;
newa->flags.pooled = 0;
newa->flags.isshared = 0;
// write data
if (!ar->flags.ptrarray) {
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 16);
// realign stream to max(data-align(array), sizeof(void*))
write_padding(s->const_data, data - ios_pos(s->const_data));
// write data and relocations
newa->data = NULL; // relocation offset
data /= sizeof(void*);
assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large");
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target
int isbitsunion = jl_array_isbitsunion(ar);
if (ar->elsize == 1 && !isbitsunion)
tot += 1;
ios_write(s->const_data, (char*)jl_array_data(ar), tot);
if (isbitsunion)
ios_write(s->const_data, jl_array_typetagdata(ar), alen);
}
else {
newa->data = (void*)tsz; // relocation offset
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target
size_t i;
for (i = 0; i < alen; i++) {
write_pointerfield(s, jl_array_ptr_ref(v, i));
}
}
}
else if (jl_typeis(v, jl_module_type)) {
jl_write_module(s, item, (jl_module_t*)v);
// will need to recreate the binding table for this
arraylist_push(&reinit_list, (void*)item);
arraylist_push(&reinit_list, (void*)5);
}
else if (jl_typeis(v, jl_task_type)) {
jl_error("Task cannot be serialized");
}
else if (jl_is_svec(v)) {
ios_write(s->s, (char*)v, sizeof(void*));
size_t i, l = jl_svec_len(v);
assert(l > 0);
for (i = 0; i < l; i++) {
write_pointerfield(s, jl_svecref(v, i));
}
}
else if (jl_is_string(v)) {
ios_write(s->s, (char*)v, sizeof(void*));
ios_write(s->s, jl_string_data(v), jl_string_len(v));
write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility
}
else if (jl_datatype_nfields(t) == 0) {
assert(t->layout->npointers == 0);
if (t->size > 0)
ios_write(s->s, (char*)v, t->size);
}
else if (jl_bigint_type && jl_typeis(v, jl_bigint_type)) {
jl_value_t *sizefield = jl_get_nth_field(v, 1);
int32_t sz = jl_unbox_int32(sizefield);
int32_t nw = (sz == 0 ? 1 : (sz < 0 ? -sz : sz));
size_t nb = nw * gmp_limb_size;
ios_write(s->s, (char*)&nw, sizeof(int32_t));
ios_write(s->s, (char*)&sz, sizeof(int32_t));
uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 8);
write_padding(s->const_data, data - ios_pos(s->const_data));
data /= sizeof(void*);
assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large");
arraylist_push(&s->relocs_list, (void*)(reloc_offset + 8)); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target
void *pdata = jl_unbox_voidpointer(jl_get_nth_field(v, 2));
ios_write(s->const_data, (char*)pdata, nb);
write_pointer(s->s);
}
else {
size_t i, nf = jl_datatype_nfields(t);
size_t tot = 0;
for (i = 0; i < nf; i++) {
size_t offset = jl_field_offset(t, i);
char *slot = (char*)v + offset;
write_padding(s->s, offset - tot);
tot = offset;
size_t fsz = jl_field_size(t, i);
if (jl_field_isptr(t, i) > 0) {
write_pointerfield(s, *(jl_value_t**)slot);
}
else if (t->mutabl && jl_is_cpointer_type(jl_field_type(t, i))) {
write_pointer(s->s);
}
else if (fsz > 0) {
ios_write(s->s, slot, fsz);
}
tot += fsz;
}
if (jl_is_method(v)) {
write_padding(s->s, sizeof(jl_method_t) - tot);
}
else if (jl_is_method_instance(v)) {
jl_method_instance_t *m = (jl_method_instance_t*)v;
jl_method_instance_t *newm = (jl_method_instance_t*)&s->s->buf[reloc_offset];
newm->invoke = NULL;
newm->specptr.fptr = NULL;
newm->functionObjectsDecls.functionObject = NULL;
newm->functionObjectsDecls.specFunctionObject = NULL;
uintptr_t fptr_id = JL_API_TRAMPOLINE;
uintptr_t specfptr_id = 0;
if (m->invoke == jl_fptr_const_return) {
fptr_id = JL_API_CONST;
}
else {
if (jl_is_method(m->def.method)) {
specfptr_id = jl_fptr_id(m->specptr.fptr);
const char *fname = m->functionObjectsDecls.functionObject;
if (specfptr_id) { // found in the table of builtins
assert(specfptr_id >= 2);
fptr_id = JL_API_BUILTIN;
}
else if (fname) {
assert(reloc_offset < INT32_MAX);
if (!strcmp(fname, "jl_fptr_args")) {
fptr_id = JL_API_BOXED;
}
else if (!strcmp(fname, "jl_fptr_sparam")) {
fptr_id = JL_API_WITH_PARAMETERS;
}
else {
int func = jl_assign_functionID(fname);
assert(func > 0);
ios_ensureroom(s->fptr_record, func * sizeof(void*));
ios_seek(s->fptr_record, (func - 1) * sizeof(void*));
write_uint32(s->fptr_record, ~reloc_offset);
#ifdef _P64
write_padding(s->fptr_record, 4);
#endif
}
fname = m->functionObjectsDecls.specFunctionObject;
if (fname) {
int cfunc = jl_assign_functionID(fname);
assert(cfunc > 0);
ios_ensureroom(s->fptr_record, cfunc * sizeof(void*));
ios_seek(s->fptr_record, (cfunc - 1) * sizeof(void*));
write_uint32(s->fptr_record, reloc_offset);
#ifdef _P64
write_padding(s->fptr_record, 4);
#endif
}
}
}
}
newm->invoke = NULL; // relocation offset
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_method_instance_t, invoke))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + fptr_id)); // relocation target
if (specfptr_id >= 2) {
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_method_instance_t, specptr.fptr))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)BuiltinFunctionRef << RELOC_TAG_OFFSET) + specfptr_id - 2)); // relocation target
}
}
else if (jl_is_datatype(v)) {
jl_datatype_t *dt = (jl_datatype_t*)v;
jl_datatype_t *newdt = (jl_datatype_t*)&s->s->buf[reloc_offset];
newdt->struct_decl = NULL;
newdt->ditype = NULL;
if (dt->layout != NULL) {
size_t nf = dt->layout->nfields;
size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
int has_padding = dt->layout->npointers && nf;
char *flddesc = (char*)dt->layout;
size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize;
uintptr_t layout_unaligned = LLT_ALIGN(ios_pos(s->const_data), sizeof(uint32_t));
uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*));
if (has_padding) {
if (layout == layout_unaligned) {
layout += sizeof(void*);
layout_unaligned = layout - sizeof(uint32_t);
}
flddesc -= sizeof(uint32_t);
fldsize += sizeof(uint32_t);
write_padding(s->const_data, layout_unaligned - ios_pos(s->const_data)); // realign stream
}
else {
write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream
}
newdt->layout = NULL; // relocation offset
layout /= sizeof(void*);
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + layout)); // relocation target
ios_write(s->const_data, flddesc, fldsize);
}
}
else if (((jl_datatype_t*)(jl_typeof(v)))->name == jl_idtable_typename) {
// will need to rehash this, later (after types are fully constructed)
arraylist_push(&reinit_list, (void*)item);
arraylist_push(&reinit_list, (void*)1);
}
else {
write_padding(s->s, t->size - tot);
}
}
}
}
static void jl_write_gv_syms(jl_serializer_state *s, jl_sym_t *v)
{
// since symbols are static, they might not have had a
// reference anywhere in the code image other than here
int32_t gv = jl_get_llvm_gv((jl_value_t*)v);
if (gv != 0) {
uintptr_t item = backref_id(s, v);
assert(item >> RELOC_TAG_OFFSET == SymbolRef);
record_gvar(s, gv, item);
}
if (v->left)
jl_write_gv_syms(s, v->left);
if (v->right)
jl_write_gv_syms(s, v->right);
}
static inline uint32_t load_uint32(uintptr_t *base)
{
uint32_t v = **(uint32_t**)base;
*base += 4;
return v;
}
static void jl_read_symbols(jl_serializer_state *s)
{
assert(deser_sym.len == nsym_tag);
uintptr_t base = (uintptr_t)&s->symbols->buf[0];
uintptr_t end = base + s->symbols->size;
while (base < end) {
uint32_t len = load_uint32(&base);
const char *str = (const char*)base;
base += len + 1;
//printf("symbol %3d: %s\n", len, str);
jl_sym_t *sym = jl_symbol_n(str, len);
arraylist_push(&deser_sym, (void*)sym);
}
}
static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset)
{
enum RefTags tag = (enum RefTags)(reloc_item >> RELOC_TAG_OFFSET);
if (tag == DataRef) {
// need to compute the final relocation offset via the layout table
assert(reloc_item < layout_table.len);
uintptr_t reloc_base = (uintptr_t)layout_table.items[reloc_item];
assert(reloc_base != 0 && "layout offset missing for relocation item");
// write reloc_offset into s->s at pos
return reloc_base + reloc_offset;
}
else {
// just write the item reloc_id directly
#ifndef JL_NDEBUG
assert(reloc_offset == 0 && "offsets for relocations to builtin objects should be precomposed in the reloc_item");
size_t offset = (reloc_item & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1));
switch (tag) {
case ConstDataRef:
break;
case TagRef:
assert(offset >= 2 && offset < deser_tag.len && deser_tag.items[offset] && "corrupt relocation item id");
break;
case SymbolRef:
assert(offset < nsym_tag && "corrupt relocation item id");
break;
case BindingRef:
assert(offset == 0 && "corrupt relocation offset");
break;
case BuiltinFunctionRef:
assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id");
break;
case FunctionRef:
assert(offset < JL_API_MAX && "unknown function pointer id");
break;
case DataRef:
default:
assert("corrupt relocation item id");
abort();
}
#endif
return reloc_item; // pre-composed relocation + offset
}
}
static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uint32_t reloc_id)
{
enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET);
size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1));
switch (tag) {
case DataRef:
assert(offset < size);
return base + offset;
case ConstDataRef:
return (uintptr_t)deser_tag.items[0] + (offset * sizeof(void*));
case TagRef:
assert(offset < deser_tag.len && deser_tag.items[offset] && "corrupt relocation item id");
return (uintptr_t)deser_tag.items[offset];
case SymbolRef:
assert(offset < deser_sym.len && deser_sym.items[offset] && "corrupt relocation item id");
return (uintptr_t)deser_sym.items[offset];
case BindingRef:
return jl_buff_tag | GC_OLD_MARKED;
case BuiltinFunctionRef:
assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID");
return (uintptr_t)id_to_fptrs[offset];
case FunctionRef:
switch ((jl_callingconv_t)offset) {
case JL_API_BOXED:
if (sysimg_fptrs.base)
return (uintptr_t)jl_fptr_args;
JL_FALLTHROUGH;
case JL_API_WITH_PARAMETERS:
if (sysimg_fptrs.base)
return (uintptr_t)jl_fptr_sparam;
JL_FALLTHROUGH;
case JL_API_TRAMPOLINE:
return (uintptr_t)jl_fptr_trampoline;
case JL_API_CONST:
return (uintptr_t)jl_fptr_const_return;
case JL_API_INTERPRETED:
return (uintptr_t)jl_fptr_interpret_call;
case JL_API_BUILTIN:
return (uintptr_t)jl_fptr_args;
case JL_API_MAX:
//default:
assert("corrupt relocation item id");
}
}
abort();
}
static void jl_write_skiplist(ios_t *s, char *base, size_t size, arraylist_t *list)
{
size_t i;
for (i = 0; i < list->len; i += 2) {
size_t pos = (size_t)list->items[i];
size_t item = (size_t)list->items[i + 1];
uintptr_t *pv = (uintptr_t*)(base + pos);
assert(pos < size && pos != 0);
*pv = get_reloc_for_item(item, *pv);
// record pos in relocations list
// TODO: save space by using delta-compression
assert(pos < UINT32_MAX);
write_uint32(s, pos);
}
write_uint32(s, 0);
}
static void jl_write_relocations(jl_serializer_state *s)
{
char *base = &s->s->buf[0];
jl_write_skiplist(s->relocs, base, s->s->size, &s->gctags_list);
jl_write_skiplist(s->relocs, base, s->s->size, &s->relocs_list);
}
static void jl_read_relocations(jl_serializer_state *s, uint8_t bits)
{
uintptr_t base = (uintptr_t)&s->s->buf[0];
size_t size = s->s->size;
while (1) {
uintptr_t val = (uintptr_t)&s->relocs->buf[s->relocs->bpos];
uint32_t offset = load_uint32(&val);
s->relocs->bpos += sizeof(uint32_t);
if (offset == 0)
break;
uintptr_t *pv = (uintptr_t*)(base + offset);
uintptr_t v = *pv;
v = get_item_for_reloc(s, base, size, v);
*pv = v | bits;
}
}
static char* sysimg_base;
static char* sysimg_relocs;
void gc_sweep_sysimg(void)
{
uintptr_t base = (uintptr_t)sysimg_base;
uintptr_t relocs = (uintptr_t)sysimg_relocs;
if (relocs == 0)
return;
while (1) {
uint32_t offset = load_uint32(&relocs);
if (offset == 0)
break;
jl_taggedvalue_t *o = (jl_taggedvalue_t*)(base + offset);
o->bits.gc = GC_OLD;
}
}
#define jl_write_value(s, v) _jl_write_value((s), (jl_value_t*)(v))
static void _jl_write_value(jl_serializer_state *s, jl_value_t *v)
{
if (v == NULL) {
write_uint32(s->s, 0);
return;
}
uintptr_t item = backref_id(s, v);
uintptr_t reloc = get_reloc_for_item(item, 0);
assert(reloc < UINT32_MAX);
write_uint32(s->s, reloc);
}
static jl_value_t *jl_read_value(jl_serializer_state *s)
{
uintptr_t base = (uintptr_t)&s->s->buf[0];
size_t size = s->s->size;
uintptr_t val = base + s->s->bpos;
uint32_t offset = load_uint32(&val);
s->s->bpos += sizeof(uint32_t);
if (offset == 0)
return NULL;
return (jl_value_t*)get_item_for_reloc(s, base, size, offset);
}
static void jl_update_all_fptrs(jl_serializer_state *s)
{
jl_sysimg_fptrs_t fvars = sysimg_fptrs;
// make these NULL now so we skip trying to restore GlobalVariable pointers later
sysimg_gvars_base = NULL;
sysimg_fptrs.base = NULL;
if (fvars.base == NULL)
return;
int sysimg_fvars_max = s->fptr_record->size / sizeof(void*);
size_t i;
uintptr_t base = (uintptr_t)&s->s->buf[0];
jl_method_instance_t **linfos = (jl_method_instance_t**)&s->fptr_record->buf[0];
uint32_t clone_idx = 0;
for (i = 0; i < sysimg_fvars_max; i++) {
uintptr_t val = (uintptr_t)&linfos[i];
uint32_t offset = load_uint32(&val);
if (offset != 0) {
int specfunc = 1;
if (offset & ((uintptr_t)1 << (8 * sizeof(uint32_t) - 1))) {
// if high bit is set, this is the func wrapper, not the specfunc
specfunc = 0;
offset = ~offset;
}
jl_method_instance_t *li = (jl_method_instance_t*)(base + offset);
uintptr_t base = (uintptr_t)fvars.base;
assert(jl_is_method(li->def.method) && li->invoke != jl_fptr_const_return);
assert(specfunc ? li->invoke != jl_fptr_trampoline : li->invoke == jl_fptr_trampoline);
linfos[i] = li;
int32_t offset = fvars.offsets[i];
for (; clone_idx < fvars.nclones; clone_idx++) {
uint32_t idx = fvars.clone_idxs[clone_idx] & jl_sysimg_val_mask;
if (idx < i)
continue;
if (idx == i)
offset = fvars.clone_offsets[clone_idx];
break;
}
void *fptr = (void*)(base + offset);
if (specfunc)
li->specptr.fptr = fptr;
else
li->invoke = (jl_callptr_t)fptr;
jl_fptr_to_llvm(fptr, li, specfunc);
}
}
jl_register_fptrs(sysimage_base, &fvars, linfos, sysimg_fvars_max);
}
static void jl_update_all_gvars(jl_serializer_state *s)
{
if (sysimg_gvars_base == NULL)
return;
size_t gvname_index = 0;
uintptr_t base = (uintptr_t)&s->s->buf[0];
size_t size = s->s->size;
uintptr_t gvars = (uintptr_t)&s->gvar_record->buf[0];
uintptr_t end = gvars + s->gvar_record->size;
while (gvars < end) {
uint32_t offset = load_uint32(&gvars);
if (offset) {
uintptr_t v = get_item_for_reloc(s, base, size, offset);
*sysimg_gvars(sysimg_gvars_base, gvname_index) = v;
}
gvname_index += 1;
}
}
static void jl_finalize_serializer(jl_serializer_state *s)
{
size_t i, l;
// save module initialization order
if (jl_module_init_order != NULL) {
l = jl_array_len(jl_module_init_order);
for (i = 0; i < l; i++) {
// verify that all these modules were saved
assert(ptrhash_get(&backref_table, jl_array_ptr_ref(jl_module_init_order, i)) != HT_NOTFOUND);
}
}
jl_write_value(s, jl_module_init_order);
// record list of reinitialization functions
l = reinit_list.len;
for (i = 0; i < l; i += 2) {
size_t item = (size_t)reinit_list.items[i];
size_t reloc_offset = (size_t)layout_table.items[item];
assert(reloc_offset != 0);
write_uint32(s->s, (uint32_t)reloc_offset);
write_uint32(s->s, (uint32_t)((uintptr_t)reinit_list.items[i + 1]));
}
write_uint32(s->s, 0);
}
void jl_typemap_rehash(jl_typemap_t *ml, int8_t offs);
static void jl_reinit_item(jl_value_t *v, int how, arraylist_t *tracee_list)
{
jl_ptls_t ptls = jl_get_ptls_states();
JL_TRY {
switch (how) {
case 1: { // rehash IdDict
jl_array_t **a = (jl_array_t**)v;
assert(jl_is_array(*a));
// Assume *a don't need a write barrier
*a = jl_idtable_rehash(*a, jl_array_len(*a));
jl_gc_wb(v, *a);
break;
}
case 2: { // reinsert module v into parent (const)
jl_module_t *mod = (jl_module_t*)v;
assert(jl_is_module(mod));
jl_binding_t *b = jl_get_binding_wr(mod->parent, mod->name, 1);
jl_declare_constant(b); // this can throw
if (b->value != NULL) {
if (!jl_is_module(b->value)) {
jl_errorf("Invalid redefinition of constant %s.",
jl_symbol_name(mod->name)); // this also throws
}
if (jl_generating_output() && jl_options.incremental) {
jl_errorf("Cannot replace module %s during incremental precompile.", jl_symbol_name(mod->name));
}
jl_printf(JL_STDERR, "WARNING: replacing module %s.\n",
jl_symbol_name(mod->name));
}
b->value = v;
jl_gc_wb_binding(b, v);
break;
}
case 3: { // rehash MethodTable
jl_methtable_t *mt = (jl_methtable_t*)v;
assert(jl_is_mtable(mt));
jl_typemap_rehash(mt->defs, 0);
// TODO: consider reverting this when we can split on Type{...} better
jl_typemap_rehash(mt->cache, 1); //(mt == jl_type_typename->mt) ? 0 : 1);
if (tracee_list)
arraylist_push(tracee_list, mt);
break;
}
case 4: { // rehash specializations tfunc
jl_method_t *m = (jl_method_t*)v;
assert(jl_is_method(m));
jl_typemap_rehash(m->specializations, 0);
break;
}
case 5: { // rebuild the binding table for module v
jl_module_t *mod = (jl_module_t*)v;
assert(jl_is_module(mod));
size_t nbindings = mod->bindings.size;
htable_new(&mod->bindings, nbindings);
struct binding {
uintptr_t tag;
jl_binding_t b;
} *b;
b = (struct binding*)&mod[1];
while (nbindings > 0) {
ptrhash_put(&mod->bindings, (char*)b->b.name, &b->b);
b += 1;
nbindings -= 1;
}
break;
}
default:
assert(0 && "corrupt deserialization state");
abort();
}
}
JL_CATCH {
jl_printf(JL_STDERR, "WARNING: error while reinitializing value ");
jl_static_show(JL_STDERR, v);
jl_printf(JL_STDERR, ":\n");
jl_static_show(JL_STDERR, ptls->exception_in_transit);
jl_printf(JL_STDERR, "\n");
}
}
static jl_array_t *jl_finalize_deserializer(jl_serializer_state *s, arraylist_t *tracee_list)
{
jl_array_t *init_order = (jl_array_t*)jl_read_value(s);
// run reinitialization functions
uintptr_t base = (uintptr_t)&s->s->buf[0];
while (1) {
size_t offset = read_uint32(s->s);
if (offset == 0)
break;
jl_value_t *v = (jl_value_t*)(base + offset);
jl_reinit_item(v, read_uint32(s->s), tracee_list);
}
return init_order;
}
// --- helper functions ---
// remove cached types not referenced in the stream
static void jl_prune_type_cache(jl_svec_t *cache)
{
size_t l = jl_svec_len(cache), ins = 0, i;
for (i = 0; i < l; i++) {
jl_value_t *ti = jl_svecref(cache, i);
if (ti == NULL)
break;
if (ptrhash_get(&backref_table, ti) != HT_NOTFOUND || jl_get_llvm_gv(ti) != 0)
jl_svecset(cache, ins++, ti);
else if (jl_is_datatype(ti)) {
jl_value_t *singleton = ((jl_datatype_t*)ti)->instance;
if (singleton && (ptrhash_get(&backref_table, singleton) != HT_NOTFOUND || jl_get_llvm_gv(singleton) != 0))
jl_svecset(cache, ins++, ti);
}
}
if (i > ins) {
memset(&jl_svec_data(cache)[ins], 0, (i - ins) * sizeof(jl_value_t*));
}
}
// --- entry points ---
static void jl_init_serializer2(int);
static void jl_cleanup_serializer2(void);
static void jl_save_system_image_to_stream(ios_t *f)
{
jl_gc_collect(1); // full
jl_gc_collect(0); // incremental (sweep finalizers)
JL_TIMING(SYSIMG_DUMP);
int en = jl_gc_enable(0);
jl_init_serializer2(1);
htable_reset(&backref_table, 250000);
arraylist_new(&reinit_list, 0);
backref_table_numel = 0;
ios_t sysimg, const_data, symbols, relocs, gvar_record, fptr_record;
ios_mem(&sysimg, 1000000);
ios_mem(&const_data, 100000);
ios_mem(&symbols, 100000);
ios_mem(&relocs, 100000);
ios_mem(&gvar_record, 100000);
ios_mem(&fptr_record, 100000);
jl_serializer_state s;
s.s = &sysimg;
s.const_data = &const_data;
s.symbols = &symbols;
s.relocs = &relocs;
s.gvar_record = &gvar_record;
s.fptr_record = &fptr_record;
s.mode = MODE_SYSTEM_IMAGE;
s.ptls = jl_get_ptls_states();
arraylist_new(&s.relocs_list, 0);
arraylist_new(&s.gctags_list, 0);
// empty!(Core.ARGS)
if (jl_core_module != NULL) {
jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS"));
if (args != NULL) {
jl_array_del_end(args, jl_array_len(args));
}
}
jl_idtable_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("IdDict")) : NULL;
jl_idtable_typename = jl_base_module ? ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_idtable_type))->name : NULL;
jl_bigint_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("BigInt")) : NULL;
if (jl_bigint_type) {
gmp_limb_size = jl_unbox_long(jl_get_global((jl_module_t*)jl_get_global(jl_base_module, jl_symbol("GMP")),
jl_symbol("BITS_PER_LIMB"))) / 8;
}
{ // step 1: record values (recursively) that need to go in the image
jl_serialize_value(&s, jl_core_module);
jl_serialize_value(&s, jl_main_module);
jl_serialize_value(&s, jl_top_module);
jl_serialize_value(&s, jl_typeinf_func);
jl_serialize_value(&s, jl_module_init_order);
// serialize method tables of builtin types
jl_serialize_value(&s, jl_type_typename->mt);
jl_serialize_value(&s, jl_intrinsic_type->name->mt);
jl_serialize_value(&s, jl_sym_type->name->mt);
jl_serialize_value(&s, jl_array_typename->mt);
jl_serialize_value(&s, jl_module_type->name->mt);
jl_prune_type_cache(jl_tuple_typename->cache);
jl_prune_type_cache(jl_tuple_typename->linearcache);
jl_prune_type_cache(jl_type_typename->cache);
uintptr_t i;
for (i = 0; i < builtin_typenames.len; i++) {
jl_serialize_value(&s, ((jl_typename_t*)builtin_typenames.items[i])->cache);
jl_serialize_value(&s, ((jl_typename_t*)builtin_typenames.items[i])->linearcache);
}
}
{ // step 2: build all the sysimg sections
write_padding(&sysimg, sizeof(uint32_t));
jl_write_values(&s);
jl_write_relocations(&s);
jl_write_gv_syms(&s, jl_get_root_symbol());
// ensure everything in deser_tag are reassociated with their GlobalValue
uintptr_t i;
for (i = 0; i < deser_tag.len; i++) {
jl_value_t *v = (jl_value_t*)deser_tag.items[i];
record_gvar(&s, jl_get_llvm_gv(v), ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + i);
}
}
// step 3: combine all of the sections into one file
write_uint32(f, sysimg.size - sizeof(uint32_t));
ios_seek(&sysimg, sizeof(uint32_t));
ios_copyall(f, &sysimg);
ios_close(&sysimg);
write_uint32(f, const_data.size);
// realign stream to max-alignment for data
write_padding(f, LLT_ALIGN(ios_pos(f), 16) - ios_pos(f));
ios_seek(&const_data, 0);
ios_copyall(f, &const_data);
ios_close(&const_data);
write_uint32(f, symbols.size);
ios_seek(&symbols, 0);
ios_copyall(f, &symbols);
ios_close(&symbols);
write_uint32(f, relocs.size);
ios_seek(&relocs, 0);
ios_copyall(f, &relocs);
ios_close(&relocs);
write_uint32(f, gvar_record.size);
ios_seek(&gvar_record, 0);
ios_copyall(f, &gvar_record);
ios_close(&gvar_record);
write_uint32(f, fptr_record.size);
ios_seek(&fptr_record, 0);
ios_copyall(f, &fptr_record);
ios_close(&fptr_record);
{ // step 4: record locations of special roots
s.s = f;
jl_finalize_serializer(&s);
jl_write_value(&s, jl_main_module);
jl_write_value(&s, jl_top_module);
jl_write_value(&s, jl_typeinf_func);
write_uint32(f, jl_typeinf_world);
jl_write_value(&s, jl_type_typename->mt);
jl_write_value(&s, jl_intrinsic_type->name->mt);
jl_write_value(&s, jl_sym_type->name->mt);
jl_write_value(&s, jl_array_typename->mt);
jl_write_value(&s, jl_module_type->name->mt);
uintptr_t i;
for (i = 0; i < builtin_typenames.len; i++) {
jl_write_value(&s, ((jl_typename_t*)builtin_typenames.items[i])->cache);
jl_write_value(&s, ((jl_typename_t*)builtin_typenames.items[i])->linearcache);
}
write_uint32(f, jl_get_t_uid_ctr());
write_uint32(f, jl_get_gs_ctr());
write_uint32(f, jl_world_counter);
}
arraylist_free(&layout_table);
arraylist_free(&reinit_list);
arraylist_free(&s.relocs_list);
arraylist_free(&s.gctags_list);
jl_cleanup_serializer2();
jl_gc_enable(en);
}
JL_DLLEXPORT ios_t *jl_create_system_image(void)
{
ios_t *f = (ios_t*)malloc(sizeof(ios_t));
ios_mem(f, 0);
jl_save_system_image_to_stream(f);
return f;
}
JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src);
JL_DLLEXPORT void jl_save_system_image(const char *fname)
{
ios_t f;
if (ios_file(&f, fname, 1, 1, 1, 1) == NULL) {
jl_errorf("cannot open system image file \"%s\" for writing", fname);
}
JL_SIGATOMIC_BEGIN();
jl_save_system_image_to_stream(&f);
ios_close(&f);
JL_SIGATOMIC_END();
}
extern void jl_get_builtins(void);
extern void jl_get_builtin_hooks(void);
extern void jl_gc_set_permalloc_region(void *start, void *end);
// Takes in a path of the form "usr/lib/julia/sys.so" (jl_restore_system_image should be passed the same string)
JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname)
{
if (jl_sysimg_handle)
return; // embedded target already called jl_set_sysimg_so
char *dot = (char*) strrchr(fname, '.');
int is_ji = (dot && !strcmp(dot, ".ji"));
// Get handle to sys.so
if (!is_ji) // .ji extension => load .ji file only
jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW, 1));
}
// Allow passing in a module handle directly, rather than a path
JL_DLLEXPORT void jl_set_sysimg_so(void *handle)
{
void* *jl_RTLD_DEFAULT_handle_pointer;
int symbol_found = jl_dlsym(handle, "jl_RTLD_DEFAULT_handle_pointer", (void **)&jl_RTLD_DEFAULT_handle_pointer, 0);
if (!symbol_found || (void*)&jl_RTLD_DEFAULT_handle != *jl_RTLD_DEFAULT_handle_pointer)
jl_error("System image file failed consistency check: maybe opened the wrong version?");
if (jl_options.cpu_target == NULL)
jl_options.cpu_target = "native";
jl_sysimg_handle = handle;
sysimg_fptrs = jl_init_processor_sysimg(handle);
}
static void jl_restore_system_image_from_stream(ios_t *f)
{
JL_TIMING(SYSIMG_LOAD);
int en = jl_gc_enable(0);
jl_init_serializer2(0);
ios_t sysimg, const_data, symbols, relocs, gvar_record, fptr_record;
jl_serializer_state s;
s.s = &sysimg;
s.const_data = &const_data;
s.symbols = &symbols;
s.relocs = &relocs;
s.gvar_record = &gvar_record;
s.fptr_record = &fptr_record;
s.mode = MODE_SYSTEM_IMAGE;
s.ptls = jl_get_ptls_states();
arraylist_new(&s.relocs_list, 0);
arraylist_new(&s.gctags_list, 0);
jl_bigint_type = jl_base_module ? jl_get_global(jl_base_module, jl_symbol("BigInt")) : NULL;
if (jl_bigint_type) {
gmp_limb_size = jl_unbox_long(jl_get_global((jl_module_t*)jl_get_global(jl_base_module, jl_symbol("GMP")),
jl_symbol("BITS_PER_LIMB"))) / 8;
}
// step 1: read section map and apply relocations
assert(ios_pos(f) == 0 && f->bm == bm_mem);
size_t sizeof_sysimg = read_uint32(f);
ios_static_buffer(&sysimg, f->buf, sizeof_sysimg + sizeof(uint32_t));
ios_skip(f, sizeof_sysimg);
size_t sizeof_constdata = read_uint32(f);
// realign stream to max-alignment for data
ios_seek(f, LLT_ALIGN(ios_pos(f), 16));
ios_static_buffer(&const_data, f->buf + f->bpos, sizeof_constdata);
ios_skip(f, sizeof_constdata);
size_t sizeof_symbols = read_uint32(f);
ios_static_buffer(&symbols, f->buf + f->bpos, sizeof_symbols);
ios_skip(f, sizeof_symbols);
size_t sizeof_relocations = read_uint32(f);
assert(!ios_eof(f));
ios_static_buffer(&relocs, f->buf + f->bpos, sizeof_relocations);
ios_skip(f, sizeof_relocations);
size_t sizeof_gvar_record = read_uint32(f);
assert(!ios_eof(f));
ios_static_buffer(&gvar_record, f->buf + f->bpos, sizeof_gvar_record);
ios_skip(f, sizeof_gvar_record);
size_t sizeof_fptr_record = read_uint32(f);
assert(!ios_eof(f));
ios_static_buffer(&fptr_record, f->buf + f->bpos, sizeof_fptr_record);
ios_skip(f, sizeof_fptr_record);
assert(!ios_eof(f));
jl_read_symbols(&s);
ios_close(&symbols);
sysimg_base = &sysimg.buf[0];
sysimg_relocs = &relocs.buf[0];
jl_gc_set_permalloc_region((void*)sysimg_base, (void*)(sysimg_base + sysimg.size));
deser_tag.items[0] = (void*)const_data.buf;
jl_read_relocations(&s, GC_OLD_MARKED); // gctags
size_t sizeof_tags = ios_pos(&relocs);
(void)sizeof_tags;
jl_read_relocations(&s, 0); // general relocs
ios_close(&relocs);
ios_close(&const_data);
ios_close(&sysimg);
if (0) {
printf("sysimg size breakdown:\n"
" sys data: %8u\n"
" isbits data: %8u\n"
" symbols: %8u\n"
" tags list: %8u\n"
" reloc list: %8u\n"
" gvar list: %8u\n"
" fptr list: %8u\n",
(unsigned)sizeof_sysimg,
(unsigned)sizeof_constdata,
(unsigned)sizeof_symbols,
(unsigned)sizeof_tags,
(unsigned)(sizeof_relocations - sizeof_tags),
(unsigned)sizeof_gvar_record,
(unsigned)sizeof_fptr_record);
}
// step 2: get special values
s.s = f;
jl_module_init_order = jl_finalize_deserializer(&s, NULL);
jl_main_module = (jl_module_t*)jl_read_value(&s);
jl_top_module = (jl_module_t*)jl_read_value(&s);
jl_typeinf_func = (jl_function_t*)jl_read_value(&s);
jl_typeinf_world = read_uint32(f);
jl_type_type_mt = (jl_methtable_t*)jl_read_value(&s);
jl_type_typename->mt = jl_type_type_mt;
jl_unionall_type->name->mt = jl_type_type_mt;
jl_uniontype_type->name->mt = jl_type_type_mt;
jl_datatype_type->name->mt = jl_type_type_mt;
jl_intrinsic_type->name->mt = (jl_methtable_t*)jl_read_value(&s);
jl_sym_type->name->mt = (jl_methtable_t*)jl_read_value(&s);
jl_array_typename->mt = (jl_methtable_t*)jl_read_value(&s);
jl_module_type->name->mt = (jl_methtable_t*)jl_read_value(&s);
uintptr_t i;
for (i = 0; i < builtin_typenames.len; i++) {
jl_typename_t *tn = (jl_typename_t*)builtin_typenames.items[i];
tn->cache = (jl_svec_t*)jl_read_value(&s);
jl_gc_wb(tn, tn->cache);
tn->linearcache = (jl_svec_t*)jl_read_value(&s);
jl_gc_wb(tn, tn->linearcache);
jl_resort_type_cache(tn->cache);
}
jl_core_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Core"));
jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Base"));
uint32_t uid_ctr = read_uint32(f);
uint32_t gs_ctr = read_uint32(f);
jl_world_counter = read_uint32(f);
jl_set_t_uid_ctr(uid_ctr);
jl_set_gs_ctr(gs_ctr);
jl_get_builtins();
jl_get_builtin_hooks();
jl_init_box_caches();
jl_update_all_gvars(&s);
ios_close(&gvar_record);
jl_update_all_fptrs(&s);
ios_close(&fptr_record);
jl_gc_reset_alloc_count();
jl_gc_enable(en);
jl_cleanup_serializer2();
}
// TODO: need to enforce that the alignment of the buffer is suitable for vectors
JL_DLLEXPORT void jl_restore_system_image(const char *fname)
{
#ifndef JL_NDEBUG
char *dot = fname ? (char*)strrchr(fname, '.') : NULL;
int is_ji = (dot && !strcmp(dot, ".ji"));
assert((is_ji || jl_sysimg_handle) && "System image file not preloaded");
#endif
if (jl_sysimg_handle) {
// load the pre-compiled sysimage from jl_sysimg_handle
jl_load_sysimg_so();
}
else {
ios_t f;
if (ios_file(&f, fname, 1, 0, 0, 0) == NULL)
jl_errorf("System image file \"%s\" not found.", fname);
ios_bufmode(&f, bm_none);
JL_SIGATOMIC_BEGIN();
ios_seek_end(&f);
size_t len = ios_pos(&f);
char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0);
ios_seek(&f, 0);
if (ios_readall(&f, sysimg, len) != len)
jl_errorf("Error reading system image file.");
ios_close(&f);
ios_static_buffer(&f, sysimg, len);
jl_restore_system_image_from_stream(&f);
ios_close(&f);
JL_SIGATOMIC_END();
}
}
JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len)
{
ios_t f;
JL_SIGATOMIC_BEGIN();
ios_static_buffer(&f, (char*)buf, len);
jl_restore_system_image_from_stream(&f);
ios_close(&f);
JL_SIGATOMIC_END();
}
// --- init ---
static void jl_init_serializer2(int for_serialize)
{
jl_ptls_t ptls = jl_get_ptls_states();
arraylist_new(&builtin_typenames, 0);
if (for_serialize) {
htable_new(&sertag_table, 0);
htable_new(&symbol_table, 0);
htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs));
htable_new(&backref_table, 0);
}
else {
arraylist_new(&deser_tag, 0);
arraylist_new(&deser_sym, 0);
}
uintptr_t i;
void *tags[] = { ptls->root_task,
jl_symbol_type, jl_ssavalue_type, jl_datatype_type, jl_slotnumber_type,
jl_simplevector_type, jl_array_type, jl_typedslot_type,
jl_expr_type, jl_globalref_type, jl_string_type,
jl_module_type, jl_tvar_type, jl_method_instance_type, jl_method_type,
jl_emptysvec, jl_emptytuple, jl_false, jl_true, jl_nothing, jl_any_type,
call_sym, invoke_sym, goto_ifnot_sym, return_sym, jl_symbol("tuple"),
unreachable_sym,
jl_linenumbernode_type, jl_lineinfonode_type,
jl_gotonode_type, jl_quotenode_type,
jl_pinode_type, jl_phinode_type, jl_phicnode_type, jl_upsilonnode_type,
jl_type_type, jl_bottom_type, jl_ref_type, jl_pointer_type,
jl_vararg_type, jl_abstractarray_type,
jl_densearray_type, jl_void_type, jl_function_type, jl_typeofbottom_type,
jl_unionall_type, jl_typename_type, jl_builtin_type, jl_code_info_type,
jl_task_type, jl_uniontype_type, jl_typetype_type, jl_abstractstring_type,
jl_array_any_type, jl_intrinsic_type, jl_abstractslot_type,
jl_methtable_type, jl_typemap_level_type, jl_typemap_entry_type,
jl_voidpointer_type, jl_newvarnode_type,
jl_array_symbol_type, jl_anytuple_type, jl_tparam0(jl_anytuple_type),
jl_emptytuple_type, jl_array_uint8_type, jl_array_int32_type,
jl_symbol_type->name, jl_ssavalue_type->name, jl_tuple_typename,
((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_ref_type))->name,
jl_pointer_typename, jl_simplevector_type->name,
jl_datatype_type->name, jl_uniontype_type->name, jl_array_typename,
jl_expr_type->name, jl_typename_type->name, jl_type_typename,
jl_methtable_type->name, jl_typemap_level_type->name, jl_typemap_entry_type->name, jl_tvar_type->name,
((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_abstractarray_type))->name,
((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_densearray_type))->name,
jl_vararg_typename, jl_void_type->name, jl_method_instance_type->name, jl_method_type->name,
jl_module_type->name, jl_function_type->name, jl_typedslot_type->name,
jl_abstractslot_type->name, jl_slotnumber_type->name,
jl_unionall_type->name, jl_intrinsic_type->name, jl_task_type->name,
jl_linenumbernode_type->name, jl_builtin_type->name,
jl_gotonode_type->name, jl_quotenode_type->name, jl_lineinfonode_type->name,
jl_pinode_type->name, jl_phinode_type->name,
jl_phicnode_type->name, jl_upsilonnode_type->name,
jl_globalref_type->name, jl_typeofbottom_type->name,
jl_string_type->name, jl_abstractstring_type->name,
jl_namedtuple_type, jl_namedtuple_typename,
jl_int32_type, jl_int64_type, jl_bool_type, jl_uint8_type,
jl_uint32_type, jl_uint64_type,
// empirical list of very common symbols
#include "common_symbols1.inc"
NULL };
arraylist_push(&deser_tag, NULL);
arraylist_push(&deser_tag, NULL);
for (i = 0; tags[i] != NULL; i++) {
void *v = tags[i];
if (!for_serialize) {
// some builtins are only rooted through a type cache or Main.Core binding,
// but were allocated young, so we force the gc to change their tag here
jl_gc_force_mark_old(ptls, (jl_value_t*)v);
}
if (jl_is_symbol(v)) {
arraylist_push(&deser_sym, v);
if (for_serialize)
ptrhash_put(&symbol_table, v, (void*)((char*)HT_NOTFOUND + ((uintptr_t)SymbolRef << RELOC_TAG_OFFSET) + deser_sym.len));
}
else {
arraylist_push(&deser_tag, v);
if (for_serialize)
ptrhash_put(&sertag_table, v, (void*)((char*)HT_NOTFOUND + ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + deser_tag.len));
}
}
assert(i + 1 == sizeof(tags) / sizeof(tags[0]));
assert(!for_serialize || ptrhash_get(&sertag_table, ptls->root_task) == (char*)HT_NOTFOUND + ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + 3);
nsym_tag = deser_sym.len;
// this also ensures all objects referenced in the code have
// references in the system image to their global variable
// since codegen knows that some integer boxes are static,
// they might not have had a reference anywhere in the code
// image other than here
#define NBOX_C 1024
for (i = 0; i < NBOX_C; i++) {
jl_value_t *v32 = jl_box_int32(i - NBOX_C / 2);
arraylist_push(&deser_tag, v32);
if (for_serialize)
ptrhash_put(&sertag_table, v32, (void*)((char*)HT_NOTFOUND + ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + deser_tag.len));
jl_value_t *v64 = jl_box_int64(i - NBOX_C / 2);
arraylist_push(&deser_tag, v64);
if (for_serialize)
ptrhash_put(&sertag_table, v64, (void*)((char*)HT_NOTFOUND + ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + deser_tag.len));
}
for (i = 0; i < 256; i++) {
jl_value_t *vu = jl_box_uint8(i);
arraylist_push(&deser_tag, vu);
if (for_serialize)
ptrhash_put(&sertag_table, vu, (void*)((char*)HT_NOTFOUND + ((uintptr_t)TagRef << RELOC_TAG_OFFSET) + deser_tag.len));
}
if (for_serialize) {
for (i = 0; id_to_fptrs[i] != NULL; i++) {
ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2));
}
}
arraylist_push(&builtin_typenames, jl_array_typename);
arraylist_push(&builtin_typenames, ((jl_datatype_t*)jl_ref_type->body)->name);
arraylist_push(&builtin_typenames, jl_pointer_typename);
arraylist_push(&builtin_typenames, jl_type_typename);
arraylist_push(&builtin_typenames, ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_abstractarray_type))->name);
arraylist_push(&builtin_typenames, ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_densearray_type))->name);
arraylist_push(&builtin_typenames, jl_tuple_typename);
arraylist_push(&builtin_typenames, jl_vararg_typename);
arraylist_push(&builtin_typenames, jl_namedtuple_typename);
}
static void jl_cleanup_serializer2(void)
{
htable_reset(&sertag_table, 0);
htable_reset(&symbol_table, 0);
htable_reset(&fptr_to_id, 0);
htable_reset(&backref_table, 0);
arraylist_free(&deser_tag);
arraylist_free(&deser_sym);
arraylist_free(&builtin_typenames);
nsym_tag = 0;
}
#ifdef __cplusplus
}
#endif