/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sts=4 et sw=4 tw=99:
 *
 * Copyright 2014 Mozilla Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "asmjs/AsmJSModule.h"

#include "mozilla/BinarySearch.h"
#include "mozilla/Compression.h"
#include "mozilla/EnumeratedRange.h"
#include "mozilla/PodOperations.h"
#include "mozilla/TaggedAnonymousMemory.h"
#include "mozilla/Vector.h"

#include "jslibmath.h"
#include "jsmath.h"
#include "jsprf.h"

#include "builtin/AtomicsObject.h"
#include "frontend/Parser.h"
#include "jit/IonCode.h"
#ifdef JS_ION_PERF
# include "jit/PerfSpewer.h"
#endif
#include "js/Class.h"
#include "js/Conversions.h"
#include "js/MemoryMetrics.h"
#include "vm/Time.h"

#include "jsobjinlines.h"

#include "frontend/ParseNode-inl.h"
#include "jit/MacroAssembler-inl.h"
#include "vm/ArrayBufferObject-inl.h"
#include "vm/Stack-inl.h"

using namespace js;
using namespace js::jit;
using namespace js::wasm;
using namespace js::frontend;
using mozilla::BinarySearch;
using mozilla::Compression::LZ4;
using mozilla::MakeEnumeratedRange;
using mozilla::MallocSizeOf;
using mozilla::PodCopy;
using mozilla::PodEqual;
using mozilla::PodZero;
using mozilla::Swap;
using JS::GenericNaN;

static uint8_t*
AllocateExecutableMemory(ExclusiveContext* cx, size_t bytes)
{
    // On most platforms, this will allocate RWX memory. On iOS, or when
    // --non-writable-jitcode is used, this will allocate RW memory. In this
    // case, DynamicallyLinkModule will reprotect the code as RX.
    unsigned permissions =
        ExecutableAllocator::initialProtectionFlags(ExecutableAllocator::Writable);
    void* p = AllocateExecutableMemory(nullptr, bytes, permissions, "asm-js-code", AsmJSPageSize);
    if (!p)
        ReportOutOfMemory(cx);
    return (uint8_t*)p;
}

AsmJSModule::AsmJSModule(ScriptSource* scriptSource, uint32_t srcStart, uint32_t srcBodyStart,
                         bool strict, bool canUseSignalHandlers)
  : srcStart_(srcStart),
    srcBodyStart_(srcBodyStart),
    scriptSource_(scriptSource),
    globalArgumentName_(nullptr),
    importArgumentName_(nullptr),
    bufferArgumentName_(nullptr),
    code_(nullptr),
    interruptExit_(nullptr),
    prevLinked_(nullptr),
    nextLinked_(nullptr),
    dynamicallyLinked_(false),
    loadedFromCache_(false),
    profilingEnabled_(false),
    interrupted_(false)
{
    mozilla::PodZero(&pod);
    pod.globalBytes_ = sInitialGlobalDataBytes;
    pod.minHeapLength_ = RoundUpToNextValidAsmJSHeapLength(0);
    pod.maxHeapLength_ = 0x80000000;
    pod.strict_ = strict;
    pod.canUseSignalHandlers_ = canUseSignalHandlers;

    // AsmJSCheckedImmediateRange should be defined to be at most the minimum
    // heap length so that offsets can be folded into bounds checks.
    MOZ_ASSERT(pod.minHeapLength_ - AsmJSCheckedImmediateRange <= pod.minHeapLength_);

    scriptSource_->incref();
}

AsmJSModule::~AsmJSModule()
{
    MOZ_ASSERT(!interrupted_);

    scriptSource_->decref();

    if (code_) {
        for (unsigned i = 0; i < numExits(); i++) {
            AsmJSModule::ExitDatum& exitDatum = exit(i).datum(*this);
            if (!exitDatum.baselineScript)
                continue;

            jit::DependentAsmJSModuleExit exit(this, i);
            exitDatum.baselineScript->removeDependentAsmJSModule(exit);
        }

        DeallocateExecutableMemory(code_, pod.totalBytes_, AsmJSPageSize);
    }

    if (prevLinked_)
        *prevLinked_ = nextLinked_;
    if (nextLinked_)
        nextLinked_->prevLinked_ = prevLinked_;
}

void
AsmJSModule::trace(JSTracer* trc)
{
    for (Global& global : globals_)
        global.trace(trc);
    for (Exit& exit : exits_) {
        if (exit.datum(*this).fun)
            TraceEdge(trc, &exit.datum(*this).fun, "asm.js imported function");
    }
    for (ExportedFunction& exp : exports_)
        exp.trace(trc);
    for (Name& name : names_)
        TraceManuallyBarrieredEdge(trc, &name.name(), "asm.js module function name");
#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
    for (ProfiledFunction& profiledFunction : profiledFunctions_)
        profiledFunction.trace(trc);
#endif
    if (globalArgumentName_)
        TraceManuallyBarrieredEdge(trc, &globalArgumentName_, "asm.js global argument name");
    if (importArgumentName_)
        TraceManuallyBarrieredEdge(trc, &importArgumentName_, "asm.js import argument name");
    if (bufferArgumentName_)
        TraceManuallyBarrieredEdge(trc, &bufferArgumentName_, "asm.js buffer argument name");
    if (maybeHeap_)
        TraceEdge(trc, &maybeHeap_, "asm.js heap");
}

void
AsmJSModule::addSizeOfMisc(MallocSizeOf mallocSizeOf, size_t* asmJSModuleCode,
                           size_t* asmJSModuleData)
{
    *asmJSModuleCode += pod.totalBytes_;
    *asmJSModuleData += mallocSizeOf(this) +
                        globals_.sizeOfExcludingThis(mallocSizeOf) +
                        exits_.sizeOfExcludingThis(mallocSizeOf) +
                        exports_.sizeOfExcludingThis(mallocSizeOf) +
                        callSites_.sizeOfExcludingThis(mallocSizeOf) +
                        codeRanges_.sizeOfExcludingThis(mallocSizeOf) +
                        names_.sizeOfExcludingThis(mallocSizeOf) +
                        heapAccesses_.sizeOfExcludingThis(mallocSizeOf) +
#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
                        profiledFunctions_.sizeOfExcludingThis(mallocSizeOf) +
#endif
                        staticLinkData_.sizeOfExcludingThis(mallocSizeOf);
}

struct CallSiteRetAddrOffset
{
    const CallSiteVector& callSites;
    explicit CallSiteRetAddrOffset(const CallSiteVector& callSites) : callSites(callSites) {}
    uint32_t operator[](size_t index) const {
        return callSites[index].returnAddressOffset();
    }
};

const CallSite*
AsmJSModule::lookupCallSite(void* returnAddress) const
{
    MOZ_ASSERT(isFinished());

    uint32_t target = ((uint8_t*)returnAddress) - code_;
    size_t lowerBound = 0;
    size_t upperBound = callSites_.length();

    size_t match;
    if (!BinarySearch(CallSiteRetAddrOffset(callSites_), lowerBound, upperBound, target, &match))
        return nullptr;

    return &callSites_[match];
}

namespace js {

// Create an ordering on CodeRange and pc offsets suitable for BinarySearch.
// Stick these in the same namespace as AsmJSModule so that argument-dependent
// lookup will find it.
bool
operator==(size_t pcOffset, const AsmJSModule::CodeRange& rhs)
{
    return pcOffset >= rhs.begin() && pcOffset < rhs.end();
}
bool
operator<=(const AsmJSModule::CodeRange& lhs, const AsmJSModule::CodeRange& rhs)
{
    return lhs.begin() <= rhs.begin();
}
bool
operator<(size_t pcOffset, const AsmJSModule::CodeRange& rhs)
{
    return pcOffset < rhs.begin();
}

} // namespace js

const AsmJSModule::CodeRange*
AsmJSModule::lookupCodeRange(void* pc) const
{
    MOZ_ASSERT(isFinished());

    uint32_t target = ((uint8_t*)pc) - code_;
    size_t lowerBound = 0;
    size_t upperBound = codeRanges_.length();

    size_t match;
    if (!BinarySearch(codeRanges_, lowerBound, upperBound, target, &match))
        return nullptr;

    return &codeRanges_[match];
}

struct HeapAccessOffset
{
    const HeapAccessVector& accesses;
    explicit HeapAccessOffset(const HeapAccessVector& accesses) : accesses(accesses) {}
    uintptr_t operator[](size_t index) const {
        return accesses[index].insnOffset();
    }
};

const HeapAccess*
AsmJSModule::lookupHeapAccess(void* pc) const
{
    MOZ_ASSERT(isFinished());
    MOZ_ASSERT(containsFunctionPC(pc));

    uint32_t target = ((uint8_t*)pc) - code_;
    size_t lowerBound = 0;
    size_t upperBound = heapAccesses_.length();

    size_t match;
    if (!BinarySearch(HeapAccessOffset(heapAccesses_), lowerBound, upperBound, target, &match))
        return nullptr;

    return &heapAccesses_[match];
}

bool
AsmJSModule::finish(ExclusiveContext* cx, TokenStream& tokenStream, MacroAssembler& masm)
{
    MOZ_ASSERT(!isFinished());

    uint32_t endBeforeCurly = tokenStream.currentToken().pos.end;
    TokenPos pos;
    if (!tokenStream.peekTokenPos(&pos, TokenStream::Operand))
        return false;
    uint32_t endAfterCurly = pos.end;
    MOZ_ASSERT(endBeforeCurly >= srcBodyStart_);
    MOZ_ASSERT(endAfterCurly >= srcBodyStart_);
    pod.srcLength_ = endBeforeCurly - srcStart_;
    pod.srcLengthWithRightBrace_ = endAfterCurly - srcStart_;

    // Start global data on a new page so JIT code may be given independent
    // protection flags.
    pod.codeBytes_ = AlignBytes(masm.bytesNeeded(), AsmJSPageSize);
    MOZ_ASSERT(pod.functionBytes_ <= pod.codeBytes_);

    // The entire region is allocated via mmap/VirtualAlloc which requires
    // units of pages.
    pod.totalBytes_ = AlignBytes(pod.codeBytes_ + pod.globalBytes_, AsmJSPageSize);

    MOZ_ASSERT(!code_);
    code_ = AllocateExecutableMemory(cx, pod.totalBytes_);
    if (!code_)
        return false;

    // Delay flushing until dynamic linking. The flush-inhibited range is set within
    // masm.executableCopy.
    AutoFlushICache afc("CheckModule", /* inhibit = */ true);

    // Copy the code from the MacroAssembler into its final resting place in the
    // AsmJSModule.
    MOZ_ASSERT(uintptr_t(code_) % AsmJSPageSize == 0);
    masm.executableCopy(code_);

    // c.f. JitCode::copyFrom
    MOZ_ASSERT(masm.jumpRelocationTableBytes() == 0);
    MOZ_ASSERT(masm.dataRelocationTableBytes() == 0);
    MOZ_ASSERT(masm.preBarrierTableBytes() == 0);
    MOZ_ASSERT(!masm.hasSelfReference());

    // Heap-access metadata used for link-time patching and fault-handling.
    heapAccesses_ = masm.extractHeapAccesses();

    // Call-site metadata used for stack unwinding.
    const CallSiteAndTargetVector& callSites = masm.callSites();
    if (!callSites_.appendAll(callSites))
        return false;

    // Absolute link metadata: absolute addresses that refer to some fixed
    // address in the address space.
    AbsoluteLinkArray& absoluteLinks = staticLinkData_.absoluteLinks;
    for (size_t i = 0; i < masm.numAsmJSAbsoluteLinks(); i++) {
        AsmJSAbsoluteLink src = masm.asmJSAbsoluteLink(i);
        if (!absoluteLinks[src.target].append(src.patchAt.offset()))
            return false;
    }

    // Relative link metadata: absolute addresses that refer to another point within
    // the asm.js module.

    // CodeLabels are used for switch cases and loads from floating-point /
    // SIMD values in the constant pool.
    for (size_t i = 0; i < masm.numCodeLabels(); i++) {
        CodeLabel cl = masm.codeLabel(i);
        RelativeLink link(RelativeLink::CodeLabel);
        link.patchAtOffset = masm.labelToPatchOffset(*cl.patchAt());
        link.targetOffset = cl.target()->offset();
        if (!staticLinkData_.relativeLinks.append(link))
            return false;
    }

#if defined(JS_CODEGEN_X86)
    // Global data accesses in x86 need to be patched with the absolute
    // address of the global. Globals are allocated sequentially after the
    // code section so we can just use an RelativeLink.
    for (size_t i = 0; i < masm.numAsmJSGlobalAccesses(); i++) {
        AsmJSGlobalAccess a = masm.asmJSGlobalAccess(i);
        RelativeLink link(RelativeLink::RawPointer);
        link.patchAtOffset = masm.labelToPatchOffset(a.patchAt);
        link.targetOffset = offsetOfGlobalData() + a.globalDataOffset;
        if (!staticLinkData_.relativeLinks.append(link))
            return false;
    }
#endif

#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    // On MIPS we need to update all the long jumps because they contain an
    // absolute adress. The values are correctly patched for the current address
    // space, but not after serialization or profiling-mode toggling.
    for (size_t i = 0; i < masm.numLongJumps(); i++) {
        size_t off = masm.longJump(i);
        RelativeLink link(RelativeLink::InstructionImmediate);
        link.patchAtOffset = off;
        link.targetOffset = Assembler::ExtractInstructionImmediate(code_ + off) - uintptr_t(code_);
        if (!staticLinkData_.relativeLinks.append(link))
            return false;
    }
#endif

#if defined(JS_CODEGEN_X64)
    // Global data accesses on x64 use rip-relative addressing and thus do
    // not need patching after deserialization.
    for (size_t i = 0; i < masm.numAsmJSGlobalAccesses(); i++) {
        AsmJSGlobalAccess a = masm.asmJSGlobalAccess(i);
        masm.patchAsmJSGlobalAccess(a.patchAt, code_, globalData(), a.globalDataOffset);
    }
#endif

    return true;
}

void
AsmJSModule::setAutoFlushICacheRange()
{
    MOZ_ASSERT(isFinished());
    AutoFlushICache::setRange(uintptr_t(code_), pod.codeBytes_);
}

static void
AsmJSReportOverRecursed()
{
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();
    ReportOverRecursed(cx);
}

static void
OnDetached()
{
    // See hasDetachedHeap comment in LinkAsmJS.
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();
    JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_OUT_OF_MEMORY);
}

static void
OnOutOfBounds()
{
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();
    JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_BAD_INDEX);
}

static void
OnImpreciseConversion()
{
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();
    JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_SIMD_FAILED_CONVERSION);
}

static bool
AsmJSHandleExecutionInterrupt()
{
    AsmJSActivation* act = JSRuntime::innermostAsmJSActivation();
    act->module().setInterrupted(true);
    bool ret = CheckForInterrupt(act->cx());
    act->module().setInterrupted(false);
    return ret;
}

static int32_t
CoerceInPlace_ToInt32(MutableHandleValue val)
{
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();

    int32_t i32;
    if (!ToInt32(cx, val, &i32))
        return false;
    val.set(Int32Value(i32));

    return true;
}

static int32_t
CoerceInPlace_ToNumber(MutableHandleValue val)
{
    JSContext* cx = JSRuntime::innermostAsmJSActivation()->cx();

    double dbl;
    if (!ToNumber(cx, val, &dbl))
        return false;
    val.set(DoubleValue(dbl));

    return true;
}

static bool
TryEnablingJit(JSContext* cx, AsmJSModule& module, HandleFunction fun, uint32_t exitIndex,
               int32_t argc, Value* argv)
{
    if (!fun->hasScript())
        return true;

    // Test if the function is JIT compiled.
    JSScript* script = fun->nonLazyScript();
    if (!script->hasBaselineScript()) {
        MOZ_ASSERT(!script->hasIonScript());
        return true;
    }

    // Don't enable jit entry when we have a pending ion builder.
    // Take the interpreter path which will link it and enable
    // the fast path on the next call.
    if (script->baselineScript()->hasPendingIonBuilder())
        return true;

    // Currently we can't rectify arguments. Therefore disabling if argc is too low.
    if (fun->nargs() > size_t(argc))
        return true;

    // Ensure the argument types are included in the argument TypeSets stored in
    // the TypeScript. This is necessary for Ion, because the FFI exit will
    // use the skip-arg-checks entry point.
    //
    // Note that the TypeScript is never discarded while the script has a
    // BaselineScript, so if those checks hold now they must hold at least until
    // the BaselineScript is discarded and when that happens the FFI exit is
    // patched back.
    if (!TypeScript::ThisTypes(script)->hasType(TypeSet::UndefinedType()))
        return true;
    for (uint32_t i = 0; i < fun->nargs(); i++) {
        StackTypeSet* typeset = TypeScript::ArgTypes(script, i);
        TypeSet::Type type = TypeSet::DoubleType();
        if (!argv[i].isDouble())
            type = TypeSet::PrimitiveType(argv[i].extractNonDoubleType());
        if (!typeset->hasType(type))
            return true;
    }

    // The exit may have become optimized while executing the FFI.
    AsmJSModule::Exit& exit = module.exit(exitIndex);
    if (exit.isOptimized(module))
        return true;

    BaselineScript* baselineScript = script->baselineScript();
    if (!baselineScript->addDependentAsmJSModule(cx, DependentAsmJSModuleExit(&module, exitIndex)))
        return false;

    exit.optimize(module, baselineScript);
    return true;
}

static bool
InvokeFromAsmJS(AsmJSActivation* activation, int32_t exitIndex, int32_t argc, Value* argv,
                MutableHandleValue rval)
{
    JSContext* cx = activation->cx();
    AsmJSModule& module = activation->module();

    RootedFunction fun(cx, module.exit(exitIndex).datum(module).fun);
    RootedValue fval(cx, ObjectValue(*fun));
    if (!Invoke(cx, UndefinedValue(), fval, argc, argv, rval))
        return false;

    return TryEnablingJit(cx, module, fun, exitIndex, argc, argv);
}

// Use an int32_t return type instead of bool since bool does not have a
// specified width and the caller is assuming a word-sized return.
static int32_t
InvokeFromAsmJS_Ignore(int32_t exitIndex, int32_t argc, Value* argv)
{
    AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation();
    JSContext* cx = activation->cx();

    RootedValue rval(cx);
    return InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval);
}

// Use an int32_t return type instead of bool since bool does not have a
// specified width and the caller is assuming a word-sized return.
static int32_t
InvokeFromAsmJS_ToInt32(int32_t exitIndex, int32_t argc, Value* argv)
{
    AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation();
    JSContext* cx = activation->cx();

    RootedValue rval(cx);
    if (!InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval))
        return false;

    int32_t i32;
    if (!ToInt32(cx, rval, &i32))
        return false;

    argv[0] = Int32Value(i32);
    return true;
}

// Use an int32_t return type instead of bool since bool does not have a
// specified width and the caller is assuming a word-sized return.
static int32_t
InvokeFromAsmJS_ToNumber(int32_t exitIndex, int32_t argc, Value* argv)
{
    AsmJSActivation* activation = JSRuntime::innermostAsmJSActivation();
    JSContext* cx = activation->cx();

    RootedValue rval(cx);
    if (!InvokeFromAsmJS(activation, exitIndex, argc, argv, &rval))
        return false;

    double dbl;
    if (!ToNumber(cx, rval, &dbl))
        return false;

    argv[0] = DoubleValue(dbl);
    return true;
}

#if defined(JS_CODEGEN_ARM)
extern "C" {

extern MOZ_EXPORT int64_t
__aeabi_idivmod(int, int);

extern MOZ_EXPORT int64_t
__aeabi_uidivmod(int, int);

}
#endif

template <class F>
static inline void*
FuncCast(F* pf)
{
    return JS_FUNC_TO_DATA_PTR(void*, pf);
}

static void*
RedirectCall(void* fun, ABIFunctionType type)
{
#ifdef JS_SIMULATOR
    fun = Simulator::RedirectNativeFunction(fun, type);
#endif
    return fun;
}

static void*
AddressOf(SymbolicAddress imm, ExclusiveContext* cx)
{
    switch (imm) {
      case SymbolicAddress::Runtime:
        return cx->runtimeAddressForJit();
      case SymbolicAddress::RuntimeInterruptUint32:
        return cx->runtimeAddressOfInterruptUint32();
      case SymbolicAddress::StackLimit:
        return cx->stackLimitAddressForJitCode(StackForUntrustedScript);
      case SymbolicAddress::ReportOverRecursed:
        return RedirectCall(FuncCast(AsmJSReportOverRecursed), Args_General0);
      case SymbolicAddress::OnDetached:
        return RedirectCall(FuncCast(OnDetached), Args_General0);
      case SymbolicAddress::OnOutOfBounds:
        return RedirectCall(FuncCast(OnOutOfBounds), Args_General0);
      case SymbolicAddress::OnImpreciseConversion:
        return RedirectCall(FuncCast(OnImpreciseConversion), Args_General0);
      case SymbolicAddress::HandleExecutionInterrupt:
        return RedirectCall(FuncCast(AsmJSHandleExecutionInterrupt), Args_General0);
      case SymbolicAddress::InvokeFromAsmJS_Ignore:
        return RedirectCall(FuncCast(InvokeFromAsmJS_Ignore), Args_General3);
      case SymbolicAddress::InvokeFromAsmJS_ToInt32:
        return RedirectCall(FuncCast(InvokeFromAsmJS_ToInt32), Args_General3);
      case SymbolicAddress::InvokeFromAsmJS_ToNumber:
        return RedirectCall(FuncCast(InvokeFromAsmJS_ToNumber), Args_General3);
      case SymbolicAddress::CoerceInPlace_ToInt32:
        return RedirectCall(FuncCast(CoerceInPlace_ToInt32), Args_General1);
      case SymbolicAddress::CoerceInPlace_ToNumber:
        return RedirectCall(FuncCast(CoerceInPlace_ToNumber), Args_General1);
      case SymbolicAddress::ToInt32:
        return RedirectCall(FuncCast<int32_t (double)>(JS::ToInt32), Args_Int_Double);
#if defined(JS_CODEGEN_ARM)
      case SymbolicAddress::aeabi_idivmod:
        return RedirectCall(FuncCast(__aeabi_idivmod), Args_General2);
      case SymbolicAddress::aeabi_uidivmod:
        return RedirectCall(FuncCast(__aeabi_uidivmod), Args_General2);
      case SymbolicAddress::AtomicCmpXchg:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t, int32_t)>(js::atomics_cmpxchg_asm_callout), Args_General4);
      case SymbolicAddress::AtomicXchg:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_xchg_asm_callout), Args_General3);
      case SymbolicAddress::AtomicFetchAdd:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_add_asm_callout), Args_General3);
      case SymbolicAddress::AtomicFetchSub:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_sub_asm_callout), Args_General3);
      case SymbolicAddress::AtomicFetchAnd:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_and_asm_callout), Args_General3);
      case SymbolicAddress::AtomicFetchOr:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_or_asm_callout), Args_General3);
      case SymbolicAddress::AtomicFetchXor:
        return RedirectCall(FuncCast<int32_t (int32_t, int32_t, int32_t)>(js::atomics_xor_asm_callout), Args_General3);
#endif
      case SymbolicAddress::ModD:
        return RedirectCall(FuncCast(NumberMod), Args_Double_DoubleDouble);
      case SymbolicAddress::SinD:
#ifdef _WIN64
        // Workaround a VS 2013 sin issue, see math_sin_uncached.
        return RedirectCall(FuncCast<double (double)>(js::math_sin_uncached), Args_Double_Double);
#else
        return RedirectCall(FuncCast<double (double)>(sin), Args_Double_Double);
#endif
      case SymbolicAddress::CosD:
        return RedirectCall(FuncCast<double (double)>(cos), Args_Double_Double);
      case SymbolicAddress::TanD:
        return RedirectCall(FuncCast<double (double)>(tan), Args_Double_Double);
      case SymbolicAddress::ASinD:
        return RedirectCall(FuncCast<double (double)>(asin), Args_Double_Double);
      case SymbolicAddress::ACosD:
        return RedirectCall(FuncCast<double (double)>(acos), Args_Double_Double);
      case SymbolicAddress::ATanD:
        return RedirectCall(FuncCast<double (double)>(atan), Args_Double_Double);
      case SymbolicAddress::CeilD:
        return RedirectCall(FuncCast<double (double)>(ceil), Args_Double_Double);
      case SymbolicAddress::CeilF:
        return RedirectCall(FuncCast<float (float)>(ceilf), Args_Float32_Float32);
      case SymbolicAddress::FloorD:
        return RedirectCall(FuncCast<double (double)>(floor), Args_Double_Double);
      case SymbolicAddress::FloorF:
        return RedirectCall(FuncCast<float (float)>(floorf), Args_Float32_Float32);
      case SymbolicAddress::ExpD:
        return RedirectCall(FuncCast<double (double)>(exp), Args_Double_Double);
      case SymbolicAddress::LogD:
        return RedirectCall(FuncCast<double (double)>(log), Args_Double_Double);
      case SymbolicAddress::PowD:
        return RedirectCall(FuncCast(ecmaPow), Args_Double_DoubleDouble);
      case SymbolicAddress::ATan2D:
        return RedirectCall(FuncCast(ecmaAtan2), Args_Double_DoubleDouble);
      case SymbolicAddress::Limit:
        break;
    }

    MOZ_CRASH("Bad SymbolicAddress");
}

void
AsmJSModule::staticallyLink(ExclusiveContext* cx)
{
    MOZ_ASSERT(isFinished());

    // Process staticLinkData_

    MOZ_ASSERT(staticLinkData_.pod.interruptExitOffset != 0);
    interruptExit_ = code_ + staticLinkData_.pod.interruptExitOffset;

    MOZ_ASSERT(staticLinkData_.pod.outOfBoundsExitOffset != 0);
    outOfBoundsExit_ = code_ + staticLinkData_.pod.outOfBoundsExitOffset;

    for (size_t i = 0; i < staticLinkData_.relativeLinks.length(); i++) {
        RelativeLink link = staticLinkData_.relativeLinks[i];
        uint8_t* patchAt = code_ + link.patchAtOffset;
        uint8_t* target = code_ + link.targetOffset;

        // In the case of long-jumps on MIPS and possibly future cases, a
        // RelativeLink is used to patch a pointer to the function entry. If
        // profiling is enabled (by cloning a module with profiling enabled),
        // the target should be the profiling entry.
        if (profilingEnabled_) {
            const CodeRange* codeRange = lookupCodeRange(target);
            if (codeRange && codeRange->isFunction() && link.targetOffset == codeRange->entry())
                target = code_ + codeRange->profilingEntry();
        }

        if (link.isRawPointerPatch())
            *(uint8_t**)(patchAt) = target;
        else
            Assembler::PatchInstructionImmediate(patchAt, PatchedImmPtr(target));
    }

    for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) {
        const OffsetVector& offsets = staticLinkData_.absoluteLinks[imm];
        for (size_t i = 0; i < offsets.length(); i++) {
            uint8_t* patchAt = code_ + offsets[i];
            void* target = AddressOf(imm, cx);

            // Builtin calls are another case where, when profiling is enabled,
            // we must point to the profiling entry.
            Builtin builtin;
            if (profilingEnabled_ && ImmediateIsBuiltin(imm, &builtin)) {
                const CodeRange* codeRange = lookupCodeRange(patchAt);
                if (codeRange->isFunction())
                    target = code_ + staticLinkData_.pod.builtinThunkOffsets[builtin];
            }

            Assembler::PatchDataWithValueCheck(CodeLocationLabel(patchAt),
                                               PatchedImmPtr(target),
                                               PatchedImmPtr((void*)-1));
        }
    }

    // Initialize global data segment

    *(double*)(globalData() + NaN64GlobalDataOffset) = GenericNaN();
    *(float*)(globalData() + NaN32GlobalDataOffset) = GenericNaN();

    for (size_t tableIndex = 0; tableIndex < staticLinkData_.funcPtrTables.length(); tableIndex++) {
        FuncPtrTable& funcPtrTable = staticLinkData_.funcPtrTables[tableIndex];
        const OffsetVector& offsets = funcPtrTable.elemOffsets();
        auto array = reinterpret_cast<void**>(globalData() + funcPtrTable.globalDataOffset());
        for (size_t elemIndex = 0; elemIndex < offsets.length(); elemIndex++) {
            uint8_t* target = code_ + offsets[elemIndex];
            if (profilingEnabled_)
                target = code_ + lookupCodeRange(target)->profilingEntry();
            array[elemIndex] = target;
        }
    }

    for (AsmJSModule::Exit& exit : exits_)
        exit.initDatum(*this);
}

void
AsmJSModule::initHeap(Handle<ArrayBufferObjectMaybeShared*> heap, JSContext* cx)
{
    MOZ_ASSERT_IF(heap->is<ArrayBufferObject>(), heap->as<ArrayBufferObject>().isAsmJS());
    MOZ_ASSERT(IsValidAsmJSHeapLength(heap->byteLength()));
    MOZ_ASSERT(dynamicallyLinked_);
    MOZ_ASSERT(!maybeHeap_);

    maybeHeap_ = heap;
    // heapDatum() may point to shared memory but that memory is only
    // accessed from maybeHeap(), which wraps it, and from
    // hasDetachedHeap(), which checks it for null.
    heapDatum() = heap->dataPointerEither().unwrap(/*safe - explained above*/);

#if defined(JS_CODEGEN_X86)
    uint8_t* heapOffset = heap->dataPointerEither().unwrap(/*safe - used for value*/);
    uint32_t heapLength = heap->byteLength();
    for (unsigned i = 0; i < heapAccesses_.length(); i++) {
        const HeapAccess& access = heapAccesses_[i];
        // An access is out-of-bounds iff
        //      ptr + offset + data-type-byte-size > heapLength
        // i.e. ptr > heapLength - data-type-byte-size - offset.
        // data-type-byte-size and offset are already included in the addend
        // so we just have to add the heap length here.
        if (access.hasLengthCheck())
            X86Encoding::AddInt32(access.patchLengthAt(code_), heapLength);
        void* addr = access.patchHeapPtrImmAt(code_);
        uint32_t disp = reinterpret_cast<uint32_t>(X86Encoding::GetPointer(addr));
        MOZ_ASSERT(disp <= INT32_MAX);
        X86Encoding::SetPointer(addr, (void*)(heapOffset + disp));
    }
#elif defined(JS_CODEGEN_X64)
    // Even with signal handling being used for most bounds checks, there may be
    // atomic operations that depend on explicit checks.
    //
    // If we have any explicit bounds checks, we need to patch the heap length
    // checks at the right places. All accesses that have been recorded are the
    // only ones that need bound checks (see also
    // CodeGeneratorX64::visitAsmJS{Load,Store,CompareExchange,Exchange,AtomicBinop}Heap)
    uint32_t heapLength = heap->byteLength();
    for (size_t i = 0; i < heapAccesses_.length(); i++) {
        const HeapAccess& access = heapAccesses_[i];
        // See comment above for x86 codegen.
        if (access.hasLengthCheck())
            X86Encoding::AddInt32(access.patchLengthAt(code_), heapLength);
    }
#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    uint32_t heapLength = heap->byteLength();
    for (unsigned i = 0; i < heapAccesses_.length(); i++) {
        jit::Assembler::UpdateBoundsCheck(heapLength,
                                          (jit::Instruction*)(heapAccesses_[i].insnOffset() + code_));
    }
#endif
}

void
AsmJSModule::restoreHeapToInitialState(ArrayBufferObjectMaybeShared* maybePrevBuffer)
{
#if defined(JS_CODEGEN_X86)
    if (maybePrevBuffer) {
        // Subtract out the base-pointer added by AsmJSModule::initHeap.
        uint8_t* ptrBase = maybePrevBuffer->dataPointerEither().unwrap(/*safe - used for value*/);
        uint32_t heapLength = maybePrevBuffer->byteLength();
        for (unsigned i = 0; i < heapAccesses_.length(); i++) {
            const HeapAccess& access = heapAccesses_[i];
            // Subtract the heap length back out, leaving the raw displacement in place.
            if (access.hasLengthCheck())
                X86Encoding::AddInt32(access.patchLengthAt(code_), -heapLength);
            void* addr = access.patchHeapPtrImmAt(code_);
            uint8_t* ptr = reinterpret_cast<uint8_t*>(X86Encoding::GetPointer(addr));
            MOZ_ASSERT(ptr >= ptrBase);
            X86Encoding::SetPointer(addr, (void*)(ptr - ptrBase));
        }
    }
#elif defined(JS_CODEGEN_X64)
    if (maybePrevBuffer) {
        uint32_t heapLength = maybePrevBuffer->byteLength();
        for (unsigned i = 0; i < heapAccesses_.length(); i++) {
            const HeapAccess& access = heapAccesses_[i];
            // See comment above for x86 codegen.
            if (access.hasLengthCheck())
                X86Encoding::AddInt32(access.patchLengthAt(code_), -heapLength);
        }
    }
#endif

    maybeHeap_ = nullptr;
    heapDatum() = nullptr;
}

void
AsmJSModule::restoreToInitialState(ArrayBufferObjectMaybeShared* maybePrevBuffer,
                                   uint8_t* prevCode,
                                   ExclusiveContext* cx)
{
#ifdef DEBUG
    // Put the absolute links back to -1 so PatchDataWithValueCheck assertions
    // in staticallyLink are valid.
    for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) {
        void* callee = AddressOf(imm, cx);

        // If we are in profiling mode, calls to builtins will have been patched
        // by setProfilingEnabled to be calls to thunks.
        Builtin builtin;
        void* profilingCallee = profilingEnabled_ && ImmediateIsBuiltin(imm, &builtin)
                                ? prevCode + staticLinkData_.pod.builtinThunkOffsets[builtin]
                                : nullptr;

        const AsmJSModule::OffsetVector& offsets = staticLinkData_.absoluteLinks[imm];
        for (size_t i = 0; i < offsets.length(); i++) {
            uint8_t* caller = code_ + offsets[i];
            void* originalValue = profilingCallee && !lookupCodeRange(caller)->isThunk()
                                  ? profilingCallee
                                  : callee;
            Assembler::PatchDataWithValueCheck(CodeLocationLabel(caller),
                                               PatchedImmPtr((void*)-1),
                                               PatchedImmPtr(originalValue));
        }
    }
#endif

    restoreHeapToInitialState(maybePrevBuffer);
}

namespace {

class MOZ_STACK_CLASS AutoMutateCode
{
    AutoWritableJitCode awjc_;
    AutoFlushICache afc_;

   public:
    AutoMutateCode(JSContext* cx, AsmJSModule& module, const char* name)
      : awjc_(cx->runtime(), module.codeBase(), module.codeBytes()),
        afc_(name)
    {
        module.setAutoFlushICacheRange();
    }
};

} // namespace

bool
AsmJSModule::detachHeap(JSContext* cx)
{
    MOZ_ASSERT(isDynamicallyLinked());
    MOZ_ASSERT(maybeHeap_);

    // Content JS should not be able to run (and detach heap) from within an
    // interrupt callback, but in case it does, fail. Otherwise, the heap can
    // change at an arbitrary instruction and break the assumption below.
    if (interrupted_) {
        JS_ReportError(cx, "attempt to detach from inside interrupt handler");
        return false;
    }

    // Even if this->active(), to reach here, the activation must have called
    // out via an FFI stub. FFI stubs check if heapDatum() is null on reentry
    // and throw an exception if so.
    MOZ_ASSERT_IF(active(), activation()->exitReason().kind() == ExitReason::Jit ||
                            activation()->exitReason().kind() == ExitReason::Slow);

    AutoMutateCode amc(cx, *this, "AsmJSModule::detachHeap");
    restoreHeapToInitialState(maybeHeap_);

    MOZ_ASSERT(hasDetachedHeap());
    return true;
}

bool
js::OnDetachAsmJSArrayBuffer(JSContext* cx, Handle<ArrayBufferObject*> buffer)
{
    for (AsmJSModule* m = cx->runtime()->linkedAsmJSModules; m; m = m->nextLinked()) {
        if (buffer == m->maybeHeapBufferObject() && !m->detachHeap(cx))
            return false;
    }
    return true;
}

static void
AsmJSModuleObject_finalize(FreeOp* fop, JSObject* obj)
{
    fop->delete_(&obj->as<AsmJSModuleObject>().module());
}

static void
AsmJSModuleObject_trace(JSTracer* trc, JSObject* obj)
{
    obj->as<AsmJSModuleObject>().module().trace(trc);
}

const Class AsmJSModuleObject::class_ = {
    "AsmJSModuleObject",
    JSCLASS_IS_ANONYMOUS | JSCLASS_DELAY_METADATA_CALLBACK |
    JSCLASS_HAS_RESERVED_SLOTS(AsmJSModuleObject::RESERVED_SLOTS),
    nullptr, /* addProperty */
    nullptr, /* delProperty */
    nullptr, /* getProperty */
    nullptr, /* setProperty */
    nullptr, /* enumerate */
    nullptr, /* resolve */
    nullptr, /* mayResolve */
    AsmJSModuleObject_finalize,
    nullptr, /* call */
    nullptr, /* hasInstance */
    nullptr, /* construct */
    AsmJSModuleObject_trace
};

AsmJSModuleObject*
AsmJSModuleObject::create(ExclusiveContext* cx, ScopedJSDeletePtr<AsmJSModule>* module)
{
    AutoSetNewObjectMetadata metadata(cx);
    JSObject* obj = NewObjectWithGivenProto(cx, &AsmJSModuleObject::class_, nullptr);
    if (!obj)
        return nullptr;
    AsmJSModuleObject* nobj = &obj->as<AsmJSModuleObject>();

    nobj->setReservedSlot(MODULE_SLOT, PrivateValue(module->forget()));

    return nobj;
}

AsmJSModule&
AsmJSModuleObject::module() const
{
    MOZ_ASSERT(is<AsmJSModuleObject>());
    return *(AsmJSModule*)getReservedSlot(MODULE_SLOT).toPrivate();
}

static inline uint8_t*
WriteBytes(uint8_t* dst, const void* src, size_t nbytes)
{
    memcpy(dst, src, nbytes);
    return dst + nbytes;
}

static inline const uint8_t*
ReadBytes(const uint8_t* src, void* dst, size_t nbytes)
{
    memcpy(dst, src, nbytes);
    return src + nbytes;
}

template <class T>
static inline uint8_t*
WriteScalar(uint8_t* dst, T t)
{
    memcpy(dst, &t, sizeof(t));
    return dst + sizeof(t);
}

template <class T>
static inline const uint8_t*
ReadScalar(const uint8_t* src, T* dst)
{
    memcpy(dst, src, sizeof(*dst));
    return src + sizeof(*dst);
}

static size_t
SerializedNameSize(PropertyName* name)
{
    size_t s = sizeof(uint32_t);
    if (name)
        s += name->length() * (name->hasLatin1Chars() ? sizeof(Latin1Char) : sizeof(char16_t));
    return s;
}

size_t
AsmJSModule::Name::serializedSize() const
{
    return SerializedNameSize(name_);
}

static uint8_t*
SerializeName(uint8_t* cursor, PropertyName* name)
{
    MOZ_ASSERT_IF(name, !name->empty());
    if (name) {
        static_assert(JSString::MAX_LENGTH <= INT32_MAX, "String length must fit in 31 bits");
        uint32_t length = name->length();
        uint32_t lengthAndEncoding = (length << 1) | uint32_t(name->hasLatin1Chars());
        cursor = WriteScalar<uint32_t>(cursor, lengthAndEncoding);
        JS::AutoCheckCannotGC nogc;
        if (name->hasLatin1Chars())
            cursor = WriteBytes(cursor, name->latin1Chars(nogc), length * sizeof(Latin1Char));
        else
            cursor = WriteBytes(cursor, name->twoByteChars(nogc), length * sizeof(char16_t));
    } else {
        cursor = WriteScalar<uint32_t>(cursor, 0);
    }
    return cursor;
}

uint8_t*
AsmJSModule::Name::serialize(uint8_t* cursor) const
{
    return SerializeName(cursor, name_);
}

template <typename CharT>
static const uint8_t*
DeserializeChars(ExclusiveContext* cx, const uint8_t* cursor, size_t length, PropertyName** name)
{
    Vector<CharT> tmp(cx);
    CharT* src;
    if ((size_t(cursor) & (sizeof(CharT) - 1)) != 0) {
        // Align 'src' for AtomizeChars.
        if (!tmp.resize(length))
            return nullptr;
        memcpy(tmp.begin(), cursor, length * sizeof(CharT));
        src = tmp.begin();
    } else {
        src = (CharT*)cursor;
    }

    JSAtom* atom = AtomizeChars(cx, src, length);
    if (!atom)
        return nullptr;

    *name = atom->asPropertyName();
    return cursor + length * sizeof(CharT);
}

static const uint8_t*
DeserializeName(ExclusiveContext* cx, const uint8_t* cursor, PropertyName** name)
{
    uint32_t lengthAndEncoding;
    cursor = ReadScalar<uint32_t>(cursor, &lengthAndEncoding);

    uint32_t length = lengthAndEncoding >> 1;
    if (length == 0) {
        *name = nullptr;
        return cursor;
    }

    bool latin1 = lengthAndEncoding & 0x1;
    return latin1
           ? DeserializeChars<Latin1Char>(cx, cursor, length, name)
           : DeserializeChars<char16_t>(cx, cursor, length, name);
}

const uint8_t*
AsmJSModule::Name::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    return DeserializeName(cx, cursor, &name_);
}

bool
AsmJSModule::Name::clone(ExclusiveContext* cx, Name* out) const
{
    out->name_ = name_;
    return true;
}

template <class T, size_t N>
size_t
SerializedVectorSize(const mozilla::Vector<T, N, SystemAllocPolicy>& vec)
{
    size_t size = sizeof(uint32_t);
    for (size_t i = 0; i < vec.length(); i++)
        size += vec[i].serializedSize();
    return size;
}

template <class T, size_t N>
uint8_t*
SerializeVector(uint8_t* cursor, const mozilla::Vector<T, N, SystemAllocPolicy>& vec)
{
    cursor = WriteScalar<uint32_t>(cursor, vec.length());
    for (size_t i = 0; i < vec.length(); i++)
        cursor = vec[i].serialize(cursor);
    return cursor;
}

template <class T, size_t N>
const uint8_t*
DeserializeVector(ExclusiveContext* cx, const uint8_t* cursor,
                  mozilla::Vector<T, N, SystemAllocPolicy>* vec)
{
    uint32_t length;
    cursor = ReadScalar<uint32_t>(cursor, &length);
    if (!vec->resize(length))
        return nullptr;
    for (size_t i = 0; i < vec->length(); i++) {
        if (!(cursor = (*vec)[i].deserialize(cx, cursor)))
            return nullptr;
    }
    return cursor;
}

template <class T, size_t N>
bool
CloneVector(ExclusiveContext* cx, const mozilla::Vector<T, N, SystemAllocPolicy>& in,
            mozilla::Vector<T, N, SystemAllocPolicy>* out)
{
    if (!out->resize(in.length()))
        return false;
    for (size_t i = 0; i < in.length(); i++) {
        if (!in[i].clone(cx, &(*out)[i]))
            return false;
    }
    return true;
}

template <class T, size_t N, class AllocPolicy>
size_t
SerializedPodVectorSize(const mozilla::Vector<T, N, AllocPolicy>& vec)
{
    return sizeof(uint32_t) +
           vec.length() * sizeof(T);
}

template <class T, size_t N, class AllocPolicy>
uint8_t*
SerializePodVector(uint8_t* cursor, const mozilla::Vector<T, N, AllocPolicy>& vec)
{
    cursor = WriteScalar<uint32_t>(cursor, vec.length());
    cursor = WriteBytes(cursor, vec.begin(), vec.length() * sizeof(T));
    return cursor;
}

template <class T, size_t N, class AllocPolicy>
const uint8_t*
DeserializePodVector(ExclusiveContext* cx, const uint8_t* cursor,
                     mozilla::Vector<T, N, AllocPolicy>* vec)
{
    uint32_t length;
    cursor = ReadScalar<uint32_t>(cursor, &length);
    if (!vec->resize(length))
        return nullptr;
    cursor = ReadBytes(cursor, vec->begin(), length * sizeof(T));
    return cursor;
}

template <class T, size_t N>
bool
ClonePodVector(ExclusiveContext* cx, const mozilla::Vector<T, N, SystemAllocPolicy>& in,
               mozilla::Vector<T, N, SystemAllocPolicy>* out)
{
    if (!out->resize(in.length()))
        return false;
    PodCopy(out->begin(), in.begin(), in.length());
    return true;
}

size_t
SerializedSigSize(const MallocSig& sig)
{
    return sizeof(ExprType) +
           SerializedPodVectorSize(sig.args());
}

uint8_t*
SerializeSig(uint8_t* cursor, const MallocSig& sig)
{
    cursor = WriteScalar<ExprType>(cursor, sig.ret());
    cursor = SerializePodVector(cursor, sig.args());
    return cursor;
}

const uint8_t*
DeserializeSig(ExclusiveContext* cx, const uint8_t* cursor, MallocSig* sig)
{
    ExprType ret;
    cursor = ReadScalar<ExprType>(cursor, &ret);

    MallocSig::ArgVector args;
    cursor = DeserializePodVector(cx, cursor, &args);
    if (!cursor)
        return nullptr;

    sig->init(Move(args), ret);
    return cursor;
}

bool
CloneSig(ExclusiveContext* cx, const MallocSig& sig, MallocSig* out)
{
    MallocSig::ArgVector args;
    if (!ClonePodVector(cx, sig.args(), &args))
        return false;

    out->init(Move(args), sig.ret());
    return true;
}

uint8_t*
AsmJSModule::Global::serialize(uint8_t* cursor) const
{
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    cursor = SerializeName(cursor, name_);
    return cursor;
}

size_t
AsmJSModule::Global::serializedSize() const
{
    return sizeof(pod) +
           SerializedNameSize(name_);
}

const uint8_t*
AsmJSModule::Global::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = ReadBytes(cursor, &pod, sizeof(pod))) &&
    (cursor = DeserializeName(cx, cursor, &name_));
    return cursor;
}

bool
AsmJSModule::Global::clone(ExclusiveContext* cx, Global* out) const
{
    *out = *this;
    return true;
}

uint8_t*
AsmJSModule::Exit::serialize(uint8_t* cursor) const
{
    cursor = SerializeSig(cursor, sig_);
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    return cursor;
}

size_t
AsmJSModule::Exit::serializedSize() const
{
    return SerializedSigSize(sig_) +
           sizeof(pod);
}

const uint8_t*
AsmJSModule::Exit::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = DeserializeSig(cx, cursor, &sig_)) &&
    (cursor = ReadBytes(cursor, &pod, sizeof(pod)));
    return cursor;
}

bool
AsmJSModule::Exit::clone(ExclusiveContext* cx, Exit* out) const
{
    out->pod = pod;
    return CloneSig(cx, sig_, &out->sig_);
}

uint8_t*
AsmJSModule::ExportedFunction::serialize(uint8_t* cursor) const
{
    cursor = SerializeName(cursor, name_);
    cursor = SerializeName(cursor, maybeFieldName_);
    cursor = SerializeSig(cursor, sig_);
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    return cursor;
}

size_t
AsmJSModule::ExportedFunction::serializedSize() const
{
    return SerializedNameSize(name_) +
           SerializedNameSize(maybeFieldName_) +
           SerializedSigSize(sig_) +
           sizeof(pod);
}

const uint8_t*
AsmJSModule::ExportedFunction::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = DeserializeName(cx, cursor, &name_)) &&
    (cursor = DeserializeName(cx, cursor, &maybeFieldName_)) &&
    (cursor = DeserializeSig(cx, cursor, &sig_)) &&
    (cursor = ReadBytes(cursor, &pod, sizeof(pod)));
    return cursor;
}

bool
AsmJSModule::ExportedFunction::clone(ExclusiveContext* cx, ExportedFunction* out) const
{
    out->name_ = name_;
    out->maybeFieldName_ = maybeFieldName_;
    out->pod = pod;
    return CloneSig(cx, sig_, &out->sig_);
}

AsmJSModule::CodeRange::CodeRange(uint32_t lineNumber, AsmJSFunctionOffsets offsets)
  : nameIndex_(UINT32_MAX),
    lineNumber_(lineNumber)
{
    PodZero(&u);  // zero padding for Valgrind
    u.kind_ = Function;

    MOZ_ASSERT(offsets.nonProfilingEntry - offsets.begin <= UINT8_MAX);
    begin_ = offsets.begin;
    u.func.beginToEntry_ = offsets.nonProfilingEntry - begin_;

    MOZ_ASSERT(offsets.nonProfilingEntry < offsets.profilingReturn);
    MOZ_ASSERT(offsets.profilingReturn - offsets.profilingJump <= UINT8_MAX);
    MOZ_ASSERT(offsets.profilingReturn - offsets.profilingEpilogue <= UINT8_MAX);
    profilingReturn_ = offsets.profilingReturn;
    u.func.profilingJumpToProfilingReturn_ = profilingReturn_ - offsets.profilingJump;
    u.func.profilingEpilogueToProfilingReturn_ = profilingReturn_ - offsets.profilingEpilogue;

    MOZ_ASSERT(offsets.nonProfilingEntry < offsets.end);
    end_ = offsets.end;
}

AsmJSModule::CodeRange::CodeRange(Kind kind, AsmJSOffsets offsets)
  : nameIndex_(0),
    lineNumber_(0),
    begin_(offsets.begin),
    profilingReturn_(0),
    end_(offsets.end)
{
    PodZero(&u);  // zero padding for Valgrind
    u.kind_ = kind;

    MOZ_ASSERT(begin_ <= end_);
    MOZ_ASSERT(u.kind_ == Entry || u.kind_ == Inline);
}

AsmJSModule::CodeRange::CodeRange(Kind kind, AsmJSProfilingOffsets offsets)
  : nameIndex_(0),
    lineNumber_(0),
    begin_(offsets.begin),
    profilingReturn_(offsets.profilingReturn),
    end_(offsets.end)
{
    PodZero(&u);  // zero padding for Valgrind
    u.kind_ = kind;

    MOZ_ASSERT(begin_ < profilingReturn_);
    MOZ_ASSERT(profilingReturn_ < end_);
    MOZ_ASSERT(u.kind_ == JitFFI || u.kind_ == SlowFFI || u.kind_ == Interrupt);
}

AsmJSModule::CodeRange::CodeRange(Builtin builtin, AsmJSProfilingOffsets offsets)
  : nameIndex_(0),
    lineNumber_(0),
    begin_(offsets.begin),
    profilingReturn_(offsets.profilingReturn),
    end_(offsets.end)
{
    PodZero(&u);  // zero padding for Valgrind
    u.kind_ = Thunk;
    u.thunk.target_ = uint16_t(builtin);

    MOZ_ASSERT(begin_ < profilingReturn_);
    MOZ_ASSERT(profilingReturn_ < end_);
}

#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
size_t
AsmJSModule::ProfiledFunction::serializedSize() const
{
    return SerializedNameSize(name) +
           sizeof(pod);
}

uint8_t*
AsmJSModule::ProfiledFunction::serialize(uint8_t* cursor) const
{
    cursor = SerializeName(cursor, name);
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    return cursor;
}

const uint8_t*
AsmJSModule::ProfiledFunction::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = DeserializeName(cx, cursor, &name)) &&
    (cursor = ReadBytes(cursor, &pod, sizeof(pod)));
    return cursor;
}
#endif

size_t
AsmJSModule::AbsoluteLinkArray::serializedSize() const
{
    size_t size = 0;
    for (const OffsetVector& offsets : *this)
        size += SerializedPodVectorSize(offsets);
    return size;
}

uint8_t*
AsmJSModule::AbsoluteLinkArray::serialize(uint8_t* cursor) const
{
    for (const OffsetVector& offsets : *this)
        cursor = SerializePodVector(cursor, offsets);
    return cursor;
}

const uint8_t*
AsmJSModule::AbsoluteLinkArray::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    for (OffsetVector& offsets : *this) {
        cursor = DeserializePodVector(cx, cursor, &offsets);
        if (!cursor)
            return nullptr;
    }
    return cursor;
}

bool
AsmJSModule::AbsoluteLinkArray::clone(ExclusiveContext* cx, AbsoluteLinkArray* out) const
{
    for (auto imm : MakeEnumeratedRange(SymbolicAddress::Limit)) {
        if (!ClonePodVector(cx, (*this)[imm], &(*out)[imm]))
            return false;
    }
    return true;
}

size_t
AsmJSModule::AbsoluteLinkArray::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const
{
    size_t size = 0;
    for (const OffsetVector& offsets : *this)
        size += offsets.sizeOfExcludingThis(mallocSizeOf);
    return size;
}

size_t
AsmJSModule::FuncPtrTable::serializedSize() const
{
    return sizeof(pod) +
           SerializedPodVectorSize(elemOffsets_);
}

uint8_t*
AsmJSModule::FuncPtrTable::serialize(uint8_t* cursor) const
{
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    cursor = SerializePodVector(cursor, elemOffsets_);
    return cursor;
}

const uint8_t*
AsmJSModule::FuncPtrTable::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = ReadBytes(cursor, &pod, sizeof(pod))) &&
    (cursor = DeserializePodVector(cx, cursor, &elemOffsets_));
    return cursor;
}

bool
AsmJSModule::FuncPtrTable::clone(ExclusiveContext* cx, FuncPtrTable* out) const
{
    out->pod = pod;
    return ClonePodVector(cx, elemOffsets_, &out->elemOffsets_);
}

size_t
AsmJSModule::FuncPtrTable::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const
{
    return elemOffsets_.sizeOfExcludingThis(mallocSizeOf);
}

size_t
AsmJSModule::StaticLinkData::serializedSize() const
{
    return sizeof(pod) +
           SerializedPodVectorSize(relativeLinks) +
           absoluteLinks.serializedSize() +
           SerializedVectorSize(funcPtrTables);
}

uint8_t*
AsmJSModule::StaticLinkData::serialize(uint8_t* cursor) const
{
    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    cursor = SerializePodVector(cursor, relativeLinks);
    cursor = absoluteLinks.serialize(cursor);
    cursor = SerializeVector(cursor, funcPtrTables);
    return cursor;
}

const uint8_t*
AsmJSModule::StaticLinkData::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    (cursor = ReadBytes(cursor, &pod, sizeof(pod))) &&
    (cursor = DeserializePodVector(cx, cursor, &relativeLinks)) &&
    (cursor = absoluteLinks.deserialize(cx, cursor)) &&
    (cursor = DeserializeVector(cx, cursor, &funcPtrTables));
    return cursor;
}

bool
AsmJSModule::StaticLinkData::clone(ExclusiveContext* cx, StaticLinkData* out) const
{
    out->pod = pod;
    return ClonePodVector(cx, relativeLinks, &out->relativeLinks) &&
           absoluteLinks.clone(cx, &out->absoluteLinks) &&
           CloneVector(cx, funcPtrTables, &out->funcPtrTables);
}

size_t
AsmJSModule::StaticLinkData::sizeOfExcludingThis(MallocSizeOf mallocSizeOf) const
{
    size_t size = relativeLinks.sizeOfExcludingThis(mallocSizeOf) +
                  absoluteLinks.sizeOfExcludingThis(mallocSizeOf) +
                  funcPtrTables.sizeOfExcludingThis(mallocSizeOf);

    for (const FuncPtrTable& table : funcPtrTables)
        size += table.sizeOfExcludingThis(mallocSizeOf);

    return size;
}

size_t
AsmJSModule::serializedSize() const
{
    return sizeof(pod) +
           pod.codeBytes_ +
           SerializedNameSize(globalArgumentName_) +
           SerializedNameSize(importArgumentName_) +
           SerializedNameSize(bufferArgumentName_) +
           SerializedVectorSize(globals_) +
           SerializedVectorSize(exits_) +
           SerializedVectorSize(exports_) +
           SerializedPodVectorSize(callSites_) +
           SerializedPodVectorSize(codeRanges_) +
           SerializedVectorSize(names_) +
           SerializedPodVectorSize(heapAccesses_) +
#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
           SerializedVectorSize(profiledFunctions_) +
#endif
           staticLinkData_.serializedSize();
}

uint8_t*
AsmJSModule::serialize(uint8_t* cursor) const
{
    MOZ_ASSERT(!dynamicallyLinked_);
    MOZ_ASSERT(!loadedFromCache_);
    MOZ_ASSERT(!profilingEnabled_);
    MOZ_ASSERT(!interrupted_);

    cursor = WriteBytes(cursor, &pod, sizeof(pod));
    cursor = WriteBytes(cursor, code_, pod.codeBytes_);
    cursor = SerializeName(cursor, globalArgumentName_);
    cursor = SerializeName(cursor, importArgumentName_);
    cursor = SerializeName(cursor, bufferArgumentName_);
    cursor = SerializeVector(cursor, globals_);
    cursor = SerializeVector(cursor, exits_);
    cursor = SerializeVector(cursor, exports_);
    cursor = SerializePodVector(cursor, callSites_);
    cursor = SerializePodVector(cursor, codeRanges_);
    cursor = SerializeVector(cursor, names_);
    cursor = SerializePodVector(cursor, heapAccesses_);
#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
    cursor = SerializeVector(cursor, profiledFunctions_);
#endif
    cursor = staticLinkData_.serialize(cursor);
    return cursor;
}

const uint8_t*
AsmJSModule::deserialize(ExclusiveContext* cx, const uint8_t* cursor)
{
    // To avoid GC-during-deserialization corner cases, prevent atoms from
    // being collected.
    AutoKeepAtoms aka(cx->perThreadData);

    (cursor = ReadBytes(cursor, &pod, sizeof(pod))) &&
    (code_ = AllocateExecutableMemory(cx, pod.totalBytes_)) &&
    (cursor = ReadBytes(cursor, code_, pod.codeBytes_)) &&
    (cursor = DeserializeName(cx, cursor, &globalArgumentName_)) &&
    (cursor = DeserializeName(cx, cursor, &importArgumentName_)) &&
    (cursor = DeserializeName(cx, cursor, &bufferArgumentName_)) &&
    (cursor = DeserializeVector(cx, cursor, &globals_)) &&
    (cursor = DeserializeVector(cx, cursor, &exits_)) &&
    (cursor = DeserializeVector(cx, cursor, &exports_)) &&
    (cursor = DeserializePodVector(cx, cursor, &callSites_)) &&
    (cursor = DeserializePodVector(cx, cursor, &codeRanges_)) &&
    (cursor = DeserializeVector(cx, cursor, &names_)) &&
    (cursor = DeserializePodVector(cx, cursor, &heapAccesses_)) &&
#if defined(MOZ_VTUNE) || defined(JS_ION_PERF)
    (cursor = DeserializeVector(cx, cursor, &profiledFunctions_)) &&
#endif
    (cursor = staticLinkData_.deserialize(cx, cursor));

    loadedFromCache_ = true;

    return cursor;
}

bool
AsmJSModule::clone(JSContext* cx, ScopedJSDeletePtr<AsmJSModule>* moduleOut) const
{
    *moduleOut = cx->new_<AsmJSModule>(scriptSource_, srcStart_, srcBodyStart_, pod.strict_,
                                       pod.canUseSignalHandlers_);
    if (!*moduleOut)
        return false;

    AsmJSModule& out = **moduleOut;

    // Mirror the order of serialize/deserialize in cloning:

    out.pod = pod;

    out.code_ = AllocateExecutableMemory(cx, pod.totalBytes_);
    if (!out.code_)
        return false;

    memcpy(out.code_, code_, pod.codeBytes_);

    out.globalArgumentName_ = globalArgumentName_;
    out.importArgumentName_ = importArgumentName_;
    out.bufferArgumentName_ = bufferArgumentName_;

    if (!CloneVector(cx, globals_, &out.globals_) ||
        !CloneVector(cx, exits_, &out.exits_) ||
        !CloneVector(cx, exports_, &out.exports_) ||
        !ClonePodVector(cx, callSites_, &out.callSites_) ||
        !ClonePodVector(cx, codeRanges_, &out.codeRanges_) ||
        !CloneVector(cx, names_, &out.names_) ||
        !ClonePodVector(cx, heapAccesses_, &out.heapAccesses_) ||
        !staticLinkData_.clone(cx, &out.staticLinkData_))
    {
        return false;
    }

    out.loadedFromCache_ = loadedFromCache_;
    out.profilingEnabled_ = profilingEnabled_;

    if (profilingEnabled_) {
        if (!out.profilingLabels_.resize(profilingLabels_.length()))
            return false;
        for (size_t i = 0; i < profilingLabels_.length(); i++) {
            out.profilingLabels_[i] = DuplicateString(cx, profilingLabels_[i].get());
            if (!out.profilingLabels_[i])
                return false;
        }
    }


    // Delay flushing until dynamic linking.
    AutoFlushICache afc("AsmJSModule::clone", /* inhibit = */ true);
    out.setAutoFlushICacheRange();

    out.restoreToInitialState(maybeHeap_, code_, cx);
    out.staticallyLink(cx);
    return true;
}

bool
AsmJSModule::changeHeap(Handle<ArrayBufferObject*> newHeap, JSContext* cx)
{
    MOZ_ASSERT(hasArrayView());

    // Content JS should not be able to run (and change heap) from within an
    // interrupt callback, but in case it does, fail to change heap. Otherwise,
    // the heap can change at every single instruction which would prevent
    // future optimizations like heap-base hoisting.
    if (interrupted_)
        return false;

    AutoMutateCode amc(cx, *this, "AsmJSModule::changeHeap");
    restoreHeapToInitialState(maybeHeap_);
    initHeap(newHeap, cx);
    return true;
}

size_t
AsmJSModule::heapLength() const
{
    MOZ_ASSERT(isDynamicallyLinked());
    return maybeHeap_ ? maybeHeap_->byteLength() : 0;
}

void
AsmJSModule::setProfilingEnabled(bool enabled, JSContext* cx)
{
    MOZ_ASSERT(isDynamicallyLinked());

    if (profilingEnabled_ == enabled)
        return;

    // When enabled, generate profiling labels for every name in names_ that is
    // the name of some Function CodeRange. This involves malloc() so do it now
    // since, once we start sampling, we'll be in a signal-handing context where
    // we cannot malloc.
    if (enabled) {
        profilingLabels_.resize(names_.length());
        const char* filename = scriptSource_->filename();
        JS::AutoCheckCannotGC nogc;
        for (size_t i = 0; i < codeRanges_.length(); i++) {
            CodeRange& cr = codeRanges_[i];
            if (!cr.isFunction())
                continue;
            unsigned lineno = cr.functionLineNumber();
            PropertyName* name = names_[cr.functionNameIndex()].name();
            profilingLabels_[cr.functionNameIndex()].reset(
                name->hasLatin1Chars()
                ? JS_smprintf("%s (%s:%u)", name->latin1Chars(nogc), filename, lineno)
                : JS_smprintf("%hs (%s:%u)", name->twoByteChars(nogc), filename, lineno));
        }
    } else {
        profilingLabels_.clear();
    }

    AutoMutateCode amc(cx, *this, "AsmJSModule::setProfilingEnabled");

    // Patch all internal (asm.js->asm.js) callsites to call the profiling
    // prologues:
    for (size_t i = 0; i < callSites_.length(); i++) {
        CallSite& cs = callSites_[i];
        if (cs.kind() != CallSite::Relative)
            continue;

        uint8_t* callerRetAddr = code_ + cs.returnAddressOffset();
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
        void* callee = X86Encoding::GetRel32Target(callerRetAddr);
#elif defined(JS_CODEGEN_ARM)
        uint8_t* caller = callerRetAddr - 4;
        Instruction* callerInsn = reinterpret_cast<Instruction*>(caller);
        BOffImm calleeOffset;
        callerInsn->as<InstBLImm>()->extractImm(&calleeOffset);
        void* callee = calleeOffset.getDest(callerInsn);
#elif defined(JS_CODEGEN_ARM64)
        MOZ_CRASH();
        void* callee = nullptr;
        (void)callerRetAddr;
#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
        uint8_t* instr = callerRetAddr - Assembler::PatchWrite_NearCallSize();
        void* callee = (void*)Assembler::ExtractInstructionImmediate(instr);
#elif defined(JS_CODEGEN_NONE)
        MOZ_CRASH();
        void* callee = nullptr;
#else
# error "Missing architecture"
#endif

        const CodeRange* codeRange = lookupCodeRange(callee);
        if (codeRange->kind() != CodeRange::Function)
            continue;

        uint8_t* profilingEntry = code_ + codeRange->profilingEntry();
        uint8_t* entry = code_ + codeRange->entry();
        MOZ_ASSERT_IF(profilingEnabled_, callee == profilingEntry);
        MOZ_ASSERT_IF(!profilingEnabled_, callee == entry);
        uint8_t* newCallee = enabled ? profilingEntry : entry;

#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
        X86Encoding::SetRel32(callerRetAddr, newCallee);
#elif defined(JS_CODEGEN_ARM)
        new (caller) InstBLImm(BOffImm(newCallee - caller), Assembler::Always);
#elif defined(JS_CODEGEN_ARM64)
        (void)newCallee;
        MOZ_CRASH();
#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
        Assembler::PatchInstructionImmediate(instr, PatchedImmPtr(newCallee));
#elif defined(JS_CODEGEN_NONE)
        MOZ_CRASH();
#else
# error "Missing architecture"
#endif
    }

    // Update all the addresses in the function-pointer tables to point to the
    // profiling prologues:
    for (FuncPtrTable& funcPtrTable : staticLinkData_.funcPtrTables) {
        auto array = reinterpret_cast<void**>(globalData() + funcPtrTable.globalDataOffset());
        for (size_t i = 0; i < funcPtrTable.elemOffsets().length(); i++) {
            void* callee = array[i];
            const CodeRange* codeRange = lookupCodeRange(callee);
            void* profilingEntry = code_ + codeRange->profilingEntry();
            void* entry = code_ + codeRange->entry();
            MOZ_ASSERT_IF(profilingEnabled_, callee == profilingEntry);
            MOZ_ASSERT_IF(!profilingEnabled_, callee == entry);
            if (enabled)
                array[i] = profilingEntry;
            else
                array[i] = entry;
        }
    }

    // Replace all the nops in all the epilogues of asm.js functions with jumps
    // to the profiling epilogues.
    for (size_t i = 0; i < codeRanges_.length(); i++) {
        CodeRange& cr = codeRanges_[i];
        if (!cr.isFunction())
            continue;
        uint8_t* jump = code_ + cr.profilingJump();
        uint8_t* profilingEpilogue = code_ + cr.profilingEpilogue();
#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
        // An unconditional jump with a 1 byte offset immediate has the opcode
        // 0x90. The offset is relative to the address of the instruction after
        // the jump. 0x66 0x90 is the canonical two-byte nop.
        ptrdiff_t jumpImmediate = profilingEpilogue - jump - 2;
        MOZ_ASSERT(jumpImmediate > 0 && jumpImmediate <= 127);
        if (enabled) {
            MOZ_ASSERT(jump[0] == 0x66);
            MOZ_ASSERT(jump[1] == 0x90);
            jump[0] = 0xeb;
            jump[1] = jumpImmediate;
        } else {
            MOZ_ASSERT(jump[0] == 0xeb);
            MOZ_ASSERT(jump[1] == jumpImmediate);
            jump[0] = 0x66;
            jump[1] = 0x90;
        }
#elif defined(JS_CODEGEN_ARM)
        if (enabled) {
            MOZ_ASSERT(reinterpret_cast<Instruction*>(jump)->is<InstNOP>());
            new (jump) InstBImm(BOffImm(profilingEpilogue - jump), Assembler::Always);
        } else {
            MOZ_ASSERT(reinterpret_cast<Instruction*>(jump)->is<InstBImm>());
            new (jump) InstNOP();
        }
#elif defined(JS_CODEGEN_ARM64)
        (void)jump;
        (void)profilingEpilogue;
        MOZ_CRASH();
#elif defined(JS_CODEGEN_MIPS32)
        Instruction* instr = (Instruction*)jump;
        if (enabled) {
            Assembler::WriteLuiOriInstructions(instr, instr->next(),
                                               ScratchRegister, (uint32_t)profilingEpilogue);
            instr[2] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr);
        } else {
            instr[0].makeNop();
            instr[1].makeNop();
            instr[2].makeNop();
        }
#elif defined(JS_CODEGEN_MIPS64)
        Instruction* instr = (Instruction*)jump;
        if (enabled) {
            Assembler::WriteLoad64Instructions(instr, ScratchRegister, (uint64_t)profilingEpilogue);
            instr[4] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr);
        } else {
            instr[0].makeNop();
            instr[1].makeNop();
            instr[2].makeNop();
            instr[3].makeNop();
            instr[4].makeNop();
        }
#elif defined(JS_CODEGEN_NONE)
        MOZ_CRASH();
#else
# error "Missing architecture"
#endif
    }

    // Replace all calls to builtins with calls to profiling thunks that push a
    // frame pointer. Since exit unwinding always starts at the caller of fp,
    // this avoids losing the innermost asm.js function.
    for (auto builtin : MakeEnumeratedRange(Builtin::Limit)) {
        auto imm = BuiltinToImmediate(builtin);
        const OffsetVector& offsets = staticLinkData_.absoluteLinks[imm];
        void* from = AddressOf(imm, nullptr);
        void* to = code_ + staticLinkData_.pod.builtinThunkOffsets[builtin];
        if (!enabled)
            Swap(from, to);
        for (size_t j = 0; j < offsets.length(); j++) {
            uint8_t* caller = code_ + offsets[j];
            const AsmJSModule::CodeRange* codeRange = lookupCodeRange(caller);
            if (codeRange->isThunk())
                continue;
            MOZ_ASSERT(codeRange->isFunction());
            Assembler::PatchDataWithValueCheck(CodeLocationLabel(caller),
                                               PatchedImmPtr(to),
                                               PatchedImmPtr(from));
        }
    }

    profilingEnabled_ = enabled;
}

static bool
GetCPUID(uint32_t* cpuId)
{
    enum Arch {
        X86 = 0x1,
        X64 = 0x2,
        ARM = 0x3,
        MIPS = 0x4,
        MIPS64 = 0x5,
        ARCH_BITS = 3
    };

#if defined(JS_CODEGEN_X86)
    MOZ_ASSERT(uint32_t(CPUInfo::GetSSEVersion()) <= (UINT32_MAX >> ARCH_BITS));
    *cpuId = X86 | (uint32_t(CPUInfo::GetSSEVersion()) << ARCH_BITS);
    return true;
#elif defined(JS_CODEGEN_X64)
    MOZ_ASSERT(uint32_t(CPUInfo::GetSSEVersion()) <= (UINT32_MAX >> ARCH_BITS));
    *cpuId = X64 | (uint32_t(CPUInfo::GetSSEVersion()) << ARCH_BITS);
    return true;
#elif defined(JS_CODEGEN_ARM)
    MOZ_ASSERT(GetARMFlags() <= (UINT32_MAX >> ARCH_BITS));
    *cpuId = ARM | (GetARMFlags() << ARCH_BITS);
    return true;
#elif defined(JS_CODEGEN_MIPS32)
    MOZ_ASSERT(GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS));
    *cpuId = MIPS | (GetMIPSFlags() << ARCH_BITS);
    return true;
#elif defined(JS_CODEGEN_MIPS64)
    MOZ_ASSERT(GetMIPSFlags() <= (UINT32_MAX >> ARCH_BITS));
    *cpuId = MIPS64 | (GetMIPSFlags() << ARCH_BITS);
    return true;
#else
    return false;
#endif
}

class MachineId
{
    uint32_t cpuId_;
    JS::BuildIdCharVector buildId_;

  public:
    bool extractCurrentState(ExclusiveContext* cx) {
        if (!cx->asmJSCacheOps().buildId)
            return false;
        if (!cx->asmJSCacheOps().buildId(&buildId_))
            return false;
        if (!GetCPUID(&cpuId_))
            return false;
        return true;
    }

    size_t serializedSize() const {
        return sizeof(uint32_t) +
               SerializedPodVectorSize(buildId_);
    }

    uint8_t* serialize(uint8_t* cursor) const {
        cursor = WriteScalar<uint32_t>(cursor, cpuId_);
        cursor = SerializePodVector(cursor, buildId_);
        return cursor;
    }

    const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) {
        (cursor = ReadScalar<uint32_t>(cursor, &cpuId_)) &&
        (cursor = DeserializePodVector(cx, cursor, &buildId_));
        return cursor;
    }

    bool operator==(const MachineId& rhs) const {
        return cpuId_ == rhs.cpuId_ &&
               buildId_.length() == rhs.buildId_.length() &&
               PodEqual(buildId_.begin(), rhs.buildId_.begin(), buildId_.length());
    }
    bool operator!=(const MachineId& rhs) const {
        return !(*this == rhs);
    }
};

struct PropertyNameWrapper
{
    PropertyName* name;

    PropertyNameWrapper()
      : name(nullptr)
    {}
    explicit PropertyNameWrapper(PropertyName* name)
      : name(name)
    {}
    size_t serializedSize() const {
        return SerializedNameSize(name);
    }
    uint8_t* serialize(uint8_t* cursor) const {
        return SerializeName(cursor, name);
    }
    const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) {
        return DeserializeName(cx, cursor, &name);
    }
};

class ModuleChars
{
  protected:
    uint32_t isFunCtor_;
    Vector<PropertyNameWrapper, 0, SystemAllocPolicy> funCtorArgs_;

  public:
    static uint32_t beginOffset(AsmJSParser& parser) {
        return parser.pc->maybeFunction->pn_pos.begin;
    }

    static uint32_t endOffset(AsmJSParser& parser) {
        TokenPos pos(0, 0);  // initialize to silence GCC warning
        MOZ_ALWAYS_TRUE(parser.tokenStream.peekTokenPos(&pos, TokenStream::Operand));
        return pos.end;
    }
};

class ModuleCharsForStore : ModuleChars
{
    uint32_t uncompressedSize_;
    uint32_t compressedSize_;
    Vector<char, 0, SystemAllocPolicy> compressedBuffer_;

  public:
    bool init(AsmJSParser& parser) {
        MOZ_ASSERT(beginOffset(parser) < endOffset(parser));

        uncompressedSize_ = (endOffset(parser) - beginOffset(parser)) * sizeof(char16_t);
        size_t maxCompressedSize = LZ4::maxCompressedSize(uncompressedSize_);
        if (maxCompressedSize < uncompressedSize_)
            return false;

        if (!compressedBuffer_.resize(maxCompressedSize))
            return false;

        const char16_t* chars = parser.tokenStream.rawCharPtrAt(beginOffset(parser));
        const char* source = reinterpret_cast<const char*>(chars);
        size_t compressedSize = LZ4::compress(source, uncompressedSize_, compressedBuffer_.begin());
        if (!compressedSize || compressedSize > UINT32_MAX)
            return false;

        compressedSize_ = compressedSize;

        // For a function statement or named function expression:
        //   function f(x,y,z) { abc }
        // the range [beginOffset, endOffset) captures the source:
        //   f(x,y,z) { abc }
        // An unnamed function expression captures the same thing, sans 'f'.
        // Since asm.js modules do not contain any free variables, equality of
        // [beginOffset, endOffset) is sufficient to guarantee identical code
        // generation, modulo MachineId.
        //
        // For functions created with 'new Function', function arguments are
        // not present in the source so we must manually explicitly serialize
        // and match the formals as a Vector of PropertyName.
        isFunCtor_ = parser.pc->isFunctionConstructorBody();
        if (isFunCtor_) {
            unsigned numArgs;
            ParseNode* arg = FunctionArgsList(parser.pc->maybeFunction, &numArgs);
            for (unsigned i = 0; i < numArgs; i++, arg = arg->pn_next) {
                if (!funCtorArgs_.append(arg->name()))
                    return false;
            }
        }

        return true;
    }

    size_t serializedSize() const {
        return sizeof(uint32_t) +
               sizeof(uint32_t) +
               compressedSize_ +
               sizeof(uint32_t) +
               (isFunCtor_ ? SerializedVectorSize(funCtorArgs_) : 0);
    }

    uint8_t* serialize(uint8_t* cursor) const {
        cursor = WriteScalar<uint32_t>(cursor, uncompressedSize_);
        cursor = WriteScalar<uint32_t>(cursor, compressedSize_);
        cursor = WriteBytes(cursor, compressedBuffer_.begin(), compressedSize_);
        cursor = WriteScalar<uint32_t>(cursor, isFunCtor_);
        if (isFunCtor_)
            cursor = SerializeVector(cursor, funCtorArgs_);
        return cursor;
    }
};

class ModuleCharsForLookup : ModuleChars
{
    Vector<char16_t, 0, SystemAllocPolicy> chars_;

  public:
    const uint8_t* deserialize(ExclusiveContext* cx, const uint8_t* cursor) {
        uint32_t uncompressedSize;
        cursor = ReadScalar<uint32_t>(cursor, &uncompressedSize);

        uint32_t compressedSize;
        cursor = ReadScalar<uint32_t>(cursor, &compressedSize);

        if (!chars_.resize(uncompressedSize / sizeof(char16_t)))
            return nullptr;

        const char* source = reinterpret_cast<const char*>(cursor);
        char* dest = reinterpret_cast<char*>(chars_.begin());
        if (!LZ4::decompress(source, dest, uncompressedSize))
            return nullptr;

        cursor += compressedSize;

        cursor = ReadScalar<uint32_t>(cursor, &isFunCtor_);
        if (isFunCtor_)
            cursor = DeserializeVector(cx, cursor, &funCtorArgs_);

        return cursor;
    }

    bool match(AsmJSParser& parser) const {
        const char16_t* parseBegin = parser.tokenStream.rawCharPtrAt(beginOffset(parser));
        const char16_t* parseLimit = parser.tokenStream.rawLimit();
        MOZ_ASSERT(parseLimit >= parseBegin);
        if (uint32_t(parseLimit - parseBegin) < chars_.length())
            return false;
        if (!PodEqual(chars_.begin(), parseBegin, chars_.length()))
            return false;
        if (isFunCtor_ != parser.pc->isFunctionConstructorBody())
            return false;
        if (isFunCtor_) {
            // For function statements, the closing } is included as the last
            // character of the matched source. For Function constructor,
            // parsing terminates with EOF which we must explicitly check. This
            // prevents
            //   new Function('"use asm"; function f() {} return f')
            // from incorrectly matching
            //   new Function('"use asm"; function f() {} return ff')
            if (parseBegin + chars_.length() != parseLimit)
                return false;
            unsigned numArgs;
            ParseNode* arg = FunctionArgsList(parser.pc->maybeFunction, &numArgs);
            if (funCtorArgs_.length() != numArgs)
                return false;
            for (unsigned i = 0; i < funCtorArgs_.length(); i++, arg = arg->pn_next) {
                if (funCtorArgs_[i].name != arg->name())
                    return false;
            }
        }
        return true;
    }
};

struct ScopedCacheEntryOpenedForWrite
{
    ExclusiveContext* cx;
    const size_t serializedSize;
    uint8_t* memory;
    intptr_t handle;

    ScopedCacheEntryOpenedForWrite(ExclusiveContext* cx, size_t serializedSize)
      : cx(cx), serializedSize(serializedSize), memory(nullptr), handle(-1)
    {}

    ~ScopedCacheEntryOpenedForWrite() {
        if (memory)
            cx->asmJSCacheOps().closeEntryForWrite(serializedSize, memory, handle);
    }
};

JS::AsmJSCacheResult
js::StoreAsmJSModuleInCache(AsmJSParser& parser,
                            const AsmJSModule& module,
                            ExclusiveContext* cx)
{
    MachineId machineId;
    if (!machineId.extractCurrentState(cx))
        return JS::AsmJSCache_InternalError;

    ModuleCharsForStore moduleChars;
    if (!moduleChars.init(parser))
        return JS::AsmJSCache_InternalError;

    size_t serializedSize = machineId.serializedSize() +
                            moduleChars.serializedSize() +
                            module.serializedSize();

    JS::OpenAsmJSCacheEntryForWriteOp open = cx->asmJSCacheOps().openEntryForWrite;
    if (!open)
        return JS::AsmJSCache_Disabled_Internal;

    const char16_t* begin = parser.tokenStream.rawCharPtrAt(ModuleChars::beginOffset(parser));
    const char16_t* end = parser.tokenStream.rawCharPtrAt(ModuleChars::endOffset(parser));
    bool installed = parser.options().installedFile;

    ScopedCacheEntryOpenedForWrite entry(cx, serializedSize);
    JS::AsmJSCacheResult openResult =
        open(cx->global(), installed, begin, end, serializedSize, &entry.memory, &entry.handle);
    if (openResult != JS::AsmJSCache_Success)
        return openResult;

    uint8_t* cursor = entry.memory;
    cursor = machineId.serialize(cursor);
    cursor = moduleChars.serialize(cursor);
    cursor = module.serialize(cursor);

    MOZ_ASSERT(cursor == entry.memory + serializedSize);
    return JS::AsmJSCache_Success;
}

struct ScopedCacheEntryOpenedForRead
{
    ExclusiveContext* cx;
    size_t serializedSize;
    const uint8_t* memory;
    intptr_t handle;

    explicit ScopedCacheEntryOpenedForRead(ExclusiveContext* cx)
      : cx(cx), serializedSize(0), memory(nullptr), handle(0)
    {}

    ~ScopedCacheEntryOpenedForRead() {
        if (memory)
            cx->asmJSCacheOps().closeEntryForRead(serializedSize, memory, handle);
    }
};

bool
js::LookupAsmJSModuleInCache(ExclusiveContext* cx,
                             AsmJSParser& parser,
                             ScopedJSDeletePtr<AsmJSModule>* moduleOut,
                             ScopedJSFreePtr<char>* compilationTimeReport)
{
    int64_t usecBefore = PRMJ_Now();

    MachineId machineId;
    if (!machineId.extractCurrentState(cx))
        return true;

    JS::OpenAsmJSCacheEntryForReadOp open = cx->asmJSCacheOps().openEntryForRead;
    if (!open)
        return true;

    const char16_t* begin = parser.tokenStream.rawCharPtrAt(ModuleChars::beginOffset(parser));
    const char16_t* limit = parser.tokenStream.rawLimit();

    ScopedCacheEntryOpenedForRead entry(cx);
    if (!open(cx->global(), begin, limit, &entry.serializedSize, &entry.memory, &entry.handle))
        return true;

    const uint8_t* cursor = entry.memory;

    MachineId cachedMachineId;
    cursor = cachedMachineId.deserialize(cx, cursor);
    if (!cursor)
        return false;
    if (machineId != cachedMachineId)
        return true;

    ModuleCharsForLookup moduleChars;
    cursor = moduleChars.deserialize(cx, cursor);
    if (!moduleChars.match(parser))
        return true;

    uint32_t srcStart = parser.pc->maybeFunction->pn_body->pn_pos.begin;
    uint32_t srcBodyStart = parser.tokenStream.currentToken().pos.end;
    bool strict = parser.pc->sc->strict() && !parser.pc->sc->hasExplicitUseStrict();

    // canUseSignalHandlers will be clobbered when deserializing and checked below
    ScopedJSDeletePtr<AsmJSModule> module(
        cx->new_<AsmJSModule>(parser.ss, srcStart, srcBodyStart, strict,
                              /* canUseSignalHandlers = */ false));
    if (!module)
        return false;

    cursor = module->deserialize(cx, cursor);
    if (!cursor)
        return false;

    bool atEnd = cursor == entry.memory + entry.serializedSize;
    MOZ_ASSERT(atEnd, "Corrupt cache file");
    if (!atEnd)
        return true;

    if (module->canUseSignalHandlers() != cx->canUseSignalHandlers())
        return true;

    if (!parser.tokenStream.advance(module->srcEndBeforeCurly()))
        return false;

    {
        // Delay flushing until dynamic linking.
        AutoFlushICache afc("LookupAsmJSModuleInCache", /* inhibit = */ true);
        module->setAutoFlushICacheRange();

        module->staticallyLink(cx);
    }

    int64_t usecAfter = PRMJ_Now();
    int ms = (usecAfter - usecBefore) / PRMJ_USEC_PER_MSEC;
    *compilationTimeReport = JS_smprintf("loaded from cache in %dms", ms);
    *moduleOut = module.forget();
    return true;
}