From 1acf9bbc23036376922bd87c0c7ef47062eb5fd2 Mon Sep 17 00:00:00 2001 From: Tooru Fujisawa Date: Sat, 19 Mar 2016 02:42:08 +0900 Subject: [PATCH] Bug 1248289 - Part 1: Inline _GetNextMapEntryForIterator intrinsic. r=jandem --- js/src/builtin/Map.js | 6 +- js/src/builtin/MapObject.cpp | 43 +++++ js/src/builtin/MapObject.h | 2 + js/src/ds/OrderedHashTable.h | 47 +++++ js/src/jit/CodeGenerator.cpp | 168 ++++++++++++++++-- js/src/jit/CodeGenerator.h | 3 + js/src/jit/InlinableNatives.h | 2 + js/src/jit/IonBuilder.h | 3 + js/src/jit/Lowering.cpp | 12 ++ js/src/jit/Lowering.h | 1 + js/src/jit/MCallOptimize.cpp | 44 +++++ js/src/jit/MIR.h | 28 +++ js/src/jit/MOpcodes.h | 1 + js/src/jit/MacroAssembler.h | 12 ++ js/src/jit/arm/MacroAssembler-arm-inl.h | 7 + js/src/jit/arm/MacroAssembler-arm.cpp | 15 ++ js/src/jit/arm/MacroAssembler-arm.h | 7 + js/src/jit/arm64/MacroAssembler-arm64-inl.h | 8 + js/src/jit/arm64/MacroAssembler-arm64.cpp | 19 +- js/src/jit/arm64/MacroAssembler-arm64.h | 4 + js/src/jit/mips32/MacroAssembler-mips32-inl.h | 7 + js/src/jit/mips32/MacroAssembler-mips32.cpp | 15 ++ js/src/jit/mips32/MacroAssembler-mips32.h | 7 + js/src/jit/mips64/MacroAssembler-mips64-inl.h | 9 + js/src/jit/mips64/MacroAssembler-mips64.cpp | 17 +- js/src/jit/mips64/MacroAssembler-mips64.h | 4 + js/src/jit/shared/LIR-shared.h | 33 ++++ js/src/jit/shared/LOpcodes-shared.h | 1 + js/src/jit/x64/MacroAssembler-x64-inl.h | 8 + js/src/jit/x64/MacroAssembler-x64.cpp | 17 +- js/src/jit/x64/MacroAssembler-x64.h | 4 + js/src/jit/x86/MacroAssembler-x86-inl.h | 7 + js/src/jit/x86/MacroAssembler-x86.cpp | 31 +++- js/src/jit/x86/MacroAssembler-x86.h | 10 ++ js/src/vm/SelfHosting.cpp | 18 +- 35 files changed, 601 insertions(+), 19 deletions(-) diff --git a/js/src/builtin/Map.js b/js/src/builtin/Map.js index cf7c35af7bf8..eaf180af0fef 100644 --- a/js/src/builtin/Map.js +++ b/js/src/builtin/Map.js @@ -47,8 +47,10 @@ function MapIteratorNext() { // Steps 8-9 (omitted). var mapIterationResultPair = iteratorTemp.mapIterationResultPair; - if (!mapIterationResultPair) - mapIterationResultPair = iteratorTemp.mapIterationResultPair = [null, null]; + if (!mapIterationResultPair) { + mapIterationResultPair = iteratorTemp.mapIterationResultPair = + _CreateMapIterationResultPair(); + } var retVal = {value: undefined, done: true}; diff --git a/js/src/builtin/MapObject.cpp b/js/src/builtin/MapObject.cpp index 89337802d8cc..a5ff86975fbc 100644 --- a/js/src/builtin/MapObject.cpp +++ b/js/src/builtin/MapObject.cpp @@ -186,7 +186,27 @@ bool MapIteratorObject::next(JSContext* cx, Handle mapIterator, HandleArrayObject resultPairObj) { + // Check invariants for inlined _GetNextMapEntryForIterator. + + // The array should be tenured, so that post-barrier can be done simply. + MOZ_ASSERT(resultPairObj->isTenured()); + + // The array elements should be fixed. + MOZ_ASSERT(resultPairObj->hasFixedElements()); MOZ_ASSERT(resultPairObj->getDenseInitializedLength() == 2); + MOZ_ASSERT(resultPairObj->getDenseCapacity() >= 2); + +#ifdef DEBUG + // The array elements should be null, so that inlined + // _GetNextMapEntryForIterator doesn't have to perform pre-barrier. + RootedValue val(cx); + if (!GetElement(cx, resultPairObj, resultPairObj, 0, &val)) + return false; + MOZ_ASSERT(val.isNull()); + if (!GetElement(cx, resultPairObj, resultPairObj, 1, &val)) + return false; + MOZ_ASSERT(val.isNull()); +#endif ValueMap::Range* range = MapIteratorObjectRange(mapIterator); if (!range || range->empty()) { @@ -213,6 +233,29 @@ MapIteratorObject::next(JSContext* cx, Handle mapIterator, return false; } +/* static */ JSObject* +MapIteratorObject::createResultPair(JSContext* cx) +{ + RootedArrayObject resultPairObj(cx, NewDenseFullyAllocatedArray(cx, 2, nullptr, TenuredObject)); + if (!resultPairObj) + return nullptr; + + Rooted proto(cx, resultPairObj->getTaggedProto()); + ObjectGroup* group = ObjectGroupCompartment::makeGroup(cx, resultPairObj->getClass(), proto); + if (!group) + return nullptr; + resultPairObj->setGroup(group); + + resultPairObj->setDenseInitializedLength(2); + resultPairObj->initDenseElement(0, NullValue()); + resultPairObj->initDenseElement(1, NullValue()); + + // See comments in MapIteratorObject::next. + AddTypePropertyId(cx, resultPairObj, JSID_VOID, TypeSet::UnknownType()); + + return resultPairObj; +} + /*** Map *****************************************************************************************/ diff --git a/js/src/builtin/MapObject.h b/js/src/builtin/MapObject.h index a3d57f7dd379..47a23ec4d5b0 100644 --- a/js/src/builtin/MapObject.h +++ b/js/src/builtin/MapObject.h @@ -164,6 +164,8 @@ class MapIteratorObject : public NativeObject static bool next(JSContext* cx, Handle mapIterator, HandleArrayObject resultPairObj); + static JSObject* createResultPair(JSContext* cx); + private: inline MapObject::IteratorKind kind() const; }; diff --git a/js/src/ds/OrderedHashTable.h b/js/src/ds/OrderedHashTable.h index 1fade2b99aac..14258907ed3f 100644 --- a/js/src/ds/OrderedHashTable.h +++ b/js/src/ds/OrderedHashTable.h @@ -459,6 +459,22 @@ class OrderedHashTable *ep = &entry; } } + + static size_t offsetOfHashTable() { + return offsetof(Range, ht); + } + static size_t offsetOfI() { + return offsetof(Range, i); + } + static size_t offsetOfCount() { + return offsetof(Range, count); + } + static size_t offsetOfPrevP() { + return offsetof(Range, prevp); + } + static size_t offsetOfNext() { + return offsetof(Range, next); + } }; Range all() { return Range(this); } @@ -506,6 +522,18 @@ class OrderedHashTable *ep = entry; } + static size_t offsetOfDataLength() { + return offsetof(OrderedHashTable, dataLength); + } + static size_t offsetOfData() { + return offsetof(OrderedHashTable, data); + } +#ifdef DEBUG + static size_t sizeofData() { + return sizeof(Data); + } +#endif + private: /* Logarithm base 2 of the number of buckets in the hash table initially. */ static uint32_t initialBucketsLog2() { return 1; } @@ -680,6 +708,13 @@ class OrderedHashMap const Key key; Value value; + + static size_t offsetOfKey() { + return offsetof(Entry, key); + } + static size_t offsetOfValue() { + return offsetof(Entry, value); + } }; private: @@ -724,6 +759,18 @@ class OrderedHashMap return; return impl.rekeyOneEntry(current, newKey, Entry(newKey, e->value)); } + + static size_t offsetOfImplDataLength() { + return Impl::offsetOfDataLength(); + } + static size_t offsetOfImplData() { + return Impl::offsetOfData(); + } +#ifdef DEBUG + static size_t sizeofImplData() { + return Impl::sizeofData(); + } +#endif }; template diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp index ab0746f4d35a..16667869c123 100644 --- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -3104,13 +3104,23 @@ class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase } }; -void -CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool) +static void +EmitPostWriteBarrier(MacroAssembler& masm, Register objreg, bool isGlobal, + AllocatableGeneralRegisterSet regs) { - saveLiveVolatile(ool->lir()); + Register runtimereg = regs.takeAny(); + masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg); - const LAllocation* obj = ool->object(); + void (*fun)(JSRuntime*, JSObject*) = isGlobal ? PostGlobalWriteBarrier : PostWriteBarrier; + masm.setupUnalignedABICall(regs.takeAny()); + masm.passABIArg(runtimereg); + masm.passABIArg(objreg); + masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun)); +} +void +CodeGenerator::emitPostWriteBarrier(const LAllocation* obj) +{ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); Register objreg; @@ -3125,15 +3135,23 @@ CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* regs.takeUnchecked(objreg); } - Register runtimereg = regs.takeAny(); - masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg); + EmitPostWriteBarrier(masm, objreg, isGlobal, regs); +} - void (*fun)(JSRuntime*, JSObject*) = isGlobal ? PostGlobalWriteBarrier : PostWriteBarrier; - masm.setupUnalignedABICall(regs.takeAny()); - masm.passABIArg(runtimereg); - masm.passABIArg(objreg); - masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun)); +void +CodeGenerator::emitPostWriteBarrier(Register objreg) +{ + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile()); + regs.takeUnchecked(objreg); + EmitPostWriteBarrier(masm, objreg, false, regs); +} +void +CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool) +{ + saveLiveVolatile(ool->lir()); + const LAllocation* obj = ool->object(); + emitPostWriteBarrier(obj); restoreLiveVolatile(ool->lir()); masm.jump(ool->rejoin()); @@ -5655,6 +5673,134 @@ CodeGenerator::visitSetArrayLength(LSetArrayLength* lir) masm.dec32(&newLength); } +static inline void +ValueMapRangeFront(MacroAssembler& masm, Register range, Register i, Register front) +{ + masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front); + masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front); + + MOZ_ASSERT(ValueMap::sizeofImplData() == 24); + masm.mulBy3(i, i); + masm.lshiftPtr(Imm32(3), i); + masm.addPtr(i, front); +} + +static inline void +ValueMapRangePopFront(MacroAssembler& masm, Register range, Register front, Register dataLength, + Register temp) +{ + Register i = temp; + + masm.add32(Imm32(1), Address(range, ValueMap::Range::offsetOfCount())); + + masm.load32(Address(range, ValueMap::Range::offsetOfI()), i); + masm.add32(Imm32(1), i); + + Label done, seek; + masm.bind(&seek); + masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done); + + MOZ_ASSERT(ValueMap::sizeofImplData() == 24); + masm.addPtr(Imm32(24), front); + + masm.branchTestMagic(Assembler::NotEqual, Address(front, ValueMap::Entry::offsetOfKey()), + JS_HASH_KEY_EMPTY, &done); + + masm.add32(Imm32(1), i); + masm.jump(&seek); + + masm.bind(&done); + masm.store32(i, Address(range, ValueMap::Range::offsetOfI())); +} + +static inline void +ValueMapRangeDestruct(MacroAssembler& masm, Register range, Register temp0, Register temp1) +{ + Register next = temp0; + Register prevp = temp1; + + masm.loadPtr(Address(range, ValueMap::Range::offsetOfNext()), next); + masm.loadPtr(Address(range, ValueMap::Range::offsetOfPrevP()), prevp); + masm.storePtr(next, Address(prevp, 0)); + + Label hasNoNext; + masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext); + + masm.storePtr(prevp, Address(next, ValueMap::Range::offsetOfPrevP())); + + masm.bind(&hasNoNext); + + masm.callFreeStub(range); +} + +void +CodeGenerator::visitGetNextMapEntryForIterator(LGetNextMapEntryForIterator* lir) +{ + Register iter = ToRegister(lir->iter()); + Register result = ToRegister(lir->result()); + Register temp = ToRegister(lir->temp0()); + Register dataLength = ToRegister(lir->temp1()); + Register range = ToRegister(lir->temp2()); + Register output = ToRegister(lir->output()); + + masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(MapIteratorObject::RangeSlot)), + range); + + Label iterDone, done; + masm.branchTestPtr(Assembler::Zero, range, range, &iterDone); + + masm.load32(Address(range, ValueMap::Range::offsetOfI()), temp); + masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), dataLength); + masm.load32(Address(dataLength, ValueMap::offsetOfImplDataLength()), dataLength); + masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone); + { + masm.push(iter); + + Register front = iter; + ValueMapRangeFront(masm, range, temp, front); + + size_t elementsOffset = NativeObject::offsetOfFixedElements(); + + Address keyAddress(front, ValueMap::Entry::offsetOfKey()); + Address valueAddress(front, ValueMap::Entry::offsetOfValue()); + masm.storeValue(keyAddress, Address(result, elementsOffset), temp); + masm.storeValue(valueAddress, Address(result, elementsOffset + sizeof(Value)), temp); + + Label keyIsNotObject, valueIsNotNurseryObject, emitBarrier; + masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject); + masm.branchValueIsNurseryObject(Assembler::Equal, keyAddress, temp, + &emitBarrier); + masm.bind(&keyIsNotObject); + masm.branchTestObject(Assembler::NotEqual, valueAddress, &valueIsNotNurseryObject); + masm.branchValueIsNurseryObject(Assembler::NotEqual, valueAddress, temp, + &valueIsNotNurseryObject); + { + masm.bind(&emitBarrier); + saveVolatile(temp); + emitPostWriteBarrier(result); + restoreVolatile(temp); + } + masm.bind(&valueIsNotNurseryObject); + + ValueMapRangePopFront(masm, range, front, dataLength, temp); + + masm.pop(iter); + masm.move32(Imm32(0), output); + } + masm.jump(&done); + { + masm.bind(&iterDone); + + ValueMapRangeDestruct(masm, range, temp, dataLength); + + masm.storeValue(PrivateValue(nullptr), + Address(iter, NativeObject::getFixedSlotOffset(MapIteratorObject::RangeSlot))); + + masm.move32(Imm32(1), output); + } + masm.bind(&done); +} + void CodeGenerator::visitTypedArrayLength(LTypedArrayLength* lir) { diff --git a/js/src/jit/CodeGenerator.h b/js/src/jit/CodeGenerator.h index 8e2b33f4c23b..971cbab8d67b 100644 --- a/js/src/jit/CodeGenerator.h +++ b/js/src/jit/CodeGenerator.h @@ -139,6 +139,8 @@ class CodeGenerator : public CodeGeneratorSpecific void visitTypeBarrierV(LTypeBarrierV* lir); void visitTypeBarrierO(LTypeBarrierO* lir); void visitMonitorTypes(LMonitorTypes* lir); + void emitPostWriteBarrier(const LAllocation* obj); + void emitPostWriteBarrier(Register objreg); template void visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode* ool); template @@ -207,6 +209,7 @@ class CodeGenerator : public CodeGeneratorSpecific void visitComputeThis(LComputeThis* lir); void visitArrayLength(LArrayLength* lir); void visitSetArrayLength(LSetArrayLength* lir); + void visitGetNextMapEntryForIterator(LGetNextMapEntryForIterator* lir); void visitTypedArrayLength(LTypedArrayLength* lir); void visitTypedArrayElements(LTypedArrayElements* lir); void visitSetDisjointTypedElements(LSetDisjointTypedElements* lir); diff --git a/js/src/jit/InlinableNatives.h b/js/src/jit/InlinableNatives.h index 2d7c46df2b83..3364bd512bf6 100644 --- a/js/src/jit/InlinableNatives.h +++ b/js/src/jit/InlinableNatives.h @@ -109,6 +109,8 @@ _(IntrinsicIsStringIterator) \ _(IntrinsicIsListIterator) \ \ + _(IntrinsicGetNextMapEntryForIterator) \ + \ _(IntrinsicIsTypedArray) \ _(IntrinsicIsPossiblyWrappedTypedArray) \ _(IntrinsicTypedArrayLength) \ diff --git a/js/src/jit/IonBuilder.h b/js/src/jit/IonBuilder.h index 74fce5a437ac..2f09796ffd05 100644 --- a/js/src/jit/IonBuilder.h +++ b/js/src/jit/IonBuilder.h @@ -841,6 +841,9 @@ class IonBuilder InliningStatus inlineUnsafeGetReservedSlot(CallInfo& callInfo, MIRType knownValueType); + // Map intrinsics. + InliningStatus inlineGetNextMapEntryForIterator(CallInfo& callInfo); + // TypedArray intrinsics. enum WrappingBehavior { AllowWrappedTypedArrays, RejectWrappedTypedArrays }; InliningStatus inlineIsTypedArrayHelper(CallInfo& callInfo, WrappingBehavior wrappingBehavior); diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp index b7a01387fd64..ed228a5dc011 100644 --- a/js/src/jit/Lowering.cpp +++ b/js/src/jit/Lowering.cpp @@ -2697,6 +2697,18 @@ LIRGenerator::visitSetArrayLength(MSetArrayLength* ins) useRegisterOrConstant(ins->index())), ins); } +void +LIRGenerator::visitGetNextMapEntryForIterator(MGetNextMapEntryForIterator* ins) +{ + MOZ_ASSERT(ins->iter()->type() == MIRType_Object); + MOZ_ASSERT(ins->result()->type() == MIRType_Object); + auto lir = new(alloc()) LGetNextMapEntryForIterator(useRegister(ins->iter()), + useRegister(ins->result()), + temp(), temp(), temp()); + define(lir, ins); + assignSafepoint(lir, ins); +} + void LIRGenerator::visitTypedArrayLength(MTypedArrayLength* ins) { diff --git a/js/src/jit/Lowering.h b/js/src/jit/Lowering.h index 3f8d34ac31e6..d2fb5c77ae1b 100644 --- a/js/src/jit/Lowering.h +++ b/js/src/jit/Lowering.h @@ -192,6 +192,7 @@ class LIRGenerator : public LIRGeneratorSpecific void visitPostWriteElementBarrier(MPostWriteElementBarrier* ins); void visitArrayLength(MArrayLength* ins); void visitSetArrayLength(MSetArrayLength* ins); + void visitGetNextMapEntryForIterator(MGetNextMapEntryForIterator* ins); void visitTypedArrayLength(MTypedArrayLength* ins); void visitTypedArrayElements(MTypedArrayElements* ins); void visitSetDisjointTypedElements(MSetDisjointTypedElements* ins); diff --git a/js/src/jit/MCallOptimize.cpp b/js/src/jit/MCallOptimize.cpp index 4708c5ee8c07..a5c736fdf3c8 100644 --- a/js/src/jit/MCallOptimize.cpp +++ b/js/src/jit/MCallOptimize.cpp @@ -260,6 +260,10 @@ IonBuilder::inlineNativeCall(CallInfo& callInfo, JSFunction* target) case InlinableNative::IntrinsicDefineDataProperty: return inlineDefineDataProperty(callInfo); + // Map intrinsics. + case InlinableNative::IntrinsicGetNextMapEntryForIterator: + return inlineGetNextMapEntryForIterator(callInfo); + // TypedArray intrinsics. case InlinableNative::IntrinsicIsTypedArray: return inlineIsTypedArray(callInfo); @@ -2190,6 +2194,46 @@ IonBuilder::inlineHasClass(CallInfo& callInfo, return InliningStatus_Inlined; } +IonBuilder::InliningStatus +IonBuilder::inlineGetNextMapEntryForIterator(CallInfo& callInfo) +{ + if (callInfo.argc() != 2 || callInfo.constructing()) { + trackOptimizationOutcome(TrackedOutcome::CantInlineNativeBadForm); + return InliningStatus_NotInlined; + } + + MDefinition* iterArg = callInfo.getArg(0); + MDefinition* resultArg = callInfo.getArg(1); + + if (iterArg->type() != MIRType_Object) + return InliningStatus_NotInlined; + + TemporaryTypeSet* iterTypes = iterArg->resultTypeSet(); + const Class* iterClasp = iterTypes ? iterTypes->getKnownClass(constraints()) : nullptr; + if (iterClasp != &MapIteratorObject::class_) + return InliningStatus_NotInlined; + + if (resultArg->type() != MIRType_Object) + return InliningStatus_NotInlined; + + TemporaryTypeSet* resultTypes = resultArg->resultTypeSet(); + const Class* resultClasp = resultTypes ? resultTypes->getKnownClass(constraints()) : nullptr; + if (resultClasp != &ArrayObject::class_) + return InliningStatus_NotInlined; + + callInfo.setImplicitlyUsedUnchecked(); + + MInstruction* next = MGetNextMapEntryForIterator::New(alloc(), iterArg, + resultArg); + current->add(next); + current->push(next); + + if (!resumeAfter(next)) + return InliningStatus_Error; + + return InliningStatus_Inlined; +} + IonBuilder::InliningStatus IonBuilder::inlineIsTypedArrayHelper(CallInfo& callInfo, WrappingBehavior wrappingBehavior) { diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h index 4127b6968879..18497f0827f1 100644 --- a/js/src/jit/MIR.h +++ b/js/src/jit/MIR.h @@ -8705,6 +8705,34 @@ class MSetArrayLength } }; +class MGetNextMapEntryForIterator + : public MBinaryInstruction, + public MixPolicy, ObjectPolicy<1> >::Data +{ + protected: + explicit MGetNextMapEntryForIterator(MDefinition* iter, MDefinition* result) + : MBinaryInstruction(iter, result) + { + setResultType(MIRType_Boolean); + } + + public: + INSTRUCTION_HEADER(GetNextMapEntryForIterator) + + static MGetNextMapEntryForIterator* New(TempAllocator& alloc, MDefinition* iter, MDefinition* result) + { + return new(alloc) MGetNextMapEntryForIterator(iter, result); + } + + MDefinition* iter() { + return getOperand(0); + } + + MDefinition* result() { + return getOperand(1); + } +}; + // Read the length of a typed array. class MTypedArrayLength : public MUnaryInstruction, diff --git a/js/src/jit/MOpcodes.h b/js/src/jit/MOpcodes.h index 026179313ff6..889adb7bc0cf 100644 --- a/js/src/jit/MOpcodes.h +++ b/js/src/jit/MOpcodes.h @@ -180,6 +180,7 @@ namespace jit { _(LoadUnboxedExpando) \ _(ArrayLength) \ _(SetArrayLength) \ + _(GetNextMapEntryForIterator) \ _(TypedArrayLength) \ _(TypedArrayElements) \ _(SetDisjointTypedElements) \ diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h index 443a17b6fae8..b023fd65be3f 100644 --- a/js/src/jit/MacroAssembler.h +++ b/js/src/jit/MacroAssembler.h @@ -849,6 +849,9 @@ class MacroAssembler : public MacroAssemblerSpecific void branchPtrInNurseryRange(Condition cond, Register ptr, Register temp, Label* label) DEFINED_ON(arm, arm64, mips_shared, x86, x64); + void branchPtrInNurseryRange(Condition cond, const Address& address, Register temp, Label* label) + DEFINED_ON(x86); + void branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, Label* label) PER_ARCH; void branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) PER_ARCH; // This function compares a Value (lhs) which is having a private pointer @@ -1006,6 +1009,8 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchTestMagic(Condition cond, const ValueOperand& value, L label) DEFINED_ON(arm, arm64, mips32, mips64, x86_shared); + inline void branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) PER_ARCH; + inline void branchTestMagicValue(Condition cond, const ValueOperand& val, JSWhyMagic why, Label* label); @@ -1032,6 +1037,13 @@ class MacroAssembler : public MacroAssemblerSpecific inline void branchPtrImpl(Condition cond, const T& lhs, const S& rhs, Label* label) DEFINED_ON(x86_shared); + template + void branchPtrInNurseryRangeImpl(Condition cond, const T& ptr, Register temp, Label* label) + DEFINED_ON(x86); + template + void branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp, Label* label) + DEFINED_ON(arm64, mips64, x64); + template inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label) DEFINED_ON(arm, arm64, x86_shared); diff --git a/js/src/jit/arm/MacroAssembler-arm-inl.h b/js/src/jit/arm/MacroAssembler-arm-inl.h index c5c8b9c086cf..21925cf0a3eb 100644 --- a/js/src/jit/arm/MacroAssembler-arm-inl.h +++ b/js/src/jit/arm/MacroAssembler-arm-inl.h @@ -1164,6 +1164,13 @@ MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) ma_b(label, cond); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + branchTestMagic(cond, valaddr, label); + branch32(cond, ToPayload(valaddr), Imm32(why), label); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp index a74544f5a83a..00844824484d 100644 --- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -5022,6 +5022,21 @@ MacroAssembler::branchPtrInNurseryRange(Condition cond, Register ptr, Register t scratch2, Imm32(nursery.numChunks()), label); } +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, + Register temp, Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label); + loadPtr(address, temp); + branchPtrInNurseryRange(cond, temp, InvalidReg, label); + + bind(&done); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h index 38f8f7615f5c..25ae99571970 100644 --- a/js/src/jit/arm/MacroAssembler-arm.h +++ b/js/src/jit/arm/MacroAssembler-arm.h @@ -845,6 +845,13 @@ class MacroAssemblerARMCompat : public MacroAssemblerARM ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd); storeValue(val, Address(scratch, dest.offset)); } + void storeValue(const Address& src, const Address& dest, Register temp) { + load32(ToType(src), temp); + store32(temp, ToType(dest)); + + load32(ToPayload(src), temp); + store32(temp, ToPayload(dest)); + } void loadValue(Address src, ValueOperand val); void loadValue(Operand dest, ValueOperand val) { diff --git a/js/src/jit/arm64/MacroAssembler-arm64-inl.h b/js/src/jit/arm64/MacroAssembler-arm64-inl.h index 2e4b4f9affd2..f0314757e553 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64-inl.h +++ b/js/src/jit/arm64/MacroAssembler-arm64-inl.h @@ -1263,6 +1263,14 @@ MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) B(label, c); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + uint64_t magic = MagicValue(why).asRawBits(); + cmpPtr(valaddr, ImmWord(magic)); + B(label, cond); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp index 09f3a13a417c..b97ff8de4584 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.cpp +++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp @@ -716,11 +716,26 @@ MacroAssembler::branchPtrInNurseryRange(Condition cond, Register ptr, Register t temp, ImmWord(nursery.nurserySize()), label); } +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, + Label* label) +{ + branchValueIsNurseryObjectImpl(cond, address, temp, label); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) { - MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + branchValueIsNurseryObjectImpl(cond, value.valueReg(), temp, label); +} + +template +void +MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp, + Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); MOZ_ASSERT(temp != ScratchReg && temp != ScratchReg2); // Both may be used internally. const Nursery& nursery = GetJitContext()->runtime->gcNursery(); @@ -733,7 +748,7 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, R Value start = ObjectValue(*reinterpret_cast(nursery.start())); movePtr(ImmWord(-ptrdiff_t(start.asRawBits())), temp); - addPtr(value.valueReg(), temp); + addPtr(value, temp); branchPtr(cond == Assembler::Equal ? Assembler::Below : Assembler::AboveOrEqual, temp, ImmWord(nursery.nurserySize()), label); } diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h index 5d54773e68c8..c04b2ea64797 100644 --- a/js/src/jit/arm64/MacroAssembler-arm64.h +++ b/js/src/jit/arm64/MacroAssembler-arm64.h @@ -279,6 +279,10 @@ class MacroAssemblerCompat : public vixl::MacroAssembler void storeValue(ValueOperand val, BaseIndex dest) { storePtr(val.valueReg(), dest); } + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } template void storeUnboxedValue(ConstantOrRegister value, MIRType valueType, const T& dest, MIRType slotType) { diff --git a/js/src/jit/mips32/MacroAssembler-mips32-inl.h b/js/src/jit/mips32/MacroAssembler-mips32-inl.h index 271d82cdfc7c..9bb6b828c716 100644 --- a/js/src/jit/mips32/MacroAssembler-mips32-inl.h +++ b/js/src/jit/mips32/MacroAssembler-mips32-inl.h @@ -387,6 +387,13 @@ MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value, L lab ma_b(value.typeReg(), ImmTag(JSVAL_TAG_MAGIC), label, cond); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + branchTestMagic(cond, valaddr, label); + branch32(cond, ToPayload(valaddr), Imm32(why), label); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/mips32/MacroAssembler-mips32.cpp b/js/src/jit/mips32/MacroAssembler-mips32.cpp index 0d731a7acc71..1201262ce2cc 100644 --- a/js/src/jit/mips32/MacroAssembler-mips32.cpp +++ b/js/src/jit/mips32/MacroAssembler-mips32.cpp @@ -2142,6 +2142,21 @@ MacroAssembler::callWithABINoProfiler(const Address& fun, MoveOp::Type result) // =============================================================== // Branch functions +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, + Register temp, Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestObject(Assembler::NotEqual, value, cond == Assembler::Equal ? &done : label); + loadPtr(address, temp); + branchPtrInNurseryRange(cond, temp, InvalidReg, label); + + bind(&done); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) diff --git a/js/src/jit/mips32/MacroAssembler-mips32.h b/js/src/jit/mips32/MacroAssembler-mips32.h index 367aea530a99..ea5c8863212f 100644 --- a/js/src/jit/mips32/MacroAssembler-mips32.h +++ b/js/src/jit/mips32/MacroAssembler-mips32.h @@ -457,6 +457,13 @@ class MacroAssemblerMIPSCompat : public MacroAssemblerMIPS void storeValue(JSValueType type, Register reg, Address dest); void storeValue(const Value& val, Address dest); void storeValue(const Value& val, BaseIndex dest); + void storeValue(const Address& src, const Address& dest, Register temp) { + load32(ToType(src), temp); + store32(temp, ToType(dest)); + + load32(ToPayload(src), temp); + store32(temp, ToPayload(dest)); + } void loadValue(Address src, ValueOperand val); void loadValue(Operand dest, ValueOperand val) { diff --git a/js/src/jit/mips64/MacroAssembler-mips64-inl.h b/js/src/jit/mips64/MacroAssembler-mips64-inl.h index 889951b85cd8..4d5991a96a05 100644 --- a/js/src/jit/mips64/MacroAssembler-mips64-inl.h +++ b/js/src/jit/mips64/MacroAssembler-mips64-inl.h @@ -346,6 +346,15 @@ MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value, L lab ma_b(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + uint64_t magic = MagicValue(why).asRawBits(); + ScratchRegisterScope scratch(*this); + loadPtr(valaddr, scratch); + ma_b(scratch, ImmWord(magic), cond, label); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/mips64/MacroAssembler-mips64.cpp b/js/src/jit/mips64/MacroAssembler-mips64.cpp index 5ceabf0f0ea3..b5f70078d0a9 100644 --- a/js/src/jit/mips64/MacroAssembler-mips64.cpp +++ b/js/src/jit/mips64/MacroAssembler-mips64.cpp @@ -2304,9 +2304,24 @@ MacroAssembler::callWithABINoProfiler(const Address& fun, MoveOp::Type result) // =============================================================== // Branch functions +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, + Label* label) +{ + branchValueIsNurseryObject(cond, address, temp, label); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) +{ + branchValueIsNurseryObject(cond, value.valueReg(), temp, label); +} + +template +void +MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp, + Label* label) { MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); @@ -2315,7 +2330,7 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Value start = ObjectValue(*reinterpret_cast(nursery.start())); movePtr(ImmWord(-ptrdiff_t(start.asRawBits())), SecondScratchReg); - addPtr(value.valueReg(), SecondScratchReg); + addPtr(value, SecondScratchReg); branchPtr(cond == Assembler::Equal ? Assembler::Below : Assembler::AboveOrEqual, SecondScratchReg, Imm32(nursery.nurserySize()), label); } diff --git a/js/src/jit/mips64/MacroAssembler-mips64.h b/js/src/jit/mips64/MacroAssembler-mips64.h index 31cbf1df7d4c..629f7cd01dd0 100644 --- a/js/src/jit/mips64/MacroAssembler-mips64.h +++ b/js/src/jit/mips64/MacroAssembler-mips64.h @@ -476,6 +476,10 @@ class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64 void storeValue(JSValueType type, Register reg, Address dest); void storeValue(const Value& val, Address dest); void storeValue(const Value& val, BaseIndex dest); + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } void loadValue(Address src, ValueOperand val); void loadValue(Operand dest, ValueOperand val) { diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h index 0c1dc1c278e3..e14aaba67fd6 100644 --- a/js/src/jit/shared/LIR-shared.h +++ b/js/src/jit/shared/LIR-shared.h @@ -4730,6 +4730,39 @@ class LSetArrayLength : public LInstructionHelper<0, 2, 0> } }; +class LGetNextMapEntryForIterator : public LInstructionHelper<1, 2, 3> +{ + public: + LIR_HEADER(GetNextMapEntryForIterator) + + explicit LGetNextMapEntryForIterator(const LAllocation& iter, const LAllocation& result, + const LDefinition& temp0, const LDefinition& temp1, + const LDefinition& temp2) + { + setOperand(0, iter); + setOperand(1, result); + setTemp(0, temp0); + setTemp(1, temp1); + setTemp(2, temp2); + } + + const LAllocation* iter() { + return getOperand(0); + } + const LAllocation* result() { + return getOperand(1); + } + const LDefinition* temp0() { + return getTemp(0); + } + const LDefinition* temp1() { + return getTemp(1); + } + const LDefinition* temp2() { + return getTemp(2); + } +}; + // Read the length of a typed array. class LTypedArrayLength : public LInstructionHelper<1, 1, 0> { diff --git a/js/src/jit/shared/LOpcodes-shared.h b/js/src/jit/shared/LOpcodes-shared.h index a2b2cef3c929..d4ffced85d8f 100644 --- a/js/src/jit/shared/LOpcodes-shared.h +++ b/js/src/jit/shared/LOpcodes-shared.h @@ -315,6 +315,7 @@ _(IteratorEnd) \ _(ArrayLength) \ _(SetArrayLength) \ + _(GetNextMapEntryForIterator) \ _(TypedArrayLength) \ _(TypedArrayElements) \ _(SetDisjointTypedElements) \ diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h index db01ce42229d..5dd9ea5148b4 100644 --- a/js/src/jit/x64/MacroAssembler-x64-inl.h +++ b/js/src/jit/x64/MacroAssembler-x64-inl.h @@ -405,6 +405,14 @@ MacroAssembler::branchTestBooleanTruthy(bool truthy, const ValueOperand& value, j(truthy ? NonZero : Zero, label); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + uint64_t magic = MagicValue(why).asRawBits(); + cmpPtr(valaddr, ImmWord(magic)); + j(cond, label); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp index 58bf83243ad5..77ccce04ac1a 100644 --- a/js/src/jit/x64/MacroAssembler-x64.cpp +++ b/js/src/jit/x64/MacroAssembler-x64.cpp @@ -445,9 +445,24 @@ MacroAssembler::branchPtrInNurseryRange(Condition cond, Register ptr, Register t scratch, Imm32(nursery.nurserySize()), label); } +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, + Label* label) +{ + branchValueIsNurseryObjectImpl(cond, address, temp, label); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) +{ + branchValueIsNurseryObjectImpl(cond, value.valueReg(), temp, label); +} + +template +void +MacroAssembler::branchValueIsNurseryObjectImpl(Condition cond, const T& value, Register temp, + Label* label) { MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); @@ -462,7 +477,7 @@ MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, R ScratchRegisterScope scratch(*this); movePtr(ImmWord(-ptrdiff_t(start.asRawBits())), scratch); - addPtr(value.valueReg(), scratch); + addPtr(value, scratch); branchPtr(cond == Assembler::Equal ? Assembler::Below : Assembler::AboveOrEqual, scratch, Imm32(nursery.nurserySize()), label); } diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h index 955e2884fcd1..060f0e101ff9 100644 --- a/js/src/jit/x64/MacroAssembler-x64.h +++ b/js/src/jit/x64/MacroAssembler-x64.h @@ -144,6 +144,10 @@ class MacroAssemblerX64 : public MacroAssemblerX86Shared void storeValue(ValueOperand val, BaseIndex dest) { storeValue(val, Operand(dest)); } + void storeValue(const Address& src, const Address& dest, Register temp) { + loadPtr(src, temp); + storePtr(temp, dest); + } void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); } diff --git a/js/src/jit/x86/MacroAssembler-x86-inl.h b/js/src/jit/x86/MacroAssembler-x86-inl.h index 649bffc8cc19..b7b116c5d03f 100644 --- a/js/src/jit/x86/MacroAssembler-x86-inl.h +++ b/js/src/jit/x86/MacroAssembler-x86-inl.h @@ -386,6 +386,13 @@ MacroAssembler::branchTestBooleanTruthy(bool truthy, const ValueOperand& value, j(truthy ? NonZero : Zero, label); } +void +MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr, JSWhyMagic why, Label* label) +{ + branchTestMagic(cond, valaddr, label); + branch32(cond, ToPayload(valaddr), Imm32(why), label); +} + //}}} check_macroassembler_style // =============================================================== diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp index 9bbd83d6ee03..2b2819d1b5f7 100644 --- a/js/src/jit/x86/MacroAssembler-x86.cpp +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -450,8 +450,23 @@ void MacroAssembler::branchPtrInNurseryRange(Condition cond, Register ptr, Register temp, Label* label) { - MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); MOZ_ASSERT(ptr != temp); + branchPtrInNurseryRangeImpl(cond, ptr, temp, label); +} + +void +MacroAssembler::branchPtrInNurseryRange(Condition cond, const Address& address, Register temp, + Label* label) +{ + branchPtrInNurseryRangeImpl(cond, address, temp, label); +} + +template +void +MacroAssembler::branchPtrInNurseryRangeImpl(Condition cond, const T& ptr, Register temp, + Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86. const Nursery& nursery = GetJitContext()->runtime->gcNursery(); @@ -461,6 +476,20 @@ MacroAssembler::branchPtrInNurseryRange(Condition cond, Register ptr, Register t temp, Imm32(nursery.nurserySize()), label); } +void +MacroAssembler::branchValueIsNurseryObject(Condition cond, const Address& address, Register temp, + Label* label) +{ + MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual); + + Label done; + + branchTestObject(Assembler::NotEqual, address, cond == Assembler::Equal ? &done : label); + branchPtrInNurseryRange(cond, address, temp, label); + + bind(&done); +} + void MacroAssembler::branchValueIsNurseryObject(Condition cond, ValueOperand value, Register temp, Label* label) diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h index 5bc1dda9a58b..d22a34ef7a9c 100644 --- a/js/src/jit/x86/MacroAssembler-x86.h +++ b/js/src/jit/x86/MacroAssembler-x86.h @@ -150,6 +150,16 @@ class MacroAssemblerX86 : public MacroAssemblerX86Shared void storeValue(ValueOperand val, BaseIndex dest) { storeValue(val, Operand(dest)); } + void storeValue(const Address& src, const Address& dest, Register temp) { + MOZ_ASSERT(src.base != temp); + MOZ_ASSERT(dest.base != temp); + + load32(ToType(src), temp); + store32(temp, ToType(dest)); + + load32(ToPayload(src), temp); + store32(temp, ToPayload(dest)); + } void loadValue(Operand src, ValueOperand val) { Operand payload = ToPayload(src); Operand type = ToType(src); diff --git a/js/src/vm/SelfHosting.cpp b/js/src/vm/SelfHosting.cpp index 2c966e5de39c..231b812e7b73 100644 --- a/js/src/vm/SelfHosting.cpp +++ b/js/src/vm/SelfHosting.cpp @@ -699,6 +699,20 @@ intrinsic_GetNextMapEntryForIterator(JSContext* cx, unsigned argc, Value* vp) return true; } +static bool +intrinsic_CreateMapIterationResultPair(JSContext* cx, unsigned argc, Value* vp) +{ + CallArgs args = CallArgsFromVp(argc, vp); + MOZ_ASSERT(args.length() == 0); + + RootedObject result(cx, MapIteratorObject::createResultPair(cx)); + if (!result) + return false; + + args.rval().setObject(*result); + return true; +} + static bool intrinsic_NewStringIterator(JSContext* cx, unsigned argc, Value* vp) { @@ -1955,7 +1969,9 @@ static const JSFunctionSpec intrinsic_functions[] = { intrinsic_IsInstanceOfBuiltin, 1,0, IntrinsicIsListIterator), - JS_FN("_GetNextMapEntryForIterator", intrinsic_GetNextMapEntryForIterator, 3,0), + JS_FN("_CreateMapIterationResultPair", intrinsic_CreateMapIterationResultPair, 0, 0), + JS_INLINABLE_FN("_GetNextMapEntryForIterator", intrinsic_GetNextMapEntryForIterator, 2,0, + IntrinsicGetNextMapEntryForIterator), JS_FN("CallMapIteratorMethodIfWrapped", CallNonGenericSelfhostedMethod>, 2,0),