Skip to content

Commit b00a531

Browse files
Bill BudgeCommit Bot
Bill Budge
authored and
Commit Bot
committed
[wasm] Move fast path of wasm Table.get/set to builtins
- Reworks the builtins WasmTableGet and WasmTableSet to do the fast path, instead of generating this inline in wasm-compiler. Change-Id: I0a47c09d6f4f6d81c7b362f6f45e95b19e3edf86 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2135864 Commit-Queue: Bill Budge <bbudge@chromium.org> Reviewed-by: Andreas Haas <ahaas@chromium.org> Cr-Commit-Position: refs/heads/master@{#67296}
1 parent 30c6bd4 commit b00a531

5 files changed

Lines changed: 110 additions & 147 deletions

File tree

‎src/builtins/builtins-wasm-gen.cc‎

Lines changed: 75 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -312,50 +312,99 @@ TF_BUILTIN(WasmTableCopy, WasmBuiltinsAssembler) {
312312
}
313313

314314
TF_BUILTIN(WasmTableGet, WasmBuiltinsAssembler) {
315-
TNode<Int32T> entry_index =
316-
UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex));
317315
TNode<WasmInstanceObject> instance = LoadInstanceFromFrame();
318-
TNode<Context> context = LoadContextFromInstance(instance);
319-
Label entry_index_out_of_range(this, Label::kDeferred);
320316

321-
TNode<BoolT> entry_index_fits_in_smi =
322-
IsValidPositiveSmi(ChangeInt32ToIntPtr(entry_index));
323-
GotoIfNot(entry_index_fits_in_smi, &entry_index_out_of_range);
317+
Label call_runtime(this, Label::kDeferred),
318+
index_out_of_range(this, Label::kDeferred);
319+
320+
TNode<IntPtrT> table_index =
321+
UncheckedCast<IntPtrT>(Parameter(Descriptor::kTableIndex));
322+
GotoIfNot(IsValidPositiveSmi(table_index), &index_out_of_range);
323+
TNode<IntPtrT> entry_index = ChangeInt32ToIntPtr(
324+
UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex)));
325+
GotoIfNot(IsValidPositiveSmi(entry_index), &index_out_of_range);
326+
327+
TNode<FixedArray> tables_array =
328+
LoadObjectField<FixedArray>(instance, WasmInstanceObject::kTablesOffset);
329+
TNode<WasmTableObject> table =
330+
CAST(LoadFixedArrayElement(tables_array, table_index));
331+
TNode<IntPtrT> entries_length =
332+
LoadAndUntagObjectField(table, WasmTableObject::kCurrentLengthOffset);
333+
GotoIfNot(IntPtrLessThan(entry_index, entries_length), &index_out_of_range);
334+
335+
TNode<FixedArray> entries_array =
336+
LoadObjectField<FixedArray>(table, WasmTableObject::kEntriesOffset);
337+
338+
TNode<Object> entry = LoadFixedArrayElement(entries_array, entry_index);
324339

325-
TNode<Smi> entry_index_smi = SmiFromInt32(entry_index);
326-
TNode<Smi> table_index_smi = CAST(Parameter(Descriptor::kTableIndex));
340+
// If the entry is our placeholder for lazy function initialization, then we
341+
// fall back to the runtime call.
342+
TNode<Map> map = LoadReceiverMap(entry);
343+
GotoIf(IsTuple2Map(map), &call_runtime);
327344

328-
TailCallRuntime(Runtime::kWasmFunctionTableGet, context, instance,
329-
table_index_smi, entry_index_smi);
345+
Return(entry);
330346

331-
BIND(&entry_index_out_of_range);
347+
BIND(&call_runtime);
348+
// Fall back to the runtime call for more complex cases.
349+
// table_index and entry_index must be in Smi range, due to checks above.
350+
TailCallRuntime(Runtime::kWasmFunctionTableGet,
351+
LoadContextFromInstance(instance), instance,
352+
SmiFromIntPtr(table_index), SmiFromIntPtr(entry_index));
353+
354+
BIND(&index_out_of_range);
332355
MessageTemplate message_id =
333356
wasm::WasmOpcodes::TrapReasonToMessageId(wasm::kTrapTableOutOfBounds);
334-
TailCallRuntime(Runtime::kThrowWasmError, context,
357+
TailCallRuntime(Runtime::kThrowWasmError, LoadContextFromInstance(instance),
335358
SmiConstant(static_cast<int>(message_id)));
336359
}
337360

338361
TF_BUILTIN(WasmTableSet, WasmBuiltinsAssembler) {
339-
TNode<Int32T> entry_index =
340-
UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex));
341362
TNode<WasmInstanceObject> instance = LoadInstanceFromFrame();
342-
TNode<Context> context = LoadContextFromInstance(instance);
343-
Label entry_index_out_of_range(this, Label::kDeferred);
344363

345-
TNode<BoolT> entry_index_fits_in_smi =
346-
IsValidPositiveSmi(ChangeInt32ToIntPtr(entry_index));
347-
GotoIfNot(entry_index_fits_in_smi, &entry_index_out_of_range);
364+
Label call_runtime(this, Label::kDeferred),
365+
index_out_of_range(this, Label::kDeferred);
366+
367+
TNode<IntPtrT> table_index =
368+
UncheckedCast<IntPtrT>(Parameter(Descriptor::kTableIndex));
369+
GotoIfNot(IsValidPositiveSmi(table_index), &index_out_of_range);
370+
TNode<IntPtrT> entry_index = ChangeInt32ToIntPtr(
371+
UncheckedCast<Int32T>(Parameter(Descriptor::kEntryIndex)));
372+
GotoIfNot(IsValidPositiveSmi(entry_index), &index_out_of_range);
348373

349-
TNode<Smi> entry_index_smi = SmiFromInt32(entry_index);
350-
TNode<Smi> table_index_smi = CAST(Parameter(Descriptor::kTableIndex));
351374
TNode<Object> value = CAST(Parameter(Descriptor::kValue));
352-
TailCallRuntime(Runtime::kWasmFunctionTableSet, context, instance,
353-
table_index_smi, entry_index_smi, value);
354375

355-
BIND(&entry_index_out_of_range);
376+
TNode<FixedArray> tables_array =
377+
LoadObjectField<FixedArray>(instance, WasmInstanceObject::kTablesOffset);
378+
TNode<WasmTableObject> table =
379+
CAST(LoadFixedArrayElement(tables_array, table_index));
380+
// Fall back to the runtime to set funcrefs, since we have to update function
381+
// dispatch tables.
382+
TNode<Smi> table_type =
383+
LoadObjectField<Smi>(table, WasmTableObject::kRawTypeOffset);
384+
GotoIf(SmiEqual(table_type, SmiConstant(wasm::ValueType::Kind::kFuncRef)),
385+
&call_runtime);
386+
387+
TNode<IntPtrT> entries_length =
388+
LoadAndUntagObjectField(table, WasmTableObject::kCurrentLengthOffset);
389+
GotoIfNot(IntPtrLessThan(entry_index, entries_length), &index_out_of_range);
390+
391+
TNode<FixedArray> entries_array =
392+
LoadObjectField<FixedArray>(table, WasmTableObject::kEntriesOffset);
393+
394+
StoreFixedArrayElement(entries_array, entry_index, value);
395+
Return(UndefinedConstant());
396+
397+
BIND(&call_runtime);
398+
// Fall back to the runtime call for more complex cases.
399+
// table_index and entry_index must be in Smi range, due to checks above.
400+
TailCallRuntime(
401+
Runtime::kWasmFunctionTableSet, LoadContextFromInstance(instance),
402+
instance, SmiFromIntPtr(table_index), SmiFromIntPtr(entry_index), value);
403+
404+
BIND(&index_out_of_range);
356405
MessageTemplate message_id =
357406
wasm::WasmOpcodes::TrapReasonToMessageId(wasm::kTrapTableOutOfBounds);
358-
TailCallRuntime(Runtime::kThrowWasmError, context,
407+
TailCallRuntime(Runtime::kThrowWasmError, LoadContextFromInstance(instance),
359408
SmiConstant(static_cast<int>(message_id)));
360409
}
361410

‎src/codegen/interface-descriptors.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1403,18 +1403,18 @@ class WasmTableCopyDescriptor final : public CallInterfaceDescriptor {
14031403
class WasmTableGetDescriptor final : public CallInterfaceDescriptor {
14041404
public:
14051405
DEFINE_PARAMETERS_NO_CONTEXT(kTableIndex, kEntryIndex)
1406-
DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result 1
1407-
MachineType::TaggedSigned(), // kTableIndex
1408-
MachineType::Int32()) // kEntryIndex
1406+
DEFINE_RESULT_AND_PARAMETER_TYPES(MachineType::AnyTagged(), // result
1407+
MachineType::IntPtr(), // kTableIndex
1408+
MachineType::Int32()) // kEntryIndex
14091409
DECLARE_DESCRIPTOR(WasmTableGetDescriptor, CallInterfaceDescriptor)
14101410
};
14111411

14121412
class WasmTableSetDescriptor final : public CallInterfaceDescriptor {
14131413
public:
14141414
DEFINE_PARAMETERS_NO_CONTEXT(kTableIndex, kEntryIndex, kValue)
1415-
DEFINE_PARAMETER_TYPES(MachineType::TaggedSigned(), // kTableIndex
1416-
MachineType::Int32(), // kEntryIndex
1417-
MachineType::AnyTagged()) // kValue
1415+
DEFINE_PARAMETER_TYPES(MachineType::IntPtr(), // kTableIndex
1416+
MachineType::Int32(), // kEntryIndex
1417+
MachineType::AnyTagged()) // kValue
14181418
DECLARE_DESCRIPTOR(WasmTableSetDescriptor, CallInterfaceDescriptor)
14191419
};
14201420

‎src/compiler/wasm-compiler.cc‎

Lines changed: 25 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -1938,6 +1938,19 @@ ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
19381938
}
19391939
}
19401940

1941+
template <typename BuiltinDescriptor>
1942+
CallDescriptor* GetBuiltinCallDescriptor(WasmGraphBuilder* builder,
1943+
StubCallMode stub_mode) {
1944+
BuiltinDescriptor interface_descriptor;
1945+
return Linkage::GetStubCallDescriptor(
1946+
builder->mcgraph()->zone(), // zone
1947+
interface_descriptor, // descriptor
1948+
interface_descriptor.GetStackParameterCount(), // stack parameter count
1949+
CallDescriptor::kNoFlags, // flags
1950+
Operator::kNoProperties, // properties
1951+
stub_mode); // stub call mode
1952+
}
1953+
19411954
} // namespace
19421955

19431956
Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
@@ -3348,114 +3361,32 @@ Node* WasmGraphBuilder::GlobalSet(uint32_t index, Node* val) {
33483361
graph()->NewNode(op, base, offset, val, effect(), control()));
33493362
}
33503363

3351-
void WasmGraphBuilder::BoundsCheckTable(uint32_t table_index, Node* entry_index,
3352-
wasm::WasmCodePosition position,
3353-
wasm::TrapReason trap_reason,
3354-
Node** base_node) {
3355-
Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
3356-
Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
3357-
3358-
int length_field_size = WasmTableObject::kCurrentLengthOffsetEnd -
3359-
WasmTableObject::kCurrentLengthOffset + 1;
3360-
Node* length_smi = gasm_->Load(
3361-
assert_size(length_field_size, MachineType::TaggedSigned()), table,
3362-
wasm::ObjectAccess::ToTagged(WasmTableObject::kCurrentLengthOffset));
3363-
Node* length = BuildChangeSmiToInt32(length_smi);
3364-
3365-
// Bounds check against the table size.
3366-
Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
3367-
entry_index, length);
3368-
TrapIfFalse(trap_reason, in_bounds, position);
3369-
3370-
if (base_node) {
3371-
int storage_field_size = WasmTableObject::kEntriesOffsetEnd -
3372-
WasmTableObject::kEntriesOffset + 1;
3373-
*base_node = gasm_->Load(
3374-
assert_size(storage_field_size, MachineType::TaggedPointer()), table,
3375-
wasm::ObjectAccess::ToTagged(WasmTableObject::kEntriesOffset));
3376-
}
3377-
}
3378-
3379-
void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index,
3380-
Node* entry_index,
3381-
wasm::WasmCodePosition position,
3382-
Node** base_node,
3383-
Node** offset_node) {
3384-
BoundsCheckTable(table_index, entry_index, position,
3385-
wasm::kTrapTableOutOfBounds, base_node);
3386-
// From the index, calculate the actual offset in the FixeArray. This
3387-
// is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3388-
// wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3389-
Node* index_times_tagged_size = graph()->NewNode(
3390-
mcgraph()->machine()->IntMul(), Uint32ToUintptr(entry_index),
3391-
mcgraph()->Int32Constant(kTaggedSize));
3392-
3393-
*offset_node = graph()->NewNode(
3394-
mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3395-
mcgraph()->IntPtrConstant(
3396-
wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3397-
}
3398-
33993364
Node* WasmGraphBuilder::TableGet(uint32_t table_index, Node* index,
34003365
wasm::WasmCodePosition position) {
3401-
if (env_->module->tables[table_index].type == wasm::kWasmAnyRef ||
3402-
env_->module->tables[table_index].type == wasm::kWasmNullRef ||
3403-
env_->module->tables[table_index].type == wasm::kWasmExnRef) {
3404-
Node* base = nullptr;
3405-
Node* offset = nullptr;
3406-
GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3407-
return gasm_->Load(MachineType::AnyTagged(), base, offset);
3408-
}
3409-
// We access funcref tables through runtime calls.
3410-
WasmTableGetDescriptor interface_descriptor;
3411-
auto call_descriptor = Linkage::GetStubCallDescriptor(
3412-
mcgraph()->zone(), // zone
3413-
interface_descriptor, // descriptor
3414-
interface_descriptor.GetStackParameterCount(), // stack parameter count
3415-
CallDescriptor::kNoFlags, // flags
3416-
Operator::kNoProperties, // properties
3417-
StubCallMode::kCallWasmRuntimeStub); // stub call mode
3366+
auto call_descriptor = GetBuiltinCallDescriptor<WasmTableGetDescriptor>(
3367+
this, StubCallMode::kCallWasmRuntimeStub);
34183368
// A direct call to a wasm runtime stub defined in this module.
34193369
// Just encode the stub index. This will be patched at relocation.
34203370
Node* call_target = mcgraph()->RelocatableIntPtrConstant(
34213371
wasm::WasmCode::kWasmTableGet, RelocInfo::WASM_STUB_CALL);
34223372

34233373
return SetEffectControl(graph()->NewNode(
34243374
mcgraph()->common()->Call(call_descriptor), call_target,
3425-
graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), index,
3426-
effect(), control()));
3375+
IntPtrConstant(table_index), index, effect(), control()));
34273376
}
34283377

34293378
Node* WasmGraphBuilder::TableSet(uint32_t table_index, Node* index, Node* val,
34303379
wasm::WasmCodePosition position) {
3431-
if (env_->module->tables[table_index].type == wasm::kWasmAnyRef ||
3432-
env_->module->tables[table_index].type == wasm::kWasmNullRef ||
3433-
env_->module->tables[table_index].type == wasm::kWasmExnRef) {
3434-
Node* base = nullptr;
3435-
Node* offset = nullptr;
3436-
GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3437-
return STORE_RAW_NODE_OFFSET(
3438-
base, offset, val, MachineRepresentation::kTagged, kFullWriteBarrier);
3439-
} else {
3440-
// We access funcref tables through runtime calls.
3441-
WasmTableSetDescriptor interface_descriptor;
3442-
auto call_descriptor = Linkage::GetStubCallDescriptor(
3443-
mcgraph()->zone(), // zone
3444-
interface_descriptor, // descriptor
3445-
interface_descriptor.GetStackParameterCount(), // stack parameter count
3446-
CallDescriptor::kNoFlags, // flags
3447-
Operator::kNoProperties, // properties
3448-
StubCallMode::kCallWasmRuntimeStub); // stub call mode
3449-
// A direct call to a wasm runtime stub defined in this module.
3450-
// Just encode the stub index. This will be patched at relocation.
3451-
Node* call_target = mcgraph()->RelocatableIntPtrConstant(
3452-
wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
3380+
auto call_descriptor = GetBuiltinCallDescriptor<WasmTableSetDescriptor>(
3381+
this, StubCallMode::kCallWasmRuntimeStub);
3382+
// A direct call to a wasm runtime stub defined in this module.
3383+
// Just encode the stub index. This will be patched at relocation.
3384+
Node* call_target = mcgraph()->RelocatableIntPtrConstant(
3385+
wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
34533386

3454-
return SetEffectControl(graph()->NewNode(
3455-
mcgraph()->common()->Call(call_descriptor), call_target,
3456-
graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
3457-
index, val, effect(), control()));
3458-
}
3387+
return SetEffectControl(graph()->NewNode(
3388+
mcgraph()->common()->Call(call_descriptor), call_target,
3389+
IntPtrConstant(table_index), index, val, effect(), control()));
34593390
}
34603391

34613392
Node* WasmGraphBuilder::CheckBoundsAndAlignment(
@@ -4029,19 +3960,6 @@ Signature<MachineRepresentation>* CreateMachineSignature(
40293960
return builder.Build();
40303961
}
40313962

4032-
template <typename BuiltinDescriptor>
4033-
CallDescriptor* GetBuiltinCallDescriptor(WasmGraphBuilder* builder,
4034-
StubCallMode stub_mode) {
4035-
BuiltinDescriptor interface_descriptor;
4036-
return Linkage::GetStubCallDescriptor(
4037-
builder->mcgraph()->zone(), // zone
4038-
interface_descriptor, // descriptor
4039-
interface_descriptor.GetStackParameterCount(), // stack parameter count
4040-
CallDescriptor::kNoFlags, // flags
4041-
Operator::kNoProperties, // properties
4042-
stub_mode); // stub call mode
4043-
}
4044-
40453963
} // namespace
40463964

40473965
void WasmGraphBuilder::AddInt64LoweringReplacement(

‎src/compiler/wasm-compiler.h‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -321,14 +321,6 @@ class WasmGraphBuilder {
321321
void GetBaseAndOffsetForImportedMutableAnyRefGlobal(
322322
const wasm::WasmGlobal& global, Node** base, Node** offset);
323323

324-
void BoundsCheckTable(uint32_t table_index, Node* index,
325-
wasm::WasmCodePosition position,
326-
wasm::TrapReason trap_reason, Node** base_node);
327-
328-
void GetTableBaseAndOffset(uint32_t table_index, Node* index,
329-
wasm::WasmCodePosition position, Node** base_node,
330-
Node** offset_node);
331-
332324
// Utilities to manipulate sets of instance cache nodes.
333325
void InitInstanceCache(WasmInstanceCacheNodes* instance_cache);
334326
void PrepareInstanceCacheForLoop(WasmInstanceCacheNodes* instance_cache,

‎src/runtime/runtime-wasm.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,8 @@ RUNTIME_FUNCTION(Runtime_WasmFunctionTableGet) {
487487
DCHECK_LT(table_index, instance->tables().length());
488488
auto table = handle(
489489
WasmTableObject::cast(instance->tables().get(table_index)), isolate);
490+
// We only use the runtime call for lazily initialized function references.
491+
DCHECK_EQ(table->type(), wasm::kWasmFuncRef);
490492

491493
if (!WasmTableObject::IsInBounds(isolate, table, entry_index)) {
492494
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);
@@ -508,6 +510,8 @@ RUNTIME_FUNCTION(Runtime_WasmFunctionTableSet) {
508510
DCHECK_LT(table_index, instance->tables().length());
509511
auto table = handle(
510512
WasmTableObject::cast(instance->tables().get(table_index)), isolate);
513+
// We only use the runtime call for function references.
514+
DCHECK_EQ(table->type(), wasm::kWasmFuncRef);
511515

512516
if (!WasmTableObject::IsInBounds(isolate, table, entry_index)) {
513517
return ThrowWasmError(isolate, MessageTemplate::kWasmTrapTableOutOfBounds);

0 commit comments

Comments
 (0)