[wasm] Implement "atomic.fence" operator.

Michael Starzinger · Commit Bot · commit 4ca8b4dd67d7 · 2019-07-19T09:14:42.000Z
This adds decoding and compilation of the "atomic.fence" operator, which is intended to preserve the synchronization guarantees of higher-level languages. Unlike other atomic operators, it does not target a particular linear memory. It may occur in modules which declare no memory, or a non-shared memory, without causing a validation error. See proposal: WebAssembly/threads#141 See discussion: WebAssembly/threads#140 R=clemensh@chromium.org TEST=cctest/test-run-wasm-atomics/RunWasmXXX_AtomicFence BUG=v8:9452 Change-Id: Ibf7e46227f7edfe5c81c097cfc15924c59614067 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1701856 Commit-Queue: Michael Starzinger <mstarzinger@chromium.org> Reviewed-by: Clemens Hammacher <clemensh@chromium.org> Reviewed-by: Deepti Gandluri <gdeepti@chromium.org> Cr-Commit-Position: refs/heads/master@{#62821}
diff --git a/src/codegen/ia32/assembler-ia32.cc b/src/codegen/ia32/assembler-ia32.cc
@@ -756,6 +756,13 @@ void Assembler::cmpxchg8b(Operand dst) {
   emit_operand(ecx, dst);
 }
 
+void Assembler::mfence() {
+  EnsureSpace ensure_space(this);
+  EMIT(0x0F);
+  EMIT(0xAE);
+  EMIT(0xF0);
+}
+
 void Assembler::lfence() {
   EnsureSpace ensure_space(this);
   EMIT(0x0F);
diff --git a/src/codegen/ia32/assembler-ia32.h b/src/codegen/ia32/assembler-ia32.h
@@ -542,6 +542,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   void cmpxchg8b(Operand dst);
 
   // Memory Fence
+  void mfence();
   void lfence();
 
   void pause();
diff --git a/src/codegen/x64/assembler-x64.cc b/src/codegen/x64/assembler-x64.cc
@@ -1258,6 +1258,13 @@ void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
   emit_operand(src, dst);
 }
 
+void Assembler::mfence() {
+  EnsureSpace ensure_space(this);
+  emit(0x0F);
+  emit(0xAE);
+  emit(0xF0);
+}
+
 void Assembler::lfence() {
   EnsureSpace ensure_space(this);
   emit(0x0F);
diff --git a/src/codegen/x64/assembler-x64.h b/src/codegen/x64/assembler-x64.h
@@ -1746,6 +1746,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   void rorxl(Register dst, Register src, byte imm8);
   void rorxl(Register dst, Operand src, byte imm8);
 
+  void mfence();
   void lfence();
   void pause();
 
diff --git a/src/compiler/backend/arm/code-generator-arm.cc b/src/compiler/backend/arm/code-generator-arm.cc
@@ -1752,6 +1752,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       }
       break;
     }
+    case kArmDmbIsh: {
+      __ dmb(ISH);
+      break;
+    }
     case kArmDsbIsb: {
       __ dsb(SY);
       __ isb(SY);
diff --git a/src/compiler/backend/arm/instruction-codes-arm.h b/src/compiler/backend/arm/instruction-codes-arm.h
@@ -126,6 +126,7 @@ namespace compiler {
   V(ArmPush)                       \
   V(ArmPoke)                       \
   V(ArmPeek)                       \
+  V(ArmDmbIsh)                     \
   V(ArmDsbIsb)                     \
   V(ArmF32x4Splat)                 \
   V(ArmF32x4ExtractLane)           \
diff --git a/src/compiler/backend/arm/instruction-scheduler-arm.cc b/src/compiler/backend/arm/instruction-scheduler-arm.cc
@@ -275,6 +275,7 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kArmStr:
     case kArmPush:
     case kArmPoke:
+    case kArmDmbIsh:
     case kArmDsbIsb:
     case kArmWord32AtomicPairStore:
     case kArmWord32AtomicPairAdd:
diff --git a/src/compiler/backend/arm/instruction-selector-arm.cc b/src/compiler/backend/arm/instruction-selector-arm.cc
@@ -2020,6 +2020,11 @@ void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
        g.UseRegister(right));
 }
 
+void InstructionSelector::VisitMemoryBarrier(Node* node) {
+  ArmOperandGenerator g(this);
+  Emit(kArmDmbIsh, g.NoOutput());
+}
+
 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
   LoadRepresentation load_rep = LoadRepresentationOf(node->op());
   ArmOperandGenerator g(this);
diff --git a/src/compiler/backend/arm64/code-generator-arm64.cc b/src/compiler/backend/arm64/code-generator-arm64.cc
@@ -1625,6 +1625,9 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     case kArm64StrCompressTagged:
       __ StoreTaggedField(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
       break;
+    case kArm64DmbIsh:
+      __ Dmb(InnerShareable, BarrierAll);
+      break;
     case kArm64DsbIsb:
       __ Dsb(FullSystem, BarrierAll);
       __ Isb();
diff --git a/src/compiler/backend/arm64/instruction-codes-arm64.h b/src/compiler/backend/arm64/instruction-codes-arm64.h
@@ -171,6 +171,7 @@ namespace compiler {
   V(Arm64CompressSigned)                    \
   V(Arm64CompressPointer)                   \
   V(Arm64CompressAny)                       \
+  V(Arm64DmbIsh)                            \
   V(Arm64DsbIsb)                            \
   V(Arm64F32x4Splat)                        \
   V(Arm64F32x4ExtractLane)                  \
diff --git a/src/compiler/backend/arm64/instruction-scheduler-arm64.cc b/src/compiler/backend/arm64/instruction-scheduler-arm64.cc
@@ -319,6 +319,7 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kArm64StrW:
     case kArm64Str:
     case kArm64StrCompressTagged:
+    case kArm64DmbIsh:
     case kArm64DsbIsb:
       return kHasSideEffect;
 
diff --git a/src/compiler/backend/arm64/instruction-selector-arm64.cc b/src/compiler/backend/arm64/instruction-selector-arm64.cc
@@ -2801,6 +2801,11 @@ void InstructionSelector::VisitFloat64Mul(Node* node) {
   return VisitRRR(this, kArm64Float64Mul, node);
 }
 
+void InstructionSelector::VisitMemoryBarrier(Node* node) {
+  Arm64OperandGenerator g(this);
+  Emit(kArm64DmbIsh, g.NoOutput());
+}
+
 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
   LoadRepresentation load_rep = LoadRepresentationOf(node->op());
   ArchOpcode opcode = kArchNop;
diff --git a/src/compiler/backend/ia32/code-generator-ia32.cc b/src/compiler/backend/ia32/code-generator-ia32.cc
@@ -1217,7 +1217,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     case kArchWordPoisonOnSpeculation:
       // TODO(860429): Remove remaining poisoning infrastructure on ia32.
       UNREACHABLE();
-    case kLFence:
+    case kIA32MFence:
+      __ mfence();
+      break;
+    case kIA32LFence:
       __ lfence();
       break;
     case kSSEFloat32Cmp:
diff --git a/src/compiler/backend/ia32/instruction-codes-ia32.h b/src/compiler/backend/ia32/instruction-codes-ia32.h
@@ -44,7 +44,8 @@ namespace compiler {
   V(IA32Tzcnt)                     \
   V(IA32Popcnt)                    \
   V(IA32Bswap)                     \
-  V(LFence)                        \
+  V(IA32MFence)                    \
+  V(IA32LFence)                    \
   V(SSEFloat32Cmp)                 \
   V(SSEFloat32Add)                 \
   V(SSEFloat32Sub)                 \
diff --git a/src/compiler/backend/ia32/instruction-scheduler-ia32.cc b/src/compiler/backend/ia32/instruction-scheduler-ia32.cc
@@ -365,7 +365,8 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kIA32PushFloat64:
     case kIA32PushSimd128:
     case kIA32Poke:
-    case kLFence:
+    case kIA32MFence:
+    case kIA32LFence:
       return kHasSideEffect;
 
     case kIA32Word32AtomicPairLoad:
diff --git a/src/compiler/backend/ia32/instruction-selector-ia32.cc b/src/compiler/backend/ia32/instruction-selector-ia32.cc
@@ -1593,6 +1593,11 @@ void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
        g.UseRegister(node->InputAt(0)));
 }
 
+void InstructionSelector::VisitMemoryBarrier(Node* node) {
+  IA32OperandGenerator g(this);
+  Emit(kIA32MFence, g.NoOutput());
+}
+
 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
   LoadRepresentation load_rep = LoadRepresentationOf(node->op());
   DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
diff --git a/src/compiler/backend/instruction-selector.cc b/src/compiler/backend/instruction-selector.cc
@@ -1080,7 +1080,8 @@ void InstructionSelector::VisitBlock(BasicBlock* block) {
         node->opcode() == IrOpcode::kCall ||
         node->opcode() == IrOpcode::kCallWithCallerSavedRegisters ||
         node->opcode() == IrOpcode::kProtectedLoad ||
-        node->opcode() == IrOpcode::kProtectedStore) {
+        node->opcode() == IrOpcode::kProtectedStore ||
+        node->opcode() == IrOpcode::kMemoryBarrier) {
       ++effect_level;
     }
   }
@@ -1740,6 +1741,8 @@ void InstructionSelector::VisitNode(Node* node) {
       MarkAsWord32(node);
       MarkPairProjectionsAsWord32(node);
       return VisitWord32PairSar(node);
+    case IrOpcode::kMemoryBarrier:
+      return VisitMemoryBarrier(node);
     case IrOpcode::kWord32AtomicLoad: {
       LoadRepresentation type = LoadRepresentationOf(node->op());
       MarkAsRepresentation(type.representation(), node);
diff --git a/src/compiler/backend/x64/code-generator-x64.cc b/src/compiler/backend/x64/code-generator-x64.cc
@@ -1051,7 +1051,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       DCHECK_EQ(i.OutputRegister(), i.InputRegister(0));
       __ andq(i.InputRegister(0), kSpeculationPoisonRegister);
       break;
-    case kLFence:
+    case kX64MFence:
+      __ mfence();
+      break;
+    case kX64LFence:
       __ lfence();
       break;
     case kArchStackSlot: {
diff --git a/src/compiler/backend/x64/instruction-codes-x64.h b/src/compiler/backend/x64/instruction-codes-x64.h
@@ -58,7 +58,8 @@ namespace compiler {
   V(X64Popcnt32)                          \
   V(X64Bswap)                             \
   V(X64Bswap32)                           \
-  V(LFence)                               \
+  V(X64MFence)                            \
+  V(X64LFence)                            \
   V(SSEFloat32Cmp)                        \
   V(SSEFloat32Add)                        \
   V(SSEFloat32Sub)                        \
diff --git a/src/compiler/backend/x64/instruction-scheduler-x64.cc b/src/compiler/backend/x64/instruction-scheduler-x64.cc
@@ -354,7 +354,8 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kX64Poke:
       return kHasSideEffect;
 
-    case kLFence:
+    case kX64MFence:
+    case kX64LFence:
       return kHasSideEffect;
 
     case kX64Word64AtomicLoadUint8:
diff --git a/src/compiler/backend/x64/instruction-selector-x64.cc b/src/compiler/backend/x64/instruction-selector-x64.cc
@@ -2371,6 +2371,11 @@ void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
        g.UseRegister(node->InputAt(0)));
 }
 
+void InstructionSelector::VisitMemoryBarrier(Node* node) {
+  X64OperandGenerator g(this);
+  Emit(kX64MFence, g.NoOutput());
+}
+
 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
   LoadRepresentation load_rep = LoadRepresentationOf(node->op());
   DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
diff --git a/src/compiler/machine-operator.cc b/src/compiler/machine-operator.cc
@@ -778,6 +778,14 @@ struct MachineOperatorGlobalCache {
   };
   Word32AtomicPairCompareExchangeOperator kWord32AtomicPairCompareExchange;
 
+  struct MemoryBarrierOperator : public Operator {
+    MemoryBarrierOperator()
+        : Operator(IrOpcode::kMemoryBarrier,
+                   Operator::kNoDeopt | Operator::kNoThrow, "MemoryBarrier", 0,
+                   1, 1, 0, 1, 0) {}
+  };
+  MemoryBarrierOperator kMemoryBarrier;
+
   // The {BitcastWordToTagged} operator must not be marked as pure (especially
   // not idempotent), because otherwise the splitting logic in the Scheduler
   // might decide to split these operators, thus potentially creating live
@@ -1041,6 +1049,10 @@ const Operator* MachineOperatorBuilder::Comment(const char* msg) {
   return new (zone_) CommentOperator(msg);
 }
 
+const Operator* MachineOperatorBuilder::MemBarrier() {
+  return &cache_.kMemoryBarrier;
+}
+
 const Operator* MachineOperatorBuilder::Word32AtomicLoad(
     LoadRepresentation rep) {
 #define LOAD(Type)                          \
diff --git a/src/compiler/machine-operator.h b/src/compiler/machine-operator.h
@@ -655,6 +655,9 @@ class V8_EXPORT_PRIVATE MachineOperatorBuilder final
   const Operator* LoadFramePointer();
   const Operator* LoadParentFramePointer();
 
+  // Memory barrier.
+  const Operator* MemBarrier();
+
   // atomic-load [base + index]
   const Operator* Word32AtomicLoad(LoadRepresentation rep);
   // atomic-load [base + index]
diff --git a/src/compiler/opcodes.h b/src/compiler/opcodes.h
@@ -714,6 +714,7 @@
   V(Word32PairSar)                          \
   V(ProtectedLoad)                          \
   V(ProtectedStore)                         \
+  V(MemoryBarrier)                          \
   V(Word32AtomicLoad)                       \
   V(Word32AtomicStore)                      \
   V(Word32AtomicExchange)                   \
diff --git a/src/compiler/verifier.cc b/src/compiler/verifier.cc
@@ -1842,6 +1842,7 @@ void Verifier::Visitor::Check(Node* node, const AllNodes& all) {
     case IrOpcode::kLoadParentFramePointer:
     case IrOpcode::kUnalignedLoad:
     case IrOpcode::kUnalignedStore:
+    case IrOpcode::kMemoryBarrier:
     case IrOpcode::kWord32AtomicLoad:
     case IrOpcode::kWord32AtomicStore:
     case IrOpcode::kWord32AtomicExchange:
diff --git a/src/compiler/wasm-compiler.cc b/src/compiler/wasm-compiler.cc
@@ -4695,6 +4695,11 @@ Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
   return SetEffect(node);
 }
 
+Node* WasmGraphBuilder::AtomicFence() {
+  return SetEffect(graph()->NewNode(mcgraph()->machine()->MemBarrier(),
+                                    Effect(), Control()));
+}
+
 #undef ATOMIC_BINOP_LIST
 #undef ATOMIC_CMP_EXCHG_LIST
 #undef ATOMIC_LOAD_LIST
diff --git a/src/compiler/wasm-compiler.h b/src/compiler/wasm-compiler.h
@@ -387,6 +387,7 @@ class WasmGraphBuilder {
   Node* AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
                  uint32_t alignment, uint32_t offset,
                  wasm::WasmCodePosition position);
+  Node* AtomicFence();
 
   // Returns a pointer to the dropped_data_segments array. Traps if the data
   // segment is active or has been dropped.
diff --git a/src/diagnostics/ia32/disasm-ia32.cc b/src/diagnostics/ia32/disasm-ia32.cc
@@ -1870,6 +1870,9 @@ int DisassemblerIA32::InstructionDecode(v8::internal::Vector<char> out_buffer,
           int mod, regop, rm;
           get_modrm(*data, &mod, &regop, &rm);
           data += PrintRightOperand(data);
+        } else if (f0byte == 0xAE && (data[2] & 0xF8) == 0xF0) {
+          AppendToBuffer("mfence");
+          data += 3;
         } else if (f0byte == 0xAE && (data[2] & 0xF8) == 0xE8) {
           AppendToBuffer("lfence");
           data += 3;
diff --git a/src/diagnostics/x64/disasm-x64.cc b/src/diagnostics/x64/disasm-x64.cc
@@ -2290,7 +2290,10 @@ int DisassemblerX64::TwoByteOpcodeInstruction(byte* data) {
       byte_size_operand_ = true;
     }
     current += PrintOperands(mnemonic, OPER_REG_OP_ORDER, current);
-  } else if (opcode == 0xAE && (*(data + 2) & 0xF8) == 0xE8) {
+  } else if (opcode == 0xAE && (data[2] & 0xF8) == 0xF0) {
+    AppendToBuffer("mfence");
+    current = data + 3;
+  } else if (opcode == 0xAE && (data[2] & 0xF8) == 0xE8) {
     AppendToBuffer("lfence");
     current = data + 3;
   } else {
diff --git a/src/wasm/baseline/liftoff-compiler.cc b/src/wasm/baseline/liftoff-compiler.cc
@@ -2007,6 +2007,9 @@ class LiftoffCompiler {
                 const MemoryAccessImmediate<validate>& imm, Value* result) {
     unsupported(decoder, kAtomics, "atomicop");
   }
+  void AtomicFence(FullDecoder* decoder) {
+    unsupported(decoder, kAtomics, "atomic.fence");
+  }
   void MemoryInit(FullDecoder* decoder,
                   const MemoryInitImmediate<validate>& imm, const Value& dst,
                   const Value& src, const Value& size) {
diff --git a/src/wasm/function-body-decoder-impl.h b/src/wasm/function-body-decoder-impl.h
diff --git a/src/wasm/graph-builder-interface.cc b/src/wasm/graph-builder-interface.cc
diff --git a/src/wasm/wasm-interpreter.cc b/src/wasm/wasm-interpreter.cc
diff --git a/src/wasm/wasm-opcodes.cc b/src/wasm/wasm-opcodes.cc
diff --git a/src/wasm/wasm-opcodes.h b/src/wasm/wasm-opcodes.h
diff --git a/src/wasm/wasm-text.cc b/src/wasm/wasm-text.cc
diff --git a/test/cctest/test-disasm-ia32.cc b/test/cctest/test-disasm-ia32.cc
diff --git a/test/cctest/test-disasm-x64.cc b/test/cctest/test-disasm-x64.cc
diff --git a/test/cctest/wasm/test-run-wasm-atomics.cc b/test/cctest/wasm/test-run-wasm-atomics.cc
diff --git a/test/common/wasm/wasm-macro-gen.h b/test/common/wasm/wasm-macro-gen.h

Original file line number	Diff line number	Diff line change
`@@ -1752,6 +1752,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(`
`1752`	`1752`	`}`
`1753`	`1753`	`break;`
`1754`	`1754`	`}`
	`1755`	`+ case kArmDmbIsh: {`
	`1756`	`+ __ dmb(ISH);`
	`1757`	`+ break;`
	`1758`	`+ }`
`1755`	`1759`	`case kArmDsbIsb: {`
`1756`	`1760`	`__ dsb(SY);`
`1757`	`1761`	`__ isb(SY);`