inline ScopedHandle() noexcept

: value(nullptr) {}

inline ~ScopedHandle() noexcept {

if (value != nullptr)

::CloseHandle(value);

}

HANDLE value;

SYSTEM_INFO systemInfo;

::GetSystemInfo(&systemInfo);

vmInfo.pageSize = Support::alignUpPowerOf2<uint32_t>(systemInfo.dwPageSize);

vmInfo.pageGranularity = systemInfo.dwAllocationGranularity;

DWORD protectFlags;

// READ|WRITE|EXECUTE.

if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))

protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;

else if (Support::test(memoryFlags, MemoryFlags::kAccessRW))

protectFlags = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? PAGE_READWRITE : PAGE_READONLY;

else

protectFlags = PAGE_NOACCESS;

// Any other flags to consider?

return protectFlags;

DWORD access = Support::test(memoryFlags, MemoryFlags::kAccessWrite) ? FILE_MAP_WRITE : FILE_MAP_READ;

if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))

access |= FILE_MAP_EXECUTE;

return access;

HardenedRuntimeFlags flags = HardenedRuntimeFlags::kNone;

if (hasDualMappingSupport())

flags |= HardenedRuntimeFlags::kDualMapping;

return flags;

*p = nullptr;

if (size == 0)

return DebugUtils::errored(kErrorInvalidArgument);

DWORD allocationType = MEM_COMMIT | MEM_RESERVE;

DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);

if (Support::test(memoryFlags, MemoryFlags::kMMapLargePages)) {

size_t lpSize = largePageSize();

// Does it make sense to call VirtualAlloc() if we failed to query large page size?

if (lpSize == 0)

return DebugUtils::errored(kErrorFeatureNotEnabled);

if (!Support::isAligned(size, lpSize))

return DebugUtils::errored(kErrorInvalidArgument);

allocationType |= MEM_LARGE_PAGES;

}

void* result = ::VirtualAlloc(nullptr, size, allocationType, protectFlags);

if (!result)

return DebugUtils::errored(kErrorOutOfMemory);

*p = result;

return kErrorOk;

DebugUtils::unused(size);

// NOTE: If the `dwFreeType` parameter is MEM_RELEASE, `size` parameter must be zero.

constexpr DWORD dwFreeType = MEM_RELEASE;

if (ASMJIT_UNLIKELY(!::VirtualFree(p, 0, dwFreeType)))

return DebugUtils::errored(kErrorInvalidArgument);

return kErrorOk;

DWORD protectFlags = protectFlagsFromMemoryFlags(memoryFlags);

DWORD oldFlags;

if (::VirtualProtect(p, size, protectFlags, &oldFlags))

return kErrorOk;

return DebugUtils::errored(kErrorInvalidArgument);

dm->rx = nullptr;

dm->rw = nullptr;

if (size == 0)

return DebugUtils::errored(kErrorInvalidArgument);

ScopedHandle handle;

handle.value = ::CreateFileMappingW(

INVALID_HANDLE_VALUE,

nullptr,

PAGE_EXECUTE_READWRITE,

(DWORD)(uint64_t(size) >> 32),

(DWORD)(size & 0xFFFFFFFFu),

nullptr);

if (ASMJIT_UNLIKELY(!handle.value))

return DebugUtils::errored(kErrorOutOfMemory);

void* ptr[2];

for (uint32_t i = 0; i < 2; i++) {

MemoryFlags accessFlags = memoryFlags & ~dualMappingFilter[i];

DWORD desiredAccess = desiredAccessFromMemoryFlags(accessFlags);

ptr[i] = ::MapViewOfFile(handle.value, desiredAccess, 0, 0, size);

if (ptr[i] == nullptr) {

if (i == 1u)

::UnmapViewOfFile(ptr[0]);

return DebugUtils::errored(kErrorOutOfMemory);

}

}

dm->rx = ptr[0];

dm->rw = ptr[1];

return kErrorOk;

DebugUtils::unused(size);

bool failed = false;

if (!::UnmapViewOfFile(dm->rx))

failed = true;

if (dm->rx != dm->rw && !UnmapViewOfFile(dm->rw))

failed = true;

if (failed)

return DebugUtils::errored(kErrorInvalidArgument);

dm->rx = nullptr;

dm->rw = nullptr;

return kErrorOk;

long ver[2];

inline long major() const noexcept { return ver[0]; }

inline long minor() const noexcept { return ver[1]; }

inline bool eq(long major, long minor) const noexcept { return ver[0] == major && ver[1] == minor; }

inline bool ge(long major, long minor) const noexcept { return ver[0] > major || (ver[0] == major && ver[1] >= minor); }

KernelVersion out {};

struct utsname buf {};

uname(&buf);

size_t i = 0;

char* p = buf.release;

while (*p && i < 2u) {

uint8_t c = uint8_t(*p);

if (c >= uint8_t('0') && c <= uint8_t('9')) {

out.ver[i] = strtol(p, &p, 10);

i++;

continue;

}

p++;

}

return out;

switch (e) {

case EACCES:

case EAGAIN:

case ENODEV:

case EPERM:

return kErrorInvalidState;

case EFBIG:

case ENOMEM:

case EOVERFLOW:

return kErrorOutOfMemory;

case EMFILE:

case ENFILE:

return kErrorTooManyHandles;

default:

return kErrorInvalidArgument;

}

int protection = 0;

if (Support::test(memoryFlags, MemoryFlags::kAccessRead)) protection |= PROT_READ;

if (Support::test(memoryFlags, MemoryFlags::kAccessWrite)) protection |= PROT_READ | PROT_WRITE;

if (Support::test(memoryFlags, MemoryFlags::kAccessExecute)) protection |= PROT_READ | PROT_EXEC;

return protection;

MemoryFlags acc = maxAccessFlagsToRegularAccessFlags(memoryFlags);

if (acc != MemoryFlags::kNone) {

#if defined(__NetBSD__) && defined(PROT_MPROTECT)

return PROT_MPROTECT(mmProtFromMemoryFlags(acc));

#elif defined(PROT_MAX)

return PROT_MAX(mmProtFromMemoryFlags(acc));

#else

return 0;

#endif

}

return 0;

uint32_t pageSize = uint32_t(::getpagesize());

vmInfo.pageSize = pageSize;

vmInfo.pageGranularity = Support::max<uint32_t>(pageSize, 65536);

#if defined(__APPLE__) && defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && ASMJIT_ARCH_X86

return 2u * 1024u * 1024u;

#elif defined(__FreeBSD__)

Support::Array<size_t, 2> pageSize;

// TODO: Does it return unsigned?

return (getpagesizes(pageSize.data(), 2) < 2) ? 0 : uint32_t(pageSize[1]);

#elif defined(__linux__)

StringTmp<128> storage;

if (OSUtils::readFile("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", storage, 16) != kErrorOk || storage.empty())

return 0u;

// The first value should be the size of the page (hpage_pmd_size).

size_t largePageSize = 0;

const char* buf = storage.data();

size_t bufSize = storage.size();

for (size_t i = 0; i < bufSize; i++) {

uint32_t digit = uint32_t(uint8_t(buf[i]) - uint8_t('0'));

if (digit >= 10u)

break;

largePageSize = largePageSize * 10 + digit;

}

if (Support::isPowerOf2(largePageSize))

return largePageSize;

else

return 0u;

#else

return 0u;

#endif

kUnknown = 0,

kDevShm = 1,

kTmpDir = 2

static std::atomic<uint32_t> cachedMfdExecSupported;

uint32_t val = cachedMfdExecSupported.load();

if (val == 0u) {

KernelVersion ver = getKernelVersion();

val = uint32_t(ver.ge(6, 3)) + 1u;

cachedMfdExecSupported.store(val);

}

return val == 2u ? uint32_t(MFD_EXEC) : uint32_t(0u);

static std::atomic<uint32_t> internalCounter;

#if defined(__GNUC__) && ASMJIT_ARCH_X86

// Use RDTSC instruction to avoid gettimeofday() as we just need some "random" bits.

uint64_t mix = __builtin_ia32_rdtsc();

#else

struct timeval tm {};

uint64_t mix = 1; // only used when gettimeofday() fails, which is unlikely.

if (gettimeofday(&tm, nullptr) == 0) {

mix = uint64_t(tm.tv_usec) ^ uint64_t(tm.tv_sec);

}

#endif

uint64_t bits = (uint64_t(stackPtr) & 0x1010505000055590u) - mix * 773703683;

bits = (bits >> 33) ^ (bits << 7) ^ (attempt * 87178291199);

return bits + uint64_t(++internalCounter) * 10619863;

enum FileType : uint32_t {

kFileTypeNone,

kFileTypeShm,

kFileTypeTmp

};

int _fd;

FileType _fileType;

StringTmp<128> _tmpName;

inline AnonymousMemory() noexcept

: _fd(-1),

_fileType(kFileTypeNone),

_tmpName() {}

inline ~AnonymousMemory() noexcept {

unlink();

close();

}

inline int fd() const noexcept { return _fd; }

Error open(bool preferTmpOverDevShm) noexcept {

#if defined(__linux__) && defined(__NR_memfd_create)

// Linux specific 'memfd_create' - if the syscall returns `ENOSYS` it means

// it's not available and we will never call it again (would be pointless).

//

// NOTE: There is also memfd_create() libc function in FreeBSD, but it internally

// uses `shm_open(SHM_ANON, ...)` so it's not needed to add support for it (it's

// not a syscall as in Linux).

// Zero initialized, if ever changed to '1' that would mean the syscall is not

// available and we must use `shm_open()` and `shm_unlink()` (or regular `open()`).

static volatile uint32_t memfd_create_not_supported;

if (!memfd_create_not_supported) {

_fd = (int)syscall(__NR_memfd_create, "vmem", MFD_CLOEXEC | getMfdExecFlag());

if (ASMJIT_LIKELY(_fd >= 0))

return kErrorOk;

int e = errno;

if (e == ENOSYS)

memfd_create_not_supported = 1;

else

return DebugUtils::errored(asmjitErrorFromErrno(e));

}

#endif // __linux__ && __NR_memfd_create

#if defined(ASMJIT_HAS_SHM_OPEN) && defined(SHM_ANON)

// Originally FreeBSD extension, apparently works in other BSDs too.

DebugUtils::unused(preferTmpOverDevShm);

_fd = ::shm_open(SHM_ANON, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);

if (ASMJIT_LIKELY(_fd >= 0))

return kErrorOk;

else

return DebugUtils::errored(asmjitErrorFromErrno(errno));

#else

// POSIX API. We have to generate somehow a unique name, so use `generateRandomBits()` helper. To prevent

// having file collisions we use `shm_open()` with flags that require creation of the file so we never open

// an existing shared memory.

static const char kShmFormat[] = "/shm-id-%016llX";

uint32_t kRetryCount = 100;

for (uint32_t i = 0; i < kRetryCount; i++) {

bool useTmp = !ASMJIT_VM_SHM_DETECT || preferTmpOverDevShm;

uint64_t bits = generateRandomBits((uintptr_t)this, i);

if (useTmp) {

_tmpName.assign(getTmpDir());

_tmpName.appendFormat(kShmFormat, (unsigned long long)bits);

_fd = ASMJIT_FILE64_API(::open)(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, 0);

if (ASMJIT_LIKELY(_fd >= 0)) {

_fileType = kFileTypeTmp;

return kErrorOk;

}

}

#if defined(ASMJIT_HAS_SHM_OPEN)

else {

_tmpName.assignFormat(kShmFormat, (unsigned long long)bits);

_fd = ::shm_open(_tmpName.data(), O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);

if (ASMJIT_LIKELY(_fd >= 0)) {

_fileType = kFileTypeShm;

return kErrorOk;

}

}

#endif

int e = errno;

if (e != EEXIST)

return DebugUtils::errored(asmjitErrorFromErrno(e));

}

return DebugUtils::errored(kErrorFailedToOpenAnonymousMemory);

#endif

}

void unlink() noexcept {

FileType type = _fileType;

_fileType = kFileTypeNone;

#ifdef ASMJIT_HAS_SHM_OPEN

if (type == kFileTypeShm) {

::shm_unlink(_tmpName.data());

return;

}

#endif

if (type == kFileTypeTmp) {

::unlink(_tmpName.data());

return;

}

}

void close() noexcept {

if (_fd >= 0) {

::close(_fd);

_fd = -1;

}

}

Error allocate(size_t size) noexcept {

// TODO: Improve this by using `posix_fallocate()` when available.

if (ASMJIT_FILE64_API(ftruncate)(_fd, off_t(size)) != 0)

return DebugUtils::errored(asmjitErrorFromErrno(errno));

return kErrorOk;

}

AnonymousMemory anonMem;

Info vmInfo = info();

ASMJIT_PROPAGATE(anonMem.open(false));

ASMJIT_PROPAGATE(anonMem.allocate(vmInfo.pageSize));

void* ptr = mmap(nullptr, vmInfo.pageSize, PROT_READ | PROT_EXEC, MAP_SHARED, anonMem.fd(), 0);

if (ptr == MAP_FAILED) {

int e = errno;

if (e == EINVAL) {

*strategyOut = AnonymousMemoryStrategy::kTmpDir;

return kErrorOk;

}

return DebugUtils::errored(asmjitErrorFromErrno(e));

}

else {

munmap(ptr, vmInfo.pageSize);

*strategyOut = AnonymousMemoryStrategy::kDevShm;

return kErrorOk;

}

#if ASMJIT_VM_SHM_DETECT

// Initially don't assume anything. It has to be tested whether '/dev/shm' was mounted with 'noexec' flag or not.

static std::atomic<uint32_t> cachedStrategy;

AnonymousMemoryStrategy strategy = static_cast<AnonymousMemoryStrategy>(cachedStrategy.load());

if (strategy == AnonymousMemoryStrategy::kUnknown) {

ASMJIT_PROPAGATE(detectAnonymousMemoryStrategy(&strategy));

cachedStrategy.store(static_cast<uint32_t>(strategy));

}

*strategyOut = strategy;

return kErrorOk;

#else

*strategyOut = AnonymousMemoryStrategy::kTmpDir;

return kErrorOk;

#endif

#if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_ARM >= 64

// OSX on AArch64 has always hardened runtime enabled.

return true;

#else

static std::atomic<uint32_t> cachedHardenedFlag;

enum HardenedFlag : uint32_t {

kHardenedFlagUnknown = 0,

kHardenedFlagDisabled = 1,

kHardenedFlagEnabled = 2

};

uint32_t flag = cachedHardenedFlag.load();

if (flag == kHardenedFlagUnknown) {

size_t pageSize = size_t(::getpagesize());

void* ptr = mmap(nullptr, pageSize, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

if (ptr == MAP_FAILED) {

flag = kHardenedFlagEnabled;

}

else {

flag = kHardenedFlagDisabled;

munmap(ptr, pageSize);

}

cachedHardenedFlag.store(flag);

}

return flag == kHardenedFlagEnabled;

#endif

#if defined(__APPLE__) && TARGET_OS_OSX && ASMJIT_ARCH_X86 == 0

// Apple platforms always use hardened runtime + MAP_JIT on non-x86 hardware:

// - https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon

return true;

#elif defined(__APPLE__) && TARGET_OS_OSX

// MAP_JIT flag required to run unsigned JIT code is only supported by kernel version 10.14+ (Mojave).

static std::atomic<uint32_t> cachedMapJitSupport;

uint32_t val = cachedMapJitSupport.load();

if (val == 0u) {

KernelVersion ver = getKernelVersion();

val = uint32_t(ver.ge(18, 0)) + 1u;

cachedMapJitSupport.store(val);

}

return val == 2u;

#else

// MAP_JIT is not available (it's only available on OSX).

return false;

#endif

#if defined(__APPLE__)

// Always use MAP_JIT flag if user asked for it (could be used for testing on non-hardened processes) and detect

// whether it must be used when the process is actually hardened (in that case it doesn't make sense to rely on

// user `memoryFlags`).

//

// MAP_JIT is not required when dual-mapping memory and is incompatible with MAP_SHARED, so it will not be

// added when the latter is enabled.

bool useMapJit = (Support::test(memoryFlags, MemoryFlags::kMMapEnableMapJit) || hasHardenedRuntime())

&& !Support::test(memoryFlags, MemoryFlags::kMapShared);

if (useMapJit)

return hasMapJitSupport() ? int(MAP_JIT) : 0;

else

return 0;

#else

DebugUtils::unused(memoryFlags);

return 0;

#endif

#if defined(ASMJIT_NO_DUAL_MAPPING)

return false;

#else

return true;

#endif

HardenedRuntimeFlags flags = HardenedRuntimeFlags::kNone;

if (hasHardenedRuntime())

flags |= HardenedRuntimeFlags::kEnabled;

if (hasMapJitSupport())

flags |= HardenedRuntimeFlags::kMapJit;

if (hasDualMappingSupport())

flags |= HardenedRuntimeFlags::kDualMapping;

return flags;

*p = nullptr;

if (size == 0)

return DebugUtils::errored(kErrorInvalidArgument);

int protection = mmProtFromMemoryFlags(memoryFlags) | mmMaxProtFromMemoryFlags(memoryFlags);

int mmFlags = mmMapJitFromMemoryFlags(memoryFlags);

mmFlags |= Support::test(memoryFlags, MemoryFlags::kMapShared) ? MAP_SHARED : MAP_PRIVATE;

if (fd == -1)

mmFlags |= MAP_ANONYMOUS;

bool useLargePages = Support::test(memoryFlags, VirtMem::MemoryFlags::kMMapLargePages);

if (useLargePages) {

#if defined(__linux__)

size_t lpSize = largePageSize();

if (lpSize == 0)

return DebugUtils::errored(kErrorFeatureNotEnabled);

if (!Support::isAligned(size, lpSize))

return DebugUtils::errored(kErrorInvalidArgument);

unsigned lpSizeLog2 = Support::ctz(lpSize);

mmFlags |= int(unsigned(MAP_HUGETLB) | (lpSizeLog2 << MAP_HUGE_SHIFT));

#else

return DebugUtils::errored(kErrorFeatureNotEnabled);

#endif // __linux__

}

void* ptr = mmap(nullptr, size, protection, mmFlags, fd, offset);

if (ptr == MAP_FAILED)

return DebugUtils::errored(asmjitErrorFromErrno(errno));

#if defined(MADV_HUGEPAGE)

if (useLargePages) {

madvise(ptr, size, MADV_HUGEPAGE);

}

#endif

*p = ptr;

return kErrorOk;

if (ASMJIT_UNLIKELY(munmap(p, size) != 0))

return DebugUtils::errored(asmjitErrorFromErrno(errno));

return kErrorOk;

int protection = mmProtFromMemoryFlags(memoryFlags);

if (mprotect(p, size, protection) == 0)

return kErrorOk;

return DebugUtils::errored(asmjitErrorFromErrno(errno));

Error err1 = unmapMemory(dm->rx, size);

Error err2 = kErrorOk;

if (dm->rx != dm->rw)

err2 = unmapMemory(dm->rw, size);

// We can report only one error, so report the first...

if (err1 || err2)

return DebugUtils::errored(err1 ? err1 : err2);

dm->rx = nullptr;

dm->rw = nullptr;

return kErrorOk;

MemoryFlags maxAccessFlags = regularAccessFlagsToMaxAccessFlags(memoryFlags);

MemoryFlags finalFlags = memoryFlags | maxAccessFlags | MemoryFlags::kMapShared;

MemoryFlags rxFlags = finalFlags & ~(MemoryFlags::kAccessWrite | MemoryFlags::kMMapMaxAccessWrite);

MemoryFlags rwFlags = finalFlags & ~(MemoryFlags::kAccessExecute);

// Allocate RW mapping.

DualMapping dm {};

ASMJIT_PROPAGATE(mapMemory(&dm.rw, size, rwFlags));

// Allocate RX mapping.

dm.rx = mremap(dm.rw, size, nullptr, size, MAP_REMAPDUP);

if (dm.rx == MAP_FAILED) {

int e = errno;

munmap(dm.rw, size);

return DebugUtils::errored(asmjitErrorFromErrno(e));

}

if (mprotect(dm.rx, size, mmProtFromMemoryFlags(rxFlags)) != 0) {

int e = errno;

unmapDualMapping(&dm, size);

return DebugUtils::errored(asmjitErrorFromErrno(e));

}

*dmOut = dm;

return kErrorOk;

switch (result) {

case KERN_PROTECTION_FAILURE:

return DebugUtils::errored(kErrorProtectionFailure);

case KERN_NO_SPACE:

return DebugUtils::errored(kErrorOutOfMemory);

case KERN_INVALID_ARGUMENT:

return DebugUtils::errored(kErrorInvalidArgument);

default:

return DebugUtils::errored(kErrorInvalidState);

}

DualMapping dm {};

MemoryFlags mmapFlags = MemoryFlags::kAccessReadWrite | (memoryFlags & MemoryFlags::kMapShared);

ASMJIT_PROPAGATE(mapMemory(&dm.rx, size, mmapFlags));

vm_prot_t curProt;

vm_prot_t maxProt;

int rwProtectFlags = VM_PROT_READ | VM_PROT_WRITE;

int rxProtectFlags = VM_PROT_READ;

if (Support::test(memoryFlags, MemoryFlags::kAccessExecute))

rxProtectFlags |= VM_PROT_EXECUTE;

kern_return_t result {};

do {

vm_map_t task = mach_task_self();

mach_vm_address_t remappedAddr {};

#if defined(VM_FLAGS_RANDOM_ADDR)

int remapFlags = VM_FLAGS_ANYWHERE | VM_FLAGS_RANDOM_ADDR;

#else

int remapFlags = VM_FLAGS_ANYWHERE;

#endif

// Try to remap the existing memory into a different address.

result = mach_vm_remap(

task, // target_task

&remappedAddr, // target_address

size, // size

0, // mask

remapFlags, // flags

task, // src_task

(mach_vm_address_t)dm.rx, // src_address

false, // copy

&curProt, // cur_protection

&maxProt, // max_protection

VM_INHERIT_DEFAULT); // inheritance

if (result != KERN_SUCCESS)

break;

dm.rw = (void*)remappedAddr;

// Now, try to change permissions of both map regions into RW and RX. The vm_protect()