static constexpr unsigned sampleSize = 128;

explicit SubjectSampler(CharSize charSize)

: m_is8Bit(charSize == CharSize::Char8)

{

}

int32_t frequency(char16_t character) const

{

if (!m_size)

return 1;

return static_cast<int32_t>(m_samples[character & BoyerMooreBitmap::mapMask]) * sampleSize / m_size;

}

void sample(StringView string)

{

unsigned half = string.length() > sampleSize ? (string.length() - sampleSize) / 2 : 0;

unsigned end = std::min(string.length(), half + sampleSize);

if (string.is8Bit()) {

auto characters8 = string.span8();

for (unsigned i = half; i < end; ++i)

add(characters8[i]);

} else {

auto characters16 = string.span16();

for (unsigned i = half; i < end; ++i)

add(characters16[i]);

}

}

void dump() const

{

dataLogLn("Sampling Results size:(", m_size, ")");

for (unsigned i = 0; i < BoyerMooreBitmap::mapSize; ++i)

dataLogLn(" [", makeString(pad(' ', 3, i)), "] ", m_samples[i]);

}

bool is8Bit() const { return m_is8Bit; }

inline void add(char16_t character)

{

++m_size;

++m_samples[character & BoyerMooreBitmap::mapMask];

}

std::array<uint8_t, BoyerMooreBitmap::mapSize> m_samples { };

uint8_t m_size { };

bool m_is8Bit { true };

{

if (!isValid()) {

out.print("isValid:(false)");

return;

}

out.print("isValid:(true),characters:(", listDump(m_characters), ")");

WTF_MAKE_NONCOPYABLE(BoyerMooreInfo);

WTF_MAKE_TZONE_ALLOCATED(BoyerMooreInfo);

static constexpr unsigned maxLength = 32;

explicit BoyerMooreInfo(CharSize charSize, unsigned length)

: m_characters(length)

, m_charSize(charSize)

{

ASSERT(this->length() <= maxLength);

}

unsigned length() const { return m_characters.size(); }

void shortenLength(unsigned length)

{

if (length <= this->length())

m_characters.shrink(length);

}

void set(unsigned index, char32_t character)

{

m_characters[index].add(m_charSize, character);

}

void setAll(unsigned index)

{

m_characters[index].setAll();

}

void addCharacters(unsigned index, const Vector<char32_t>& characters)

{

m_characters[index].addCharacters(m_charSize, characters);

}

void addRanges(unsigned index, const Vector<CharacterRange>& range)

{

m_characters[index].addRanges(m_charSize, range);

}

static UniqueRef<BoyerMooreInfo> create(CharSize charSize, unsigned length)

{

return makeUniqueRef<BoyerMooreInfo>(charSize, length);

}

std::optional<std::tuple<unsigned, unsigned>> findWorthwhileCharacterSequenceForLookahead(const SubjectSampler&) const;

std::tuple<BoyerMooreBitmap::Map, BoyerMooreFastCandidates> createCandidateBitmap(unsigned begin, unsigned end) const;

void dump(PrintStream&) const;

std::tuple<int32_t, unsigned, unsigned> findBestCharacterSequence(const SubjectSampler&, unsigned numberOfCandidatesLimit) const;

Vector<BoyerMooreBitmap> m_characters;

CharSize m_charSize;

{

int32_t biggestPoint = INT32_MIN;

unsigned beginResult = 0;

unsigned endResult = 0;

for (unsigned index = 0; index < length();) {

while (index < length() && m_characters[index].count() > numberOfCandidatesLimit)

++index;

if (index == length())

break;

unsigned begin = index;

BoyerMooreBitmap::Map map { };

for (; index < length() && m_characters[index].count() <= numberOfCandidatesLimit; ++index)

map.merge(m_characters[index].map());

int32_t frequency = 0;

map.forEachSetBit([&](unsigned index) {

frequency += sampler.frequency(index);

});

// Cutoff at 50%. If we could encounter the character more than 50%, then BM search would be useless probably.

int32_t matchingProbability = (BoyerMooreBitmap::mapSize / 2) - frequency;

int32_t point = (index - begin) * matchingProbability;

if (point > biggestPoint) {

biggestPoint = point;

beginResult = begin;

endResult = index;

}

}

return std::tuple { biggestPoint, beginResult, endResult };

{

// If candiates-per-character becomes larger, then sequence is not profitable since this sequence will match against

// too many characters. But if we limit candiates-per-character smaller, it is possible that we only find very short

// character sequence. We start with low limit, then enlarging the limit to find more and more profitable

// character sequence.

int32_t biggestPoint = INT32_MIN;

unsigned begin = 0;

unsigned end = 0;

constexpr unsigned maxCandidatesPerCharacter = 32;

static_assert(maxCandidatesPerCharacter < BoyerMooreBitmap::mapSize);

for (unsigned limit = 4; limit < maxCandidatesPerCharacter; limit *= 2) {

auto [newPoint, newBegin, newEnd] = findBestCharacterSequence(sampler, limit);

if (newPoint > biggestPoint) {

biggestPoint = newPoint;

begin = newBegin;

end = newEnd;

}

}

if (biggestPoint < 0)

return std::nullopt;

return std::tuple { begin, end };

{

BoyerMooreBitmap::Map map { };

BoyerMooreFastCandidates charactersFastPath;

for (unsigned index = begin; index < end; ++index) {

auto& bmBitmap = m_characters[index];

map.merge(bmBitmap.map());

charactersFastPath.merge(bmBitmap.charactersFastPath());

}

return std::tuple { WTF::move(map), WTF::move(charactersFastPath) };

{

out.println("BoyerMooreInfo size:(", m_characters.size(), ")");

unsigned index = 0;

for (auto& map : m_characters)

out.println(" [", makeString(pad(' ', 3, index++)), "] ", map);

WTF_MAKE_TZONE_ALLOCATED(MaskedAlternativeInfo);

static constexpr unsigned maxAlternatives = 2; // Support 2 alternatives initially

static constexpr unsigned patternBytes = 4; // Match first 4 bytes

static constexpr unsigned thresholdForDistinctCharacters = 16;

// Per-alternative pattern data

struct Alternative {

uint32_t chars { 0 }; // First 4 bytes as little-endian uint32

uint32_t mask { 0 }; // Mask for case-insensitive matching (0xFF = exact, 0xDF = case-insensitive)

};

std::array<Alternative, maxAlternatives> alternatives { };

unsigned numAlternatives { 0 };

uint32_t minPatternLength { 0 }; // Minimum length across alternatives

// Compute mask and char value for a character class that makes all members equivalent

// Returns false if the class is too complex (e.g., ranges spanning many bits, inverted class)

static bool computeMaskForCharacterClass(const CharacterClass& charClass, bool ignoreCase, uint8_t& outChar, uint8_t& outMask)

{

// Don't handle inverted classes or classes with ranges for now

// Only handle simple character sets like [acg] or [cgt]

if (charClass.m_matches.isEmpty())

return false;

if (!charClass.m_ranges.isEmpty())

return false;

if (!charClass.m_matchesUnicode.isEmpty() || !charClass.m_rangesUnicode.isEmpty())

return false;

// Collect all characters (and their case variants if ignoreCase)

Vector<uint8_t, 32> chars;

for (char32_t ch : charClass.m_matches) {

if (!isASCII(ch))

return false; // ASCII only

if (ignoreCase && isASCIIAlpha(ch)) {

chars.append(toASCIILower(static_cast<uint8_t>(ch)));

chars.append(toASCIIUpper(static_cast<uint8_t>(ch)));

} else

chars.append(static_cast<uint8_t>(ch));

if (chars.size() > thresholdForDistinctCharacters)

return false;

}

if (chars.isEmpty() || chars.size() > thresholdForDistinctCharacters)

return false;

// Compute XOR of all differing bits

uint8_t differingBits = 0;

for (unsigned i = 1; i < chars.size(); ++i)

differingBits |= (chars[0] ^ chars[i]);

// Mask clears all differing bits

outMask = ~differingBits;

// The character value is any character ANDed with the mask (they all produce the same result)

outChar = chars[0] & outMask;

return true;

}

// Create from a disjunction with exactly 2 fixed alternatives

// Returns invalid info if the pattern doesn't qualify for this optimization

static std::optional<MaskedAlternativeInfo> create(const PatternDisjunction& disjunction, bool ignoreCase, CharSize charSize)

{

MaskedAlternativeInfo info;

// Only support Latin1 (8-bit) for now

if (charSize != CharSize::Char8)

return std::nullopt;

// Need exactly 2 alternatives

auto& alternatives = disjunction.m_alternatives;

if (alternatives.size() != maxAlternatives)

return std::nullopt;

// Both alternatives must have at least 4 characters and be fixed-size

info.minPatternLength = UINT32_MAX;

for (unsigned i = 0; i < maxAlternatives; ++i) {

const PatternAlternative* alt = alternatives[i].get();

if (!alt->m_hasFixedSize)

return std::nullopt;

if (alt->m_minimumSize < patternBytes)

return std::nullopt;

info.minPatternLength = std::min(info.minPatternLength, alt->m_minimumSize);

// Extract first 4 characters - can be simple characters or character classes

uint32_t chars = 0;

uint32_t mask = 0;

unsigned charIndex = 0;

for (const PatternTerm& term : alt->m_terms) {

if (charIndex >= patternBytes)

break;

if (term.quantityType != QuantifierType::FixedCount || term.quantityMinCount != 1)

return std::nullopt; // No quantifiers

uint8_t byteChar = 0;

uint8_t byteMask = 0xFF;

if (term.type == PatternTerm::Type::PatternCharacter) {

char32_t ch = term.patternCharacter;

if (!isASCII(ch))

return std::nullopt; // ASCII only for now

byteChar = static_cast<uint8_t>(ch);

// Mask: 0xDF for case-insensitive letters, 0xFF for exact match

if (ignoreCase && isASCIIAlpha(ch))

byteMask = 0xDF; // Clear bit 5 to normalize case

} else if (term.type == PatternTerm::Type::CharacterClass) {

// Handle character class by computing a mask that makes all members equivalent

if (term.m_invert)

return std::nullopt; // Don't handle inverted classes

if (!computeMaskForCharacterClass(*term.characterClass, ignoreCase, byteChar, byteMask))

return std::nullopt; // Class too complex

} else

return std::nullopt; // Unsupported term type

// Build the 4-byte pattern (little-endian)

chars |= static_cast<uint32_t>(byteChar) << (charIndex * 8);

mask |= static_cast<uint32_t>(byteMask) << (charIndex * 8);

++charIndex;

}

if (charIndex < patternBytes)

return std::nullopt; // Not enough characters

info.alternatives[i].chars = chars;

info.alternatives[i].mask = mask;

}

info.numAlternatives = 2;

return info;

}

{

MacroAssembler::JumpList slowCases;

MacroAssembler::JumpList isBMP;

MacroAssembler::JumpList done;

YarrJITDefaultRegisters regs;

if (resultReg != regs.regT0)

jit.swap(regs.regT0, resultReg);

// Check if we can read two UTF-16 characters at once.

jit.add64(MacroAssembler::TrustedImm32(4), regs.regUnicodeInputAndTrail, regs.unicodeAndSubpatternIdTemp);

slowCases.append(jit.branchPtr(MacroAssembler::Above, regs.unicodeAndSubpatternIdTemp, regs.endOfStringAddress));

// Load and try to process two UTF-16 characters.

// If they are a proper surrogate pair, compute the non-BMP codepoint.

jit.load32(MacroAssembler::Address(regs.regUnicodeInputAndTrail), resultReg);

#if CPU(ARM64)

jit.and32AndSetFlags(surrogateTagMask, resultReg, regs.unicodeAndSubpatternIdTemp);

isBMP.append(jit.branch(MacroAssembler::Zero));

#else

jit.and32(surrogateTagMask, resultReg, regs.unicodeAndSubpatternIdTemp);

isBMP.append(jit.branch32(MacroAssembler::Equal, regs.unicodeAndSubpatternIdTemp, MacroAssembler::TrustedImm32(0)));

#endif

slowCases.append(jit.branch32(MacroAssembler::NotEqual, regs.unicodeAndSubpatternIdTemp, surrogatePairTags));

// Create the UTF32 character from the surrogate pair.

#if CPU(ARM64)

jit.urshift32(resultReg, MacroAssembler::TrustedImm32(16), regs.unicodeAndSubpatternIdTemp);

jit.insertBitField32(resultReg, MacroAssembler::TrustedImm32(10), MacroAssembler::TrustedImm32(10), regs.unicodeAndSubpatternIdTemp);

jit.add32(MacroAssembler::TrustedImm32(0x10000), regs.unicodeAndSubpatternIdTemp, resultReg);

#else

jit.and32(MacroAssembler::TrustedImm32(0xffff), resultReg, regs.unicodeAndSubpatternIdTemp);

jit.lshift32(MacroAssembler::TrustedImm32(10), regs.unicodeAndSubpatternIdTemp);

jit.urshift32(resultReg, MacroAssembler::TrustedImm32(16), resultReg);

jit.getEffectiveAddress(MacroAssembler::BaseIndex(regs.unicodeAndSubpatternIdTemp, resultReg, MacroAssembler::TimesOne, -U16_SURROGATE_OFFSET), resultReg);

#endif

#if ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP)

if (useNonBMPOptimization == TryReadUnicodeCharGenFirstNonBMPOptimization::UseOptimization)

jit.move(MacroAssembler::TrustedImm32(1), regs.firstCharacterAdditionalReadSize);

#endif

done.append(jit.jump());

isBMP.link(jit);

jit.and32(MacroAssembler::TrustedImm32(0xffff), resultReg);

done.append(jit.jump());

slowCases.link(&jit);

#if ENABLE(YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP)

if constexpr (useNonBMPOptimization == TryReadUnicodeCharGenFirstNonBMPOptimization::UseOptimization)

jit.nearCallThunk(CodeLocationLabel { vm.getCTIStub(tryReadUnicodeCharIncForNonBMPSlowThunkGenerator).template retaggedCode<NoPtrTag>() });

else

#endif

jit.nearCallThunk(CodeLocationLabel { vm.getCTIStub(tryReadUnicodeCharSlowThunkGenerator).template retaggedCode<NoPtrTag>() });

done.link(&jit);

if (resultReg != regs.regT0)

jit.swap(regs.regT0, resultReg);

{

MacroAssembler::JumpList bmpOnly;

MacroAssembler::JumpList isBMP;

MacroAssembler::JumpList notSurrogatePair;

MacroAssembler::JumpList checkForDanglingSurrogates;

MacroAssembler::JumpList bmpDone;

MacroAssembler::JumpList haveResult;

YarrJITDefaultRegisters regs;

// This code generator is used to build two variations of a character reader that handles Unicode non-BMP surrogate pairs.

// This code generator is used to build thunks or as inline code. Its "calling convention" is unconventional.

// It assumes the following registers are already populated with these values:

// regs.regUnicodeInputAndTrail is the string address to start reading from.

// regs.input contains the pointer of the beginning of the string.

// regs.endOfStringAddress contains the address one past the end of the string.

// For architectures that put the surrogate masks and tags in registers,

// regs.surrogateTagMask contains 0xdc00dc00 and regs.surrogatePairTags contains 0xdc00d800.

// When the YARR_JIT_UNICODE_CAN_INCREMENT_INDEX_FOR_NON_BMP optimization is enabled and used,

// regs.firstCharacterAdditionalReadSize is used to advance 2 characters when we read a non-BMP codepoint.

// regs.unicodeAndSubpatternIdTemp is used as a temporary.

// The result is returned via regs.regT0.

auto resultReg = regs.regT0;

// Check if we can read two UTF-16 characters at once.

jit.add64(MacroAssembler::TrustedImm32(4), regs.regUnicodeInputAndTrail, regs.unicodeAndSubpatternIdTemp);

bmpOnly.append(jit.branchPtr(MacroAssembler::Above, regs.unicodeAndSubpatternIdTemp, regs.endOfStringAddress));

// Load and try to process two UTF-16 characters.

// If they are a proper surrogate pair, compute the non-BMP codepoint.

jit.load32(MacroAssembler::Address(regs.regUnicodeInputAndTrail), resultReg);

#if CPU(ARM64)

jit.and32AndSetFlags(surrogateTagMask, resultReg, regs.unicodeAndSubpatternIdTemp);

isBMP.append(jit.branch(MacroAssembler::Zero));

#else

jit.and32(surrogateTagMask, resultReg, regs.unicodeAndSubpatternIdTemp);

isBMP.append(jit.branch32(MacroAssembler::Equal, regs.unicodeAndSubpatternIdTemp, MacroAssembler::TrustedImm32(0)));

#endif

// if it is surrogate pair, we already handled it in the inlined code.

// Check if we can return the dangling surrogate or if it is part of a valid pair where the leading surrogate

// that is offset one character before the load pointer.

jit.and32(MacroAssembler::TrustedImm32(0xffff), regs.unicodeAndSubpatternIdTemp);

// If it is a leading surrogate, the check above proved that it wasn't followed by a trailing surrogate.

// If so fall through, otherwise perform other dangling checks.

checkForDanglingSurrogates.append(jit.branch32(MacroAssembler::Equal, regs.unicodeAndSubpatternIdTemp, MacroAssembler::TrustedImm32(0xdc00)));

isBMP.link(jit);

jit.and32(MacroAssembler::TrustedImm32(0xffff), resultReg);

jit.ret();

checkForDanglingSurrogates.link(&jit);

// Remove the second character that we loaded.

jit.and32(MacroAssembler::TrustedImm32(0xffff), resultReg);

MacroAssembler::Label checkForDanglingSurrogatesLabel(&jit);

// Can ew read the prior character?

jit.subPtr(MacroAssembler::TrustedImm32(2), regs.regUnicodeInputAndTrail);

// If not, we branch to return the dangling surrogate.

bmpDone.append(jit.branchPtr(MacroAssembler::Below, regs.regUnicodeInputAndTrail, regs.input));

// Load the prior character and check if it is a leading surrogate.

jit.load16Unaligned(MacroAssembler::Address(regs.regUnicodeInputAndTrail), regs.regUnicodeInputAndTrail);

jit.and32(surrogateTagMask, regs.regUnicodeInputAndTrail, regs.unicodeAndSubpatternIdTemp);

// It wasn't a leading surrogate, so return the original dangling surrogate.

bmpDone.append(jit.branch32(MacroAssembler::NotEqual, regs.unicodeAndSubpatternIdTemp, MacroAssembler::TrustedImm32(0xd800)));

// The prior characters was a leading surrogate, Ecma262 says that this is an error, so return the error code point.

jit.move(MacroAssembler::TrustedImm32(errorCodePoint), resultReg);

bmpDone.append(jit.jump());

bmpOnly.link(&jit);

// Can't read two characters, then just read one.

jit.load16Unaligned(MacroAssembler::Address(regs.regUnicodeInputAndTrail), resultReg);

// Is the character a trailing surrogate?

jit.and32(surrogateTagMask, resultReg, regs.unicodeAndSubpatternIdTemp);

// If so, branch back to handle the possibility that we loaded the second surrogate of a proper pair.

jit.branch32(MacroAssembler::Equal, regs.unicodeAndSubpatternIdTemp, MacroAssembler::TrustedImm32(0xdc00)).linkTo(checkForDanglingSurrogatesLabel, jit);

bmpDone.link(&jit);

haveResult.link(&jit);

class MatchTargets {

public:

enum class PreferredTarget : uint8_t {

NoPreference = 0,

PreferMatchSucceeded = 1,

MatchFailFallThrough = PreferMatchSucceeded,

PreferMatchFailed = 2,

MatchSuccessFallThrough = PreferMatchFailed

};

MatchTargets(PreferredTarget preferredTarget = PreferredTarget::NoPreference)

: m_preferredTarget(preferredTarget)

{ }

MatchTargets(MacroAssembler::JumpList& matchDest)

: m_matchSucceededTargets(&matchDest)

, m_preferredTarget(PreferredTarget::PreferMatchSucceeded)

{ }

MatchTargets(MacroAssembler::JumpList& compareDest, PreferredTarget preferredTarget)

: m_preferredTarget(preferredTarget)

{

if (preferredTarget == PreferredTarget::PreferMatchFailed)

m_matchFailedTargets = &compareDest;

else

m_matchSucceededTargets = &compareDest;

}

MatchTargets(MacroAssembler::JumpList& matchDest, MacroAssembler::JumpList& failDest, PreferredTarget preferredTarget = PreferredTarget::NoPreference)

: m_matchSucceededTargets(&matchDest)

, m_matchFailedTargets(&failDest)

, m_preferredTarget(preferredTarget)

{ }

PreferredTarget preferredTarget()

{

return m_preferredTarget;

}

bool hasSucceedTarget()

{

return m_matchSucceededTargets != nullptr;

}

bool hasFailedTarget()

{

return m_matchFailedTargets != nullptr;

}

MacroAssembler::JumpList& matchSucceeded() { return *m_matchSucceededTargets; }

MacroAssembler::JumpList& matchFailed() { return *m_matchFailedTargets; }

void appendSucceeded(MacroAssembler::Jump jump)

{

ASSERT(m_matchSucceededTargets != nullptr);

m_matchSucceededTargets->append(jump);

}

void appendFailed(MacroAssembler::Jump jump)

{

ASSERT(m_matchFailedTargets != nullptr);

m_matchFailedTargets->append(jump);

}

private:

MacroAssembler::JumpList* m_matchSucceededTargets { nullptr };

MacroAssembler::JumpList* m_matchFailedTargets { nullptr };

PreferredTarget m_preferredTarget;

};

#if ENABLE(YARR_JIT_ALL_PARENS_EXPRESSIONS)

struct ParenContextSizes {

size_t m_numSubpatterns;

size_t m_numDuplicateNamedCaptures;

size_t m_frameSlots;

ParenContextSizes(size_t numSubpatterns, size_t numDuplicateNamedCaptures, size_t frameSlots)

: m_numSubpatterns(numSubpatterns)

, m_numDuplicateNamedCaptures(numDuplicateNamedCaptures)

, m_frameSlots(frameSlots)

{

}

size_t numSubpatterns() { return m_numSubpatterns; }

size_t numDuplicateNamedCaptures() { return m_numDuplicateNamedCaptures; }

size_t frameSlots() { return m_frameSlots; }

};

struct ParenContext {

struct ParenContext* next;

struct BeginAndMatchAmount {

uint32_t begin;

uint32_t matchAmount;

} beginAndMatchAmount;

uint32_t end;

uintptr_t returnAddress;

struct Subpatterns {

unsigned start;

unsigned end;

} subpatterns[0];

unsigned duplicateNamedCaptures[0];

uintptr_t frameSlots[0];

static size_t sizeFor(ParenContextSizes& parenContextSizes)

{

return sizeof(ParenContext) + sizeof(Subpatterns) * parenContextSizes.numSubpatterns() + sizeof(unsigned) * (parenContextSizes.numDuplicateNamedCaptures()) + sizeof(uintptr_t) * parenContextSizes.frameSlots();

}

static constexpr ptrdiff_t nextOffset()

{

return offsetof(ParenContext, next);

}

static constexpr ptrdiff_t beginOffset()

{

return offsetof(ParenContext, beginAndMatchAmount) + offsetof(BeginAndMatchAmount, begin);

}

static constexpr ptrdiff_t matchAmountOffset()

{

return offsetof(ParenContext, beginAndMatchAmount) + offsetof(BeginAndMatchAmount, matchAmount);

}

static constexpr ptrdiff_t endOffset()

{

return offsetof(ParenContext, end);

}

static constexpr ptrdiff_t returnAddressOffset()

{

return offsetof(ParenContext, returnAddress);

}

static constexpr ptrdiff_t subpatternOffset(size_t subpattern)

{

return offsetof(ParenContext, subpatterns) + (subpattern - 1) * sizeof(Subpatterns);

}

static constexpr ptrdiff_t duplicateNamedCaptureOffset(ParenContextSizes& parenContextSizes, size_t namedCapture)

{

return offsetof(ParenContext, subpatterns) + (parenContextSizes.numSubpatterns()) * sizeof(Subpatterns) + (namedCapture - 1) * sizeof(unsigned);

}

static ptrdiff_t savedFrameOffset(ParenContextSizes& parenContextSizes)

{

return offsetof(ParenContext, subpatterns) + (parenContextSizes.numSubpatterns()) * sizeof(Subpatterns) + (parenContextSizes.numDuplicateNamedCaptures()) * sizeof(unsigned);

}

};

void allocateParenContext(MacroAssembler::RegisterID result)

{

m_hitMatchLimit.append(m_jit.branchSub32(MacroAssembler::Zero, MacroAssembler::TrustedImm32(1), m_regs.remainingMatchCount));

// Try to allocate from freelist first.

MacroAssembler::Jump allocateFromStack;

if (m_regs.freelistRegister != InvalidGPRReg) {

allocateFromStack = m_jit.branchTestPtr(MacroAssembler::Zero, m_regs.freelistRegister);

m_jit.move(m_regs.freelistRegister, result);

m_jit.loadPtr(MacroAssembler::Address(m_regs.freelistRegister, ParenContext::nextOffset()), m_regs.freelistRegister);

} else {

m_jit.loadPtr(MacroAssembler::Address(m_regs.matchingContext, MatchingContextHolder::offsetOfFreeList()), result);

allocateFromStack = m_jit.branchTestPtr(MacroAssembler::Zero, result);

m_jit.transferPtr(MacroAssembler::Address(result, ParenContext::nextOffset()), MacroAssembler::Address(m_regs.matchingContext, MatchingContextHolder::offsetOfFreeList()));

}

auto done = m_jit.jump();

// Freelist is null, allocate from stack.

allocateFromStack.link(&m_jit);

size_t parenContextSize = WTF::roundUpToMultipleOf<stackAlignmentBytes()>(ParenContext::sizeFor(m_parenContextSizes));

m_jit.subPtr(MacroAssembler::stackPointerRegister, MacroAssembler::TrustedImm32(static_cast<int32_t>(parenContextSize)), result);

m_abortExecution.append(m_jit.branchPtr(MacroAssembler::Above, MacroAssembler::Address(m_regs.matchingContext, MatchingContextHolder::offsetOfStackLimit()), result));

m_jit.move(result, MacroAssembler::stackPointerRegister);

done.link(&m_jit);

}

void freeParenContext(MacroAssembler::RegisterID headPtrRegister)

{

if (m_regs.freelistRegister != InvalidGPRReg) {

m_jit.storePtr(m_regs.freelistRegister, MacroAssembler::Address(headPtrRegister, ParenContext::nextOffset()));

m_jit.move(headPtrRegister, m_regs.freelistRegister);

} else {

m_jit.transferPtr(MacroAssembler::Address(m_regs.matchingContext, MatchingContextHolder::offsetOfFreeList()), MacroAssembler::Address(headPtrRegister, ParenContext::nextOffset()));

m_jit.storePtr(headPtrRegister, MacroAssembler::Address(m_regs.matchingContext, MatchingContextHolder::offsetOfFreeList()));

}

}

void storeBeginAndMatchAmountToParenContext(MacroAssembler::RegisterID beginGPR, MacroAssembler::RegisterID matchAmountGPR, MacroAssembler::RegisterID parenContextGPR)

{

static_assert(ParenContext::beginOffset() + 4 == ParenContext::matchAmountOffset());

m_jit.storePair32(beginGPR, matchAmountGPR, parenContextGPR, MacroAssembler::TrustedImm32(ParenContext::beginOffset()));

}

void loadBeginAndMatchAmountFromParenContext(MacroAssembler::RegisterID parenContextGPR, MacroAssembler::RegisterID beginGPR, MacroAssembler::RegisterID matchAmountGPR)

{

static_assert(ParenContext::beginOffset() + 4 == ParenContext::matchAmountOffset());

m_jit.loadPair32(parenContextGPR, MacroAssembler::TrustedImm32(ParenContext::beginOffset()), beginGPR, matchAmountGPR);

}

void saveParenContext(MacroAssembler::RegisterID parenContextReg, MacroAssembler::RegisterID tempReg, unsigned firstSubpattern, unsigned lastSubpattern, unsigned subpatternBaseFrameLocation, bool clearCapturesAfterSave = true, bool useBeginIndexFromFrame = false)

{

BitVector duplicateNamedCaptureGroups;

bool hasNamedCaptures = m_pattern.hasDuplicateNamedCaptureGroups();

// For FixedCount saved at END, use frame.beginIndex (iteration start) instead of m_regs.index (current position).

// This is needed because content's backtrack expects beginIndex to be the iteration start.

// Also we store m_regs.index to endOffset so we save both begin and end.

if (useBeginIndexFromFrame) {

loadFromFrame(subpatternBaseFrameLocation + BackTrackInfoParentheses::beginIndex(), tempReg);

m_jit.store32(tempReg, MacroAssembler::Address(parenContextReg, ParenContext::beginOffset()));

loadFromFrame(subpatternBaseFrameLocation + BackTrackInfoParentheses::matchAmountIndex(), tempReg);

m_jit.store32(tempReg, MacroAssembler::Address(parenContextReg, ParenContext::matchAmountOffset()));

m_jit.store32(m_regs.index, MacroAssembler::Address(parenContextReg, ParenContext::endOffset()));

} else {

loadFromFrame(subpatternBaseFrameLocation + BackTrackInfoParentheses::matchAmountIndex(), tempReg);

storeBeginAndMatchAmountToParenContext(m_regs.index, tempReg, parenContextReg);

}

loadFromFrame(subpatternBaseFrameLocation + BackTrackInfoParentheses::returnAddressIndex(), tempReg);

m_jit.storePtr(tempReg, MacroAssembler::Address(parenContextReg, ParenContext::returnAddressOffset()));

if (shouldRecordSubpatterns()) {

for (unsigned subpattern = firstSubpattern; subpattern <= lastSubpattern; subpattern++) {

static_assert(is64Bit());

m_jit.transfer64(subpatternStartAddress(subpattern), MacroAssembler::Address(parenContextReg, ParenContext::subpatternOffset(subpattern)));

if (hasNamedCaptures) {

unsigned duplicateNamedGroup = m_pattern.m_duplicateNamedGroupForSubpatternId[subpattern];

if (duplicateNamedGroup)

duplicateNamedCaptureGroups.set(duplicateNamedGroup);

}

// For Greedy/NonGreedy, clear captures after saving at BEGIN (before iteration runs).

// For FixedCount, we save at END and should NOT clear (iteration already set them).

if (clearCapturesAfterSave)

clearSubpattern(subpattern);

}

for (unsigned duplicateNamedGroupId : duplicateNamedCaptureGroups) {

loadDuplicateNamedGroupSubpatternId(duplicateNamedGroupId, tempReg);

m_jit.store32(tempReg, MacroAssembler::Address(parenContextReg, ParenContext::duplicateNamedCaptureOffset(m_parenContextSizes, duplicateNamedGroupId)));

if (clearCapturesAfterSave)

storeDuplicateNamedGroupSubpatternId(duplicateNamedGroupId, 0);

}

}

subpatternBaseFrameLocation += YarrStackSpaceForBackTrackInfoParentheses;

for (unsigned frameLocation = subpatternBaseFrameLocation; frameLocation < m_parenContextSizes.frameSlots(); frameLocation++) {

loadFromFrame(frameLocation, tempReg);

m_jit.storePtr(tempReg, MacroAssembler::Address(parenContextReg, ParenContext::savedFrameOffset(m_parenContextSizes) + frameLocation * sizeof(uintptr_t)));

}

}

void restoreParenContext(MacroAssembler::RegisterID parenContextReg, MacroAssembler::RegisterID tempReg, unsigned firstSubpattern, unsigned lastSubpattern, unsigned subpatternBaseFrameLocation)

{

BitVector duplicateNamedCaptureGroups;

bool hasNamedCaptures = m_pattern.hasDuplicateNamedCaptureGroups();

loadBeginAndMatchAmountFromParenContext(parenContextReg, m_regs.index, tempReg);

storeToFrame(m_regs.index, subpatternBaseFrameLocation + BackTrackInfoParentheses::beginIndex());

storeToFrame(tempReg, subpatternBaseFrameLocation + BackTrackInfoParentheses::matchAmountIndex());

m_jit.loadPtr(MacroAssembler::Address(parenContextReg, ParenContext::returnAddressOffset()), tempReg);

storeToFrame(tempReg, subpatternBaseFrameLocation + BackTrackInfoParentheses::returnAddressIndex());

if (shouldRecordSubpatterns()) {

for (unsigned subpattern = firstSubpattern; subpattern <= lastSubpattern; subpattern++) {

static_assert(is64Bit());

m_jit.transfer64(MacroAssembler::Address(parenContextReg, ParenContext::subpatternOffset(subpattern)), subpatternStartAddress(subpattern));

if (hasNamedCaptures) {

unsigned duplicateNamedGroup = m_pattern.m_duplicateNamedGroupForSubpatternId[subpattern];

if (duplicateNamedGroup)

duplicateNamedCaptureGroups.set(duplicateNamedGroup);

}

}

for (unsigned duplicateNamedGroupId : duplicateNamedCaptureGroups) {

m_jit.load32(MacroAssembler::Address(parenContextReg, ParenContext::duplicateNamedCaptureOffset(m_parenContextSizes, duplicateNamedGroupId)), tempReg);

storeDuplicateNamedGroupSubpatternIdFromReg(duplicateNamedGroupId, tempReg);

}

}

subpatternBaseFrameLocation += YarrStackSpaceForBackTrackInfoParentheses;

for (unsigned frameLocation = subpatternBaseFrameLocation; frameLocation < m_parenContextSizes.frameSlots(); frameLocation++) {

m_jit.loadPtr(MacroAssembler::Address(parenContextReg, ParenContext::savedFrameOffset(m_parenContextSizes) + frameLocation * sizeof(uintptr_t)), tempReg);

storeToFrame(tempReg, frameLocation);

}

}

#endif

void optimizeAlternative(PatternAlternative* alternative)

{

if (!alternative->m_terms.size())

return;

for (unsigned i = 0; i < alternative->m_terms.size() - 1; ++i) {

PatternTerm& term = alternative->m_terms[i];

PatternTerm& nextTerm = alternative->m_terms[i + 1];

// We can move BMP only character classes after fixed character terms.

// When not decoding surrogate pairs (Char8 mode), only swap if the character class

// has no non-BMP characters and is not inverted. Otherwise the swapped pattern could

// be passed to byteCodeCompilePattern (on JIT allocation failure) and then executed

// against a Char16 string where the changed term order would cause misreads of

// surrogate pairs. An inverted class like [^a] has BMP-only class data but can match

// non-BMP characters (variable width in UTF-16), so it must not be swapped either.

if ((term.type == PatternTerm::Type::CharacterClass)

&& (term.quantityType == QuantifierType::FixedCount)

&& !term.m_invert

&& ((!m_decodeSurrogatePairs && !term.characterClass->hasNonBMPCharacters()) || term.characterClass->hasOneCharacterSize())

&& (nextTerm.type == PatternTerm::Type::PatternCharacter)

&& (nextTerm.quantityType == QuantifierType::FixedCount)) {

PatternTerm termCopy = term;

alternative->m_terms[i] = nextTerm;

alternative->m_terms[i + 1] = termCopy;

}

}

}

constexpr static unsigned MaximumCharacterClassSizeForBitTest = 8 * sizeof(UCPURegister);

using CharacterBitSet = WTF::BitSet<MaximumCharacterClassSizeForBitTest>;

void matchCharacterClassByBitTest(MacroAssembler::RegisterID character, MacroAssembler::RegisterID scratch, MacroAssembler::JumpList& matchDest, char32_t min, char32_t max, CharacterBitSet mask)

{

switch (mask.count()) {

case 0:

return;

case 1:

case 2:

case 3:

case 4:

// If the set is small enough, still defer to a series of branches.

mask.forEachSetBit([&](size_t value) {

matchDest.append(m_jit.branch32(MacroAssembler::Equal, character, MacroAssembler::TrustedImm32(min + value)));

return IterationStatus::Continue;

});

break;

default: {

// Otherwise, actually perform the bit test.

#if CPU(REGISTER64)

m_jit.sub32(character, MacroAssembler::Imm32(static_cast<unsigned>(min)), scratch);

MacroAssembler::Jump notInVector = m_jit.branch32(MacroAssembler::Above, scratch, MacroAssembler::TrustedImm32(max - min));

m_jit.lshift64(MacroAssembler::TrustedImm32(1), scratch, scratch);

matchDest.append(m_jit.branchTest64(MacroAssembler::NonZero, scratch, MacroAssembler::TrustedImm64(mask.storage()[0])));

#else

m_jit.sub32(character, MacroAssembler::Imm32(static_cast<unsigned>(min)), scratch);

MacroAssembler::Jump notInVector = m_jit.branch32(MacroAssembler::Above, scratch, MacroAssembler::TrustedImm32(max - min));

m_jit.lshift32(MacroAssembler::TrustedImm32(1), scratch, scratch);

matchDest.append(m_jit.branchTest32(MacroAssembler::NonZero, scratch, MacroAssembler::TrustedImm32(mask.storage()[0])));

#endif

notInVector.link(&m_jit);

}

}

}

void matchCharacterClassSet(MacroAssembler::RegisterID character, MacroAssembler::RegisterID scratch, MacroAssembler::JumpList& matchDest, std::span<const char32_t> matches)

{

if (matches.empty())

return;

if (matches.size() == 1) {

matchDest.append(m_jit.branch32(MacroAssembler::Equal, character, MacroAssembler::Imm32(static_cast<unsigned>(matches.front()))));

return;

}

// If we have multiple matches close together (not necessarily contiguous), we

// can try a biased bitmask - subtract the minimum match from the character,

// then see if it's present in a precomputed mask. We keep the bitset size quite

// small in order to keep it easy to materialize - this approach lets us avoid

// a load or lookup table in favor of just masking against an immediate.

char32_t min = matches.front();

char32_t max = matches.back();

ASSERT(max > min);

if ((max - min) < MaximumCharacterClassSizeForBitTest) {

CharacterBitSet mask;

for (char32_t character : matches)

mask.set(character - min);

matchCharacterClassByBitTest(character, scratch, matchDest, min, max, mask);

return;

}

// We have too many matches to handle in a single set, but we may be able to

// recursively group some of our matches together. Worst case, we just do a

// character-by-character match. Greedily matching is potentially suboptimal,

// but I doubt worth spending time doing better.