static const int kEntryDist = 1;

uint16_t value = chr & (kChunkBits - 1);

unsigned int low = 0;

unsigned int high = size - 1;

while (high != low) {

unsigned int mid = low + ((high - low) >> 1);

uchar current_value = GetEntry(TableGet<kEntryDist>(table, mid));

// If we've found an entry less than or equal to this one, and the

// next one is not also less than this one, we've arrived.

if ((current_value <= value) &&

(mid + 1 == size ||

GetEntry(TableGet<kEntryDist>(table, mid + 1)) > value)) {

low = mid;

break;

} else if (current_value < value) {

low = mid + 1;

} else if (current_value > value) {

// If we've just checked the bottom-most value and it's not

// the one we're looking for, we're done.

if (mid == 0) break;

high = mid - 1;

}

}

int32_t field = TableGet<kEntryDist>(table, low);

uchar entry = GetEntry(field);

bool is_start = IsStart(field);

return (entry == value) || (entry < value && is_start);

const MultiCharacterSpecialCase<kW>* multi_chars,

uchar chr, uchar next, uchar* result,

bool* allow_caching_ptr) {

static const int kEntryDist = 2;

uint16_t key = chr & (kChunkBits - 1);

uint16_t chunk_start = chr - key;

unsigned int low = 0;

unsigned int high = size - 1;

while (high != low) {

unsigned int mid = low + ((high - low) >> 1);

uchar current_value = GetEntry(TableGet<kEntryDist>(table, mid));

// If we've found an entry less than or equal to this one, and the next one

// is not also less than this one, we've arrived.

if ((current_value <= key) &&

(mid + 1 == size ||

GetEntry(TableGet<kEntryDist>(table, mid + 1)) > key)) {

low = mid;

break;

} else if (current_value < key) {

low = mid + 1;

} else if (current_value > key) {

// If we've just checked the bottom-most value and it's not

// the one we're looking for, we're done.

if (mid == 0) break;

high = mid - 1;

}

}

int32_t field = TableGet<kEntryDist>(table, low);

uchar entry = GetEntry(field);

bool is_start = IsStart(field);

bool found = (entry == key) || (entry < key && is_start);

if (found) {

int32_t value = table[2 * low + 1];

if (value == 0) {

// 0 means not present

return 0;

} else if ((value & 3) == 0) {

// Low bits 0 means a constant offset from the given character.

if (ranges_are_linear) {

result[0] = chr + (value >> 2);

} else {

result[0] = entry + chunk_start + (value >> 2);

}

return 1;

} else if ((value & 3) == 1) {

// Low bits 1 means a special case mapping

if (allow_caching_ptr) *allow_caching_ptr = false;

const MultiCharacterSpecialCase<kW>& mapping = multi_chars[value >> 2];

int length = 0;

for (length = 0; length < kW; length++) {

uchar mapped = mapping.chars[length];

if (mapped == MultiCharacterSpecialCase<kW>::kEndOfEncoding) break;

if (ranges_are_linear) {

result[length] = mapped + (key - entry);

} else {

result[length] = mapped;

}

}

return length;

} else {

// Low bits 2 means a really really special case

if (allow_caching_ptr) *allow_caching_ptr = false;

// The cases of this switch are defined in unicode.py in the

// really_special_cases mapping.

switch (value >> 2) {

case 1:

// Really special case 1: upper case sigma. This letter

// converts to two different lower case sigmas depending on

// whether or not it occurs at the end of a word.

if (next != 0 && Letter::Is(next)) {

result[0] = 0x03C3;

} else {

result[0] = 0x03C2;

}

return 1;

default:

return 0;

}

return -1;

}

} else {

return 0;

}

size_t* cursor) {

DCHECK_GT(max_length, 0);

DCHECK_GT(str[0], kMaxOneByteChar);

State state = State::kAccept;

Utf8IncrementalBuffer buffer = 0;

uchar t;

const uint8_t* start = str;

const uint8_t* end = str + max_length;

do {

t = ValueOfIncremental(&str, &state, &buffer);

} while (str < end && t == kIncomplete);

*cursor += str - start;

return (state == State::kAccept) ? t : kBadChar;

if (*state == State::kAccept) {

return kBufferEmpty;

} else {

DCHECK_GT(*state, State::kAccept);

*state = State::kAccept;

return kBadChar;

}

State state = State::kAccept;

Utf8IncrementalBuffer throw_away = 0;

for (size_t i = 0; i < length && state != State::kReject; i++) {

Utf8DfaDecoder::Decode(bytes[i], &state, &throw_away);

}

return state == State::kAccept;

uint16_t* dest_code_units,

size_t length) {

// U+FFFD (REPLACEMENT CHARACTER)

constexpr uint16_t kReplacement = 0xFFFD;

for (size_t i = 0; i < length; i++) {

const uint16_t source_code_unit = source_code_units[i];

const size_t copy_index = i;

uint16_t dest_code_unit = source_code_unit;

if (IsLeadSurrogate(source_code_unit)) {

// The current code unit is a leading surrogate. If it's not followed by a

// trailing surrogate, replace it with the replacement character.

if (i == length - 1 || !IsTrailSurrogate(source_code_units[i + 1])) {

dest_code_unit = kReplacement;

} else {

// Copy the paired trailing surrogate. The paired leading surrogate will

// be copied below.

++i;

dest_code_units[i] = source_code_units[i];

}

} else if (IsTrailSurrogate(source_code_unit)) {

// All paired trailing surrogates are skipped above, so this branch is

// only for those that are unpaired.

dest_code_unit = kReplacement;

}

dest_code_units[copy_index] = dest_code_unit;

}

using State = GeneralizedUtf8DfaDecoder::State;

auto state = State::kAccept;

uint32_t current = 0;

uint32_t previous = 0;

for (size_t i = 0; i < length; i++) {

GeneralizedUtf8DfaDecoder::Decode(bytes[i], &state, &current);

if (state == State::kReject) return false;

if (state == State::kAccept) {

if (Utf16::IsTrailSurrogate(current) &&

Utf16::IsLeadSurrogate(previous)) {

return false;

}

previous = current;

current = 0;

}

}

return state == State::kAccept;

std::vector<size_t>* surrogate_offsets) {

// A surrogate codepoint is encoded in a three-byte sequence:

//

// 0xED [0xA0,0xBF] [0x80,0xBF]

//

// If the first byte is 0xED, you already have a 50% chance of the value being

// a surrogate; you just have to check the second byte. (There are

// three-byte non-surrogates starting with 0xED whose second byte is in

// [0x80,0x9F].) Could speed this up with SWAR; most likely case is that no

// byte in the array is 0xED.

const uint8_t kWtf8SurrogateFirstByte = 0xED;

const uint8_t kWtf8SurrogateSecondByteHighBit = 0x20;

for (size_t i = 0; i < wtf8.size(); i++) {

if (wtf8[i] == kWtf8SurrogateFirstByte &&

(wtf8[i + 1] & kWtf8SurrogateSecondByteHighBit)) {

// Record the byte offset of the encoded surrogate.

surrogate_offsets->push_back(i);

}

}

int chunk_index = c >> 13;

switch (chunk_index) {

case 0:

return LookupPredicate(kUppercaseTable0, kUppercaseTable0Size, c);

case 1:

return LookupPredicate(kUppercaseTable1, kUppercaseTable1Size, c);

case 5:

return LookupPredicate(kUppercaseTable5, kUppercaseTable5Size, c);

case 7:

return LookupPredicate(kUppercaseTable7, kUppercaseTable7Size, c);

default:

return false;

}

int chunk_index = c >> 13;

switch (chunk_index) {

case 0:

return LookupPredicate(kLetterTable0, kLetterTable0Size, c);

case 1:

return LookupPredicate(kLetterTable1, kLetterTable1Size, c);

case 2:

return LookupPredicate(kLetterTable2, kLetterTable2Size, c);

case 3:

return LookupPredicate(kLetterTable3, kLetterTable3Size, c);

case 4:

return LookupPredicate(kLetterTable4, kLetterTable4Size, c);

case 5:

return LookupPredicate(kLetterTable5, kLetterTable5Size, c);

case 6:

return LookupPredicate(kLetterTable6, kLetterTable6Size, c);

case 7:

return LookupPredicate(kLetterTable7, kLetterTable7Size, c);

default:

return false;

}

int chunk_index = c >> 13;

switch (chunk_index) {

case 0:

return LookupPredicate(kID_StartTable0, kID_StartTable0Size, c);

case 1:

return LookupPredicate(kID_StartTable1, kID_StartTable1Size, c);

case 2:

return LookupPredicate(kID_StartTable2, kID_StartTable2Size, c);

case 3:

return LookupPredicate(kID_StartTable3, kID_StartTable3Size, c);

case 4:

return LookupPredicate(kID_StartTable4, kID_StartTable4Size, c);

case 5:

return LookupPredicate(kID_StartTable5, kID_StartTable5Size, c);

case 6:

return LookupPredicate(kID_StartTable6, kID_StartTable6Size, c);

case 7:

return LookupPredicate(kID_StartTable7, kID_StartTable7Size, c);

default:

return false;

}

int chunk_index = c >> 13;

switch (chunk_index) {

case 0:

return LookupPredicate(kID_ContinueTable0, kID_ContinueTable0Size, c);

case 1:

return LookupPredicate(kID_ContinueTable1, kID_ContinueTable1Size, c);

case 5:

return LookupPredicate(kID_ContinueTable5, kID_ContinueTable5Size, c);

case 7:

return LookupPredicate(kID_ContinueTable7, kID_ContinueTable7Size, c);

default:

return false;

}

int chunk_index = c >> 13;

switch (chunk_index) {

case 0:

return LookupPredicate(kWhiteSpaceTable0, kWhiteSpaceTable0Size, c);

case 1:

return LookupPredicate(kWhiteSpaceTable1, kWhiteSpaceTable1Size, c);

case 7:

return LookupPredicate(kWhiteSpaceTable7, kWhiteSpaceTable7Size, c);

default:

return false;

}