DEFAULT = 0x0,

NEON = 0x1,

AVX2 = 0x4,

SSE42 = 0x8,

PCLMULQDQ = 0x10,

BMI1 = 0x20,

BMI2 = 0x40

uint32_t *edx) {

#if defined(_MSC_VER)

int cpu_info[4];

__cpuid(cpu_info, *eax);

*eax = cpu_info[0];

*ebx = cpu_info[1];

*ecx = cpu_info[2];

*edx = cpu_info[3];

#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)

uint32_t level = *eax;

__get_cpuid(level, eax, ebx, ecx, edx);

#else

uint32_t a = *eax, b, c = *ecx, d;

asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));

*eax = a;

*ebx = b;

*ecx = c;

*edx = d;

#endif

uint32_t eax, ebx, ecx, edx;

uint32_t host_isa = 0x0;

// ECX for EAX=0x7

eax = 0x7;

ecx = 0x0;

cpuid(&eax, &ebx, &ecx, &edx);

if (ebx & cpuid_avx2_bit) {

host_isa |= instruction_set::AVX2;

}

if (ebx & cpuid_bmi1_bit) {

host_isa |= instruction_set::BMI1;

}

if (ebx & cpuid_bmi2_bit) {

host_isa |= instruction_set::BMI2;

}

// EBX for EAX=0x1

eax = 0x1;

cpuid(&eax, &ebx, &ecx, &edx);

if (ecx & cpuid_sse42_bit) {

host_isa |= instruction_set::SSE42;

}

if (ecx & cpuid_pclmulqdq_bit) {

host_isa |= instruction_set::PCLMULQDQ;

}

return host_isa;

public:

really_inline implementation() : simdjson::implementation(

"haswell",

"Intel/AMD AVX2",

instruction_set::AVX2 | instruction_set::PCLMULQDQ | instruction_set::BMI1 | instruction_set::BMI2

) {}

WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;

WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser, size_t &next_json) const noexcept final;

public:

really_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", instruction_set::SSE42 | instruction_set::PCLMULQDQ) {}

WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;

WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser, size_t &next_json) const noexcept final;

public:

really_inline implementation() : simdjson::implementation("arm64", "ARM NEON", instruction_set::NEON) {}

WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;

WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser, size_t &next_json) const noexcept final;

public:

really_inline implementation() : simdjson::implementation(

"fallback",

"Generic fallback implementation",

0

) {}

WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;

WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, parser &parser, bool streaming) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser) const noexcept final;

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, parser &parser, size_t &next_json) const noexcept final;

public:

const std::string &name() const noexcept final { return set_best()->name(); }

const std::string &description() const noexcept final { return set_best()->description(); }

uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); }

WARN_UNUSED error_code parse(const uint8_t *buf, size_t len, dom::parser &parser) const noexcept final {

return set_best()->parse(buf, len, parser);

}

WARN_UNUSED error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final {

return set_best()->minify(buf, len, dst, dst_len);

}

WARN_UNUSED error_code stage1(const uint8_t *buf, size_t len, dom::parser &parser, bool streaming) const noexcept final {

return set_best()->stage1(buf, len, parser, streaming);

}

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, dom::parser &parser) const noexcept final {

return set_best()->stage2(buf, len, parser);

}

WARN_UNUSED error_code stage2(const uint8_t *buf, size_t len, dom::parser &parser, size_t &next_json) const noexcept final {

return set_best()->stage2(buf, len, parser, next_json);

}

really_inline detect_best_supported_implementation_on_first_use() noexcept : implementation("best_supported_detector", "Detects the best supported implementation and sets it", 0) {}

private:

const implementation *set_best() const noexcept;

public:

WARN_UNUSED error_code parse(const uint8_t *, size_t, parser &) const noexcept final {

return UNSUPPORTED_ARCHITECTURE;

}

WARN_UNUSED error_code minify(const uint8_t *, size_t, uint8_t *, size_t &) const noexcept final {

return UNSUPPORTED_ARCHITECTURE;

}

WARN_UNUSED error_code stage1(const uint8_t *, size_t, parser &, bool) const noexcept final {

return UNSUPPORTED_ARCHITECTURE;

}

WARN_UNUSED error_code stage2(const uint8_t *, size_t, parser &) const noexcept final {

return UNSUPPORTED_ARCHITECTURE;

}

WARN_UNUSED error_code stage2(const uint8_t *, size_t, parser &, size_t &) const noexcept final {

return UNSUPPORTED_ARCHITECTURE;

}

unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {}

auto force_implementation = available_implementations[force_implementation_name];

if (!force_implementation) {

fprintf(stderr, "SIMDJSON_FORCE_IMPLEMENTATION environment variable set to '%s', which is not a supported implementation name!\n", force_implementation_name);

abort();

}

return active_implementation = force_implementation;

/////////////

// We could do this with PMULL, but it is apparently slow.

//

//#ifdef __ARM_FEATURE_CRYPTO // some ARM processors lack this extension

//return vmull_p64(-1ULL, bitmask);

//#else

// Analysis by @sebpop:

// When diffing the assembly for src/stage1_find_marks.cpp I see that the eors are all spread out

// in between other vector code, so effectively the extra cycles of the sequence do not matter

// because the GPR units are idle otherwise and the critical path is on the FP side.

// Also the PMULL requires two extra fmovs: GPR->FP (3 cycles in N1, 5 cycles in A72 )

// and FP->GPR (2 cycles on N1 and 5 cycles on A72.)

///////////

bitmask ^= bitmask << 1;

bitmask ^= bitmask << 2;

bitmask ^= bitmask << 4;

bitmask ^= bitmask << 8;

bitmask ^= bitmask << 16;

bitmask ^= bitmask << 32;

return bitmask;

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

unsigned long ret;

// Search the mask data from least significant bit (LSB)

// to the most significant bit (MSB) for a set bit (1).

_BitScanForward64(&ret, input_num);

return (int)ret;

#else // SIMDJSON_REGULAR_VISUAL_STUDIO

return __builtin_ctzll(input_num);

#endif // SIMDJSON_REGULAR_VISUAL_STUDIO

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

unsigned long leading_zero = 0;

// Search the mask data from most significant bit (MSB)

// to least significant bit (LSB) for a set bit (1).

if (_BitScanReverse64(&leading_zero, input_num))

return (int)(63 - leading_zero);

else

return 64;

#else

return __builtin_clzll(input_num);

#endif// SIMDJSON_REGULAR_VISUAL_STUDIO

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

*result = value1 + value2;

return *result < value1;

#else

return __builtin_uaddll_overflow(value1, value2,

(unsigned long long *)result);

#endif

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

*result = value1 * value2;

return !!__umulh(value1, value2);

#else

return __builtin_umulll_overflow(value1, value2, (unsigned long long *)result);

#endif

uint8_t x5, uint8_t x6, uint8_t x7, uint8_t x8,

uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12,

uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) {

// Doing a load like so end ups generating worse code.

// uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,

// x9, x10,x11,x12,x13,x14,x15,x16};

// return vld1q_u8(array);

uint8x16_t x{};

// incredibly, Visual Studio does not allow x[0] = x1

x = vsetq_lane_u8(x1, x, 0);

x = vsetq_lane_u8(x2, x, 1);

x = vsetq_lane_u8(x3, x, 2);

x = vsetq_lane_u8(x4, x, 3);

x = vsetq_lane_u8(x5, x, 4);

x = vsetq_lane_u8(x6, x, 5);

x = vsetq_lane_u8(x7, x, 6);

x = vsetq_lane_u8(x8, x, 7);

x = vsetq_lane_u8(x9, x, 8);

x = vsetq_lane_u8(x10, x, 9);

x = vsetq_lane_u8(x11, x, 10);

x = vsetq_lane_u8(x12, x, 11);

x = vsetq_lane_u8(x13, x, 12);

x = vsetq_lane_u8(x14, x, 13);

x = vsetq_lane_u8(x15, x, 14);

x = vsetq_lane_u8(x16, x, 15);

return x;

int8_t x5, int8_t x6, int8_t x7, int8_t x8,

int8_t x9, int8_t x10, int8_t x11, int8_t x12,

int8_t x13, int8_t x14, int8_t x15, int8_t x16) {

// Doing a load like so end ups generating worse code.

// int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,

// x9, x10,x11,x12,x13,x14,x15,x16};

// return vld1q_s8(array);

int8x16_t x{};

// incredibly, Visual Studio does not allow x[0] = x1

x = vsetq_lane_s8(x1, x, 0);

x = vsetq_lane_s8(x2, x, 1);

x = vsetq_lane_s8(x3, x, 2);

x = vsetq_lane_s8(x4, x, 3);

x = vsetq_lane_s8(x5, x, 4);

x = vsetq_lane_s8(x6, x, 5);

x = vsetq_lane_s8(x7, x, 6);

x = vsetq_lane_s8(x8, x, 7);

x = vsetq_lane_s8(x9, x, 8);

x = vsetq_lane_s8(x10, x, 9);

x = vsetq_lane_s8(x11, x, 10);

x = vsetq_lane_s8(x12, x, 11);

x = vsetq_lane_s8(x13, x, 12);

x = vsetq_lane_s8(x14, x, 13);

x = vsetq_lane_s8(x15, x, 14);

x = vsetq_lane_s8(x16, x, 15);

return x;

uint8x16_t value;

static const int SIZE = sizeof(value);

// Conversion from/to SIMD register

really_inline base_u8(const uint8x16_t _value) : value(_value) {}

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

const uint8x16_t bit_mask = make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,

0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);

#else

const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,

0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};

#endif

auto minput = *this & bit_mask;

uint8x16_t tmp = vpaddq_u8(minput, minput);

tmp = vpaddq_u8(tmp, tmp);

tmp = vpaddq_u8(tmp, tmp);

return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);

uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7,

uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15

) : simd8(make_uint8x16_t(

v0, v1, v2, v3, v4, v5, v6, v7,

v8, v9, v10,v11,v12,v13,v14,v15

uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7,

uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15

) : simd8(uint8x16_t{

v0, v1, v2, v3, v4, v5, v6, v7,

v8, v9, v10,v11,v12,v13,v14,v15

uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7,

uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15

) {

return simd8<uint8_t>(

v0, v1, v2, v3, v4, v5, v6, v7,

v8, v9, v10,v11,v12,v13,v14,v15

);

// this particular implementation was inspired by work done by @animetosho

// we do it in two steps, first 8 bytes and then second 8 bytes

uint8_t mask1 = uint8_t(mask); // least significant 8 bits

uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits

// next line just loads the 64-bit values thintable_epi8[mask1] and

// thintable_epi8[mask2] into a 128-bit register, using only

// two instructions on most compilers.

uint64x2_t shufmask64 = {thintable_epi8[mask1], thintable_epi8[mask2]};

uint8x16_t shufmask = vreinterpretq_u8_u64(shufmask64);

// we increment by 0x08 the second half of the mask

#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO

uint8x16_t inc = make_uint8x16_t(0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08);

#else

uint8x16_t inc = {0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08};

#endif