h2o · kazuho · Jun 30, 2022 · Apr 30, 2022 · Apr 30, 2022 · Apr 30, 2022
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
       matrix:
         include:
           - name: "Linux / OpenSSL 1.1.0"
-            command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604'
+            command: make -f misc/docker-ci.mk CMAKE_ARGS='-DOPENSSL_ROOT_DIR=-DOPENSSL_ROOT_DIR=/opt/openssl-1.1.0 -DWITH_FUSION=OFF' CONTAINER_NAME='h2oserver/h2o-ci:ubuntu1604'
           - name: "Linux / OpenSSL 1.1.1"
             command: make -f misc/docker-ci.mk
           - name: "Linux / OpenSSL 1.1.1 + ASan & UBSan"

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -148,14 +148,14 @@ ENDIF ()
 
 IF (WITH_FUSION)
     ADD_LIBRARY(picotls-fusion lib/fusion.c)
-    SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul")
+    SET_TARGET_PROPERTIES(picotls-fusion PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq")
     TARGET_LINK_LIBRARIES(picotls-fusion picotls-core)
     ADD_EXECUTABLE(test-fusion.t
         deps/picotest/picotest.c
         lib/picotls.c
         t/fusion.c)
     TARGET_LINK_LIBRARIES(test-fusion.t picotls-minicrypto)
-    SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul")
+    SET_TARGET_PROPERTIES(test-fusion.t PROPERTIES COMPILE_FLAGS "-mavx2 -maes -mpclmul -mvaes -mvpclmulqdq")
     IF (WITH_DTRACE)
         ADD_DEPENDENCIES(test-fusion.t generate-picotls-probes)
     ENDIF ()

diff --git a/include/picotls.h b/include/picotls.h
@@ -70,6 +70,10 @@ extern "C" {
 #define PTLS_FUZZ_HANDSHAKE 0
 #endif
 
+#ifndef PTLS_SIZEOF_CACHE_LINE
+#define PTLS_SIZEOF_CACHE_LINE 64
+#endif
+
 #define PTLS_HELLO_RANDOM_SIZE 32
 
 #define PTLS_AES128_KEY_SIZE 16
@@ -392,6 +396,10 @@ typedef const struct st_ptls_aead_algorithm_t {
      * size of the tag
      */
     size_t tag_size;
+    /**
+     * if encrypted bytes are going to be written using non-temporal store instructions (i.e., skip cache)
+     */
+    unsigned non_temporal : 1;
     /**
      * size of memory allocated for ptls_aead_context_t. AEAD implementations can set this value to something greater than
      * sizeof(ptls_aead_context_t) and stuff additional data at the bottom of the struct.

diff --git a/include/picotls/fusion.h b/include/picotls/fusion.h
@@ -28,19 +28,24 @@ extern "C" {
 
 #include <stddef.h>
 #include <emmintrin.h>
+#include <immintrin.h>
 #include "../picotls.h"
 
 #define PTLS_FUSION_AES128_ROUNDS 10
 #define PTLS_FUSION_AES256_ROUNDS 14
 
 typedef struct ptls_fusion_aesecb_context {
-    __m128i keys[PTLS_FUSION_AES256_ROUNDS + 1];
+    union {
+        __m128i m128[PTLS_FUSION_AES256_ROUNDS + 1];
+        __m256i m256[PTLS_FUSION_AES256_ROUNDS + 1];
+    } keys;
     unsigned rounds;
-} ptls_fusion_aesecb_context_t;
+    uint8_t aesni256;
+} __attribute__((aligned(32))) ptls_fusion_aesecb_context_t;
 
 typedef struct ptls_fusion_aesgcm_context ptls_fusion_aesgcm_context_t;
 
-void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size);
+void ptls_fusion_aesecb_init(ptls_fusion_aesecb_context_t *ctx, int is_enc, const void *key, size_t key_size, int avx256);
 void ptls_fusion_aesecb_dispose(ptls_fusion_aesecb_context_t *ctx);
 void ptls_fusion_aesecb_encrypt(ptls_fusion_aesecb_context_t *ctx, void *dst, const void *src);
 
@@ -84,8 +89,15 @@ void ptls_fusion_aesgcm_encrypt(ptls_fusion_aesgcm_context_t *ctx, void *output,
 int ptls_fusion_aesgcm_decrypt(ptls_fusion_aesgcm_context_t *ctx, void *output, const void *input, size_t inlen, __m128i ctr,
                                const void *aad, size_t aadlen, const void *tag);
 
+/**
+ * A boolean flag indicating if vaes and vpclmulqdq (256-bit crypto instructions) should be used. This flag is set automatically
+ * when `ptls_fusion_is_supported_by_cpu` is called. Users can update the flag to enforce behavior. Engines that do not have support
+ * for these 256-bit instructions will continue using the 128-bit ones, even when this flag is set.
+ */
+extern int ptls_fusion_can_aesni256;
 extern ptls_cipher_algorithm_t ptls_fusion_aes128ctr, ptls_fusion_aes256ctr;
 extern ptls_aead_algorithm_t ptls_fusion_aes128gcm, ptls_fusion_aes256gcm;
+extern ptls_aead_algorithm_t ptls_non_temporal_aes128gcm, ptls_non_temporal_aes256gcm;
 
 /**
  * Returns a boolean indicating if fusion can be used.

diff --git a/lib/cifra/aes128.c b/lib/cifra/aes128.c
@@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes128gcm = {"AES128-GCM",
                                                    PTLS_AES128_KEY_SIZE,
                                                    PTLS_AESGCM_IV_SIZE,
                                                    PTLS_AESGCM_TAG_SIZE,
+                                                   0,
                                                    sizeof(struct aesgcm_context_t),
                                                    aead_aes128gcm_setup_crypto};
 ptls_cipher_suite_t ptls_minicrypto_aes128gcmsha256 = {.id = PTLS_CIPHER_SUITE_AES_128_GCM_SHA256,

diff --git a/lib/cifra/aes256.c b/lib/cifra/aes256.c
@@ -55,6 +55,7 @@ ptls_aead_algorithm_t ptls_minicrypto_aes256gcm = {"AES256-GCM",
                                                    PTLS_AES256_KEY_SIZE,
                                                    PTLS_AESGCM_IV_SIZE,
                                                    PTLS_AESGCM_TAG_SIZE,
+                                                   0,
                                                    sizeof(struct aesgcm_context_t),
                                                    aead_aes256gcm_setup_crypto};
 ptls_cipher_suite_t ptls_minicrypto_aes256gcmsha384 = {.id = PTLS_CIPHER_SUITE_AES_256_GCM_SHA384,

diff --git a/lib/cifra/chacha20.c b/lib/cifra/chacha20.c
@@ -226,6 +226,7 @@ ptls_aead_algorithm_t ptls_minicrypto_chacha20poly1305 = {"CHACHA20-POLY1305",
                                                           PTLS_CHACHA20_KEY_SIZE,
                                                           PTLS_CHACHA20POLY1305_IV_SIZE,
                                                           PTLS_CHACHA20POLY1305_TAG_SIZE,
+                                                          0,
                                                           sizeof(struct chacha20poly1305_context_t),
                                                           aead_chacha20poly1305_setup_crypto};
 ptls_cipher_suite_t ptls_minicrypto_chacha20poly1305sha256 = {.id = PTLS_CIPHER_SUITE_CHACHA20_POLY1305_SHA256,