nghttp, nghttpd, nghttpx: Add ktls support

.github/workflows/build.yml

@@ -110,7 +110,7 @@ jobs:
 
         git clone --depth 1 -b openssl-3.0.3+quic https://github.com/quictls/openssl
         cd openssl
-        ./config enable-tls1_3 --prefix=$PWD/build --libdir=$PWD/build/lib
+        ./config enable-tls1_3 enable-ktls --prefix=$PWD/build --libdir=$PWD/build/lib
         make -j$(nproc)
         make install_sw
     - name: Build nghttp3

gennghttpxfun.py

@@ -199,6 +199,7 @@ OPTIONS = [
     "worker-process-grace-shutdown-period",
     "frontend-quic-initial-rtt",
     "require-http-scheme",
+    "tls-ktls",
 ]
 
 LOGVARS = [

src/HttpServer.cc

@@ -113,7 +113,8 @@ Config::Config()
       early_response(false),
       hexdump(false),
       echo_upload(false),
-      no_content_length(false) {}
+      no_content_length(false),
+      ktls(false) {}
 
 Config::~Config() {}
 
@@ -2122,6 +2123,12 @@ int HttpServer::run() {
                     SSL_OP_SINGLE_ECDH_USE | SSL_OP_NO_TICKET |
                     SSL_OP_CIPHER_SERVER_PREFERENCE;
 
+#ifdef SSL_OP_ENABLE_KTLS
+    if (config_->ktls) {
+      ssl_opts |= SSL_OP_ENABLE_KTLS;
+    }
+#endif // SSL_OP_ENABLE_KTLS
+
     SSL_CTX_set_options(ssl_ctx, ssl_opts);
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);

src/HttpServer.h

@@ -82,6 +82,7 @@ struct Config {
   bool hexdump;
   bool echo_upload;
   bool no_content_length;
+  bool ktls;
   Config();
   ~Config();
 };

src/nghttp.cc

@@ -122,7 +122,8 @@ Config::Config()
       hexdump(false),
       no_push(false),
       expect_continue(false),
-      verify_peer(true) {
+      verify_peer(true),
+      ktls(false) {
   nghttp2_option_new(&http2_option);
   nghttp2_option_set_peer_max_concurrent_streams(http2_option,
                                                  peer_max_concurrent_streams);
@@ -2280,6 +2281,12 @@ int communicate(
                     SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION |
                     SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION;
 
+#ifdef SSL_OP_ENABLE_KTLS
+    if (config.ktls) {
+      ssl_opts |= SSL_OP_ENABLE_KTLS;
+    }
+#endif // SSL_OP_ENABLE_KTLS
+
     SSL_CTX_set_options(ssl_ctx, ssl_opts);
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
@@ -2748,6 +2755,7 @@ Options:
   -y, --no-verify-peer
               Suppress  warning  on  server  certificate  verification
               failure.
+  --ktls      Enable ktls.
   --version   Display version information and exit.
   -h, --help  Display this help and exit.
 
@@ -2803,6 +2811,7 @@ int main(int argc, char **argv) {
         {"max-concurrent-streams", required_argument, &flag, 12},
         {"expect-continue", no_argument, &flag, 13},
         {"encoder-header-table-size", required_argument, &flag, 14},
+        {"ktls", no_argument, &flag, 15},
         {nullptr, 0, nullptr, 0}};
     int option_index = 0;
     int c =
@@ -3030,6 +3039,10 @@ int main(int argc, char **argv) {
         config.encoder_header_table_size = n;
         break;
       }
+      case 15:
+        // ktls option
+        config.ktls = true;
+        break;
       }
       break;
     default:

src/nghttp.h

@@ -97,6 +97,7 @@ struct Config {
   bool no_push;
   bool expect_continue;
   bool verify_peer;
+  bool ktls;
 };
 
 enum class RequestState { INITIAL, ON_REQUEST, ON_RESPONSE, ON_COMPLETE };

src/nghttpd.cc

@@ -178,6 +178,7 @@ Options:
       << config.mime_types_file << R"(
   --no-content-length
               Don't send content-length header field.
+  --ktls      Enable ktls.
   --version   Display version information and exit.
   -h, --help  Display this help and exit.
 
@@ -228,6 +229,7 @@ int main(int argc, char **argv) {
         {"mime-types-file", required_argument, &flag, 9},
         {"no-content-length", no_argument, &flag, 10},
         {"encoder-header-table-size", required_argument, &flag, 11},
+        {"ktls", no_argument, &flag, 12},
         {nullptr, 0, nullptr, 0}};
     int option_index = 0;
     int c = getopt_long(argc, argv, "DVb:c:d:ehm:n:p:va:w:W:", long_options,
@@ -407,6 +409,10 @@ int main(int argc, char **argv) {
         config.encoder_header_table_size = n;
         break;
       }
+      case 12:
+        // tls option
+        config.ktls = true;
+        break;
       }
       break;
     default:

src/shrpx.cc

@@ -2921,6 +2921,7 @@ SSL/TLS:
               accepts.
               Default: )"
       << util::utos_unit(config->tls.max_early_data) << R"(
+  --tls-ktls  Enable ktls.
 
 HTTP/2:
   -c, --frontend-http2-max-concurrent-streams=<N>
@@ -4263,6 +4264,7 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT.c_str(), required_argument, &flag,
          190},
         {SHRPX_OPT_REQUIRE_HTTP_SCHEME.c_str(), no_argument, &flag, 191},
+        {SHRPX_OPT_TLS_KTLS.c_str(), no_argument, &flag, 192},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -5172,6 +5174,10 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_REQUIRE_HTTP_SCHEME,
                              StringRef::from_lit("yes"));
         break;
+      case 192:
+        // --tls-ktls
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_KTLS, StringRef::from_lit("yes"));
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -1903,6 +1903,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_FASTOPEN;
       }
       break;
+    case 's':
+      if (util::strieq_l("tls-ktl", name, 7)) {
+        return SHRPX_OPTID_TLS_KTLS;
+      }
+      break;
     case 't':
       if (util::strieq_l("npn-lis", name, 7)) {
         return SHRPX_OPTID_NPN_LIST;
@@ -4188,6 +4193,9 @@ int parse_config(Config *config, int optid, const StringRef &opt,
   case SHRPX_OPTID_REQUIRE_HTTP_SCHEME:
     config->http.require_http_scheme = util::strieq_l("yes", optarg);
     return 0;
+  case SHRPX_OPTID_TLS_KTLS:
+    config->tls.ktls = util::strieq_l("yes", optarg);
+    return 0;
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -404,6 +404,7 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT =
     StringRef::from_lit("frontend-quic-initial-rtt");
 constexpr auto SHRPX_OPT_REQUIRE_HTTP_SCHEME =
     StringRef::from_lit("require-http-scheme");
+constexpr auto SHRPX_OPT_TLS_KTLS = StringRef::from_lit("tls-ktls");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -783,6 +784,7 @@ struct TLSConfig {
   // true if forwarding requests included in TLS early data should not
   // be postponed until TLS handshake finishes.
   bool no_postpone_early_data;
+  bool ktls;
 };
 
 #ifdef ENABLE_HTTP3
@@ -1332,6 +1334,7 @@ enum {
   SHRPX_OPTID_SYSLOG_FACILITY,
   SHRPX_OPTID_TLS_DYN_REC_IDLE_TIMEOUT,
   SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD,
+  SHRPX_OPTID_TLS_KTLS,
   SHRPX_OPTID_TLS_MAX_EARLY_DATA,
   SHRPX_OPTID_TLS_MAX_PROTO_VERSION,
   SHRPX_OPTID_TLS_MIN_PROTO_VERSION,

src/shrpx_connection.cc

@@ -312,8 +312,8 @@ BIO_METHOD *create_bio_method() {
 void Connection::set_ssl(SSL *ssl) {
   tls.ssl = ssl;
 
-  if (proto != Proto::HTTP3) {
+  auto &tlsconf = get_config()->tls;
-    auto &tlsconf = get_config()->tls;
+  if (proto != Proto::HTTP3 && !tlsconf.session_cache.memcached.host.empty()) {
     auto bio = BIO_new(tlsconf.bio_method);
     BIO_set_data(bio, this);
     SSL_set_bio(tls.ssl, bio, bio);
@@ -336,6 +336,12 @@ int Connection::tls_handshake() {
   wlimit.stopw();
   ev_timer_stop(loop, &wt);
 
+  auto &tlsconf = get_config()->tls;
+
+  if (tlsconf.session_cache.memcached.host.empty()) {
+    return tls_handshake_simple();
+  }
+
   std::array<uint8_t, 16_k> buf;
 
   if (ev_is_active(&rev)) {
@@ -397,10 +403,6 @@ int Connection::tls_handshake() {
 
   ERR_clear_error();
 
-#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
-  auto &tlsconf = get_config()->tls;
-#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
-
 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (!tls.server_handshake || tls.early_data_finish) {
     rv = SSL_do_handshake(tls.ssl);
@@ -592,6 +594,158 @@ int Connection::tls_handshake() {
   return write_tls_pending_handshake();
 }
 
+int Connection::tls_handshake_simple() {
+  wlimit.stopw();
+  ev_timer_stop(loop, &wt);
+
+  if (tls.initial_handshake_done) {
+    return write_tls_pending_handshake();
+  }
+
+  if (SSL_get_fd(tls.ssl) == -1) {
+    SSL_set_fd(tls.ssl, fd);
+  }
+
+  int rv;
+#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
+  auto &tlsconf = get_config()->tls;
+  std::array<uint8_t, 16_k> buf;
+#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
+
+  ERR_clear_error();
+
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
+  if (!tls.server_handshake || tls.early_data_finish) {
+    rv = SSL_do_handshake(tls.ssl);
+  } else {
+    for (;;) {
+      size_t nread;
+
+      rv = SSL_read_early_data(tls.ssl, buf.data(), buf.size(), &nread);
+      if (rv == SSL_READ_EARLY_DATA_ERROR) {
+        // If we have early data, and server sends ServerHello, assume
+        // that handshake is completed in server side, and start
+        // processing request.  If we don't exit handshake code here,
+        // server waits for EndOfEarlyData and Finished message from
+        // client, which voids the purpose of 0-RTT data.  The left
+        // over of handshake is done through write_tls or read_tls.
+        if (tlsconf.no_postpone_early_data && tls.earlybuf.rleft()) {
+          rv = 1;
+        }
+
+        break;
+      }
+
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: read early data " << nread << " bytes";
+      }
+
+      tls.earlybuf.append(buf.data(), nread);
+
+      if (rv == SSL_READ_EARLY_DATA_FINISH) {
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "tls: read all early data; total "
+                    << tls.earlybuf.rleft() << " bytes";
+        }
+        tls.early_data_finish = true;
+        // The same reason stated above.
+        if (tlsconf.no_postpone_early_data && tls.earlybuf.rleft()) {
+          rv = 1;
+        } else {
+          ERR_clear_error();
+          rv = SSL_do_handshake(tls.ssl);
+        }
+        break;
+      }
+    }
+  }
+#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
+  rv = SSL_do_handshake(tls.ssl);
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
+
+  if (rv <= 0) {
+    auto err = SSL_get_error(tls.ssl, rv);
+    switch (err) {
+    case SSL_ERROR_WANT_READ:
+      if (read_buffer_full(tls.rbuf)) {
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "tls: handshake message is too large";
+        }
+        return -1;
+      }
+      break;
+    case SSL_ERROR_WANT_WRITE:
+      wlimit.startw();
+      ev_timer_again(loop, &wt);
+      break;
+    case SSL_ERROR_SSL: {
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: handshake libssl error: "
+                  << ERR_error_string(ERR_get_error(), nullptr);
+      }
+      return SHRPX_ERR_NETWORK;
+    }
+    default:
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: handshake libssl error " << err;
+      }
+      return SHRPX_ERR_NETWORK;
+    }
+  }
+
+  if (rv != 1) {
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "tls: handshake is still in progress";
+    }
+    return SHRPX_ERR_INPROGRESS;
+  }
+
+#ifdef OPENSSL_IS_BORINGSSL
+  if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) &&
+      SSL_in_init(tls.ssl)) {
+    auto nread = SSL_read(tls.ssl, buf.data(), buf.size());
+    if (nread <= 0) {
+      auto err = SSL_get_error(tls.ssl, nread);
+      switch (err) {
+      case SSL_ERROR_WANT_READ:
+      case SSL_ERROR_WANT_WRITE:
+        break;
+      case SSL_ERROR_ZERO_RETURN:
+        return SHRPX_ERR_EOF;
+      case SSL_ERROR_SSL:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: "
+                    << ERR_error_string(ERR_get_error(), nullptr);
+        }
+        return SHRPX_ERR_NETWORK;
+      default:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: SSL_get_error returned " << err;
+        }
+        return SHRPX_ERR_NETWORK;
+      }
+    } else {
+      tls.earlybuf.append(buf.data(), nread);
+    }
+
+    if (SSL_in_init(tls.ssl)) {
+      return SHRPX_ERR_INPROGRESS;
+    }
+  }
+#endif // OPENSSL_IS_BORINGSSL
+
+  // Handshake was done
+
+  rv = check_http2_requirement();
+  if (rv != 0) {
+    return -1;
+  }
+
+  tls.initial_handshake_done = true;
+
+  return write_tls_pending_handshake();
+}
+
 int Connection::write_tls_pending_handshake() {
   // Send handshake data left in the buffer
   while (tls.wbuf.rleft()) {

src/shrpx_connection.h

@@ -109,6 +109,7 @@ struct Connection {
   void prepare_server_handshake();
 
   int tls_handshake();
+  int tls_handshake_simple();
   int write_tls_pending_handshake();
 
   int check_http2_requirement();

src/shrpx_tls.cc

@@ -933,25 +933,31 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
     DIE();
   }
 
-  constexpr auto ssl_opts =
+  auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) |
-      (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | SSL_OP_NO_SSLv2 |
+                  SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION |
-      SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION |
+                  SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION |
-      SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE |
+                  SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE |
-      SSL_OP_SINGLE_DH_USE |
+                  SSL_OP_CIPHER_SERVER_PREFERENCE
-      SSL_OP_CIPHER_SERVER_PREFERENCE
 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
-      // The reason for disabling built-in anti-replay in OpenSSL is
+                  // The reason for disabling built-in anti-replay in
-      // that it only works if client gets back to the same server.
+                  // OpenSSL is that it only works if client gets back
-      // The freshness check described in
+                  // to the same server.  The freshness check
-      // https://tools.ietf.org/html/rfc8446#section-8.3 is still
+                  // described in
-      // performed.
+                  // https://tools.ietf.org/html/rfc8446#section-8.3
-      | SSL_OP_NO_ANTI_REPLAY
+                  // is still performed.
+                  | SSL_OP_NO_ANTI_REPLAY
 #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
       ;
 
   auto config = mod_config();
   auto &tlsconf = config->tls;
 
+#ifdef SSL_OP_ENABLE_KTLS
+  if (tlsconf.ktls) {
+    ssl_opts |= SSL_OP_ENABLE_KTLS;
+  }
+#endif // SSL_OP_ENABLE_KTLS
+
   SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask);
 
   if (nghttp2::tls::ssl_ctx_set_proto_versions(
@@ -1700,13 +1706,18 @@ SSL_CTX *create_ssl_client_context(
     DIE();
   }
 
-  constexpr auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) |
+  auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) |
-                            SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 |
+                  SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION |
-                            SSL_OP_NO_COMPRESSION |
+                  SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION;
-                            SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION;
 
   auto &tlsconf = get_config()->tls;
 
+#ifdef SSL_OP_ENABLE_KTLS
+  if (tlsconf.ktls) {
+    ssl_opts |= SSL_OP_ENABLE_KTLS;
+  }
+#endif // SSL_OP_ENABLE_KTLS
+
   SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask);
 
   SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_CLIENT |