tls: Add rx inline crypto offload

This patch completes the generic infrastructure to offload TLS crypto to a
network device. It enables the kernel to skip decryption and
authentication of some skbs marked as decrypted by the NIC. In the fast
path, all packets received are decrypted by the NIC and the performance
is comparable to plain TCP.

This infrastructure doesn't require a TCP offload engine. Instead, the
NIC only decrypts packets that contain the expected TCP sequence number.
Out-Of-Order TCP packets are provided unmodified. As a result, at the
worst case a received TLS record consists of both plaintext and ciphertext
packets. These partially decrypted records must be reencrypted,
only to be decrypted.

The notable differences between SW KTLS Rx and this offload are as
follows:
1. Partial decryption - Software must handle the case of a TLS record
that was only partially decrypted by HW. This can happen due to packet
reordering.
2. Resynchronization - tls_read_size calls the device driver to
resynchronize HW after HW lost track of TLS record framing in
the TCP stream.

Signed-off-by: Boris Pismenny <borisp@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 20 insertions, 12 deletions

diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index 301f22430469..b09867c8b817 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -51,15 +51,6 @@ enum {
         TLSV6,
         TLS_NUM_PROTS,
 };
-enum {
-        TLS_BASE,
-        TLS_SW,
-#ifdef CONFIG_TLS_DEVICE
-        TLS_HW,
-#endif
-        TLS_HW_RECORD,
-        TLS_NUM_CONFIG,
-};
 
 static struct proto *saved_tcpv6_prot;
 static DEFINE_MUTEX(tcpv6_prot_mutex);
@@ -290,7 +281,10 @@ static void tls_sk_proto_close(struct sock *sk, long timeout)
         }
 
 #ifdef CONFIG_TLS_DEVICE
-        if (ctx->tx_conf != TLS_HW) {
+        if (ctx->rx_conf == TLS_HW)
+                tls_device_offload_cleanup_rx(sk);
+
+        if (ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW) {
 #else
         {
 #endif
@@ -470,8 +464,16 @@ static int do_tls_setsockopt_conf(struct sock *sk, char __user *optval,
                         conf = TLS_SW;
                 }
         } else {
-                rc = tls_set_sw_offload(sk, ctx, 0);
-                conf = TLS_SW;
+#ifdef CONFIG_TLS_DEVICE
+                rc = tls_set_device_offload_rx(sk, ctx);
+                conf = TLS_HW;
+                if (rc) {
+#else
+                {
+#endif
+                        rc = tls_set_sw_offload(sk, ctx, 0);
+                        conf = TLS_SW;
+                }
         }
 
         if (rc)
@@ -629,6 +631,12 @@ static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
         prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
         prot[TLS_HW][TLS_SW].sendmsg            = tls_device_sendmsg;
         prot[TLS_HW][TLS_SW].sendpage           = tls_device_sendpage;
+
+        prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW];
+
+        prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW];
+
+        prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
 #endif
 
         prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;