[RFC net-next 08/18] tun: run offloaded XDP program in Tx path

[Prashant Bhole]

run offloaded XDP program as soon as packet is removed from the ptr
ring. Since this is XDP in Tx path, the traditional handling of
XDP actions XDP_TX/REDIRECT isn't valid. For this reason we call
do_xdp_generic_core instead of do_xdp_generic. do_xdp_generic_core
just runs the program and leaves the action handling to us.

Signed-off-by: Prashant Bhole <address@hidden>
---
 drivers/net/tun.c | 149 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 146 insertions(+), 3 deletions(-)

diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index ecb49101b0b5..466ea69f00ee 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -131,6 +131,7 @@ struct tap_filter {
 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
  * to max number of VCPUs in guest. */
 #define MAX_TAP_QUEUES 256
+#define MAX_TAP_BATCH 64
 #define MAX_TAP_FLOWS  4096
 
 #define TUN_FLOW_EXPIRE (3 * HZ)
@@ -2156,6 +2157,109 @@ static ssize_t tun_put_user(struct tun_struct *tun,
        return total;
 }
 
+static struct sk_buff *tun_prepare_xdp_skb(struct sk_buff *skb)
+{
+       struct sk_buff *nskb;
+
+       if (skb_shared(skb) || skb_cloned(skb)) {
+               nskb = skb_copy(skb, GFP_ATOMIC);
+               consume_skb(skb);
+               return nskb;
+       }
+
+       return skb;
+}
+
+static u32 tun_do_xdp_offload_generic(struct tun_struct *tun,
+                                     struct sk_buff *skb)
+{
+       struct tun_prog *xdp_prog;
+       struct xdp_buff xdp;
+       u32 act = XDP_PASS;
+
+       xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+       if (xdp_prog) {
+               skb = tun_prepare_xdp_skb(skb);
+               if (!skb) {
+                       act = XDP_DROP;
+                       kfree_skb(skb);
+                       goto drop;
+               }
+
+               act = do_xdp_generic_core(skb, &xdp, xdp_prog->prog);
+               switch (act) {
+               case XDP_TX:
+                       /*
+                        * Rx path generic XDP will be called in this path
+                        */
+                       netif_receive_skb(skb);
+                       break;
+               case XDP_PASS:
+                       break;
+               case XDP_REDIRECT:
+                       /*
+                        * Since we are not handling this case yet, let's free
+                        * skb here. In case of XDP_DROP/XDP_ABORTED, the skb
+                        * was already freed in do_xdp_generic_core()
+                        */
+                       kfree_skb(skb);
+                       /* fall through */
+               default:
+                       bpf_warn_invalid_xdp_action(act);
+                       /* fall through */
+               case XDP_ABORTED:
+                       trace_xdp_exception(tun->dev, xdp_prog->prog, act);
+                       /* fall through */
+               case XDP_DROP:
+                       goto drop;
+               }
+       }
+
+       return act;
+drop:
+       this_cpu_inc(tun->pcpu_stats->tx_dropped);
+       return act;
+}
+
+static u32 tun_do_xdp_offload(struct tun_struct *tun, struct tun_file *tfile,
+                             struct xdp_frame *frame)
+{
+       struct tun_prog *xdp_prog;
+       struct tun_page tpage;
+       struct xdp_buff xdp;
+       u32 act = XDP_PASS;
+       int flush = 0;
+
+       xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+       if (xdp_prog) {
+               xdp.data_hard_start = frame->data - frame->headroom;
+               xdp.data = frame->data;
+               xdp.data_end = xdp.data + frame->len;
+               xdp.data_meta = xdp.data - frame->metasize;
+
+               act = bpf_prog_run_xdp(xdp_prog->prog, &xdp);
+               switch (act) {
+               case XDP_PASS:
+                       break;
+               case XDP_TX:
+                       /* fall through */
+               case XDP_REDIRECT:
+                       /* fall through */
+               default:
+                       bpf_warn_invalid_xdp_action(act);
+                       /* fall through */
+               case XDP_ABORTED:
+                       trace_xdp_exception(tun->dev, xdp_prog->prog, act);
+                       /* fall through */
+               case XDP_DROP:
+                       xdp_return_frame_rx_napi(frame);
+                       break;
+               }
+       }
+
+       return act;
+}
+
 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
 {
        DECLARE_WAITQUEUE(wait, current);
@@ -2574,6 +2678,47 @@ static int tun_sendmsg(struct socket *sock, struct 
msghdr *m, size_t total_len)
        return ret;
 }
 
+static int tun_consume_packets(struct tun_file *tfile, void **ptr_array, int n)
+{
+       struct tun_prog *xdp_prog;
+       struct xdp_frame *frame;
+       struct tun_struct *tun;
+       int i, num_ptrs;
+       int pkt_cnt = 0;
+       void *pkts[MAX_TAP_BATCH];
+       void *ptr;
+       u32 act;
+
+       if (unlikely(!tfile))
+               return 0;
+
+       if (n > MAX_TAP_BATCH)
+               n = MAX_TAP_BATCH;
+
+       rcu_read_lock();
+       tun = rcu_dereference(tfile->tun);
+       if (unlikely(!tun))
+               return 0;
+       xdp_prog = rcu_dereference(tun->offloaded_xdp_prog);
+
+       num_ptrs = ptr_ring_consume_batched(&tfile->tx_ring, pkts, n);
+       for (i = 0; i < num_ptrs; i++) {
+               ptr = pkts[i];
+               if (tun_is_xdp_frame(ptr)) {
+                       frame = tun_ptr_to_xdp(ptr);
+                       act = tun_do_xdp_offload(tun, tfile, frame);
+               } else {
+                       act = tun_do_xdp_offload_generic(tun, ptr);
+               }
+
+               if (act == XDP_PASS)
+                       ptr_array[pkt_cnt++] = ptr;
+       }
+
+       rcu_read_unlock();
+       return pkt_cnt;
+}
+
 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
                       int flags)
 {
@@ -2594,9 +2739,7 @@ static int tun_recvmsg(struct socket *sock, struct msghdr 
*m, size_t total_len,
                        ptr = ctl->ptr;
                        break;
                case TUN_MSG_CONSUME_PKTS:
-                       ret = ptr_ring_consume_batched(&tfile->tx_ring,
-                                                      ctl->ptr,
-                                                      ctl->num);
+                       ret = tun_consume_packets(tfile, ctl->ptr, ctl->num);
                        goto out;
                case TUN_MSG_UNCONSUME_PKTS:
                        ptr_ring_unconsume(&tfile->tx_ring, ctl->ptr,
-- 
2.20.1