From aa01b90243b003ac6188b02180c5b95b4fac421d Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:08:40 +0000
Subject: [PATCH 01/32] Try setting flags to 0.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index e6acb3b..afbd741 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -215,7 +215,7 @@ int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 **/
 int setup_socket(const char *dev, __u32 xdp_flags, __u16 thread_id)
 {
-    flags = xdp_flags;
+    flags = 0;
     int ret;
     int xsks_map_fd;
     void *packet_buffer;

From f9f5584bd1ec82ef486fd1db16f878b91e4579ea Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:16:56 +0000
Subject: [PATCH 02/32] Try setting bind flag XDP_COPY.

---
 src/af_xdp.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index afbd741..3a4974c 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -117,10 +117,10 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     }
 
     xsk_info->umem = umem;
-    xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
+    xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;\
     xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
     xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = 0;
+    xsk_cfg.bind_flags = XDP_COPY;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 
@@ -215,7 +215,7 @@ int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 **/
 int setup_socket(const char *dev, __u32 xdp_flags, __u16 thread_id)
 {
-    flags = 0;
+    flags = xdp_flags;
     int ret;
     int xsks_map_fd;
     void *packet_buffer;

From 38192c19e49a8ef8c84f94f1283cda785ddbcf1c Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:20:16 +0000
Subject: [PATCH 03/32] Pass buffer differently.

---
 src/sequence.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sequence.c b/src/sequence.c
index 3e7a2be..de51a9f 100644
--- a/src/sequence.c
+++ b/src/sequence.c
@@ -656,7 +656,7 @@ void *thread_hdl(void *temp)
         __u16 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
         int ret;
 
-        if ((ret = send_packet(ti->id, &buffer, pckt_len, ti->cmd.verbose)) != 0)
+        if ((ret = send_packet(ti->id, buffer, pckt_len, ti->cmd.verbose)) != 0)
         {
             fprintf(stderr, "ERROR - Could not send packet on AF_XDP socket (%d) :: %s.\n", ti->id, strerror(errno));
         }

From 96611d77c4e972a133e3db9d22665412b95fc7b5 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:24:32 +0000
Subject: [PATCH 04/32] Pass buffer differently.

---
 src/sequence.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sequence.c b/src/sequence.c
index de51a9f..94c728e 100644
--- a/src/sequence.c
+++ b/src/sequence.c
@@ -656,7 +656,7 @@ void *thread_hdl(void *temp)
         __u16 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
         int ret;
 
-        if ((ret = send_packet(ti->id, buffer, pckt_len, ti->cmd.verbose)) != 0)
+        if ((ret = send_packet(ti->id, (void *)buffer[0], pckt_len, ti->cmd.verbose)) != 0)
         {
             fprintf(stderr, "ERROR - Could not send packet on AF_XDP socket (%d) :: %s.\n", ti->id, strerror(errno));
         }

From afccd0430f507d4327650901777a310c71d01707 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:25:26 +0000
Subject: [PATCH 05/32] Pass buffer differently.

---
 src/sequence.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sequence.c b/src/sequence.c
index 94c728e..1c3be9a 100644
--- a/src/sequence.c
+++ b/src/sequence.c
@@ -656,7 +656,7 @@ void *thread_hdl(void *temp)
         __u16 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
         int ret;
 
-        if ((ret = send_packet(ti->id, (void *)buffer[0], pckt_len, ti->cmd.verbose)) != 0)
+        if ((ret = send_packet(ti->id, (void *)&buffer[0], pckt_len, ti->cmd.verbose)) != 0)
         {
             fprintf(stderr, "ERROR - Could not send packet on AF_XDP socket (%d) :: %s.\n", ti->id, strerror(errno));
         }

From b5db38ca2bc6b2d88cd55607facb04dcd8eaa755 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:28:58 +0000
Subject: [PATCH 06/32] Add debugging into send_packet().

---
 src/af_xdp.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 3a4974c..416b816 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -15,6 +15,7 @@
 #include <linux/types.h>
 #include <sys/socket.h>
 #include <linux/if_link.h>
+#include <linux/if_ether.h>
 #include <bpf.h>
 #include <xsk.h>
 
@@ -176,6 +177,10 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 **/
 int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 {
+    struct ethhdr *eth = pckt;
+
+    printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
+    
     __u32 tx_idx = 0;
 
     int ret = xsk_ring_prod__reserve(&xsk_socket[thread_id]->tx, 1, &tx_idx);

From c129afbae0969ae98bfee3ac21818e7e93a2ab67 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:35:16 +0000
Subject: [PATCH 07/32] Addmore  debugging into send_packet().

---
 src/af_xdp.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 416b816..90a1e5f 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -177,10 +177,12 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 **/
 int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 {
-    struct ethhdr *eth = pckt;
+    __u64 addr = (__u64)pckt;
+
+    struct ethhdr *eth = addr;
 
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
-    
+
     __u32 tx_idx = 0;
 
     int ret = xsk_ring_prod__reserve(&xsk_socket[thread_id]->tx, 1, &tx_idx);

From eafc806b967cb3a24497e486af734c7e004d2898 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:36:44 +0000
Subject: [PATCH 08/32] Cast as ethhdr *.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 90a1e5f..e754c1d 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -179,7 +179,7 @@ int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 {
     __u64 addr = (__u64)pckt;
 
-    struct ethhdr *eth = addr;
+    struct ethhdr *eth = (struct ethhdr *)addr;
 
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
 

From 428d9e0d24ff8feb72c1522866c73112bde459da Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:40:29 +0000
Subject: [PATCH 09/32] Send buffer char differently.

---
 src/sequence.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/sequence.c b/src/sequence.c
index 1c3be9a..01a8d8c 100644
--- a/src/sequence.c
+++ b/src/sequence.c
@@ -656,7 +656,7 @@ void *thread_hdl(void *temp)
         __u16 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
         int ret;
 
-        if ((ret = send_packet(ti->id, (void *)&buffer[0], pckt_len, ti->cmd.verbose)) != 0)
+        if ((ret = send_packet(ti->id, (void *)buffer, pckt_len, ti->cmd.verbose)) != 0)
         {
             fprintf(stderr, "ERROR - Could not send packet on AF_XDP socket (%d) :: %s.\n", ti->id, strerror(errno));
         }

From 4e536611bc0448c3a323a41f4024062ad0af0754 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 16:55:44 +0000
Subject: [PATCH 10/32] Try changing flags.

---
 src/af_xdp.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index e754c1d..2f1c6df 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -119,9 +119,9 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;\
-    xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
+    xsk_cfg.libbpf_flags = 0;
     xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = XDP_COPY;
+    xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From d1316cd347244eb8dbf409fe58297573ba54ee36 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:04:01 +0000
Subject: [PATCH 11/32] Try adding more debugging for TX.

---
 src/af_xdp.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 2f1c6df..b630dbf 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -43,6 +43,7 @@ static void complete_tx(struct xsk_socket_info *xsk)
         return;
     }
 
+    printf("sendto() executed.\n");
     sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
 
     /* Collect/free completed TX buffers */
@@ -50,6 +51,8 @@ static void complete_tx(struct xsk_socket_info *xsk)
 
     if (completed > 0) 
     {
+        printf("completed > 0.\n");
+
         for (int i = 0; i < completed; i++)
         {
             xsk_free_umem_frame(xsk, *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx_cq++));
@@ -57,6 +60,7 @@ static void complete_tx(struct xsk_socket_info *xsk)
 
         xsk_ring_cons__release(&xsk->umem->cq, completed);
         xsk->outstanding_tx -= completed < xsk->outstanding_tx ? completed : xsk->outstanding_tx;
+        printf("New outstanding TX => %u.\n", xsk->outstanding_tx);
     }
 }
 
@@ -118,7 +122,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     }
 
     xsk_info->umem = umem;
-    xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;\
+    xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
     xsk_cfg.xdp_flags = flags;
     xsk_cfg.bind_flags = 0;
@@ -203,7 +207,7 @@ int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
 
     if (verbose)
     {
-        printf("AF_XDP Info :: Address => %llu. Length => %u. Outstanding TX => %u.\n", (__u64)pckt, length, xsk_socket[thread_id]->outstanding_tx);
+        printf("AF_XDP Info :: Address => %llu. Length => %u. Outstanding TX => %u. TX Index => %u.\n", (__u64)pckt, length, xsk_socket[thread_id]->outstanding_tx, tx_idx);
     }
 
     complete_tx(xsk_socket[thread_id]);

From 556f4df8d826de212b9784e91b1c513d24fe9ba7 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:06:32 +0000
Subject: [PATCH 12/32] Try XDP_COPY again.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index b630dbf..ecf72e0 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -125,7 +125,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
     xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = 0;
+    xsk_cfg.bind_flags = XDP_COPY;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From a362c433bd45532cc24bde754795f064ab1b8545 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:07:31 +0000
Subject: [PATCH 13/32] Try XDP_ZEROCOPY.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index ecf72e0..a51d2c6 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -125,7 +125,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
     xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = XDP_COPY;
+    xsk_cfg.bind_flags = XDP_ZEROCOPY;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From 3069bae0be0c3a709068680dd87c1c1cafbf312b Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:08:23 +0000
Subject: [PATCH 14/32] Try SKB mode.

---
 src/af_xdp.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index a51d2c6..95ef0d6 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -124,8 +124,8 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
-    xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = XDP_ZEROCOPY;
+    xsk_cfg.xdp_flags = XDP_FLAGS_SKB_MODE;
+    xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From e114f347e800bfd3f25a2f226dcf7b3ec06bd591 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:09:23 +0000
Subject: [PATCH 15/32] Try update if not exists.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 95ef0d6..c1ac118 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -124,7 +124,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
-    xsk_cfg.xdp_flags = XDP_FLAGS_SKB_MODE;
+    xsk_cfg.xdp_flags = XDP_FLAGS_SKB_MODE | XDP_FLAGS_UPDATE_IF_NOEXIST;
     xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);

From e87c667a176eea65cff94660d3911fd8f2241f07 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:11:26 +0000
Subject: [PATCH 16/32] Try DRV mode.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index c1ac118..3130b18 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -124,7 +124,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
-    xsk_cfg.xdp_flags = XDP_FLAGS_SKB_MODE | XDP_FLAGS_UPDATE_IF_NOEXIST;
+    xsk_cfg.xdp_flags = XDP_FLAGS_DRV_MODE | XDP_FLAGS_UPDATE_IF_NOEXIST;
     xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);

From 58f36fcf32cc3c8b678bb38190a573f370e41ac5 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:12:30 +0000
Subject: [PATCH 17/32] Try no XDP flags.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 3130b18..09e39d3 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -124,7 +124,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = 0;
-    xsk_cfg.xdp_flags = XDP_FLAGS_DRV_MODE | XDP_FLAGS_UPDATE_IF_NOEXIST;
+    xsk_cfg.xdp_flags = 0;
     xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);

From b99d8063fbec8c75337cdd4fec0a37e54baf4c8b Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 17:14:24 +0000
Subject: [PATCH 18/32] Try NEW LIBBPF FLAGS.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 09e39d3..05cce7d 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -123,7 +123,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
-    xsk_cfg.libbpf_flags = 0;
+    xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
     xsk_cfg.xdp_flags = 0;
     xsk_cfg.bind_flags = 0;
 

From ed70843e2e8f3c4934715d6986dd6abfc5c3aabd Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:31:26 +0000
Subject: [PATCH 19/32] Try XDP_USE_NEED_WAKEUP bind flag.

---
 src/af_xdp.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 05cce7d..f1b0863 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -49,7 +49,7 @@ static void complete_tx(struct xsk_socket_info *xsk)
     /* Collect/free completed TX buffers */
     completed = xsk_ring_cons__peek(&xsk->umem->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx_cq);
 
-    if (completed > 0) 
+    if (completed > 0)
     {
         printf("completed > 0.\n");
 
@@ -125,7 +125,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
     xsk_cfg.xdp_flags = 0;
-    xsk_cfg.bind_flags = 0;
+    xsk_cfg.bind_flags = XDP_USE_NEED_WAKEUP;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From ef10d3c8fe6359932344a248cb5c83f13fbe27e3 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:32:32 +0000
Subject: [PATCH 20/32] Remove XDP_USE_NEED_WAKEUP bind flag.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index f1b0863..edc9106 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -125,7 +125,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
     xsk_cfg.xdp_flags = 0;
-    xsk_cfg.bind_flags = XDP_USE_NEED_WAKEUP;
+    xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From 0b0448712735bd06f292a3abe85b3336a143cc3c Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:56:36 +0000
Subject: [PATCH 21/32] Try filling the completion ring instead of fill ring.

---
 src/af_xdp.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index edc9106..59b6cd4 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -124,7 +124,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem = umem;
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
-    xsk_cfg.xdp_flags = 0;
+    xsk_cfg.xdp_flags = flags;
     xsk_cfg.bind_flags = 0;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
@@ -145,7 +145,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem_frame_free = NUM_FRAMES;
 
     // Stuff the receive path with buffers, we assume we have enough.
-    ret = xsk_ring_prod__reserve(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
+    ret = xsk_ring_prod__reserve(&xsk_info->umem->cq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
 
     if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
     {
@@ -156,10 +156,10 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 
     for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
     {
-        *xsk_ring_prod__fill_addr(&xsk_info->umem->fq, idx++) = xsk_alloc_umem_frame(xsk_info);
+        *xsk_ring_prod__fill_addr(&xsk_info->umem->cq, idx++) = xsk_alloc_umem_frame(xsk_info);
     }
 
-    xsk_ring_prod__submit(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
+    xsk_ring_prod__submit(&xsk_info->umem->cq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
 
     return xsk_info;
 

From 2c46938bfb55b68023cb9d6125c3af01f82ea050 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:57:45 +0000
Subject: [PATCH 22/32] I guess we don't have to fill the completion ring like
 the fill ring?

---
 src/af_xdp.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 59b6cd4..8ea6630 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -145,7 +145,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_info->umem_frame_free = NUM_FRAMES;
 
     // Stuff the receive path with buffers, we assume we have enough.
-    ret = xsk_ring_prod__reserve(&xsk_info->umem->cq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
+    ret = xsk_ring_prod__reserve(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
 
     if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
     {
@@ -156,10 +156,10 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 
     for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
     {
-        *xsk_ring_prod__fill_addr(&xsk_info->umem->cq, idx++) = xsk_alloc_umem_frame(xsk_info);
+        *xsk_ring_prod__fill_addr(&xsk_info->umem->fq, idx++) = xsk_alloc_umem_frame(xsk_info);
     }
 
-    xsk_ring_prod__submit(&xsk_info->umem->cq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
+    xsk_ring_prod__submit(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
 
     return xsk_info;
 

From 009de99b5b89cbfd11a53cddf509854cbc5edf20 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:58:14 +0000
Subject: [PATCH 23/32] Try removing anything related to fill ring.

---
 src/af_xdp.c | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 8ea6630..20543ad 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -144,23 +144,6 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
 
     xsk_info->umem_frame_free = NUM_FRAMES;
 
-    // Stuff the receive path with buffers, we assume we have enough.
-    ret = xsk_ring_prod__reserve(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
-
-    if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
-    {
-        fprintf(stderr, "ret != XSK_RING_PROD__DEFAULT_NUM_DESCS :: Error.\n");
-
-        goto error_exit;
-    }
-
-    for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
-    {
-        *xsk_ring_prod__fill_addr(&xsk_info->umem->fq, idx++) = xsk_alloc_umem_frame(xsk_info);
-    }
-
-    xsk_ring_prod__submit(&xsk_info->umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
-
     return xsk_info;
 
     error_exit:

From 30549cb3eda105bc342af3e0d730c623ebaa34ca Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 18:59:35 +0000
Subject: [PATCH 24/32] Packet length should be 32-bits.

---
 src/af_xdp.c   | 2 +-
 src/sequence.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 20543ad..faf986f 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -162,7 +162,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
  * 
  * @return Returns 0 on success and -1 on failure.
 **/
-int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose)
+int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
 {
     __u64 addr = (__u64)pckt;
 
diff --git a/src/sequence.c b/src/sequence.c
index 01a8d8c..32c6690 100644
--- a/src/sequence.c
+++ b/src/sequence.c
@@ -653,7 +653,7 @@ void *thread_hdl(void *temp)
         }
 
         // Attempt to send packet.
-        __u16 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
+        __u32 pckt_len = ntohs(iph->tot_len) + sizeof(struct ethhdr);
         int ret;
 
         if ((ret = send_packet(ti->id, (void *)buffer, pckt_len, ti->cmd.verbose)) != 0)

From b2e11053a425b2c80d297f2f55bffd2cc25e54de Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:00:07 +0000
Subject: [PATCH 25/32] Packet length should be 32-bits in header.

---
 src/af_xdp.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.h b/src/af_xdp.h
index 624c8ca..6851b34 100644
--- a/src/af_xdp.h
+++ b/src/af_xdp.h
@@ -41,6 +41,6 @@ struct xsk_socket_info
 };
 
 
-int send_packet(int thread_id, void *pckt, __u16 length, __u8 verbose);
+int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose);
 int setup_socket(const char *dev, __u32 xdp_flags, __u16 thread_id);
 void cleanup_socket(__u16 id);
\ No newline at end of file

From 135e741eb5fcf744df18ecd353ca4d5a201012fe Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:03:49 +0000
Subject: [PATCH 26/32] Add more debugging.

---
 src/af_xdp.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index faf986f..bd36e7b 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -16,9 +16,12 @@
 #include <sys/socket.h>
 #include <linux/if_link.h>
 #include <linux/if_ether.h>
+#include <linux/ip.h>
 #include <bpf.h>
 #include <xsk.h>
 
+#include <arpa/inet.h>
+
 #include "af_xdp.h"
 
 __u32 flags = XDP_FLAGS_DRV_MODE | XDP_FLAGS_UPDATE_IF_NOEXIST;
@@ -168,7 +171,10 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
 
     struct ethhdr *eth = (struct ethhdr *)addr;
 
+    struct iphdr *iph = (struct iphdr *)eth + 1;
+
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
+    printf("%u => %u.\n", ntohl(iph->saddr), ntohl(iph->daddr));
 
     __u32 tx_idx = 0;
 

From eaa838e8062a2d273030c69234f5c5890ceb94f9 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:05:35 +0000
Subject: [PATCH 27/32] Oops.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index bd36e7b..79a1f31 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -174,7 +174,7 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
     struct iphdr *iph = (struct iphdr *)eth + 1;
 
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
-    printf("%u => %u.\n", ntohl(iph->saddr), ntohl(iph->daddr));
+    printf("%u => %u.\n", iph->saddr, iph->daddr);
 
     __u32 tx_idx = 0;
 

From 10fe3098be9283c9c3878d84142da09907f04734 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:06:40 +0000
Subject: [PATCH 28/32] Oops.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 79a1f31..13684f8 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -171,7 +171,7 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
 
     struct ethhdr *eth = (struct ethhdr *)addr;
 
-    struct iphdr *iph = (struct iphdr *)eth + 1;
+    struct iphdr *iph = (struct iphdr *)(addr + sizeof(struct ethhdr));
 
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
     printf("%u => %u.\n", iph->saddr, iph->daddr);

From cfe7f53a9fd17c8f7414522bfb652e57e746230c Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:11:15 +0000
Subject: [PATCH 29/32] Just want to try something.

---
 src/af_xdp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 13684f8..4769d51 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -171,7 +171,7 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
 
     struct ethhdr *eth = (struct ethhdr *)addr;
 
-    struct iphdr *iph = (struct iphdr *)(addr + sizeof(struct ethhdr));
+    struct iphdr *iph = (struct iphdr *)(eth + 1);
 
     printf("%hhx:%hhx:%hhx:%hhx:%hhx:%hhx => %hhx:%hhx:%hhx:%hhx:%hhx:%hhx.\n", eth->h_source[0], eth->h_source[1], eth->h_source[2], eth->h_source[3], eth->h_source[4], eth->h_source[5], eth->h_dest[0], eth->h_dest[1], eth->h_dest[2], eth->h_dest[3], eth->h_dest[4], eth->h_dest[5]);
     printf("%u => %u.\n", iph->saddr, iph->daddr);

From aced9909fbc65b1e0bf2b03756c6cb94061f8844 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:18:54 +0000
Subject: [PATCH 30/32] Try wakeup mode differently.

---
 src/af_xdp.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 4769d51..7b4c046 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -46,8 +46,12 @@ static void complete_tx(struct xsk_socket_info *xsk)
         return;
     }
 
-    printf("sendto() executed.\n");
-    sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
+    if (xsk_ring_prod__needs_wakeup(&xsk->tx))
+    {
+        int fd = xsk_socket__fd(xsk->xsk);
+        printf("sendto() executed with FD %d.\n", fd);
+        sendto(fd, NULL, 0, MSG_DONTWAIT, NULL, 0);
+    }
 
     /* Collect/free completed TX buffers */
     completed = xsk_ring_cons__peek(&xsk->umem->cq, XSK_RING_CONS__DEFAULT_NUM_DESCS, &idx_cq);
@@ -128,7 +132,7 @@ static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
     xsk_cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
     xsk_cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
     xsk_cfg.xdp_flags = flags;
-    xsk_cfg.bind_flags = 0;
+    xsk_cfg.bind_flags = XDP_USE_NEED_WAKEUP;
 
     ret = xsk_socket__create(&xsk_info->xsk, dev, queue_id, umem->umem, NULL, &xsk_info->tx, &xsk_cfg);
 

From 77187194c45fef0e9fd833c075278a466a80e9f1 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:34:39 +0000
Subject: [PATCH 31/32] Try something a little new.

---
 src/af_xdp.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index 7b4c046..f925b0d 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -193,7 +193,12 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
         return -1;
     }
 
-    xsk_ring_prod__tx_desc(&xsk_socket[thread_id]->tx, tx_idx)->addr = (__u64)pckt;
+    // Insert packet into umem.
+    __u64 addrat = xsk_socket[thread_id]->outstanding_tx * FRAME_SIZE;
+
+    memcpy(xsk_umem__get_data(xsk_socket[thread_id]->umem, addrat), pckt, length);
+
+    xsk_ring_prod__tx_desc(&xsk_socket[thread_id]->tx, tx_idx)->addr = addrat;
     xsk_ring_prod__tx_desc(&xsk_socket[thread_id]->tx, tx_idx)->len = length;
     xsk_ring_prod__submit(&xsk_socket[thread_id]->tx, 1);
     xsk_socket[thread_id]->outstanding_tx++;

From 9a1f736dbf5dd6f6f193ceb7190447bb6d186924 Mon Sep 17 00:00:00 2001
From: gamemann <gamemann@gflclan.com>
Date: Wed, 1 Dec 2021 19:49:41 +0000
Subject: [PATCH 32/32] Use buffer instead.

---
 src/af_xdp.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/af_xdp.c b/src/af_xdp.c
index f925b0d..78605d4 100644
--- a/src/af_xdp.c
+++ b/src/af_xdp.c
@@ -60,13 +60,8 @@ static void complete_tx(struct xsk_socket_info *xsk)
     {
         printf("completed > 0.\n");
 
-        for (int i = 0; i < completed; i++)
-        {
-            xsk_free_umem_frame(xsk, *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx_cq++));
-        }
-
         xsk_ring_cons__release(&xsk->umem->cq, completed);
-        xsk->outstanding_tx -= completed < xsk->outstanding_tx ? completed : xsk->outstanding_tx;
+        xsk->outstanding_tx -= completed;
         printf("New outstanding TX => %u.\n", xsk->outstanding_tx);
     }
 }
@@ -196,7 +191,7 @@ int send_packet(int thread_id, void *pckt, __u32 length, __u8 verbose)
     // Insert packet into umem.
     __u64 addrat = xsk_socket[thread_id]->outstanding_tx * FRAME_SIZE;
 
-    memcpy(xsk_umem__get_data(xsk_socket[thread_id]->umem, addrat), pckt, length);
+    memcpy(xsk_umem__get_data(xsk_socket[thread_id]->umem->buffer, addrat), pckt, length);
 
     xsk_ring_prod__tx_desc(&xsk_socket[thread_id]->tx, tx_idx)->addr = addrat;
     xsk_ring_prod__tx_desc(&xsk_socket[thread_id]->tx, tx_idx)->len = length;