Initial commit: dpdk-ws-client source code
This commit is contained in:
wxj
Executable
+948
@@ -0,0 +1,948 @@
|
||||
/**
|
||||
* tcp_core.c — Production-grade DPDK TCP/IP stack core
|
||||
*
|
||||
* Contains:
|
||||
* - Hash-table-based TCB management (O(1) lookup)
|
||||
* - Enhanced TCP packet encoding (with option bytes)
|
||||
* - ARP / ICMP helpers
|
||||
* - pkt_process loop with per-TCB timer maintenance
|
||||
* - Port initialization
|
||||
*/
|
||||
|
||||
#include "tcp_core.h"
|
||||
#include <sched.h>
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Globals
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
int gDpdkPortId = 0;
|
||||
uint32_t gLocalIp = MAKE_IPV4_ADDR(192, 168, 0, 115);
|
||||
uint8_t gSrcMac[RTE_ETHER_ADDR_LEN];
|
||||
uint8_t gDefaultArpMac[RTE_ETHER_ADDR_LEN] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
|
||||
|
||||
/* Latency trace buffer */
|
||||
struct lat_trace g_lat_trace[LAT_TRACE_SIZE];
|
||||
int g_lat_trace_idx = 0;
|
||||
|
||||
static struct ng_tcp_table *g_tcp_tables[MAX_WORKERS + 1];
|
||||
static struct inout_ring *g_rings[MAX_WORKERS + 1];
|
||||
static int g_num_workers = DEFAULT_WORKERS;
|
||||
|
||||
/* Fragment mempools (replaces rte_malloc for hot-path allocations) */
|
||||
struct rte_mempool *g_frag_pool = NULL;
|
||||
struct rte_mempool *g_ofo_pool = NULL;
|
||||
|
||||
/* fd → TCB index array (O(1) lookup from socket fd) — shared across all workers */
|
||||
static struct ng_tcp_stream *fd_to_tcb[MAX_FD_ARRAY];
|
||||
|
||||
/* Current worker context (set by worker thread at startup, saved/restored by main) */
|
||||
static int g_cur_worker = 0;
|
||||
|
||||
/* ── Multi-worker API ──────────────────────────────────────────── */
|
||||
|
||||
void ng_set_num_workers(int n) { if (n >= 1 && n <= MAX_WORKERS) g_num_workers = n; }
|
||||
int ng_get_num_workers(void) { return g_num_workers; }
|
||||
void ng_thread_set_worker(int w) { g_cur_worker = w; }
|
||||
int ng_thread_get_worker(void) { return g_cur_worker; }
|
||||
|
||||
/* Forward declarations for lazy init */
|
||||
static void tcpInstanceInit(int wid);
|
||||
static void ringInstanceInit(int wid);
|
||||
|
||||
struct ng_tcp_table *tcpInstance(void)
|
||||
{
|
||||
int wid = g_cur_worker;
|
||||
if (g_tcp_tables[wid] == NULL) tcpInstanceInit(wid);
|
||||
return g_tcp_tables[wid];
|
||||
}
|
||||
|
||||
struct inout_ring *ringInstance(void)
|
||||
{
|
||||
int wid = g_cur_worker;
|
||||
if (g_rings[wid] == NULL) ringInstanceInit(wid);
|
||||
return g_rings[wid];
|
||||
}
|
||||
|
||||
/* Direct access for main thread to any worker's instances */
|
||||
struct ng_tcp_table *tcpInstanceFor(int wid)
|
||||
{
|
||||
if (wid < 0 || wid > MAX_WORKERS) return NULL;
|
||||
if (g_tcp_tables[wid] == NULL) tcpInstanceInit(wid);
|
||||
return g_tcp_tables[wid];
|
||||
}
|
||||
|
||||
struct inout_ring *ringInstanceFor(int wid)
|
||||
{
|
||||
if (wid < 0 || wid > MAX_WORKERS) return NULL;
|
||||
if (g_rings[wid] == NULL) ringInstanceInit(wid);
|
||||
return g_rings[wid];
|
||||
}
|
||||
|
||||
/* Compute which worker should handle a given 4-tuple flow */
|
||||
int ng_compute_worker(uint32_t sip, uint32_t dip, uint16_t sport, uint16_t dport)
|
||||
{
|
||||
uint32_t h = sip ^ dip;
|
||||
h ^= ((uint32_t)sport << 16) | dport;
|
||||
h ^= (h >> 16);
|
||||
return (int)(h % g_num_workers) + 1; /* workers are 1..N */
|
||||
}
|
||||
|
||||
/* ── Init helpers (lazy allocation) ────────────────────────────── */
|
||||
|
||||
static void tcpInstanceInit(int wid)
|
||||
{
|
||||
g_tcp_tables[wid] = rte_malloc("ng_tcp_table", sizeof(struct ng_tcp_table), 0);
|
||||
if (g_tcp_tables[wid] == NULL)
|
||||
rte_exit(EXIT_FAILURE, "tcpInstance[%d] malloc failed\n", wid);
|
||||
memset(g_tcp_tables[wid], 0, sizeof(struct ng_tcp_table));
|
||||
}
|
||||
|
||||
static void ringInstanceInit(int wid)
|
||||
{
|
||||
g_rings[wid] = rte_malloc("inout_ring", sizeof(struct inout_ring), 0);
|
||||
memset(g_rings[wid], 0, sizeof(struct inout_ring));
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* FD ↔ TCB index (O(1))
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
void ng_tcp_fd_set(int fd, struct ng_tcp_stream *s)
|
||||
{
|
||||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||||
fd_to_tcb[fd] = s;
|
||||
}
|
||||
|
||||
void ng_tcp_fd_clear(int fd)
|
||||
{
|
||||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||||
fd_to_tcb[fd] = NULL;
|
||||
}
|
||||
|
||||
struct ng_tcp_stream *ng_tcp_fd_get(int fd)
|
||||
{
|
||||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||||
return fd_to_tcb[fd];
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Hash table operations
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
struct ng_tcp_stream *ng_tcp_stream_search(uint32_t sip, uint32_t dip,
|
||||
uint16_t sport, uint16_t dport)
|
||||
{
|
||||
uint32_t h_fwd = tcp_hash4(sip, dip, sport, dport);
|
||||
uint32_t h_rev = tcp_hash4(dip, sip, dport, sport);
|
||||
struct ng_tcp_table *table = tcpInstance();
|
||||
struct ng_tcp_stream *s;
|
||||
|
||||
/* Search forward hash bucket */
|
||||
for (s = table->htable[h_fwd]; s != NULL; s = s->ht_next)
|
||||
{
|
||||
if ((s->sip == sip && s->dip == dip &&
|
||||
s->sport == sport && s->dport == dport) ||
|
||||
(s->sip == dip && s->dip == sip &&
|
||||
s->sport == dport && s->dport == sport))
|
||||
return s;
|
||||
}
|
||||
|
||||
/* If reverse bucket is different, search it too */
|
||||
if (h_rev != h_fwd)
|
||||
{
|
||||
for (s = table->htable[h_rev]; s != NULL; s = s->ht_next)
|
||||
{
|
||||
if ((s->sip == sip && s->dip == dip &&
|
||||
s->sport == sport && s->dport == dport) ||
|
||||
(s->sip == dip && s->dip == sip &&
|
||||
s->sport == dport && s->dport == sport))
|
||||
return s;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct ng_tcp_stream *ng_tcp_stream_search_listener(uint16_t dport)
|
||||
{
|
||||
struct ng_tcp_table *table = tcpInstance();
|
||||
struct ng_tcp_stream *s;
|
||||
for (s = table->listeners; s != NULL; s = s->ht_next)
|
||||
{
|
||||
if (s->dport == dport && s->status == NG_TCP_STATUS_LISTEN)
|
||||
return s;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void ng_tcp_hash_insert(struct ng_tcp_stream *stream)
|
||||
{
|
||||
uint32_t h = tcp_hash4(stream->sip, stream->dip,
|
||||
stream->sport, stream->dport);
|
||||
struct ng_tcp_table *table = tcpInstance();
|
||||
stream->ht_next = table->htable[h];
|
||||
table->htable[h] = stream;
|
||||
table->count++;
|
||||
}
|
||||
|
||||
void ng_tcp_hash_remove(struct ng_tcp_stream *stream)
|
||||
{
|
||||
uint32_t h = tcp_hash4(stream->sip, stream->dip,
|
||||
stream->sport, stream->dport);
|
||||
struct ng_tcp_table *table = tcpInstance();
|
||||
struct ng_tcp_stream **pp;
|
||||
|
||||
for (pp = &table->htable[h]; *pp != NULL; pp = &(*pp)->ht_next)
|
||||
{
|
||||
if (*pp == stream)
|
||||
{
|
||||
*pp = stream->ht_next;
|
||||
table->count--;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Stream create / destroy
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
struct ng_tcp_stream *ng_tcp_stream_create(uint32_t sip, uint32_t dip,
|
||||
uint16_t sport, uint16_t dport)
|
||||
{
|
||||
struct ng_tcp_stream *s = rte_malloc("ng_tcp_stream",
|
||||
sizeof(struct ng_tcp_stream), 0);
|
||||
if (s == NULL)
|
||||
return NULL;
|
||||
memset(s, 0, sizeof(struct ng_tcp_stream));
|
||||
|
||||
s->sip = sip;
|
||||
s->dip = dip;
|
||||
s->sport = sport;
|
||||
s->dport = dport;
|
||||
s->protocol = IPPROTO_TCP;
|
||||
s->fd = -1;
|
||||
s->status = NG_TCP_STATUS_CLOSED;
|
||||
s->worker_id = ng_compute_worker(sip, dip, sport, dport);
|
||||
|
||||
/* Sequence numbers (random ISN) */
|
||||
uint32_t seed = time(NULL);
|
||||
s->iss = rand_r(&seed) % TCP_MAX_SEQ;
|
||||
s->snd_una = s->iss;
|
||||
s->snd_nxt = s->iss;
|
||||
s->rcv_nxt = 0;
|
||||
|
||||
/* Congestion control defaults */
|
||||
s->cwnd = TCP_INITIAL_CWND;
|
||||
s->ssthresh = TCP_INITIAL_WINDOW * 128; /* effectively unbounded */
|
||||
s->mss = TCP_MSS_DEFAULT;
|
||||
s->rcv_mss = TCP_MSS_DEFAULT;
|
||||
|
||||
/* Timers */
|
||||
s->rto = TCP_RTO_INITIAL;
|
||||
s->srtt = 0;
|
||||
s->rttvar = 0;
|
||||
|
||||
/* Send buffer */
|
||||
s->snd_buf = rte_malloc("snd_buf", TCP_SENDBUF_SIZE, 0);
|
||||
if (s->snd_buf == NULL)
|
||||
{
|
||||
rte_free(s);
|
||||
return NULL;
|
||||
}
|
||||
memset(s->snd_buf, 0, TCP_SENDBUF_SIZE);
|
||||
|
||||
/* Receive buffer accounting */
|
||||
s->rcv_buf_max = TCP_RECVBUF_SIZE;
|
||||
s->rcv_buf_avail = TCP_RECVBUF_SIZE;
|
||||
s->rcv_wnd = TCP_INITIAL_WINDOW;
|
||||
|
||||
/* Rings */
|
||||
s->sndbuf = rte_ring_create("tcp_snd", RING_SIZE,
|
||||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||||
s->rcvbuf = rte_ring_create("tcp_rcv", RING_SIZE,
|
||||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||||
s->retransmit_ring = rte_ring_create("tcp_rtx", RING_SIZE,
|
||||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||||
|
||||
/* Condition variable */
|
||||
pthread_condattr_t cattr;
|
||||
pthread_condattr_init(&cattr);
|
||||
pthread_cond_init(&s->cond, &cattr);
|
||||
pthread_condattr_destroy(&cattr);
|
||||
|
||||
pthread_mutexattr_t mattr;
|
||||
pthread_mutexattr_init(&mattr);
|
||||
pthread_mutex_init(&s->mutex, &mattr);
|
||||
pthread_mutexattr_destroy(&mattr);
|
||||
|
||||
rte_memcpy(s->localmac, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||||
|
||||
/* Insert into hash table */
|
||||
ng_tcp_hash_insert(s);
|
||||
|
||||
/* Defer active-list add until state is set (caller does ng_tcp_stream_activate) */
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
/* forward declarations for active list helpers */
|
||||
static void ng_tcp_active_add(struct ng_tcp_stream *s);
|
||||
static void ng_tcp_active_remove(struct ng_tcp_stream *s);
|
||||
static int ng_tcp_is_active(struct ng_tcp_stream *s);
|
||||
|
||||
void ng_tcp_stream_destroy(struct ng_tcp_stream *stream)
|
||||
{
|
||||
if (stream == NULL)
|
||||
return;
|
||||
|
||||
/* Remove from active list */
|
||||
ng_tcp_active_remove(stream);
|
||||
|
||||
/* Remove from hash table */
|
||||
ng_tcp_hash_remove(stream);
|
||||
|
||||
/* Clear fd mapping */
|
||||
if (stream->fd >= 0)
|
||||
ng_tcp_fd_clear(stream->fd);
|
||||
|
||||
/* Free out-of-order segments */
|
||||
struct ofo_segment *ofo = stream->ofo_head;
|
||||
while (ofo)
|
||||
{
|
||||
struct ofo_segment *next = ofo->next;
|
||||
ofo_free(ofo);
|
||||
ofo = next;
|
||||
}
|
||||
|
||||
/* Free rings */
|
||||
if (stream->sndbuf)
|
||||
rte_ring_free(stream->sndbuf);
|
||||
if (stream->rcvbuf)
|
||||
rte_ring_free(stream->rcvbuf);
|
||||
if (stream->retransmit_ring)
|
||||
rte_ring_free(stream->retransmit_ring);
|
||||
|
||||
/* Free send buffer */
|
||||
if (stream->snd_buf)
|
||||
rte_free(stream->snd_buf);
|
||||
|
||||
/* Destroy synchronization primitives */
|
||||
pthread_mutex_destroy(&stream->mutex);
|
||||
pthread_cond_destroy(&stream->cond);
|
||||
|
||||
rte_free(stream);
|
||||
}
|
||||
|
||||
void ng_tcp_stream_set_status(struct ng_tcp_stream *stream, NG_TCP_STATUS s)
|
||||
{
|
||||
stream->status = s;
|
||||
}
|
||||
|
||||
/* Active list: track connections that need timer check + output processing */
|
||||
static void ng_tcp_active_add(struct ng_tcp_stream *s)
|
||||
{
|
||||
struct ng_tcp_table *t = tcpInstance();
|
||||
if (s->active_next || s->active_prev || t->active_head == s) return;
|
||||
s->active_next = t->active_head;
|
||||
s->active_prev = NULL;
|
||||
if (t->active_head) t->active_head->active_prev = s;
|
||||
t->active_head = s;
|
||||
}
|
||||
|
||||
static void ng_tcp_active_remove(struct ng_tcp_stream *s)
|
||||
{
|
||||
struct ng_tcp_table *t = tcpInstance();
|
||||
if (s->active_prev) s->active_prev->active_next = s->active_next;
|
||||
if (s->active_next) s->active_next->active_prev = s->active_prev;
|
||||
if (t->active_head == s) t->active_head = s->active_next;
|
||||
s->active_next = s->active_prev = NULL;
|
||||
}
|
||||
|
||||
static int ng_tcp_is_active(struct ng_tcp_stream *s)
|
||||
{
|
||||
switch (s->status) {
|
||||
case NG_TCP_STATUS_SYN_SENT:
|
||||
case NG_TCP_STATUS_SYN_RCVD:
|
||||
case NG_TCP_STATUS_ESTABLISHED:
|
||||
case NG_TCP_STATUS_CLOSE_WAIT:
|
||||
case NG_TCP_STATUS_FIN_WAIT_1:
|
||||
case NG_TCP_STATUS_FIN_WAIT_2:
|
||||
case NG_TCP_STATUS_LAST_ACK:
|
||||
case NG_TCP_STATUS_TIME_WAIT: /* needed for timer-based cleanup */
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Call when TCB enters active state (after nconnect or accept) */
|
||||
void ng_tcp_stream_activate(struct ng_tcp_stream *s)
|
||||
{
|
||||
if (ng_tcp_is_active(s))
|
||||
ng_tcp_active_add(s);
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* TCP ACK helper (pure ACK — no data, no options for now)
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
void ng_tcp_send_ack(struct ng_tcp_stream *stream)
|
||||
{
|
||||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||||
if (f == NULL)
|
||||
return;
|
||||
|
||||
f->sport = stream->sport;
|
||||
f->dport = stream->dport;
|
||||
f->seqnum = stream->snd_nxt;
|
||||
f->acknum = stream->rcv_nxt;
|
||||
f->tcp_flags = RTE_TCP_ACK_FLAG;
|
||||
f->windows = (stream->rcv_buf_avail > 65535) ? 65535
|
||||
: (uint16_t)stream->rcv_buf_avail;
|
||||
f->hdrlen_off = 0x50;
|
||||
|
||||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||||
}
|
||||
|
||||
void ng_tcp_send_rst(struct ng_tcp_stream *stream, uint32_t seq, uint32_t ack)
|
||||
{
|
||||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||||
if (f == NULL)
|
||||
return;
|
||||
|
||||
f->sport = stream->sport;
|
||||
f->dport = stream->dport;
|
||||
f->seqnum = seq;
|
||||
f->acknum = 0;
|
||||
f->tcp_flags = RTE_TCP_RST_FLAG;
|
||||
f->windows = 0;
|
||||
f->hdrlen_off = 0x50;
|
||||
|
||||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||||
stream->stat_rst_sent++;
|
||||
}
|
||||
|
||||
void ng_tcp_send_fin(struct ng_tcp_stream *stream)
|
||||
{
|
||||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||||
if (f == NULL)
|
||||
return;
|
||||
|
||||
f->sport = stream->sport;
|
||||
f->dport = stream->dport;
|
||||
f->seqnum = stream->snd_nxt;
|
||||
f->acknum = stream->rcv_nxt;
|
||||
f->tcp_flags = RTE_TCP_FIN_FLAG | RTE_TCP_ACK_FLAG;
|
||||
f->windows = (stream->rcv_buf_avail > 65535) ? 65535
|
||||
: (uint16_t)stream->rcv_buf_avail;
|
||||
f->hdrlen_off = 0x50;
|
||||
|
||||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* TCP packet encoding (with option bytes in fragment->option[])
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static int ng_encode_tcp_pkt(uint8_t *msg, uint32_t sip, uint32_t dip,
|
||||
uint8_t *srcmac, uint8_t *dstmac,
|
||||
struct ng_tcp_fragment *fragment)
|
||||
{
|
||||
int opt_bytes = fragment->optlen * sizeof(uint32_t);
|
||||
unsigned total_len = fragment->length +
|
||||
sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) +
|
||||
sizeof(struct rte_tcp_hdr) + opt_bytes;
|
||||
|
||||
/* Ethernet */
|
||||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||||
rte_memcpy(eth->src_addr.addr_bytes, srcmac, RTE_ETHER_ADDR_LEN);
|
||||
rte_memcpy(eth->dst_addr.addr_bytes, dstmac, RTE_ETHER_ADDR_LEN);
|
||||
eth->ether_type = htons(RTE_ETHER_TYPE_IPV4);
|
||||
|
||||
/* IP */
|
||||
struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1);
|
||||
ip->version_ihl = 0x45;
|
||||
ip->type_of_service = 0;
|
||||
ip->total_length = htons(total_len - sizeof(struct rte_ether_hdr));
|
||||
ip->packet_id = 0;
|
||||
ip->fragment_offset = 0;
|
||||
ip->time_to_live = 64;
|
||||
ip->next_proto_id = IPPROTO_TCP;
|
||||
ip->src_addr = sip;
|
||||
ip->dst_addr = dip;
|
||||
ip->hdr_checksum = 0;
|
||||
ip->hdr_checksum = rte_ipv4_cksum(ip);
|
||||
|
||||
/* TCP */
|
||||
struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *)(ip + 1);
|
||||
tcp->src_port = fragment->sport;
|
||||
tcp->dst_port = fragment->dport;
|
||||
tcp->sent_seq = htonl(fragment->seqnum);
|
||||
tcp->recv_ack = htonl(fragment->acknum);
|
||||
tcp->data_off = fragment->hdrlen_off; /* already includes options */
|
||||
tcp->rx_win = fragment->windows;
|
||||
tcp->tcp_urp = fragment->tcp_urp;
|
||||
tcp->tcp_flags = fragment->tcp_flags;
|
||||
|
||||
/* TCP options */
|
||||
if (opt_bytes > 0)
|
||||
{
|
||||
uint8_t *optptr = (uint8_t *)(tcp + 1);
|
||||
uint8_t *raw = (uint8_t *)fragment->option;
|
||||
rte_memcpy(optptr, raw, opt_bytes);
|
||||
}
|
||||
|
||||
/* Payload */
|
||||
if (fragment->length > 0)
|
||||
{
|
||||
uint8_t *payload = (uint8_t *)(tcp + 1) + opt_bytes;
|
||||
rte_memcpy(payload, fragment->data, fragment->length);
|
||||
}
|
||||
|
||||
/* TCP checksum */
|
||||
tcp->cksum = 0;
|
||||
tcp->cksum = rte_ipv4_udptcp_cksum(ip, tcp);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct rte_mbuf *ng_tcp_pkt(struct rte_mempool *mbuf_pool,
|
||||
uint32_t sip, uint32_t dip,
|
||||
uint8_t *srcmac, uint8_t *dstmac,
|
||||
struct ng_tcp_fragment *fragment)
|
||||
{
|
||||
int opt_bytes = fragment->optlen * sizeof(uint32_t);
|
||||
unsigned total_len = fragment->length +
|
||||
sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) +
|
||||
sizeof(struct rte_tcp_hdr) + opt_bytes;
|
||||
|
||||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||||
if (!mbuf)
|
||||
return NULL;
|
||||
|
||||
mbuf->pkt_len = total_len;
|
||||
mbuf->data_len = total_len;
|
||||
|
||||
uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||||
ng_encode_tcp_pkt(pktdata, sip, dip, srcmac, dstmac, fragment);
|
||||
|
||||
return mbuf;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Note: ng_tcp_out() removed. Output is handled in pkt_process()
|
||||
* via active_head linked list (Phase 2+3), avoiding O(256) scan.
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* ARP / ICMP packet helpers
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static int ng_encode_arp_pkt(uint8_t *msg, uint16_t opcode,
|
||||
uint8_t *dst_mac, uint32_t sip, uint32_t dip)
|
||||
{
|
||||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||||
rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||||
rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||||
eth->ether_type = htons(RTE_ETHER_TYPE_ARP);
|
||||
|
||||
struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
|
||||
arp->arp_hardware = htons(1);
|
||||
arp->arp_protocol = htons(RTE_ETHER_TYPE_IPV4);
|
||||
arp->arp_hlen = RTE_ETHER_ADDR_LEN;
|
||||
arp->arp_plen = sizeof(uint32_t);
|
||||
arp->arp_opcode = htons(opcode);
|
||||
|
||||
rte_memcpy(arp->arp_data.arp_sha.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||||
rte_memcpy(arp->arp_data.arp_tha.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||||
arp->arp_data.arp_sip = sip;
|
||||
arp->arp_data.arp_tip = dip;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct rte_mbuf *ng_send_arp(struct rte_mempool *mbuf_pool,
|
||||
uint16_t opcode, uint8_t *dst_mac,
|
||||
uint32_t sip, uint32_t dip)
|
||||
{
|
||||
const unsigned total_len = sizeof(struct rte_ether_hdr) +
|
||||
sizeof(struct rte_arp_hdr);
|
||||
|
||||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||||
if (!mbuf)
|
||||
return NULL;
|
||||
|
||||
mbuf->pkt_len = total_len;
|
||||
mbuf->data_len = total_len;
|
||||
|
||||
uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||||
ng_encode_arp_pkt(pktdata, opcode, dst_mac, sip, dip);
|
||||
|
||||
return mbuf;
|
||||
}
|
||||
|
||||
static struct rte_mbuf *ng_send_icmp(struct rte_mempool *mbuf_pool,
|
||||
uint8_t *dst_mac, uint32_t sip, uint32_t dip,
|
||||
uint16_t id, uint16_t seq)
|
||||
{
|
||||
const unsigned total_len = sizeof(struct rte_ether_hdr) +
|
||||
sizeof(struct rte_ipv4_hdr) +
|
||||
sizeof(struct rte_icmp_hdr);
|
||||
|
||||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||||
if (!mbuf)
|
||||
return NULL;
|
||||
|
||||
mbuf->pkt_len = total_len;
|
||||
mbuf->data_len = total_len;
|
||||
|
||||
uint8_t *msg = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||||
rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||||
rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||||
eth->ether_type = htons(RTE_ETHER_TYPE_IPV4);
|
||||
|
||||
struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1);
|
||||
ip->version_ihl = 0x45;
|
||||
ip->type_of_service = 0;
|
||||
ip->total_length = htons(sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_icmp_hdr));
|
||||
ip->packet_id = 0;
|
||||
ip->fragment_offset = 0;
|
||||
ip->time_to_live = 64;
|
||||
ip->next_proto_id = IPPROTO_ICMP;
|
||||
ip->src_addr = sip;
|
||||
ip->dst_addr = dip;
|
||||
ip->hdr_checksum = 0;
|
||||
ip->hdr_checksum = rte_ipv4_cksum(ip);
|
||||
|
||||
struct rte_icmp_hdr *icmp = (struct rte_icmp_hdr *)(ip + 1);
|
||||
icmp->icmp_type = RTE_ICMP_TYPE_ECHO_REPLY;
|
||||
icmp->icmp_code = 0;
|
||||
icmp->icmp_ident = id;
|
||||
icmp->icmp_seq_nb = seq;
|
||||
icmp->icmp_cksum = 0;
|
||||
icmp->icmp_cksum = rte_ipv4_udptcp_cksum(ip, (const void *)icmp);
|
||||
|
||||
return mbuf;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* ng_tcp_process — main incoming packet dispatch
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
int ng_tcp_process(struct rte_mbuf *tcpmbuf)
|
||||
{
|
||||
if (!tcpmbuf) { printf("[BUG] ng_tcp_process: NULL mbuf\n"); return 0; }
|
||||
struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(tcpmbuf,
|
||||
struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
|
||||
struct rte_tcp_hdr *tcphdr = (struct rte_tcp_hdr *)(iphdr + 1);
|
||||
|
||||
int tcplen = ntohs(iphdr->total_length) - sizeof(struct rte_ipv4_hdr);
|
||||
if (tcplen < (int)sizeof(struct rte_tcp_hdr))
|
||||
return 0; /* truncated */
|
||||
|
||||
/* Checksum validation */
|
||||
uint16_t cksum = rte_ipv4_udptcp_cksum(iphdr, (const void *)tcphdr);
|
||||
if (cksum != 0 && cksum != 0xFFFF)
|
||||
return 0; /* bad checksum — drop silently */
|
||||
|
||||
uint32_t sip = iphdr->src_addr;
|
||||
uint32_t dip = iphdr->dst_addr;
|
||||
uint16_t sport = tcphdr->src_port;
|
||||
uint16_t dport = tcphdr->dst_port;
|
||||
uint8_t flags = tcphdr->tcp_flags;
|
||||
|
||||
/* Look up existing stream */
|
||||
struct ng_tcp_stream *stream = ng_tcp_stream_search(sip, dip, sport, dport);
|
||||
|
||||
if (stream == NULL)
|
||||
{
|
||||
/* Try listener on SYN */
|
||||
if (flags & RTE_TCP_SYN_FLAG)
|
||||
{
|
||||
struct ng_tcp_stream *listener = ng_tcp_stream_search_listener(dport);
|
||||
if (listener)
|
||||
{
|
||||
ng_tcp_handle_listen(listener, tcphdr, iphdr, tcplen);
|
||||
}
|
||||
}
|
||||
/* If RST and no stream, ignore */
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Update last receive timestamp */
|
||||
stream->last_rcv_tsc = rte_get_timer_cycles();
|
||||
|
||||
/* Parse TCP options */
|
||||
uint8_t hdrlen = (tcphdr->data_off & 0xF0) >> 2;
|
||||
int optlen = hdrlen - sizeof(struct rte_tcp_hdr);
|
||||
struct tcp_options opts;
|
||||
memset(&opts, 0, sizeof(opts));
|
||||
|
||||
if (optlen > 0)
|
||||
{
|
||||
uint8_t *optptr = (uint8_t *)(tcphdr + 1);
|
||||
ng_tcp_parse_options(optptr, optlen, &opts);
|
||||
}
|
||||
|
||||
/* Dispatch by state */
|
||||
switch (stream->status)
|
||||
{
|
||||
case NG_TCP_STATUS_LISTEN:
|
||||
ng_tcp_handle_listen(stream, tcphdr, iphdr, tcplen);
|
||||
break;
|
||||
|
||||
case NG_TCP_STATUS_SYN_SENT:
|
||||
ng_tcp_handle_synsent(stream, tcphdr, iphdr, tcplen);
|
||||
break;
|
||||
|
||||
case NG_TCP_STATUS_SYN_RCVD:
|
||||
ng_tcp_handle_synrcvd(stream, tcphdr, iphdr, tcplen);
|
||||
break;
|
||||
|
||||
case NG_TCP_STATUS_ESTABLISHED:
|
||||
case NG_TCP_STATUS_FIN_WAIT_1:
|
||||
case NG_TCP_STATUS_FIN_WAIT_2:
|
||||
case NG_TCP_STATUS_CLOSING:
|
||||
case NG_TCP_STATUS_TIME_WAIT:
|
||||
case NG_TCP_STATUS_CLOSE_WAIT:
|
||||
case NG_TCP_STATUS_LAST_ACK:
|
||||
ng_tcp_input(stream, tcphdr, iphdr, tcplen);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
stream->stat_seg_rcvd++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* pkt_process — protocol processing loop (lcore 1..N)
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
int pkt_process(void *arg)
|
||||
{
|
||||
struct worker_args *wa = (struct worker_args *)arg;
|
||||
struct rte_mempool *mbuf_pool = wa->mbuf_pool;
|
||||
int worker_id = wa->worker_id;
|
||||
|
||||
/* Set thread-local worker for tcpInstance()/ringInstance() routing */
|
||||
ng_thread_set_worker(worker_id);
|
||||
|
||||
struct sched_param sp = { .sched_priority = 90 };
|
||||
sched_setscheduler(0, SCHED_FIFO, &sp);
|
||||
|
||||
struct inout_ring *ring = ringInstance();
|
||||
printf("[WORKER %d] pkt_process started on lcore %d\n", worker_id, rte_lcore_id());
|
||||
|
||||
|
||||
while (1)
|
||||
{
|
||||
|
||||
/* ── Phase 1: RX processing ── */
|
||||
struct rte_mbuf *mbufs[BURST_SIZE];
|
||||
if (!ring || !ring->in) { printf("\[BUG\] pkt_process: NULL ring\n"); rte_delay_ms(1000); continue; }
|
||||
unsigned num_recvd = rte_ring_mc_dequeue_burst(ring->in,
|
||||
(void **)mbufs, BURST_SIZE, NULL);
|
||||
|
||||
for (unsigned i = 0; i < num_recvd; i++)
|
||||
{
|
||||
struct rte_ether_hdr *ehdr = rte_pktmbuf_mtod(mbufs[i],
|
||||
struct rte_ether_hdr *);
|
||||
|
||||
/* ── ARP ── */
|
||||
if (ehdr->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
|
||||
{
|
||||
struct rte_arp_hdr *ahdr = rte_pktmbuf_mtod_offset(mbufs[i],
|
||||
struct rte_arp_hdr *, sizeof(struct rte_ether_hdr));
|
||||
|
||||
if (ahdr->arp_data.arp_tip == gLocalIp)
|
||||
{
|
||||
if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REQUEST))
|
||||
{
|
||||
struct rte_mbuf *ab = ng_send_arp(mbuf_pool,
|
||||
RTE_ARP_OP_REPLY, ahdr->arp_data.arp_sha.addr_bytes,
|
||||
ahdr->arp_data.arp_tip, ahdr->arp_data.arp_sip);
|
||||
if (ab)
|
||||
rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL);
|
||||
}
|
||||
else if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REPLY))
|
||||
{
|
||||
uint8_t *hw = ng_get_dst_macaddr(ahdr->arp_data.arp_sip);
|
||||
if (hw == NULL)
|
||||
ng_arp_table_insert(ahdr->arp_data.arp_sip,
|
||||
ahdr->arp_data.arp_sha.addr_bytes,
|
||||
ARP_ENTRY_STATUS_DYNAMIC);
|
||||
}
|
||||
}
|
||||
rte_pktmbuf_free(mbufs[i]);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ehdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
|
||||
continue;
|
||||
|
||||
struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(mbufs[i],
|
||||
struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
|
||||
|
||||
/* TCP */
|
||||
if (iphdr->next_proto_id == IPPROTO_TCP)
|
||||
{
|
||||
lat_trace_add(2); /* pkt_process picked up TCP packet */
|
||||
ng_tcp_process(mbufs[i]);
|
||||
}
|
||||
|
||||
/* ICMP echo */
|
||||
if (iphdr->next_proto_id == IPPROTO_ICMP)
|
||||
{
|
||||
struct rte_icmp_hdr *icmphdr = (struct rte_icmp_hdr *)(iphdr + 1);
|
||||
if (icmphdr->icmp_type == RTE_ICMP_TYPE_ECHO_REQUEST)
|
||||
{
|
||||
struct rte_mbuf *txbuf = ng_send_icmp(mbuf_pool,
|
||||
ehdr->src_addr.addr_bytes,
|
||||
iphdr->dst_addr, iphdr->src_addr,
|
||||
icmphdr->icmp_ident, icmphdr->icmp_seq_nb);
|
||||
if (txbuf)
|
||||
rte_ring_mp_enqueue_burst(ring->out, (void **)&txbuf, 1, NULL);
|
||||
rte_pktmbuf_free(mbufs[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Phase 2+3: Timer + Output (only iterate active connections) ── */
|
||||
{
|
||||
uint64_t now = rte_get_timer_cycles();
|
||||
struct ng_tcp_table *table = tcpInstance();
|
||||
struct ng_tcp_stream *s = table->active_head;
|
||||
static int dbg_active = 0;
|
||||
if (s && !dbg_active) { printf("[WORKER %d] active_head non-null, processing\n", worker_id); dbg_active = 1; }
|
||||
while (s)
|
||||
{
|
||||
struct ng_tcp_stream *next = s->active_next;
|
||||
|
||||
/* Timer check */
|
||||
ng_tcp_check_timers(s);
|
||||
if (s->status == NG_TCP_STATUS_TIME_WAIT)
|
||||
{
|
||||
uint64_t elapsed_us = (now - s->timewait_start) * 1000000 / rte_get_timer_hz();
|
||||
if (elapsed_us >= TCP_TIME_WAIT_USEC)
|
||||
{
|
||||
ng_tcp_stream_destroy(s);
|
||||
s = next;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* Output: send pending fragments for this connection */
|
||||
struct ng_tcp_fragment *f = NULL;
|
||||
if (rte_ring_mc_dequeue(s->sndbuf, (void **)&f) >= 0 && f)
|
||||
{
|
||||
uint8_t *dstmac = s->dst_mac_valid ? s->dst_mac
|
||||
: ng_get_dst_macaddr(s->dip);
|
||||
if (dstmac == NULL)
|
||||
{
|
||||
uint64_t ts = rte_get_timer_cycles();
|
||||
uint64_t arp_int = rte_get_timer_hz() / 10;
|
||||
if (s->last_arp_tsc == 0 || ts - s->last_arp_tsc > arp_int) {
|
||||
s->last_arp_tsc = ts;
|
||||
struct rte_mbuf *ab = ng_send_arp(mbuf_pool,
|
||||
RTE_ARP_OP_REQUEST, gDefaultArpMac, s->sip, s->dip);
|
||||
if (ab) rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL);
|
||||
}
|
||||
rte_ring_mp_enqueue(s->sndbuf, f);
|
||||
}
|
||||
else
|
||||
{
|
||||
struct rte_mbuf *mbuf = ng_tcp_pkt(mbuf_pool,
|
||||
s->sip, s->dip, s->localmac, dstmac, f);
|
||||
if (mbuf) {
|
||||
rte_ring_mp_enqueue_burst(ring->out, (void **)&mbuf, 1, NULL);
|
||||
s->stat_seg_sent++;
|
||||
}
|
||||
if (f->length > 0)
|
||||
rte_ring_mp_enqueue(s->retransmit_ring, f);
|
||||
else
|
||||
frag_free(f);
|
||||
}
|
||||
}
|
||||
|
||||
/* Remove from active list if no longer active */
|
||||
if (!ng_tcp_is_active(s))
|
||||
ng_tcp_active_remove(s);
|
||||
|
||||
s = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Fragment mempool initialization
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
void ng_frag_pool_init(void)
|
||||
{
|
||||
unsigned socket_id = rte_socket_id();
|
||||
|
||||
/* Fragment pool: struct + max MSS payload (MP/MC safe, lock-free) */
|
||||
char name_frag[64];
|
||||
snprintf(name_frag, sizeof(name_frag), "frag_pool_%u", socket_id);
|
||||
g_frag_pool = rte_mempool_create(name_frag,
|
||||
FRAG_POOL_SIZE, FRAG_ELT_SIZE, 256, 0,
|
||||
NULL, NULL, NULL, NULL, socket_id, 0);
|
||||
if (g_frag_pool == NULL)
|
||||
rte_exit(EXIT_FAILURE, "frag_pool create failed\n");
|
||||
|
||||
/* OFO segment pool */
|
||||
char name_ofo[64];
|
||||
snprintf(name_ofo, sizeof(name_ofo), "ofo_pool_%u", socket_id);
|
||||
g_ofo_pool = rte_mempool_create(name_ofo,
|
||||
OFO_POOL_SIZE, OFO_ELT_SIZE, 64, 0,
|
||||
NULL, NULL, NULL, NULL, socket_id, 0);
|
||||
if (g_ofo_pool == NULL)
|
||||
rte_exit(EXIT_FAILURE, "ofo_pool create failed\n");
|
||||
|
||||
printf("[POOL] Fragment pool: %u elts × %u bytes, OFO pool: %u elts × %u bytes\n",
|
||||
FRAG_POOL_SIZE, FRAG_ELT_SIZE, OFO_POOL_SIZE, OFO_ELT_SIZE);
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════
|
||||
* Port initialization
|
||||
* ══════════════════════════════════════════════════════════════════ */
|
||||
|
||||
void ng_init_port(struct rte_mempool *mbuf_pool)
|
||||
{
|
||||
const int num_rx_queues = 1;
|
||||
const int num_tx_queues = 1;
|
||||
|
||||
struct rte_eth_conf port_conf = {
|
||||
.rxmode = {.mtu = RTE_ETHER_MTU},
|
||||
};
|
||||
|
||||
if (rte_eth_dev_configure(gDpdkPortId, num_rx_queues, num_tx_queues,
|
||||
&port_conf) < 0)
|
||||
rte_exit(EXIT_FAILURE, "Cannot configure port=%d\n", gDpdkPortId);
|
||||
|
||||
if (rte_eth_rx_queue_setup(gDpdkPortId, 0, 128,
|
||||
rte_eth_dev_socket_id(gDpdkPortId),
|
||||
NULL, mbuf_pool) < 0)
|
||||
rte_exit(EXIT_FAILURE, "rx_queue_setup failed\n");
|
||||
|
||||
if (rte_eth_tx_queue_setup(gDpdkPortId, 0, 512,
|
||||
rte_eth_dev_socket_id(gDpdkPortId),
|
||||
NULL) < 0)
|
||||
rte_exit(EXIT_FAILURE, "tx_queue_setup failed\n");
|
||||
|
||||
if (rte_eth_dev_start(gDpdkPortId) < 0)
|
||||
rte_exit(EXIT_FAILURE, "dev_start failed\n");
|
||||
|
||||
rte_eth_promiscuous_enable(gDpdkPortId);
|
||||
|
||||
printf("[DPDK] Port %d initialized\n", gDpdkPortId);
|
||||
}
|
||||
Reference in New Issue
Block a user