wxjandClaude
d57a0260e8
Keep P0-P4 optimizations: - Fragment mempool (frag_alloc/frag_free) - WS frame alloca() dynamic stack - Fragment read_offset (eliminate memmove) - nclose busy-poll (replace cond_timedwait) - Remove dead ng_tcp_out() Revert experimental multi-core changes (moved to feature/multi-core): - Worker arrays, TLS context switching - RX hash distribution, TX multi-ring collection Co-Authored-By: Claude <noreply@anthropic.com>
893 lines
33 KiB
C
Executable File
893 lines
33 KiB
C
Executable File
/**
|
||
* tcp_core.c — Production-grade DPDK TCP/IP stack core
|
||
*
|
||
* Contains:
|
||
* - Hash-table-based TCB management (O(1) lookup)
|
||
* - Enhanced TCP packet encoding (with option bytes)
|
||
* - ARP / ICMP helpers
|
||
* - pkt_process loop with per-TCB timer maintenance
|
||
* - Port initialization
|
||
*/
|
||
|
||
#include "tcp_core.h"
|
||
#include <sched.h>
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Globals
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
int gDpdkPortId = 0;
|
||
uint32_t gLocalIp = MAKE_IPV4_ADDR(192, 168, 0, 115);
|
||
uint8_t gSrcMac[RTE_ETHER_ADDR_LEN];
|
||
uint8_t gDefaultArpMac[RTE_ETHER_ADDR_LEN] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
|
||
|
||
/* Latency trace buffer */
|
||
struct lat_trace g_lat_trace[LAT_TRACE_SIZE];
|
||
int g_lat_trace_idx = 0;
|
||
|
||
static struct ng_tcp_table *tInst = NULL;
|
||
static struct inout_ring *rInst = NULL;
|
||
|
||
/* Fragment mempools (replaces rte_malloc for hot-path allocations) */
|
||
struct rte_mempool *g_frag_pool = NULL;
|
||
struct rte_mempool *g_ofo_pool = NULL;
|
||
|
||
/* fd → TCB index array (O(1) lookup from socket fd) */
|
||
static struct ng_tcp_stream *fd_to_tcb[MAX_FD_ARRAY];
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* TCP table singleton
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
struct ng_tcp_table *tcpInstance(void)
|
||
{
|
||
if (tInst == NULL)
|
||
{
|
||
tInst = rte_malloc("ng_tcp_table", sizeof(struct ng_tcp_table), 0);
|
||
if (tInst == NULL)
|
||
rte_exit(EXIT_FAILURE, "tcpInstance malloc failed\n");
|
||
memset(tInst, 0, sizeof(struct ng_tcp_table));
|
||
}
|
||
return tInst;
|
||
}
|
||
|
||
struct inout_ring *ringInstance(void)
|
||
{
|
||
if (rInst == NULL)
|
||
{
|
||
rInst = rte_malloc("inout_ring", sizeof(struct inout_ring), 0);
|
||
memset(rInst, 0, sizeof(struct inout_ring));
|
||
}
|
||
return rInst;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* FD ↔ TCB index (O(1))
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
void ng_tcp_fd_set(int fd, struct ng_tcp_stream *s)
|
||
{
|
||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||
fd_to_tcb[fd] = s;
|
||
}
|
||
|
||
void ng_tcp_fd_clear(int fd)
|
||
{
|
||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||
fd_to_tcb[fd] = NULL;
|
||
}
|
||
|
||
struct ng_tcp_stream *ng_tcp_fd_get(int fd)
|
||
{
|
||
if (fd >= 0 && fd < MAX_FD_ARRAY)
|
||
return fd_to_tcb[fd];
|
||
return NULL;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Hash table operations
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
struct ng_tcp_stream *ng_tcp_stream_search(uint32_t sip, uint32_t dip,
|
||
uint16_t sport, uint16_t dport)
|
||
{
|
||
uint32_t h_fwd = tcp_hash4(sip, dip, sport, dport);
|
||
uint32_t h_rev = tcp_hash4(dip, sip, dport, sport);
|
||
struct ng_tcp_table *table = tcpInstance();
|
||
struct ng_tcp_stream *s;
|
||
|
||
/* Search forward hash bucket */
|
||
for (s = table->htable[h_fwd]; s != NULL; s = s->ht_next)
|
||
{
|
||
if ((s->sip == sip && s->dip == dip &&
|
||
s->sport == sport && s->dport == dport) ||
|
||
(s->sip == dip && s->dip == sip &&
|
||
s->sport == dport && s->dport == sport))
|
||
return s;
|
||
}
|
||
|
||
/* If reverse bucket is different, search it too */
|
||
if (h_rev != h_fwd)
|
||
{
|
||
for (s = table->htable[h_rev]; s != NULL; s = s->ht_next)
|
||
{
|
||
if ((s->sip == sip && s->dip == dip &&
|
||
s->sport == sport && s->dport == dport) ||
|
||
(s->sip == dip && s->dip == sip &&
|
||
s->sport == dport && s->dport == sport))
|
||
return s;
|
||
}
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
struct ng_tcp_stream *ng_tcp_stream_search_listener(uint16_t dport)
|
||
{
|
||
struct ng_tcp_table *table = tcpInstance();
|
||
struct ng_tcp_stream *s;
|
||
for (s = table->listeners; s != NULL; s = s->ht_next)
|
||
{
|
||
if (s->dport == dport && s->status == NG_TCP_STATUS_LISTEN)
|
||
return s;
|
||
}
|
||
return NULL;
|
||
}
|
||
|
||
void ng_tcp_hash_insert(struct ng_tcp_stream *stream)
|
||
{
|
||
uint32_t h = tcp_hash4(stream->sip, stream->dip,
|
||
stream->sport, stream->dport);
|
||
struct ng_tcp_table *table = tcpInstance();
|
||
stream->ht_next = table->htable[h];
|
||
table->htable[h] = stream;
|
||
table->count++;
|
||
}
|
||
|
||
void ng_tcp_hash_remove(struct ng_tcp_stream *stream)
|
||
{
|
||
uint32_t h = tcp_hash4(stream->sip, stream->dip,
|
||
stream->sport, stream->dport);
|
||
struct ng_tcp_table *table = tcpInstance();
|
||
struct ng_tcp_stream **pp;
|
||
|
||
for (pp = &table->htable[h]; *pp != NULL; pp = &(*pp)->ht_next)
|
||
{
|
||
if (*pp == stream)
|
||
{
|
||
*pp = stream->ht_next;
|
||
table->count--;
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Stream create / destroy
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
struct ng_tcp_stream *ng_tcp_stream_create(uint32_t sip, uint32_t dip,
|
||
uint16_t sport, uint16_t dport)
|
||
{
|
||
struct ng_tcp_stream *s = rte_malloc("ng_tcp_stream",
|
||
sizeof(struct ng_tcp_stream), 0);
|
||
if (s == NULL)
|
||
return NULL;
|
||
memset(s, 0, sizeof(struct ng_tcp_stream));
|
||
|
||
s->sip = sip;
|
||
s->dip = dip;
|
||
s->sport = sport;
|
||
s->dport = dport;
|
||
s->protocol = IPPROTO_TCP;
|
||
s->fd = -1;
|
||
s->status = NG_TCP_STATUS_CLOSED;
|
||
|
||
/* Sequence numbers (random ISN) */
|
||
uint32_t seed = time(NULL);
|
||
s->iss = rand_r(&seed) % TCP_MAX_SEQ;
|
||
s->snd_una = s->iss;
|
||
s->snd_nxt = s->iss;
|
||
s->rcv_nxt = 0;
|
||
|
||
/* Congestion control defaults */
|
||
s->cwnd = TCP_INITIAL_CWND;
|
||
s->ssthresh = TCP_INITIAL_WINDOW * 128; /* effectively unbounded */
|
||
s->mss = TCP_MSS_DEFAULT;
|
||
s->rcv_mss = TCP_MSS_DEFAULT;
|
||
|
||
/* Timers */
|
||
s->rto = TCP_RTO_INITIAL;
|
||
s->srtt = 0;
|
||
s->rttvar = 0;
|
||
|
||
/* Send buffer */
|
||
s->snd_buf = rte_malloc("snd_buf", TCP_SENDBUF_SIZE, 0);
|
||
if (s->snd_buf == NULL)
|
||
{
|
||
rte_free(s);
|
||
return NULL;
|
||
}
|
||
memset(s->snd_buf, 0, TCP_SENDBUF_SIZE);
|
||
|
||
/* Receive buffer accounting */
|
||
s->rcv_buf_max = TCP_RECVBUF_SIZE;
|
||
s->rcv_buf_avail = TCP_RECVBUF_SIZE;
|
||
s->rcv_wnd = TCP_INITIAL_WINDOW;
|
||
|
||
/* Rings */
|
||
s->sndbuf = rte_ring_create("tcp_snd", RING_SIZE,
|
||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||
s->rcvbuf = rte_ring_create("tcp_rcv", RING_SIZE,
|
||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||
s->retransmit_ring = rte_ring_create("tcp_rtx", RING_SIZE,
|
||
rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ);
|
||
|
||
/* Condition variable */
|
||
pthread_condattr_t cattr;
|
||
pthread_condattr_init(&cattr);
|
||
pthread_cond_init(&s->cond, &cattr);
|
||
pthread_condattr_destroy(&cattr);
|
||
|
||
pthread_mutexattr_t mattr;
|
||
pthread_mutexattr_init(&mattr);
|
||
pthread_mutex_init(&s->mutex, &mattr);
|
||
pthread_mutexattr_destroy(&mattr);
|
||
|
||
rte_memcpy(s->localmac, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||
|
||
/* Insert into hash table */
|
||
ng_tcp_hash_insert(s);
|
||
|
||
/* Defer active-list add until state is set (caller does ng_tcp_stream_activate) */
|
||
|
||
return s;
|
||
}
|
||
|
||
/* forward declarations for active list helpers */
|
||
static void ng_tcp_active_add(struct ng_tcp_stream *s);
|
||
static void ng_tcp_active_remove(struct ng_tcp_stream *s);
|
||
static int ng_tcp_is_active(struct ng_tcp_stream *s);
|
||
|
||
void ng_tcp_stream_destroy(struct ng_tcp_stream *stream)
|
||
{
|
||
if (stream == NULL)
|
||
return;
|
||
|
||
/* Remove from active list */
|
||
ng_tcp_active_remove(stream);
|
||
|
||
/* Remove from hash table */
|
||
ng_tcp_hash_remove(stream);
|
||
|
||
/* Clear fd mapping */
|
||
if (stream->fd >= 0)
|
||
ng_tcp_fd_clear(stream->fd);
|
||
|
||
/* Free out-of-order segments */
|
||
struct ofo_segment *ofo = stream->ofo_head;
|
||
while (ofo)
|
||
{
|
||
struct ofo_segment *next = ofo->next;
|
||
ofo_free(ofo);
|
||
ofo = next;
|
||
}
|
||
|
||
/* Free rings */
|
||
if (stream->sndbuf)
|
||
rte_ring_free(stream->sndbuf);
|
||
if (stream->rcvbuf)
|
||
rte_ring_free(stream->rcvbuf);
|
||
if (stream->retransmit_ring)
|
||
rte_ring_free(stream->retransmit_ring);
|
||
|
||
/* Free send buffer */
|
||
if (stream->snd_buf)
|
||
rte_free(stream->snd_buf);
|
||
|
||
/* Destroy synchronization primitives */
|
||
pthread_mutex_destroy(&stream->mutex);
|
||
pthread_cond_destroy(&stream->cond);
|
||
|
||
rte_free(stream);
|
||
}
|
||
|
||
void ng_tcp_stream_set_status(struct ng_tcp_stream *stream, NG_TCP_STATUS s)
|
||
{
|
||
stream->status = s;
|
||
}
|
||
|
||
/* Active list: track connections that need timer check + output processing */
|
||
static void ng_tcp_active_add(struct ng_tcp_stream *s)
|
||
{
|
||
struct ng_tcp_table *t = tcpInstance();
|
||
if (s->active_next || s->active_prev || t->active_head == s) return;
|
||
s->active_next = t->active_head;
|
||
s->active_prev = NULL;
|
||
if (t->active_head) t->active_head->active_prev = s;
|
||
t->active_head = s;
|
||
}
|
||
|
||
static void ng_tcp_active_remove(struct ng_tcp_stream *s)
|
||
{
|
||
struct ng_tcp_table *t = tcpInstance();
|
||
if (s->active_prev) s->active_prev->active_next = s->active_next;
|
||
if (s->active_next) s->active_next->active_prev = s->active_prev;
|
||
if (t->active_head == s) t->active_head = s->active_next;
|
||
s->active_next = s->active_prev = NULL;
|
||
}
|
||
|
||
static int ng_tcp_is_active(struct ng_tcp_stream *s)
|
||
{
|
||
switch (s->status) {
|
||
case NG_TCP_STATUS_SYN_SENT:
|
||
case NG_TCP_STATUS_SYN_RCVD:
|
||
case NG_TCP_STATUS_ESTABLISHED:
|
||
case NG_TCP_STATUS_CLOSE_WAIT:
|
||
case NG_TCP_STATUS_FIN_WAIT_1:
|
||
case NG_TCP_STATUS_FIN_WAIT_2:
|
||
case NG_TCP_STATUS_LAST_ACK:
|
||
case NG_TCP_STATUS_TIME_WAIT: /* needed for timer-based cleanup */
|
||
return 1;
|
||
default:
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/* Call when TCB enters active state (after nconnect or accept) */
|
||
void ng_tcp_stream_activate(struct ng_tcp_stream *s)
|
||
{
|
||
if (ng_tcp_is_active(s))
|
||
ng_tcp_active_add(s);
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* TCP ACK helper (pure ACK — no data, no options for now)
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
void ng_tcp_send_ack(struct ng_tcp_stream *stream)
|
||
{
|
||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||
if (f == NULL)
|
||
return;
|
||
|
||
f->sport = stream->sport;
|
||
f->dport = stream->dport;
|
||
f->seqnum = stream->snd_nxt;
|
||
f->acknum = stream->rcv_nxt;
|
||
f->tcp_flags = RTE_TCP_ACK_FLAG;
|
||
f->windows = (stream->rcv_buf_avail > 65535) ? 65535
|
||
: (uint16_t)stream->rcv_buf_avail;
|
||
f->hdrlen_off = 0x50;
|
||
|
||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||
}
|
||
|
||
void ng_tcp_send_rst(struct ng_tcp_stream *stream, uint32_t seq, uint32_t ack)
|
||
{
|
||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||
if (f == NULL)
|
||
return;
|
||
|
||
f->sport = stream->sport;
|
||
f->dport = stream->dport;
|
||
f->seqnum = seq;
|
||
f->acknum = 0;
|
||
f->tcp_flags = RTE_TCP_RST_FLAG;
|
||
f->windows = 0;
|
||
f->hdrlen_off = 0x50;
|
||
|
||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||
stream->stat_rst_sent++;
|
||
}
|
||
|
||
void ng_tcp_send_fin(struct ng_tcp_stream *stream)
|
||
{
|
||
struct ng_tcp_fragment *f = frag_alloc(0);
|
||
if (f == NULL)
|
||
return;
|
||
|
||
f->sport = stream->sport;
|
||
f->dport = stream->dport;
|
||
f->seqnum = stream->snd_nxt;
|
||
f->acknum = stream->rcv_nxt;
|
||
f->tcp_flags = RTE_TCP_FIN_FLAG | RTE_TCP_ACK_FLAG;
|
||
f->windows = (stream->rcv_buf_avail > 65535) ? 65535
|
||
: (uint16_t)stream->rcv_buf_avail;
|
||
f->hdrlen_off = 0x50;
|
||
|
||
rte_ring_mp_enqueue(stream->sndbuf, f);
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* TCP packet encoding (with option bytes in fragment->option[])
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
static int ng_encode_tcp_pkt(uint8_t *msg, uint32_t sip, uint32_t dip,
|
||
uint8_t *srcmac, uint8_t *dstmac,
|
||
struct ng_tcp_fragment *fragment)
|
||
{
|
||
int opt_bytes = fragment->optlen * sizeof(uint32_t);
|
||
unsigned total_len = fragment->length +
|
||
sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) +
|
||
sizeof(struct rte_tcp_hdr) + opt_bytes;
|
||
|
||
/* Ethernet */
|
||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||
rte_memcpy(eth->src_addr.addr_bytes, srcmac, RTE_ETHER_ADDR_LEN);
|
||
rte_memcpy(eth->dst_addr.addr_bytes, dstmac, RTE_ETHER_ADDR_LEN);
|
||
eth->ether_type = htons(RTE_ETHER_TYPE_IPV4);
|
||
|
||
/* IP */
|
||
struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1);
|
||
ip->version_ihl = 0x45;
|
||
ip->type_of_service = 0;
|
||
ip->total_length = htons(total_len - sizeof(struct rte_ether_hdr));
|
||
ip->packet_id = 0;
|
||
ip->fragment_offset = 0;
|
||
ip->time_to_live = 64;
|
||
ip->next_proto_id = IPPROTO_TCP;
|
||
ip->src_addr = sip;
|
||
ip->dst_addr = dip;
|
||
ip->hdr_checksum = 0;
|
||
ip->hdr_checksum = rte_ipv4_cksum(ip);
|
||
|
||
/* TCP */
|
||
struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *)(ip + 1);
|
||
tcp->src_port = fragment->sport;
|
||
tcp->dst_port = fragment->dport;
|
||
tcp->sent_seq = htonl(fragment->seqnum);
|
||
tcp->recv_ack = htonl(fragment->acknum);
|
||
tcp->data_off = fragment->hdrlen_off; /* already includes options */
|
||
tcp->rx_win = fragment->windows;
|
||
tcp->tcp_urp = fragment->tcp_urp;
|
||
tcp->tcp_flags = fragment->tcp_flags;
|
||
|
||
/* TCP options */
|
||
if (opt_bytes > 0)
|
||
{
|
||
uint8_t *optptr = (uint8_t *)(tcp + 1);
|
||
uint8_t *raw = (uint8_t *)fragment->option;
|
||
rte_memcpy(optptr, raw, opt_bytes);
|
||
}
|
||
|
||
/* Payload */
|
||
if (fragment->length > 0)
|
||
{
|
||
uint8_t *payload = (uint8_t *)(tcp + 1) + opt_bytes;
|
||
rte_memcpy(payload, fragment->data, fragment->length);
|
||
}
|
||
|
||
/* TCP checksum */
|
||
tcp->cksum = 0;
|
||
tcp->cksum = rte_ipv4_udptcp_cksum(ip, tcp);
|
||
|
||
return 0;
|
||
}
|
||
|
||
struct rte_mbuf *ng_tcp_pkt(struct rte_mempool *mbuf_pool,
|
||
uint32_t sip, uint32_t dip,
|
||
uint8_t *srcmac, uint8_t *dstmac,
|
||
struct ng_tcp_fragment *fragment)
|
||
{
|
||
int opt_bytes = fragment->optlen * sizeof(uint32_t);
|
||
unsigned total_len = fragment->length +
|
||
sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) +
|
||
sizeof(struct rte_tcp_hdr) + opt_bytes;
|
||
|
||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||
if (!mbuf)
|
||
return NULL;
|
||
|
||
mbuf->pkt_len = total_len;
|
||
mbuf->data_len = total_len;
|
||
|
||
uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||
ng_encode_tcp_pkt(pktdata, sip, dip, srcmac, dstmac, fragment);
|
||
|
||
return mbuf;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Note: ng_tcp_out() removed. Output is handled in pkt_process()
|
||
* via active_head linked list (Phase 2+3), avoiding O(256) scan.
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* ARP / ICMP packet helpers
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
static int ng_encode_arp_pkt(uint8_t *msg, uint16_t opcode,
|
||
uint8_t *dst_mac, uint32_t sip, uint32_t dip)
|
||
{
|
||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||
rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||
rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||
eth->ether_type = htons(RTE_ETHER_TYPE_ARP);
|
||
|
||
struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1);
|
||
arp->arp_hardware = htons(1);
|
||
arp->arp_protocol = htons(RTE_ETHER_TYPE_IPV4);
|
||
arp->arp_hlen = RTE_ETHER_ADDR_LEN;
|
||
arp->arp_plen = sizeof(uint32_t);
|
||
arp->arp_opcode = htons(opcode);
|
||
|
||
rte_memcpy(arp->arp_data.arp_sha.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||
rte_memcpy(arp->arp_data.arp_tha.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||
arp->arp_data.arp_sip = sip;
|
||
arp->arp_data.arp_tip = dip;
|
||
|
||
return 0;
|
||
}
|
||
|
||
struct rte_mbuf *ng_send_arp(struct rte_mempool *mbuf_pool,
|
||
uint16_t opcode, uint8_t *dst_mac,
|
||
uint32_t sip, uint32_t dip)
|
||
{
|
||
const unsigned total_len = sizeof(struct rte_ether_hdr) +
|
||
sizeof(struct rte_arp_hdr);
|
||
|
||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||
if (!mbuf)
|
||
return NULL;
|
||
|
||
mbuf->pkt_len = total_len;
|
||
mbuf->data_len = total_len;
|
||
|
||
uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||
ng_encode_arp_pkt(pktdata, opcode, dst_mac, sip, dip);
|
||
|
||
return mbuf;
|
||
}
|
||
|
||
static struct rte_mbuf *ng_send_icmp(struct rte_mempool *mbuf_pool,
|
||
uint8_t *dst_mac, uint32_t sip, uint32_t dip,
|
||
uint16_t id, uint16_t seq)
|
||
{
|
||
const unsigned total_len = sizeof(struct rte_ether_hdr) +
|
||
sizeof(struct rte_ipv4_hdr) +
|
||
sizeof(struct rte_icmp_hdr);
|
||
|
||
struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool);
|
||
if (!mbuf)
|
||
return NULL;
|
||
|
||
mbuf->pkt_len = total_len;
|
||
mbuf->data_len = total_len;
|
||
|
||
uint8_t *msg = rte_pktmbuf_mtod(mbuf, uint8_t *);
|
||
struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg;
|
||
rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN);
|
||
rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN);
|
||
eth->ether_type = htons(RTE_ETHER_TYPE_IPV4);
|
||
|
||
struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1);
|
||
ip->version_ihl = 0x45;
|
||
ip->type_of_service = 0;
|
||
ip->total_length = htons(sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_icmp_hdr));
|
||
ip->packet_id = 0;
|
||
ip->fragment_offset = 0;
|
||
ip->time_to_live = 64;
|
||
ip->next_proto_id = IPPROTO_ICMP;
|
||
ip->src_addr = sip;
|
||
ip->dst_addr = dip;
|
||
ip->hdr_checksum = 0;
|
||
ip->hdr_checksum = rte_ipv4_cksum(ip);
|
||
|
||
struct rte_icmp_hdr *icmp = (struct rte_icmp_hdr *)(ip + 1);
|
||
icmp->icmp_type = RTE_ICMP_TYPE_ECHO_REPLY;
|
||
icmp->icmp_code = 0;
|
||
icmp->icmp_ident = id;
|
||
icmp->icmp_seq_nb = seq;
|
||
icmp->icmp_cksum = 0;
|
||
icmp->icmp_cksum = rte_ipv4_udptcp_cksum(ip, (const void *)icmp);
|
||
|
||
return mbuf;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* ng_tcp_process — main incoming packet dispatch
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
int ng_tcp_process(struct rte_mbuf *tcpmbuf)
|
||
{
|
||
struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(tcpmbuf,
|
||
struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
|
||
struct rte_tcp_hdr *tcphdr = (struct rte_tcp_hdr *)(iphdr + 1);
|
||
|
||
int tcplen = ntohs(iphdr->total_length) - sizeof(struct rte_ipv4_hdr);
|
||
if (tcplen < (int)sizeof(struct rte_tcp_hdr))
|
||
return 0; /* truncated */
|
||
|
||
/* Checksum validation */
|
||
uint16_t cksum = rte_ipv4_udptcp_cksum(iphdr, (const void *)tcphdr);
|
||
if (cksum != 0 && cksum != 0xFFFF)
|
||
return 0; /* bad checksum — drop silently */
|
||
|
||
uint32_t sip = iphdr->src_addr;
|
||
uint32_t dip = iphdr->dst_addr;
|
||
uint16_t sport = tcphdr->src_port;
|
||
uint16_t dport = tcphdr->dst_port;
|
||
uint8_t flags = tcphdr->tcp_flags;
|
||
|
||
/* Look up existing stream */
|
||
struct ng_tcp_stream *stream = ng_tcp_stream_search(sip, dip, sport, dport);
|
||
|
||
if (stream == NULL)
|
||
{
|
||
/* Try listener on SYN */
|
||
if (flags & RTE_TCP_SYN_FLAG)
|
||
{
|
||
struct ng_tcp_stream *listener = ng_tcp_stream_search_listener(dport);
|
||
if (listener)
|
||
{
|
||
ng_tcp_handle_listen(listener, tcphdr, iphdr, tcplen);
|
||
}
|
||
}
|
||
/* If RST and no stream, ignore */
|
||
return 0;
|
||
}
|
||
|
||
/* Update last receive timestamp */
|
||
stream->last_rcv_tsc = rte_get_timer_cycles();
|
||
|
||
/* Parse TCP options */
|
||
uint8_t hdrlen = (tcphdr->data_off & 0xF0) >> 2;
|
||
int optlen = hdrlen - sizeof(struct rte_tcp_hdr);
|
||
struct tcp_options opts;
|
||
memset(&opts, 0, sizeof(opts));
|
||
|
||
if (optlen > 0)
|
||
{
|
||
uint8_t *optptr = (uint8_t *)(tcphdr + 1);
|
||
ng_tcp_parse_options(optptr, optlen, &opts);
|
||
}
|
||
|
||
/* Dispatch by state */
|
||
switch (stream->status)
|
||
{
|
||
case NG_TCP_STATUS_LISTEN:
|
||
ng_tcp_handle_listen(stream, tcphdr, iphdr, tcplen);
|
||
break;
|
||
|
||
case NG_TCP_STATUS_SYN_SENT:
|
||
ng_tcp_handle_synsent(stream, tcphdr, iphdr, tcplen);
|
||
break;
|
||
|
||
case NG_TCP_STATUS_SYN_RCVD:
|
||
ng_tcp_handle_synrcvd(stream, tcphdr, iphdr, tcplen);
|
||
break;
|
||
|
||
case NG_TCP_STATUS_ESTABLISHED:
|
||
case NG_TCP_STATUS_FIN_WAIT_1:
|
||
case NG_TCP_STATUS_FIN_WAIT_2:
|
||
case NG_TCP_STATUS_CLOSING:
|
||
case NG_TCP_STATUS_TIME_WAIT:
|
||
case NG_TCP_STATUS_CLOSE_WAIT:
|
||
case NG_TCP_STATUS_LAST_ACK:
|
||
ng_tcp_input(stream, tcphdr, iphdr, tcplen);
|
||
break;
|
||
|
||
default:
|
||
break;
|
||
}
|
||
|
||
stream->stat_seg_rcvd++;
|
||
return 0;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* pkt_process — protocol processing loop (lcore 1..N)
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
int pkt_process(void *arg)
|
||
{
|
||
struct sched_param sp = { .sched_priority = 90 };
|
||
sched_setscheduler(0, SCHED_FIFO, &sp);
|
||
struct rte_mempool *mbuf_pool = (struct rte_mempool *)arg;
|
||
struct inout_ring *ring = ringInstance();
|
||
|
||
|
||
while (1)
|
||
{
|
||
|
||
/* ── Phase 1: RX processing ── */
|
||
struct rte_mbuf *mbufs[BURST_SIZE];
|
||
unsigned num_recvd = rte_ring_mc_dequeue_burst(ring->in,
|
||
(void **)mbufs, BURST_SIZE, NULL);
|
||
|
||
for (unsigned i = 0; i < num_recvd; i++)
|
||
{
|
||
struct rte_ether_hdr *ehdr = rte_pktmbuf_mtod(mbufs[i],
|
||
struct rte_ether_hdr *);
|
||
|
||
/* ── ARP ── */
|
||
if (ehdr->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
|
||
{
|
||
struct rte_arp_hdr *ahdr = rte_pktmbuf_mtod_offset(mbufs[i],
|
||
struct rte_arp_hdr *, sizeof(struct rte_ether_hdr));
|
||
|
||
if (ahdr->arp_data.arp_tip == gLocalIp)
|
||
{
|
||
if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REQUEST))
|
||
{
|
||
struct rte_mbuf *ab = ng_send_arp(mbuf_pool,
|
||
RTE_ARP_OP_REPLY, ahdr->arp_data.arp_sha.addr_bytes,
|
||
ahdr->arp_data.arp_tip, ahdr->arp_data.arp_sip);
|
||
if (ab)
|
||
rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL);
|
||
}
|
||
else if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REPLY))
|
||
{
|
||
uint8_t *hw = ng_get_dst_macaddr(ahdr->arp_data.arp_sip);
|
||
if (hw == NULL)
|
||
ng_arp_table_insert(ahdr->arp_data.arp_sip,
|
||
ahdr->arp_data.arp_sha.addr_bytes,
|
||
ARP_ENTRY_STATUS_DYNAMIC);
|
||
}
|
||
}
|
||
rte_pktmbuf_free(mbufs[i]);
|
||
continue;
|
||
}
|
||
|
||
if (ehdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
|
||
continue;
|
||
|
||
struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(mbufs[i],
|
||
struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr));
|
||
|
||
/* TCP */
|
||
if (iphdr->next_proto_id == IPPROTO_TCP)
|
||
{
|
||
lat_trace_add(2); /* pkt_process picked up TCP packet */
|
||
ng_tcp_process(mbufs[i]);
|
||
}
|
||
|
||
/* ICMP echo */
|
||
if (iphdr->next_proto_id == IPPROTO_ICMP)
|
||
{
|
||
struct rte_icmp_hdr *icmphdr = (struct rte_icmp_hdr *)(iphdr + 1);
|
||
if (icmphdr->icmp_type == RTE_ICMP_TYPE_ECHO_REQUEST)
|
||
{
|
||
struct rte_mbuf *txbuf = ng_send_icmp(mbuf_pool,
|
||
ehdr->src_addr.addr_bytes,
|
||
iphdr->dst_addr, iphdr->src_addr,
|
||
icmphdr->icmp_ident, icmphdr->icmp_seq_nb);
|
||
if (txbuf)
|
||
rte_ring_mp_enqueue_burst(ring->out, (void **)&txbuf, 1, NULL);
|
||
rte_pktmbuf_free(mbufs[i]);
|
||
}
|
||
}
|
||
}
|
||
|
||
/* ── Phase 2+3: Timer + Output (only iterate active connections) ── */
|
||
{
|
||
uint64_t now = rte_get_timer_cycles();
|
||
struct ng_tcp_table *table = tcpInstance();
|
||
struct ng_tcp_stream *s = table->active_head;
|
||
while (s)
|
||
{
|
||
struct ng_tcp_stream *next = s->active_next;
|
||
|
||
/* Timer check */
|
||
ng_tcp_check_timers(s);
|
||
if (s->status == NG_TCP_STATUS_TIME_WAIT)
|
||
{
|
||
uint64_t elapsed_us = (now - s->timewait_start) * 1000000 / rte_get_timer_hz();
|
||
if (elapsed_us >= TCP_TIME_WAIT_USEC)
|
||
{
|
||
ng_tcp_stream_destroy(s);
|
||
s = next;
|
||
continue;
|
||
}
|
||
}
|
||
|
||
/* Output: send pending fragments for this connection */
|
||
struct ng_tcp_fragment *f = NULL;
|
||
if (rte_ring_mc_dequeue(s->sndbuf, (void **)&f) >= 0 && f)
|
||
{
|
||
uint8_t *dstmac = s->dst_mac_valid ? s->dst_mac
|
||
: ng_get_dst_macaddr(s->dip);
|
||
if (dstmac == NULL)
|
||
{
|
||
uint64_t ts = rte_get_timer_cycles();
|
||
uint64_t arp_int = rte_get_timer_hz() / 10;
|
||
if (s->last_arp_tsc == 0 || ts - s->last_arp_tsc > arp_int) {
|
||
s->last_arp_tsc = ts;
|
||
struct rte_mbuf *ab = ng_send_arp(mbuf_pool,
|
||
RTE_ARP_OP_REQUEST, gDefaultArpMac, s->sip, s->dip);
|
||
if (ab) rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL);
|
||
}
|
||
rte_ring_mp_enqueue(s->sndbuf, f);
|
||
}
|
||
else
|
||
{
|
||
struct rte_mbuf *mbuf = ng_tcp_pkt(mbuf_pool,
|
||
s->sip, s->dip, s->localmac, dstmac, f);
|
||
if (mbuf) {
|
||
rte_ring_mp_enqueue_burst(ring->out, (void **)&mbuf, 1, NULL);
|
||
s->stat_seg_sent++;
|
||
}
|
||
if (f->length > 0)
|
||
rte_ring_mp_enqueue(s->retransmit_ring, f);
|
||
else
|
||
frag_free(f);
|
||
}
|
||
}
|
||
|
||
/* Remove from active list if no longer active */
|
||
if (!ng_tcp_is_active(s))
|
||
ng_tcp_active_remove(s);
|
||
|
||
s = next;
|
||
}
|
||
}
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Fragment mempool initialization
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
void ng_frag_pool_init(void)
|
||
{
|
||
unsigned socket_id = rte_socket_id();
|
||
|
||
/* Fragment pool: struct + max MSS payload (MP/MC safe, lock-free) */
|
||
char name_frag[64];
|
||
snprintf(name_frag, sizeof(name_frag), "frag_pool_%u", socket_id);
|
||
g_frag_pool = rte_mempool_create(name_frag,
|
||
FRAG_POOL_SIZE, FRAG_ELT_SIZE, 256, 0,
|
||
NULL, NULL, NULL, NULL, socket_id, 0);
|
||
if (g_frag_pool == NULL)
|
||
rte_exit(EXIT_FAILURE, "frag_pool create failed\n");
|
||
|
||
/* OFO segment pool */
|
||
char name_ofo[64];
|
||
snprintf(name_ofo, sizeof(name_ofo), "ofo_pool_%u", socket_id);
|
||
g_ofo_pool = rte_mempool_create(name_ofo,
|
||
OFO_POOL_SIZE, OFO_ELT_SIZE, 64, 0,
|
||
NULL, NULL, NULL, NULL, socket_id, 0);
|
||
if (g_ofo_pool == NULL)
|
||
rte_exit(EXIT_FAILURE, "ofo_pool create failed\n");
|
||
|
||
printf("[POOL] Fragment pool: %u elts × %u bytes, OFO pool: %u elts × %u bytes\n",
|
||
FRAG_POOL_SIZE, FRAG_ELT_SIZE, OFO_POOL_SIZE, OFO_ELT_SIZE);
|
||
}
|
||
|
||
/* ══════════════════════════════════════════════════════════════════
|
||
* Port initialization
|
||
* ══════════════════════════════════════════════════════════════════ */
|
||
|
||
void ng_init_port(struct rte_mempool *mbuf_pool)
|
||
{
|
||
const int num_rx_queues = 1;
|
||
const int num_tx_queues = 1;
|
||
|
||
struct rte_eth_conf port_conf = {
|
||
.rxmode = {.mtu = RTE_ETHER_MTU},
|
||
};
|
||
|
||
if (rte_eth_dev_configure(gDpdkPortId, num_rx_queues, num_tx_queues,
|
||
&port_conf) < 0)
|
||
rte_exit(EXIT_FAILURE, "Cannot configure port=%d\n", gDpdkPortId);
|
||
|
||
if (rte_eth_rx_queue_setup(gDpdkPortId, 0, 128,
|
||
rte_eth_dev_socket_id(gDpdkPortId),
|
||
NULL, mbuf_pool) < 0)
|
||
rte_exit(EXIT_FAILURE, "rx_queue_setup failed\n");
|
||
|
||
if (rte_eth_tx_queue_setup(gDpdkPortId, 0, 512,
|
||
rte_eth_dev_socket_id(gDpdkPortId),
|
||
NULL) < 0)
|
||
rte_exit(EXIT_FAILURE, "tx_queue_setup failed\n");
|
||
|
||
if (rte_eth_dev_start(gDpdkPortId) < 0)
|
||
rte_exit(EXIT_FAILURE, "dev_start failed\n");
|
||
|
||
rte_eth_promiscuous_enable(gDpdkPortId);
|
||
|
||
printf("[DPDK] Port %d initialized\n", gDpdkPortId);
|
||
}
|