/** * tcp_core.c — Production-grade DPDK TCP/IP stack core * * Contains: * - Hash-table-based TCB management (O(1) lookup) * - Enhanced TCP packet encoding (with option bytes) * - ARP / ICMP helpers * - pkt_process loop with per-TCB timer maintenance * - Port initialization */ #include "tcp_core.h" #include /* ══════════════════════════════════════════════════════════════════ * Globals * ══════════════════════════════════════════════════════════════════ */ int gDpdkPortId = 0; uint32_t gLocalIp = MAKE_IPV4_ADDR(192, 168, 0, 115); uint8_t gSrcMac[RTE_ETHER_ADDR_LEN]; uint8_t gDefaultArpMac[RTE_ETHER_ADDR_LEN] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; /* Latency trace buffer */ struct lat_trace g_lat_trace[LAT_TRACE_SIZE]; int g_lat_trace_idx = 0; static struct ng_tcp_table *tInst = NULL; static struct inout_ring *rInst = NULL; /* Fragment mempools (replaces rte_malloc for hot-path allocations) */ struct rte_mempool *g_frag_pool = NULL; struct rte_mempool *g_ofo_pool = NULL; /* fd → TCB index array (O(1) lookup from socket fd) */ static struct ng_tcp_stream *fd_to_tcb[MAX_FD_ARRAY]; /* ══════════════════════════════════════════════════════════════════ * TCP table singleton * ══════════════════════════════════════════════════════════════════ */ struct ng_tcp_table *tcpInstance(void) { if (tInst == NULL) { tInst = rte_malloc("ng_tcp_table", sizeof(struct ng_tcp_table), 0); if (tInst == NULL) rte_exit(EXIT_FAILURE, "tcpInstance malloc failed\n"); memset(tInst, 0, sizeof(struct ng_tcp_table)); } return tInst; } struct inout_ring *ringInstance(void) { if (rInst == NULL) { rInst = rte_malloc("inout_ring", sizeof(struct inout_ring), 0); memset(rInst, 0, sizeof(struct inout_ring)); } return rInst; } /* ══════════════════════════════════════════════════════════════════ * FD ↔ TCB index (O(1)) * ══════════════════════════════════════════════════════════════════ */ void ng_tcp_fd_set(int fd, struct ng_tcp_stream *s) { if (fd >= 0 && fd < MAX_FD_ARRAY) fd_to_tcb[fd] = s; } void ng_tcp_fd_clear(int fd) { if (fd >= 0 && fd < MAX_FD_ARRAY) fd_to_tcb[fd] = NULL; } struct ng_tcp_stream *ng_tcp_fd_get(int fd) { if (fd >= 0 && fd < MAX_FD_ARRAY) return fd_to_tcb[fd]; return NULL; } /* ══════════════════════════════════════════════════════════════════ * Hash table operations * ══════════════════════════════════════════════════════════════════ */ struct ng_tcp_stream *ng_tcp_stream_search(uint32_t sip, uint32_t dip, uint16_t sport, uint16_t dport) { uint32_t h_fwd = tcp_hash4(sip, dip, sport, dport); uint32_t h_rev = tcp_hash4(dip, sip, dport, sport); struct ng_tcp_table *table = tcpInstance(); struct ng_tcp_stream *s; /* Search forward hash bucket */ for (s = table->htable[h_fwd]; s != NULL; s = s->ht_next) { if ((s->sip == sip && s->dip == dip && s->sport == sport && s->dport == dport) || (s->sip == dip && s->dip == sip && s->sport == dport && s->dport == sport)) return s; } /* If reverse bucket is different, search it too */ if (h_rev != h_fwd) { for (s = table->htable[h_rev]; s != NULL; s = s->ht_next) { if ((s->sip == sip && s->dip == dip && s->sport == sport && s->dport == dport) || (s->sip == dip && s->dip == sip && s->sport == dport && s->dport == sport)) return s; } } return NULL; } struct ng_tcp_stream *ng_tcp_stream_search_listener(uint16_t dport) { struct ng_tcp_table *table = tcpInstance(); struct ng_tcp_stream *s; for (s = table->listeners; s != NULL; s = s->ht_next) { if (s->dport == dport && s->status == NG_TCP_STATUS_LISTEN) return s; } return NULL; } void ng_tcp_hash_insert(struct ng_tcp_stream *stream) { uint32_t h = tcp_hash4(stream->sip, stream->dip, stream->sport, stream->dport); struct ng_tcp_table *table = tcpInstance(); stream->ht_next = table->htable[h]; table->htable[h] = stream; table->count++; } void ng_tcp_hash_remove(struct ng_tcp_stream *stream) { uint32_t h = tcp_hash4(stream->sip, stream->dip, stream->sport, stream->dport); struct ng_tcp_table *table = tcpInstance(); struct ng_tcp_stream **pp; for (pp = &table->htable[h]; *pp != NULL; pp = &(*pp)->ht_next) { if (*pp == stream) { *pp = stream->ht_next; table->count--; return; } } } /* ══════════════════════════════════════════════════════════════════ * Stream create / destroy * ══════════════════════════════════════════════════════════════════ */ struct ng_tcp_stream *ng_tcp_stream_create(uint32_t sip, uint32_t dip, uint16_t sport, uint16_t dport) { struct ng_tcp_stream *s = rte_malloc("ng_tcp_stream", sizeof(struct ng_tcp_stream), 0); if (s == NULL) return NULL; memset(s, 0, sizeof(struct ng_tcp_stream)); s->sip = sip; s->dip = dip; s->sport = sport; s->dport = dport; s->protocol = IPPROTO_TCP; s->fd = -1; s->status = NG_TCP_STATUS_CLOSED; /* Sequence numbers (random ISN) */ uint32_t seed = time(NULL); s->iss = rand_r(&seed) % TCP_MAX_SEQ; s->snd_una = s->iss; s->snd_nxt = s->iss; s->rcv_nxt = 0; /* Congestion control defaults */ s->cwnd = TCP_INITIAL_CWND; s->ssthresh = TCP_INITIAL_WINDOW * 128; /* effectively unbounded */ s->mss = TCP_MSS_DEFAULT; s->rcv_mss = TCP_MSS_DEFAULT; /* Timers */ s->rto = TCP_RTO_INITIAL; s->srtt = 0; s->rttvar = 0; /* Send buffer */ s->snd_buf = rte_malloc("snd_buf", TCP_SENDBUF_SIZE, 0); if (s->snd_buf == NULL) { rte_free(s); return NULL; } memset(s->snd_buf, 0, TCP_SENDBUF_SIZE); /* Receive buffer accounting */ s->rcv_buf_max = TCP_RECVBUF_SIZE; s->rcv_buf_avail = TCP_RECVBUF_SIZE; s->rcv_wnd = TCP_INITIAL_WINDOW; /* Rings */ s->sndbuf = rte_ring_create("tcp_snd", RING_SIZE, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ); s->rcvbuf = rte_ring_create("tcp_rcv", RING_SIZE, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ); s->retransmit_ring = rte_ring_create("tcp_rtx", RING_SIZE, rte_socket_id(), RING_F_SP_ENQ | RING_F_SC_DEQ); /* Condition variable */ pthread_condattr_t cattr; pthread_condattr_init(&cattr); pthread_cond_init(&s->cond, &cattr); pthread_condattr_destroy(&cattr); pthread_mutexattr_t mattr; pthread_mutexattr_init(&mattr); pthread_mutex_init(&s->mutex, &mattr); pthread_mutexattr_destroy(&mattr); rte_memcpy(s->localmac, gSrcMac, RTE_ETHER_ADDR_LEN); /* Insert into hash table */ ng_tcp_hash_insert(s); /* Defer active-list add until state is set (caller does ng_tcp_stream_activate) */ return s; } /* forward declarations for active list helpers */ static void ng_tcp_active_add(struct ng_tcp_stream *s); static void ng_tcp_active_remove(struct ng_tcp_stream *s); static int ng_tcp_is_active(struct ng_tcp_stream *s); void ng_tcp_stream_destroy(struct ng_tcp_stream *stream) { if (stream == NULL) return; /* Remove from active list */ ng_tcp_active_remove(stream); /* Remove from hash table */ ng_tcp_hash_remove(stream); /* Clear fd mapping */ if (stream->fd >= 0) ng_tcp_fd_clear(stream->fd); /* Free out-of-order segments */ struct ofo_segment *ofo = stream->ofo_head; while (ofo) { struct ofo_segment *next = ofo->next; ofo_free(ofo); ofo = next; } /* Free rings */ if (stream->sndbuf) rte_ring_free(stream->sndbuf); if (stream->rcvbuf) rte_ring_free(stream->rcvbuf); if (stream->retransmit_ring) rte_ring_free(stream->retransmit_ring); /* Free send buffer */ if (stream->snd_buf) rte_free(stream->snd_buf); /* Destroy synchronization primitives */ pthread_mutex_destroy(&stream->mutex); pthread_cond_destroy(&stream->cond); rte_free(stream); } void ng_tcp_stream_set_status(struct ng_tcp_stream *stream, NG_TCP_STATUS s) { stream->status = s; } /* Active list: track connections that need timer check + output processing */ static void ng_tcp_active_add(struct ng_tcp_stream *s) { struct ng_tcp_table *t = tcpInstance(); if (s->active_next || s->active_prev || t->active_head == s) return; s->active_next = t->active_head; s->active_prev = NULL; if (t->active_head) t->active_head->active_prev = s; t->active_head = s; } static void ng_tcp_active_remove(struct ng_tcp_stream *s) { struct ng_tcp_table *t = tcpInstance(); if (s->active_prev) s->active_prev->active_next = s->active_next; if (s->active_next) s->active_next->active_prev = s->active_prev; if (t->active_head == s) t->active_head = s->active_next; s->active_next = s->active_prev = NULL; } static int ng_tcp_is_active(struct ng_tcp_stream *s) { switch (s->status) { case NG_TCP_STATUS_SYN_SENT: case NG_TCP_STATUS_SYN_RCVD: case NG_TCP_STATUS_ESTABLISHED: case NG_TCP_STATUS_CLOSE_WAIT: case NG_TCP_STATUS_FIN_WAIT_1: case NG_TCP_STATUS_FIN_WAIT_2: case NG_TCP_STATUS_LAST_ACK: case NG_TCP_STATUS_TIME_WAIT: /* needed for timer-based cleanup */ return 1; default: return 0; } } /* Call when TCB enters active state (after nconnect or accept) */ void ng_tcp_stream_activate(struct ng_tcp_stream *s) { if (ng_tcp_is_active(s)) ng_tcp_active_add(s); } /* ══════════════════════════════════════════════════════════════════ * TCP ACK helper (pure ACK — no data, no options for now) * ══════════════════════════════════════════════════════════════════ */ void ng_tcp_send_ack(struct ng_tcp_stream *stream) { struct ng_tcp_fragment *f = frag_alloc(0); if (f == NULL) return; f->sport = stream->sport; f->dport = stream->dport; f->seqnum = stream->snd_nxt; f->acknum = stream->rcv_nxt; f->tcp_flags = RTE_TCP_ACK_FLAG; f->windows = (stream->rcv_buf_avail > 65535) ? 65535 : (uint16_t)stream->rcv_buf_avail; f->hdrlen_off = 0x50; rte_ring_mp_enqueue(stream->sndbuf, f); } void ng_tcp_send_rst(struct ng_tcp_stream *stream, uint32_t seq, uint32_t ack) { struct ng_tcp_fragment *f = frag_alloc(0); if (f == NULL) return; f->sport = stream->sport; f->dport = stream->dport; f->seqnum = seq; f->acknum = 0; f->tcp_flags = RTE_TCP_RST_FLAG; f->windows = 0; f->hdrlen_off = 0x50; rte_ring_mp_enqueue(stream->sndbuf, f); stream->stat_rst_sent++; } void ng_tcp_send_fin(struct ng_tcp_stream *stream) { struct ng_tcp_fragment *f = frag_alloc(0); if (f == NULL) return; f->sport = stream->sport; f->dport = stream->dport; f->seqnum = stream->snd_nxt; f->acknum = stream->rcv_nxt; f->tcp_flags = RTE_TCP_FIN_FLAG | RTE_TCP_ACK_FLAG; f->windows = (stream->rcv_buf_avail > 65535) ? 65535 : (uint16_t)stream->rcv_buf_avail; f->hdrlen_off = 0x50; rte_ring_mp_enqueue(stream->sndbuf, f); } /* ══════════════════════════════════════════════════════════════════ * TCP packet encoding (with option bytes in fragment->option[]) * ══════════════════════════════════════════════════════════════════ */ static int ng_encode_tcp_pkt(uint8_t *msg, uint32_t sip, uint32_t dip, uint8_t *srcmac, uint8_t *dstmac, struct ng_tcp_fragment *fragment) { int opt_bytes = fragment->optlen * sizeof(uint32_t); unsigned total_len = fragment->length + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_tcp_hdr) + opt_bytes; /* Ethernet */ struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg; rte_memcpy(eth->src_addr.addr_bytes, srcmac, RTE_ETHER_ADDR_LEN); rte_memcpy(eth->dst_addr.addr_bytes, dstmac, RTE_ETHER_ADDR_LEN); eth->ether_type = htons(RTE_ETHER_TYPE_IPV4); /* IP */ struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1); ip->version_ihl = 0x45; ip->type_of_service = 0; ip->total_length = htons(total_len - sizeof(struct rte_ether_hdr)); ip->packet_id = 0; ip->fragment_offset = 0; ip->time_to_live = 64; ip->next_proto_id = IPPROTO_TCP; ip->src_addr = sip; ip->dst_addr = dip; ip->hdr_checksum = 0; ip->hdr_checksum = rte_ipv4_cksum(ip); /* TCP */ struct rte_tcp_hdr *tcp = (struct rte_tcp_hdr *)(ip + 1); tcp->src_port = fragment->sport; tcp->dst_port = fragment->dport; tcp->sent_seq = htonl(fragment->seqnum); tcp->recv_ack = htonl(fragment->acknum); tcp->data_off = fragment->hdrlen_off; /* already includes options */ tcp->rx_win = fragment->windows; tcp->tcp_urp = fragment->tcp_urp; tcp->tcp_flags = fragment->tcp_flags; /* TCP options */ if (opt_bytes > 0) { uint8_t *optptr = (uint8_t *)(tcp + 1); uint8_t *raw = (uint8_t *)fragment->option; rte_memcpy(optptr, raw, opt_bytes); } /* Payload */ if (fragment->length > 0) { uint8_t *payload = (uint8_t *)(tcp + 1) + opt_bytes; rte_memcpy(payload, fragment->data, fragment->length); } /* TCP checksum */ tcp->cksum = 0; tcp->cksum = rte_ipv4_udptcp_cksum(ip, tcp); return 0; } struct rte_mbuf *ng_tcp_pkt(struct rte_mempool *mbuf_pool, uint32_t sip, uint32_t dip, uint8_t *srcmac, uint8_t *dstmac, struct ng_tcp_fragment *fragment) { int opt_bytes = fragment->optlen * sizeof(uint32_t); unsigned total_len = fragment->length + sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_tcp_hdr) + opt_bytes; struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool); if (!mbuf) return NULL; mbuf->pkt_len = total_len; mbuf->data_len = total_len; uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *); ng_encode_tcp_pkt(pktdata, sip, dip, srcmac, dstmac, fragment); return mbuf; } /* ══════════════════════════════════════════════════════════════════ * Note: ng_tcp_out() removed. Output is handled in pkt_process() * via active_head linked list (Phase 2+3), avoiding O(256) scan. * ══════════════════════════════════════════════════════════════════ */ /* ══════════════════════════════════════════════════════════════════ * ARP / ICMP packet helpers * ══════════════════════════════════════════════════════════════════ */ static int ng_encode_arp_pkt(uint8_t *msg, uint16_t opcode, uint8_t *dst_mac, uint32_t sip, uint32_t dip) { struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg; rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN); rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN); eth->ether_type = htons(RTE_ETHER_TYPE_ARP); struct rte_arp_hdr *arp = (struct rte_arp_hdr *)(eth + 1); arp->arp_hardware = htons(1); arp->arp_protocol = htons(RTE_ETHER_TYPE_IPV4); arp->arp_hlen = RTE_ETHER_ADDR_LEN; arp->arp_plen = sizeof(uint32_t); arp->arp_opcode = htons(opcode); rte_memcpy(arp->arp_data.arp_sha.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN); rte_memcpy(arp->arp_data.arp_tha.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN); arp->arp_data.arp_sip = sip; arp->arp_data.arp_tip = dip; return 0; } struct rte_mbuf *ng_send_arp(struct rte_mempool *mbuf_pool, uint16_t opcode, uint8_t *dst_mac, uint32_t sip, uint32_t dip) { const unsigned total_len = sizeof(struct rte_ether_hdr) + sizeof(struct rte_arp_hdr); struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool); if (!mbuf) return NULL; mbuf->pkt_len = total_len; mbuf->data_len = total_len; uint8_t *pktdata = rte_pktmbuf_mtod(mbuf, uint8_t *); ng_encode_arp_pkt(pktdata, opcode, dst_mac, sip, dip); return mbuf; } static struct rte_mbuf *ng_send_icmp(struct rte_mempool *mbuf_pool, uint8_t *dst_mac, uint32_t sip, uint32_t dip, uint16_t id, uint16_t seq) { const unsigned total_len = sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_icmp_hdr); struct rte_mbuf *mbuf = rte_pktmbuf_alloc(mbuf_pool); if (!mbuf) return NULL; mbuf->pkt_len = total_len; mbuf->data_len = total_len; uint8_t *msg = rte_pktmbuf_mtod(mbuf, uint8_t *); struct rte_ether_hdr *eth = (struct rte_ether_hdr *)msg; rte_memcpy(eth->src_addr.addr_bytes, gSrcMac, RTE_ETHER_ADDR_LEN); rte_memcpy(eth->dst_addr.addr_bytes, dst_mac, RTE_ETHER_ADDR_LEN); eth->ether_type = htons(RTE_ETHER_TYPE_IPV4); struct rte_ipv4_hdr *ip = (struct rte_ipv4_hdr *)(eth + 1); ip->version_ihl = 0x45; ip->type_of_service = 0; ip->total_length = htons(sizeof(struct rte_ipv4_hdr) + sizeof(struct rte_icmp_hdr)); ip->packet_id = 0; ip->fragment_offset = 0; ip->time_to_live = 64; ip->next_proto_id = IPPROTO_ICMP; ip->src_addr = sip; ip->dst_addr = dip; ip->hdr_checksum = 0; ip->hdr_checksum = rte_ipv4_cksum(ip); struct rte_icmp_hdr *icmp = (struct rte_icmp_hdr *)(ip + 1); icmp->icmp_type = RTE_ICMP_TYPE_ECHO_REPLY; icmp->icmp_code = 0; icmp->icmp_ident = id; icmp->icmp_seq_nb = seq; icmp->icmp_cksum = 0; icmp->icmp_cksum = rte_ipv4_udptcp_cksum(ip, (const void *)icmp); return mbuf; } /* ══════════════════════════════════════════════════════════════════ * ng_tcp_process — main incoming packet dispatch * ══════════════════════════════════════════════════════════════════ */ int ng_tcp_process(struct rte_mbuf *tcpmbuf) { struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(tcpmbuf, struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); struct rte_tcp_hdr *tcphdr = (struct rte_tcp_hdr *)(iphdr + 1); int tcplen = ntohs(iphdr->total_length) - sizeof(struct rte_ipv4_hdr); if (tcplen < (int)sizeof(struct rte_tcp_hdr)) return 0; /* truncated */ /* Checksum validation */ uint16_t cksum = rte_ipv4_udptcp_cksum(iphdr, (const void *)tcphdr); if (cksum != 0 && cksum != 0xFFFF) return 0; /* bad checksum — drop silently */ uint32_t sip = iphdr->src_addr; uint32_t dip = iphdr->dst_addr; uint16_t sport = tcphdr->src_port; uint16_t dport = tcphdr->dst_port; uint8_t flags = tcphdr->tcp_flags; /* Look up existing stream */ struct ng_tcp_stream *stream = ng_tcp_stream_search(sip, dip, sport, dport); if (stream == NULL) { /* Try listener on SYN */ if (flags & RTE_TCP_SYN_FLAG) { struct ng_tcp_stream *listener = ng_tcp_stream_search_listener(dport); if (listener) { ng_tcp_handle_listen(listener, tcphdr, iphdr, tcplen); } } /* If RST and no stream, ignore */ return 0; } /* Update last receive timestamp */ stream->last_rcv_tsc = rte_get_timer_cycles(); /* Parse TCP options */ uint8_t hdrlen = (tcphdr->data_off & 0xF0) >> 2; int optlen = hdrlen - sizeof(struct rte_tcp_hdr); struct tcp_options opts; memset(&opts, 0, sizeof(opts)); if (optlen > 0) { uint8_t *optptr = (uint8_t *)(tcphdr + 1); ng_tcp_parse_options(optptr, optlen, &opts); } /* Dispatch by state */ switch (stream->status) { case NG_TCP_STATUS_LISTEN: ng_tcp_handle_listen(stream, tcphdr, iphdr, tcplen); break; case NG_TCP_STATUS_SYN_SENT: ng_tcp_handle_synsent(stream, tcphdr, iphdr, tcplen); break; case NG_TCP_STATUS_SYN_RCVD: ng_tcp_handle_synrcvd(stream, tcphdr, iphdr, tcplen); break; case NG_TCP_STATUS_ESTABLISHED: case NG_TCP_STATUS_FIN_WAIT_1: case NG_TCP_STATUS_FIN_WAIT_2: case NG_TCP_STATUS_CLOSING: case NG_TCP_STATUS_TIME_WAIT: case NG_TCP_STATUS_CLOSE_WAIT: case NG_TCP_STATUS_LAST_ACK: ng_tcp_input(stream, tcphdr, iphdr, tcplen); break; default: break; } stream->stat_seg_rcvd++; return 0; } /* ══════════════════════════════════════════════════════════════════ * pkt_process — protocol processing loop (lcore 1..N) * ══════════════════════════════════════════════════════════════════ */ int pkt_process(void *arg) { struct sched_param sp = { .sched_priority = 90 }; sched_setscheduler(0, SCHED_FIFO, &sp); struct rte_mempool *mbuf_pool = (struct rte_mempool *)arg; struct inout_ring *ring = ringInstance(); while (1) { /* ── Phase 1: RX processing ── */ struct rte_mbuf *mbufs[BURST_SIZE]; unsigned num_recvd = rte_ring_mc_dequeue_burst(ring->in, (void **)mbufs, BURST_SIZE, NULL); for (unsigned i = 0; i < num_recvd; i++) { struct rte_ether_hdr *ehdr = rte_pktmbuf_mtod(mbufs[i], struct rte_ether_hdr *); /* ── ARP ── */ if (ehdr->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) { struct rte_arp_hdr *ahdr = rte_pktmbuf_mtod_offset(mbufs[i], struct rte_arp_hdr *, sizeof(struct rte_ether_hdr)); if (ahdr->arp_data.arp_tip == gLocalIp) { if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REQUEST)) { struct rte_mbuf *ab = ng_send_arp(mbuf_pool, RTE_ARP_OP_REPLY, ahdr->arp_data.arp_sha.addr_bytes, ahdr->arp_data.arp_tip, ahdr->arp_data.arp_sip); if (ab) rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL); } else if (ahdr->arp_opcode == rte_cpu_to_be_16(RTE_ARP_OP_REPLY)) { uint8_t *hw = ng_get_dst_macaddr(ahdr->arp_data.arp_sip); if (hw == NULL) ng_arp_table_insert(ahdr->arp_data.arp_sip, ahdr->arp_data.arp_sha.addr_bytes, ARP_ENTRY_STATUS_DYNAMIC); } } rte_pktmbuf_free(mbufs[i]); continue; } if (ehdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) continue; struct rte_ipv4_hdr *iphdr = rte_pktmbuf_mtod_offset(mbufs[i], struct rte_ipv4_hdr *, sizeof(struct rte_ether_hdr)); /* TCP */ if (iphdr->next_proto_id == IPPROTO_TCP) { lat_trace_add(2); /* pkt_process picked up TCP packet */ ng_tcp_process(mbufs[i]); } /* ICMP echo */ if (iphdr->next_proto_id == IPPROTO_ICMP) { struct rte_icmp_hdr *icmphdr = (struct rte_icmp_hdr *)(iphdr + 1); if (icmphdr->icmp_type == RTE_ICMP_TYPE_ECHO_REQUEST) { struct rte_mbuf *txbuf = ng_send_icmp(mbuf_pool, ehdr->src_addr.addr_bytes, iphdr->dst_addr, iphdr->src_addr, icmphdr->icmp_ident, icmphdr->icmp_seq_nb); if (txbuf) rte_ring_mp_enqueue_burst(ring->out, (void **)&txbuf, 1, NULL); rte_pktmbuf_free(mbufs[i]); } } } /* ── Phase 2+3: Timer + Output (only iterate active connections) ── */ { uint64_t now = rte_get_timer_cycles(); struct ng_tcp_table *table = tcpInstance(); struct ng_tcp_stream *s = table->active_head; while (s) { struct ng_tcp_stream *next = s->active_next; /* Timer check */ ng_tcp_check_timers(s); if (s->status == NG_TCP_STATUS_TIME_WAIT) { uint64_t elapsed_us = (now - s->timewait_start) * 1000000 / rte_get_timer_hz(); if (elapsed_us >= TCP_TIME_WAIT_USEC) { ng_tcp_stream_destroy(s); s = next; continue; } } /* Output: send pending fragments for this connection */ struct ng_tcp_fragment *f = NULL; if (rte_ring_mc_dequeue(s->sndbuf, (void **)&f) >= 0 && f) { uint8_t *dstmac = s->dst_mac_valid ? s->dst_mac : ng_get_dst_macaddr(s->dip); if (dstmac == NULL) { uint64_t ts = rte_get_timer_cycles(); uint64_t arp_int = rte_get_timer_hz() / 10; if (s->last_arp_tsc == 0 || ts - s->last_arp_tsc > arp_int) { s->last_arp_tsc = ts; struct rte_mbuf *ab = ng_send_arp(mbuf_pool, RTE_ARP_OP_REQUEST, gDefaultArpMac, s->sip, s->dip); if (ab) rte_ring_mp_enqueue_burst(ring->out, (void **)&ab, 1, NULL); } rte_ring_mp_enqueue(s->sndbuf, f); } else { struct rte_mbuf *mbuf = ng_tcp_pkt(mbuf_pool, s->sip, s->dip, s->localmac, dstmac, f); if (mbuf) { rte_ring_mp_enqueue_burst(ring->out, (void **)&mbuf, 1, NULL); s->stat_seg_sent++; } if (f->length > 0) rte_ring_mp_enqueue(s->retransmit_ring, f); else frag_free(f); } } /* Remove from active list if no longer active */ if (!ng_tcp_is_active(s)) ng_tcp_active_remove(s); s = next; } } } return 0; } /* ══════════════════════════════════════════════════════════════════ * Fragment mempool initialization * ══════════════════════════════════════════════════════════════════ */ void ng_frag_pool_init(void) { unsigned socket_id = rte_socket_id(); /* Fragment pool: struct + max MSS payload (MP/MC safe, lock-free) */ char name_frag[64]; snprintf(name_frag, sizeof(name_frag), "frag_pool_%u", socket_id); g_frag_pool = rte_mempool_create(name_frag, FRAG_POOL_SIZE, FRAG_ELT_SIZE, 256, 0, NULL, NULL, NULL, NULL, socket_id, 0); if (g_frag_pool == NULL) rte_exit(EXIT_FAILURE, "frag_pool create failed\n"); /* OFO segment pool */ char name_ofo[64]; snprintf(name_ofo, sizeof(name_ofo), "ofo_pool_%u", socket_id); g_ofo_pool = rte_mempool_create(name_ofo, OFO_POOL_SIZE, OFO_ELT_SIZE, 64, 0, NULL, NULL, NULL, NULL, socket_id, 0); if (g_ofo_pool == NULL) rte_exit(EXIT_FAILURE, "ofo_pool create failed\n"); printf("[POOL] Fragment pool: %u elts × %u bytes, OFO pool: %u elts × %u bytes\n", FRAG_POOL_SIZE, FRAG_ELT_SIZE, OFO_POOL_SIZE, OFO_ELT_SIZE); } /* ══════════════════════════════════════════════════════════════════ * Port initialization * ══════════════════════════════════════════════════════════════════ */ void ng_init_port(struct rte_mempool *mbuf_pool) { const int num_rx_queues = 1; const int num_tx_queues = 1; struct rte_eth_conf port_conf = { .rxmode = {.mtu = RTE_ETHER_MTU}, }; if (rte_eth_dev_configure(gDpdkPortId, num_rx_queues, num_tx_queues, &port_conf) < 0) rte_exit(EXIT_FAILURE, "Cannot configure port=%d\n", gDpdkPortId); if (rte_eth_rx_queue_setup(gDpdkPortId, 0, 128, rte_eth_dev_socket_id(gDpdkPortId), NULL, mbuf_pool) < 0) rte_exit(EXIT_FAILURE, "rx_queue_setup failed\n"); if (rte_eth_tx_queue_setup(gDpdkPortId, 0, 512, rte_eth_dev_socket_id(gDpdkPortId), NULL) < 0) rte_exit(EXIT_FAILURE, "tx_queue_setup failed\n"); if (rte_eth_dev_start(gDpdkPortId) < 0) rte_exit(EXIT_FAILURE, "dev_start failed\n"); rte_eth_promiscuous_enable(gDpdkPortId); printf("[DPDK] Port %d initialized\n", gDpdkPortId); }