Skip to content

Commit bf46b29

Browse files
MikeWang000000MikeWang000000
committed
fix: Ensure packet order
1 parent 3d7b5ce commit bf46b29

File tree

4 files changed

+124
-34
lines changed

4 files changed

+124
-34
lines changed

‎src/ipv4ipt.c‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,6 @@ int fh_ipt4_setup(void)
172172
"--tcp-flags", "SYN,FIN,RST", "SYN", "-j", "NFQUEUE",
173173
"--queue-bypass", "--queue-num", nfqnum_str, NULL}};
174174

175-
E("ERROR: iptables rules is under development, please use nft.");
176-
return -1;
177-
178175
ipt_cmds_cnt = sizeof(ipt_cmds) / sizeof(*ipt_cmds);
179176

180177
res = snprintf(xmark_str, sizeof(xmark_str), "%" PRIu32 "/%" PRIu32,

‎src/ipv6ipt.c‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,6 @@ int fh_ipt6_setup(void)
165165
"--tcp-flags", "SYN,FIN,RST", "SYN", "-j", "NFQUEUE",
166166
"--queue-bypass", "--queue-num", nfqnum_str, NULL}};
167167

168-
E("ERROR: iptables rules is under development, please use nft.");
169-
return -1;
170-
171168
ipt_cmds_cnt = sizeof(ipt_cmds) / sizeof(*ipt_cmds);
172169

173170
res = snprintf(xmark_str, sizeof(xmark_str), "%" PRIu32 "/%" PRIu32,

‎src/nfqueue.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ static int callback(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg,
4747
struct nfq_data *nfa, void *data)
4848
{
4949
uint32_t pkt_id, iifindex, oifindex;
50-
int res, pkt_len, modified;
50+
int verdict, pkt_len, modified;
5151
struct nfqnl_msg_packet_hdr *ph;
5252
unsigned char *pkt_data;
5353
struct nfqnl_msg_packet_hw *hwph;
@@ -98,16 +98,18 @@ static int callback(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg,
9898
memset(sll.sll_addr, 0, sizeof(sll.sll_addr));
9999
}
100100

101-
res = fh_rawsend_handle(&sll, pkt_data, pkt_len, &modified);
102-
if (res < 0) {
101+
verdict = fh_rawsend_handle(&sll, pkt_data, pkt_len, &modified);
102+
if (verdict < 0) {
103103
EE(T(fh_rawsend_handle));
104104
goto ret_accept;
105105
}
106106

107-
if (modified) {
108-
return nfq_set_verdict(qh, pkt_id, NF_ACCEPT, pkt_len, pkt_data);
107+
if (modified && verdict != NF_DROP) {
108+
return nfq_set_verdict(qh, pkt_id, verdict, pkt_len, pkt_data);
109109
}
110110

111+
return nfq_set_verdict(qh, pkt_id, verdict, 0, NULL);
112+
111113
ret_accept:
112114
return nfq_set_verdict(qh, pkt_id, NF_ACCEPT, 0, NULL);
113115
}

‎src/rawsend.c‎

Lines changed: 117 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@
2626
#include <unistd.h>
2727
#include <arpa/inet.h>
2828
#include <net/ethernet.h>
29+
#include <net/if.h>
2930
#include <netinet/tcp.h>
3031
#include <sys/socket.h>
3132
#include <linux/if_packet.h>
33+
#include <linux/netfilter.h>
3234
#include <libnetfilter_queue/libnetfilter_queue_tcp.h>
3335

3436
#include "globvar.h"
@@ -146,9 +148,69 @@ static int remove_tfo_cookie(uint16_t ethertype, uint8_t *pkt,
146148
}
147149

148150

151+
/*
152+
This is a workaround for iptables since it does not allow us to intercept
153+
packets after POSTROUTING SNAT, which means the SNATed source address is
154+
unknown.
155+
Instead of using an AF_PACKET socket, we create a temporary AF_INET or
156+
AF_INET6 raw socket, so that the packet gets SNATed correctly.
157+
*/
158+
static int sendto_snat(struct sockaddr_ll *sll, struct sockaddr *daddr,
159+
uint8_t *pkt_buff, int pkt_len)
160+
{
161+
int res, ret, sock_fd;
162+
ssize_t nbytes;
163+
char *iface, iface_buf[IF_NAMESIZE];
164+
165+
ret = -1;
166+
167+
iface = if_indextoname(sll->sll_ifindex, iface_buf);
168+
if (!iface) {
169+
E("ERROR: if_indextoname(): %s", strerror(errno));
170+
return -1;
171+
}
172+
173+
sock_fd = socket(daddr->sa_family, SOCK_RAW, IPPROTO_RAW);
174+
if (sock_fd < 0) {
175+
E("ERROR: socket(): %s", strerror(errno));
176+
return -1;
177+
}
178+
179+
res = setsockopt(sock_fd, SOL_SOCKET, SO_BINDTODEVICE, iface,
180+
strlen(iface));
181+
if (res < 0) {
182+
E("ERROR: setsockopt(): SO_BINDTODEVICE: %s", strerror(errno));
183+
goto close_socket;
184+
}
185+
186+
res = setsockopt(sock_fd, SOL_SOCKET, SO_MARK, &g_ctx.fwmark,
187+
sizeof(g_ctx.fwmark));
188+
if (res < 0) {
189+
E("ERROR: setsockopt(): SO_MARK: %s", strerror(errno));
190+
goto close_socket;
191+
}
192+
193+
nbytes = sendto(sock_fd, pkt_buff, pkt_len, 0, daddr,
194+
daddr->sa_family == AF_INET6 ? sizeof(struct sockaddr_in6)
195+
: sizeof(struct sockaddr_in));
196+
if (nbytes < 0) {
197+
E("ERROR: sendto(): %s", strerror(errno));
198+
goto close_socket;
199+
}
200+
201+
ret = nbytes;
202+
203+
close_socket:
204+
close(sock_fd);
205+
206+
return ret;
207+
}
208+
209+
149210
static int send_payload(struct sockaddr_ll *sll, struct sockaddr *saddr,
150211
struct sockaddr *daddr, uint8_t ttl, uint16_t sport_be,
151-
uint16_t dport_be, uint32_t seq_be, uint32_t ackseq_be)
212+
uint16_t dport_be, uint32_t seq_be, uint32_t ackseq_be,
213+
int need_snat)
152214
{
153215
int pkt_len;
154216
ssize_t nbytes;
@@ -175,11 +237,19 @@ static int send_payload(struct sockaddr_ll *sll, struct sockaddr *saddr,
175237
return -1;
176238
}
177239

178-
nbytes = sendto(sockfd, pkt_buff, pkt_len, 0, (struct sockaddr *) sll,
179-
sizeof(*sll));
180-
if (nbytes < 0) {
181-
E("ERROR: sendto(): %s", strerror(errno));
182-
return -1;
240+
if (need_snat) {
241+
nbytes = sendto_snat(sll, daddr, pkt_buff, pkt_len);
242+
if (nbytes < 0) {
243+
E(T(sendto_snat));
244+
return -1;
245+
}
246+
} else {
247+
nbytes = sendto(sockfd, pkt_buff, pkt_len, 0, (struct sockaddr *) sll,
248+
sizeof(*sll));
249+
if (nbytes < 0) {
250+
E("ERROR: sendto(): %s", strerror(errno));
251+
return -1;
252+
}
183253
}
184254

185255
return 0;
@@ -258,6 +328,7 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
258328
char src_ip[INET6_ADDRSTRLEN], dst_ip[INET6_ADDRSTRLEN];
259329
struct sockaddr_storage saddr_store, daddr_store;
260330
struct sockaddr *saddr, *daddr;
331+
ssize_t nbytes;
261332

262333
*modified = 0;
263334

@@ -292,14 +363,14 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
292363
if (tcp_payload_len > 0) {
293364
E_INFO("%s:%u ===PAYLOAD(?)===> %s:%u", src_ip, ntohs(tcph->source),
294365
dst_ip, ntohs(tcph->dest));
295-
return 0;
366+
return NF_ACCEPT;
296367
} else if (sll->sll_pkttype == PACKET_HOST && tcph->syn && tcph->ack) {
297368
sll->sll_pkttype = 0;
298369

299370
if (!g_ctx.outbound) {
300371
E_INFO("%s:%u ===SYN-ACK(?)===> %s:%u", src_ip,
301372
ntohs(tcph->source), dst_ip, ntohs(tcph->dest));
302-
return 0;
373+
return NF_ACCEPT;
303374
}
304375

305376
E_INFO("%s:%u ===SYN-ACK===> %s:%u", src_ip, ntohs(tcph->source),
@@ -316,7 +387,7 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
316387
if (hop <= g_ctx.ttl) {
317388
E_INFO("%s:%u ===LOCAL(?)===> %s:%u", src_ip,
318389
ntohs(tcph->source), dst_ip, ntohs(tcph->dest));
319-
return 0;
390+
return NF_ACCEPT;
320391
}
321392
snd_ttl = calc_snd_ttl(hop);
322393
}
@@ -325,7 +396,7 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
325396

326397
for (i = 0; i < g_ctx.repeat; i++) {
327398
res = send_payload(sll, daddr, saddr, snd_ttl, tcph->dest,
328-
tcph->source, tcph->ack_seq, ack_new);
399+
tcph->source, tcph->ack_seq, ack_new, 0);
329400
if (res < 0) {
330401
E(T(send_payload));
331402
return -1;
@@ -334,7 +405,7 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
334405
E_INFO("%s:%u <===FAKE(*)=== %s:%u", src_ip, ntohs(tcph->source),
335406
dst_ip, ntohs(tcph->dest));
336407

337-
return 0;
408+
return NF_ACCEPT;
338409
} else if (sll->sll_pkttype == PACKET_OUTGOING && tcph->syn && tcph->ack) {
339410
sll->sll_pkttype = 0;
340411

@@ -343,12 +414,9 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
343414
if (!g_ctx.inbound || srcinfo_unavail) {
344415
E_INFO("%s:%u <===SYN-ACK(?)=== %s:%u", dst_ip, ntohs(tcph->dest),
345416
src_ip, ntohs(tcph->source));
346-
return 0;
417+
return NF_ACCEPT;
347418
}
348419

349-
E_INFO("%s:%u <===SYN-ACK=== %s:%u", dst_ip, ntohs(tcph->dest), src_ip,
350-
ntohs(tcph->source));
351-
352420
seq_new = ntohl(tcph->seq);
353421
seq_new++;
354422
seq_new = htonl(seq_new);
@@ -360,7 +428,7 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
360428
if (hop <= g_ctx.ttl) {
361429
E_INFO("%s:%u <===LOCAL(?)=== %s:%u", src_ip,
362430
ntohs(tcph->source), dst_ip, ntohs(tcph->dest));
363-
return 0;
431+
return NF_ACCEPT;
364432
}
365433
snd_ttl = calc_snd_ttl(hop);
366434
}
@@ -369,7 +437,8 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
369437

370438
for (i = 0; i < g_ctx.repeat; i++) {
371439
res = send_payload(sll, saddr, daddr, snd_ttl, tcph->source,
372-
tcph->dest, seq_new, tcph->ack_seq);
440+
tcph->dest, seq_new, tcph->ack_seq,
441+
g_ctx.use_iptables /* needs SNAT */);
373442
if (res < 0) {
374443
E(T(send_payload));
375444
return -1;
@@ -378,12 +447,37 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
378447
E_INFO("%s:%u <===FAKE(*)=== %s:%u", dst_ip, ntohs(tcph->dest), src_ip,
379448
ntohs(tcph->source));
380449

381-
return 0;
450+
/*
451+
We send the original packet using a raw socket and discard the
452+
current processing flow.
453+
This ensures that the SYN-ACK is transmitted after the payload.
454+
Although this deliberately causes a TCP out-of-order situation,
455+
it guarantees that our payload is always sent before the client's
456+
packet.
457+
*/
458+
if (g_ctx.use_iptables) {
459+
nbytes = sendto_snat(sll, daddr, pkt_data, pkt_len);
460+
if (nbytes < 0) {
461+
E(T(sendto_snat));
462+
return -1;
463+
}
464+
} else {
465+
nbytes = sendto(sockfd, pkt_data, pkt_len, 0,
466+
(struct sockaddr *) sll, sizeof(*sll));
467+
if (nbytes < 0) {
468+
E("ERROR: sendto(): %s", strerror(errno));
469+
return -1;
470+
}
471+
}
472+
473+
E_INFO("%s:%u <===SYN-ACK=== %s:%u", dst_ip, ntohs(tcph->dest), src_ip,
474+
ntohs(tcph->source));
475+
return NF_DROP;
382476
} else if (sll->sll_pkttype == PACKET_HOST && tcph->syn) {
383477
if (!g_ctx.inbound) {
384478
E_INFO("%s:%u ===SYN(?)===> %s:%u", src_ip, ntohs(tcph->source),
385479
dst_ip, ntohs(tcph->dest));
386-
return 0;
480+
return NF_ACCEPT;
387481
}
388482

389483
*modified = !remove_tfo_cookie(ethertype, pkt_data, tcph);
@@ -401,12 +495,12 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
401495
return -1;
402496
}
403497

404-
return 0;
498+
return NF_ACCEPT;
405499
} else if (sll->sll_pkttype == PACKET_OUTGOING && tcph->syn) {
406500
if (!g_ctx.outbound) {
407501
E_INFO("%s:%u <===SYN(?)=== %s:%u", dst_ip, ntohs(tcph->dest),
408502
src_ip, ntohs(tcph->source));
409-
return 0;
503+
return NF_ACCEPT;
410504
}
411505

412506
*modified = !remove_tfo_cookie(ethertype, pkt_data, tcph);
@@ -418,10 +512,10 @@ int fh_rawsend_handle(struct sockaddr_ll *sll, uint8_t *pkt_data, int pkt_len,
418512
ntohs(tcph->source));
419513
}
420514

421-
return 0;
515+
return NF_ACCEPT;
422516
} else {
423517
E_INFO("%s:%u ===(?)=== %s:%u", src_ip, ntohs(tcph->source), dst_ip,
424518
ntohs(tcph->dest));
425-
return 0;
519+
return NF_ACCEPT;
426520
}
427521
}

0 commit comments

Comments
 (0)