Merge branch 'dev'

Author: jfb8856606

README.md

@@ -22,7 +22,7 @@ To deal with the increasingly severe DDoS attacks the authorized DNS server of T
 
 After several months of development and testing, DKDNS, high-performance DNS server based on DPDK officially released in October 2013. It's capable of achieving up to 11 million QPS with a single 10GE port and 18.2 million QPS with two 10GE ports. And then we developed a user-space TCP/IP stack called F-Stack that can process 0.6 million RPS with a single 10GE port.
 
-With the fast growth of Tencent Cloud more and more of our services needed higher network access performance. Meanwhile, F-Stack was continuing to improve, being driven by our business growth, and, ultimately developed into a general network access framework. But our initial TCP/IP stack couldn't meet the needs of these services. Continuing to develop and maintain a complete high performance network stack would have been too expensive. After evaluating several plans; we finally determined to port FreeBSD's (11.0 stable) TCP/IP stack into F-Stack. Not only does this allow us to stop reinventing the wheel, we can take advantage of the the improvements the FreeBSD community will bring in the future. Thanks to [libplebnet](https://gitorious.org/freebsd/kmm-sandbox/commit/fa8a11970bc0ed092692736f175925766bebf6af?p=freebsd:kmm-sandbox.git;a=tree;f=lib/libplebnet;h=ae446dba0b4f8593b69b339ea667e12d5b709cfb;hb=refs/heads/work/svn_trunk_libplebnet) and [libuinet](https://github.com/pkelsey/libuinet) this work became a lot easier.
+With the fast growth of Tencent Cloud more and more of our services needed higher network access performance. Meanwhile, F-Stack was continuing to improve, being driven by our business growth, and, ultimately developed into a general network access framework. But our initial TCP/IP stack couldn't meet the needs of these services. Continuing to develop and maintain a complete high performance network stack would have been too expensive. After evaluating several plans; we finally determined to port FreeBSD's (13.0 stable) TCP/IP stack into F-Stack. Not only does this allow us to stop reinventing the wheel, we can take advantage of the the improvements the FreeBSD community will bring in the future. Thanks to [libplebnet](https://gitorious.org/freebsd/kmm-sandbox/commit/fa8a11970bc0ed092692736f175925766bebf6af?p=freebsd:kmm-sandbox.git;a=tree;f=lib/libplebnet;h=ae446dba0b4f8593b69b339ea667e12d5b709cfb;hb=refs/heads/work/svn_trunk_libplebnet) and [libuinet](https://github.com/pkelsey/libuinet) this work became a lot easier.
 
 With the rapid development of all kinds of applications, in order to help different APPs quick and easily use F-Stack, F-Stack has integrated Nginx, Redis and other commonly used APPs, and a micro thread framework, and provides a standard Epoll/Kqueue interface.
 
@@ -50,7 +50,8 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
     cd f-stack
     # Compile DPDK
     cd dpdk/
-    meson -Denable_kmods=true build
+    # re-enable kni now, to remove kni later
+    meson -Denable_kmods=true -Ddisable_libs=flow_classify build
     ninja -C build
     ninja -C build install
 
@@ -122,7 +123,7 @@ Currently, besides authorized DNS server of DNSPod, there are various products i
 
 #### Nginx
 
-    cd app/nginx-1.16.1
+    cd app/nginx-1.25.2
     bash ./configure --prefix=/usr/local/nginx_fstack --with-ff_module
     make
     make install

VERSION

@@ -1 +1 @@
-1.23
+1.24

adapter/syscall/ff_hook_syscall.c

@@ -419,7 +419,7 @@ ff_hook_getsockname(int fd, struct sockaddr *name,
     SYSCALL(FF_SO_GETSOCKNAME, args);
 
     if (ret == 0) {
-        socklen_t cplen = *namelen ? *sh_namelen > *namelen
+        socklen_t cplen = *sh_namelen > *namelen ? *namelen
             : *sh_namelen;
         rte_memcpy(name, sh_name, cplen);
         *namelen = *sh_namelen;
@@ -475,7 +475,7 @@ ff_hook_getpeername(int fd, struct sockaddr *name,
     SYSCALL(FF_SO_GETPEERNAME, args);
 
     if (ret == 0) {
-        socklen_t cplen = *namelen ? *sh_namelen > *namelen
+        socklen_t cplen = *sh_namelen > *namelen ? *namelen
             : *sh_namelen;
         rte_memcpy(name, sh_name, cplen);
         *namelen = *sh_namelen;
@@ -794,7 +794,7 @@ ff_hook_recvfrom(int fd, void *buf, size_t len, int flags,
     if (ret >= 0) {
         rte_memcpy(buf, sh_buf, ret);
         if (from) {
-            socklen_t cplen = *fromlen ? *sh_fromlen > *fromlen
+            socklen_t cplen = *sh_fromlen > *fromlen ? *fromlen
                 : *sh_fromlen;
             rte_memcpy(from, sh_from, cplen);
             *fromlen = *sh_fromlen;
@@ -1542,6 +1542,18 @@ ff_hook_close(int fd)
 
     SYSCALL(FF_SO_CLOSE, args);
 
+#ifdef FF_KERNEL_EVENT
+    if (ret == 0 && fstack_kernel_fd_map[fd]) {
+        int kernel_fd_ret = ff_linux_close(fstack_kernel_fd_map[fd]);
+        if (kernel_fd_ret < 0) {
+            ERR_LOG("fstack_kernel_fd_map[%d]=%d, ff_linux_close returns %d, errno=%d\n",
+                fd, fstack_kernel_fd_map[fd], kernel_fd_ret, errno);
+        } else {
+            fstack_kernel_fd_map[fd] = 0;
+        }
+    }
+#endif
+
     RETURN_NOFREE();
 }
 

app/nginx-1.25.2/src/event/modules/ngx_ff_module.c

@@ -545,13 +545,12 @@ kevent(int kq, const struct kevent *changelist, int nchanges,
     return ff_kevent(kq, changelist, nchanges, eventlist, nevents, timeout);
 }
 
-/*
- * It is need to modify the definition, such as Ubuntu 22.04 or later.
- *
- * int(struct timeval * restrict,  void * restrict)
- */
 int
+#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 31)
+gettimeofday(struct timeval *tv, void *tz)
+#else
 gettimeofday(struct timeval *tv, struct timezone *tz)
+#endif
 {
     if (unlikely(inited == 0)) {
         return SYSCALL(gettimeofday)(tv, tz);

app/nginx-1.25.2/src/event/ngx_event_connect.c

@@ -353,10 +353,20 @@ ngx_event_connect_peer(ngx_peer_connection_t *pc)
 
 #if (NGX_HAVE_TRANSPARENT_PROXY)
 
+#if (NGX_HAVE_FSTACK)
+extern int is_fstack_fd(int sockfd);
+#ifndef IP_BINDANY
+#define IP_BINDANY	24
+#endif
+#endif
+
 static ngx_int_t
 ngx_event_connect_set_transparent(ngx_peer_connection_t *pc, ngx_socket_t s)
 {
     int  value;
+#if defined(NGX_HAVE_FSTACK)
+    int optname;
+#endif
 
     value = 1;
 
@@ -376,23 +386,31 @@ ngx_event_connect_set_transparent(ngx_peer_connection_t *pc, ngx_socket_t s)
 
     case AF_INET:
 
-#if defined(IP_TRANSPARENT)
+#if defined(NGX_HAVE_FSTACK)
+        /****
+        FreeBSD define IP_BINDANY in freebsd/netinet/in.h
+        Fstack should only support IP_BINDANY.
+        ****/
+        if(is_fstack_fd(s)){
+            optname = IP_BINDANY;
+        } else {
+            optname = IP_TRANSPARENT;
+        }
 
-        if (setsockopt(s, IPPROTO_IP, IP_TRANSPARENT,
+        if (setsockopt(s, IPPROTO_IP, optname,
                        (const void *) &value, sizeof(int)) == -1)
         {
             ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
-            "setsockopt(IP_TRANSPARENT) failed");
+                   "setsockopt(IP_BINDANY/IP_TRANSPARENT) failed");
             return NGX_ERROR;
         }
 
-#elif defined(IP_BINDANY)
-
-        if (setsockopt(s, IPPROTO_IP, IP_BINDANY,
+#elif defined(IP_TRANSPARENT)
+        if (setsockopt(s, IPPROTO_IP, IP_TRANSPARENT,
                        (const void *) &value, sizeof(int)) == -1)
         {
             ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno,
-                   "setsockopt(IP_BINDANY) failed");
+            "setsockopt(IP_TRANSPARENT) failed");
             return NGX_ERROR;
         }
 

config.ini

@@ -24,6 +24,11 @@ tso=0
 # HW vlan strip, default: enabled.
 vlan_strip=1
 
+# Set [vlanN]'s addrs like [portN] later
+# the format is same as port_list
+# Set vlan filter id, to enable L3/L4 RSS below vlan hdr is not enable after f-stack-1.22.
+vlan_filter=1,2,4-6
+
 # sleep when no pkts incomming
 # unit: microseconds
 idle_sleep=0
@@ -98,13 +103,19 @@ gateway=192.168.1.1
 #gateway6=ff::01
 
 # Multi virtual IPv4/IPv6 net addr, Optional parameters.
-#	`vip_ifname`: default `f-stack-x`
-#	`vip_addr`: Separated by semicolons, MAX number 64;
-#		    Only support netmask 255.255.255.255, broadcast x.x.x.255 now, hard code in `ff_veth_setvaddr`.
-#	`vip_addr6`: Separated by semicolons, MAX number 64.
-#	`vip_prefix_len`: All addr6 use the same prefix now, default 64.
+#       `vip_ifname`: default `f-stack-x`
+#       `vip_addr`: Separated by semicolons, MAX number 64;
+#                   Only support netmask 255.255.255.255, broadcast x.x.x.255 now, hard code in `ff_veth_setvaddr`.
+#       `ipfw_pr`: Set simplest policy routing,  Optional parameters.
+#                  Such as the cmd `ff_ipfw -P 0 add 100 setfib 0 ip from 192.168.0.0/24 to any out`
+#                  can set parameter`192.168.0.0 255.255.255.0`, cidr and netmask separated by space.
+#                  Multi cidr separated by semicolons.
+#                  IPv4 only now, and if you want set more complex policy routing, should use tool `ff_ipfw`.
+#       `vip_addr6`: Separated by semicolons, MAX number 64.
+#       `vip_prefix_len`: All addr6 use the same prefix now, default 64.
 #vip_ifname=lo0
-#vip_addr=192.168.1.3;192.168.1.4;192.168.1.5;192.168.1.6
+#vip_addr=192.168.0.3;192.168.0.4;192.168.0.5;192.168.0.6
+#ipfw_pr=192.168.0.0 255.255.255.0;192.168.10.0 255.255.255.0
 #vip_addr6=ff::03;ff::04;ff::05;ff::06;ff::07
 #vip_prefix_len=64
 
@@ -117,6 +128,55 @@ gateway=192.168.1.1
 # the format is same as port_list
 #slave_port_list=0,1
 
+# Vlan config section, Must set after all [portN]
+# NOTE1: Must enable dpdk.vlan_filter first, and match it.
+# NOTE2: If enable vlan config, all [PortN] config will be ignored!
+#[vlan1]
+#portid=0
+#addr=192.169.0.2
+#netmask=255.255.255.0
+#broadcast=192.169.0.255
+#gateway=192.169.0.1
+#
+#vip_addr=192.169.0.3;192.169.0.4;192.169.0.5;192.169.0.6
+#ipfw_pr=192.169.0.0 255.255.255.0;192.169.10.0 255.255.255.0
+#
+#[vlan2]
+#portid=0
+#addr=192.169.1.2
+#netmask=255.255.255.0
+#broadcast=192.169.1.255
+#gateway=192.169.1.1
+#
+#vip_addr=192.169.1.3;192.169.1.4;192.169.1.5;192.169.1.6
+#ipfw_pr=192.169.1.0 255.255.255.0;192.169.11.0 255.255.255.0
+#
+#[vlan4]
+#portid=0
+#addr=192.169.2.2
+#netmask=255.255.255.0
+#broadcast=192.169.2.255
+#gateway=192.169.2.1
+#
+#vip_addr=192.169.2.3;192.169.2.4;192.169.2.5;192.169.2.6
+#ipfw_pr=192.169.2.0 255.255.255.0;192.169.12.0 255.255.255.0
+#
+#[vlan5]
+#portid=0
+#addr=192.169.3.2
+#netmask=255.255.255.0
+#broadcast=192.169.3.255
+#gateway=192.169.3.1
+#
+#addr6=fe::32
+#prefix_len=64
+#gateway6=fe::31
+#
+#vip_addr=192.169.3.3;192.169.3.4;192.169.3.5;192.169.3.6
+#ipfw_pr=192.169.3.0 255.255.255.0;192.169.13.0 255.255.255.0
+#vip_addr6=fe::33;fe::34;fe::35;fe::36;fe::37
+#vip_prefix_len=64
+
 # Vdev config section
 # orrespond to dpdk.nb_vdev's index: vdev0, vdev1...
 #    iface : Shouldn't set always.
@@ -151,12 +211,22 @@ gateway=192.168.1.1
 # all packets that do not belong to the following tcp_port and udp_port
 # will transmit to kernel; if method=accept, all packets that belong to
 # the following tcp_port and udp_port will transmit to kernel.
+#	type: exception path type, 0 means kni(must set meson -Ddisable_libs=flow_classif to re-enable kni in DPDK first), 1 means virtio_user(linux only)
 #[kni]
+#type=1
 #enable=1
 #method=reject
 # The format is same as port_list
 #tcp_port=80,443
 #udp_port=53
+# KNI ratelimit value, default: 0, means disable ratelimit.
+# example:
+# The total speed limit for a single process entering the kni ring is 10,000 QPS,
+# 1000 QPS for general packets, 9000 QPS for console packets (ospf/arp, etc.)
+# The total speed limit for kni forwarding to the kernel is 20,000 QPS.
+#console_packets_ratelimit=0
+#general_packets_ratelimit=0
+#kernel_packets_ratelimit=0
 
 # FreeBSD network performance tuning configurations.
 # Most native FreeBSD configurations are supported.
@@ -184,6 +254,8 @@ kern.features.inet6=1
 kern.ipc.somaxconn=32768
 kern.ipc.maxsockbuf=16777216
 
+net.add_addr_allfibs=1
+
 net.link.ether.inet.maxhold=5
 
 net.inet.tcp.fast_finwait2_recycle=1

doc/F-Stack_API_Reference.md

@@ -49,6 +49,12 @@ However, it is  supported only before F-Stack is started.
   The ioctl() function manipulates the underlying device parameters of special files.
   more info see man ioctl.
 
+#### ff_stop_run
+
+	void ff_stop_run();
+
+  Stop the infinite poll loop started by `ff_run`.
+
 ### Network API
 
 #### ff_socket