F-Stack实现UDP服务端、客户端,并进行吞吐量测试的实现

Table of Contents

怎么个加速法?

我的网络拓扑

开始编码

服务端

server.c

common.h

Makefile

配置文件server.ini

客户端代码

client.c

common.h

common.c

Makefile

运行结果


最近在做DPDK相关的网口加速的内容,开始采用DPDK+VPP的方式,但鉴于VPP的部署需要花费一定的时间,遂采用F-Stack,关于F-Stack,本文不做介绍,只说明它是腾讯开发的。

鉴于“一图胜过千言”,本文将以图+简短的话为例

怎么个加速法?

F-Stack不走内核协议栈,通过移植BSD的协议栈和DPDK进行联合,组成一套完整的用户态协议栈。

我准备用两个万兆网卡加速,如下图

然后在采用一个网卡运行服务端,一个网卡运行客户端,

也可以总结成这样

可是最后都初始化ff_init后法宝不成功,最终我确定了一个方案

也就是万兆网卡运行客户端进行收发包,内核协议栈控制的千兆网卡运行客户端收发包。

我的网络拓扑

一开始我的拓扑是下面这样

我从我的虚拟机进行对左侧X86的控制,左面服务器运行客户端和服务端,但是好像哪里不对,因为我要让数据像下面这样走

当然是少了一根线,可以有两种连接方法

我踩用第一种,如下

开始编码

https://download.csdn.net/download/Rong_Toa/12631153

服务端

server.c

/**
 *  测试F-Stack的UDP发包速率
 *  作者:荣涛 <rongtao@sylincom.com>
 *  时间:2020年7月16日10:08:34
 */
#include "common.h"

#include <strings.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <assert.h>

#include <sys/socket.h>
#include <netinet/ip.h>
#include <sys/epoll.h>


int sockfd;

int epfd;
struct epoll_event ev;
struct epoll_event events[MAX_EVENTS];

char buf[MAXLINE] = {"100000000"};



int udpsocket_server()
{
    int sockfd;
    
    struct sockaddr_in servaddr;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;    
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(PORT);
    
    if((sockfd = ff_socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
        return -1;
    }

    if(ff_bind(sockfd, (struct linux_sockaddr *)&servaddr, sizeof(servaddr))) {
        return -1;
    }
    
    return sockfd;
}

long int direct_sendto_addr(int sockfd, const char *dst_ip, int port, long int n_pkg)
{
    
	struct sockaddr_in cliaddr;
    int n = 0;
    
    long int npkg = 0, nbyte=0;
    
	socklen_t len;
    len = sizeof(cliaddr);
    struct timeval tvbrfore, tvafter;
    
    bzero(&cliaddr, sizeof(struct sockaddr_in));
    cliaddr.sin_family = AF_INET;
    cliaddr.sin_port = htons(port);
    inet_pton(AF_INET, dst_ip, &cliaddr.sin_addr);

    /* 统计时间 */
    gettimeval(&tvbrfore);

     /* 循环发包,测速率 */
    while(1) {
        if((n = ff_sendto(sockfd, buf, MAXLINE, 0, (struct linux_sockaddr *)&cliaddr, len)) < 0)
        {
            log_warn("sendto error");
            exit(1);
        } else {
            nbyte += n;
            if(npkg++ > n_pkg) {
                break;
            }
        }
    }
    /* 统计时间 */
    gettimeval(&tvafter);
    
    /* 输出此段时间内的速率 */
    statistic_throughput("Sendto", &tvbrfore, &tvafter, nbyte, npkg-1);
}

int server_loop(void *arg)
{
    int n;
	socklen_t len;
    
	struct sockaddr_in cliaddr;
    struct timeval tvbrfore, tvafter;

    
#if HAVE_FSTACK!=1
    while(1) {
        log_warn("Wait for a Client epoll_wait...\n");
#endif
    
    /* Wait for events to happen */
    int nevents = ff_epoll_wait(epfd,  events, MAX_EVENTS, -1);
    int i;
    len = sizeof(cliaddr);
        
#if HAVE_FSTACK!=1
    log_warn("nevents = %d\n", nevents);
#endif
    for (i = 0; i < nevents; ++i) {
        /* 如果是UDP的服务端, 接收一个来自客户端的消息,
            获取客户端地址,用于向客户端发送数据测发送速率 */
        if (events[i].data.fd == sockfd) {
    
    
            if((n = ff_recvfrom(sockfd, buf, MAXLINE, 0,  (struct linux_sockaddr *)&cliaddr, &len)) < 0)
            {
                log_warn("recvfrom error\n");
            } else {
                log_warn("Server recv   %s\n", buf);
                log_warn("Client Family %d\n", cliaddr.sin_family);
                log_warn("Client Port   %u\n", ntohs(cliaddr.sin_port));
                log_warn("Client Addr   %s\n", inet_ntoa(cliaddr.sin_addr));

                
                
                long int npkg = 0, nbyte=0;
                long int Nkpg = atol(buf);
#if HAVE_FSTACK==1
                Nkpg = Nkpg>1000000?Nkpg:1000000;
#else
                Nkpg = Nkpg>10000?Nkpg:10000;
#endif
                
                log_warn("Ready Send N Pkg %ld\n", Nkpg);

                /* 告诉我向客户端发了啥 */
#define MY_ID   "[FSTACK-DPDK][RongTao Test][牛逼了老哥]"
                strcpy(buf, MY_ID);

                /* 统计时间 */
                gettimeval(&tvbrfore);
                
                /* 循环发包,测速率 */
                while(1) {
                    if((n = ff_sendto(sockfd, buf, MAXLINE, 0, (struct linux_sockaddr *)&cliaddr, len)) < 0)
                    {
                        log_warn("sendto error");
                        exit(1);
                    } else {
                        nbyte += n;
                        if(npkg++ > Nkpg) {
                            break;
                        }
                    }
                }
                /* 统计时间 */
                gettimeval(&tvafter);
                
                /* 输出此段时间内的速率 */
                statistic_throughput("Sendto", &tvbrfore, &tvafter, nbyte, npkg-1);
            }
        }
    }

#if HAVE_FSTACK!=1
    }
#endif

}

int main(int argc, char *argv[])
{
    /* 绑核 */
    setaffinity(7);

#if HAVE_FSTACK==1

    ff_init(argc, argv);
#endif

    log_warn("Init socket.\n");
    
    sockfd = udpsocket_server();

    assert((epfd = ff_epoll_create(10)) > 0);
    ev.data.fd = sockfd;
    ev.events = EPOLLIN;
    ff_epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev);

    log_warn("Sockfd %d Wait for a Client...\n", sockfd);
    
#if 1

#if HAVE_FSTACK==1
    log_warn("Ready to ff_run.\n");
    ff_run(server_loop, NULL);
#else
    server_loop(NULL);
#endif
#else //直接发送 
    direct_sendto_addr(sockfd, "10.170.6.66", PORT, 10000);
#endif
}

common.h

/**
 *  测试F-Stack的UDP发包速率
 *  作者:荣涛 <rongtao@sylincom.com>
 *  时间:2020年7月16日10:08:34
 */
#ifndef __COMMON_H
#define __COMMON_H 1

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysinfo.h>
#include <time.h>
#include <stdint.h>


#define __USE_GNU
#include <sched.h>
#include <ctype.h>
#include <string.h>

#include <stdarg.h>
#include <libgen.h>

#include "ff_config.h"
#include "ff_api.h"
#include "ff_epoll.h"

/* 使用 fstack 进行UDP发包速率测试=1 
   使用 linux 进行UDP发包速率测试=1 
*/
#define HAVE_FSTACK 1

#define PORT 2152

#define MAXLINE 1500

#define MAX_EVENTS 10


#define TEST_ADDR1    "10.170.7.169"
#define TEST_ADDR2    "10.170.7.170"


/**
 *  打印log debug信息
 *  作者: 荣涛
 *  时间: s2020年7月15日10:12:24
 */
enum {
    __LV_INFO,
    __LV_WARNING,
    __LV_ERR,
    __LV_DEBUG,
};

#define LOG_DEBUG 1
#ifdef LOG_DEBUG
#define log_info(fmt...) ___debug_log(__LV_INFO, __FILE__, __func__ ,__LINE__, fmt)
#define log_warn(fmt...) ___debug_log(__LV_WARNING, __FILE__, __func__ ,__LINE__, fmt)
#define log_error(fmt...) ___debug_log(__LV_ERR, __FILE__, __func__ ,__LINE__, fmt)
#define log_debg(fmt...) ___debug_log(__LV_DEBUG, __FILE__, __func__ ,__LINE__, fmt)
#else
#define log_info(fmt...) 
#define log_warn(fmt...)
#define log_error(fmt...)
#define log_debg(fmt...)
#define log_errorno(i_errno)
#endif



#if HAVE_FSTACK!=1
#define ff_socket socket
#define ff_bind bind
#define ff_recvfrom recvfrom
#define ff_sendto sendto
#define linux_sockaddr sockaddr
#define ff_epoll_wait epoll_wait
#define ff_epoll_create epoll_create
#define ff_epoll_ctl epoll_ctl
#endif


static inline int ___debug_log(int level, char *file, const char *func, int line, char *fmt, ...)
{
    

    va_list av;
    va_start(av, fmt);

    switch(level) {
        case __LV_INFO:
            printf(" [%sINFO%s][%s:%s:%d]: ","\033[1;36m","\033[0m", basename(file), func, line);
            break;
        case __LV_WARNING:
            printf(" [%sWARN%s][%s:%s:%d]: ","\033[1;35m","\033[0m", basename(file), func, line);
            break;
        case __LV_ERR:
            printf("[%sERROR%s][%s:%s:%d]: ","\033[1;31m","\033[0m", basename(file), func, line);
            break;
        case __LV_DEBUG:
            printf("[%sDEBUG%s][%s:%s:%d]: ","\033[1m",   "\033[0m", basename(file), func, line);
            break;
    }
    
    int i = vprintf(fmt, av);

    va_end(av);

    return i;
}


static inline long int gettimeval(struct timeval *tv)
{
    gettimeofday(tv, NULL);    
}
static inline void statistic_throughput(char *description, 
            struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg
)
{
//    printf("\t -- before time: %ld, %ld\n", before->tv_sec, before->tv_usec);
//    printf("\t --  after time: %ld, %ld\n", after->tv_sec, after->tv_usec);
    printf("-- %s: Total %.3lf Mbps, bytes = %ld(bits:%ld), npkg = %ld.\n", 
                            description?description:"Unknown Description", 
                            8*bytes*1.0/((after->tv_sec-before->tv_sec)*1000000
    						            +after->tv_usec-before->tv_usec), bytes, bytes*8, npkg);
}


static void setaffinity(long int ncpu)
{
//    long int ncpu = sysconf (_SC_NPROCESSORS_ONLN);
    
    cpu_set_t cpuset;

	CPU_ZERO(&cpuset);
    
    CPU_SET(ncpu>1?ncpu-1:1, &cpuset);
    
    int ret = sched_setaffinity(getpid(), sizeof(cpuset), &cpuset);
    
    log_warn("setaffinity ret = %d\n", ret);
    
    int j;
    for(j=0;j<CPU_SETSIZE; j++)
    {
        if(CPU_ISSET(j, &cpuset))
        {
            printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\n", CPU_SETSIZE, j, cpuset);
            CPU_CLR(j, &cpuset);
            printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\n", CPU_SETSIZE, j, cpuset);
        }
    }
    
    ret = sched_getaffinity(getpid(), sizeof(cpuset), &cpuset);

    for(j=0;j<CPU_SETSIZE; j++)
    {
        if(CPU_ISSET(j, &cpuset))
        {
            printf("CPU_SETSIZE = %d, j = %d, cpuset = %d\n", CPU_SETSIZE, j, cpuset);
        }
    }
}




#endif /*__COMMON_H*/

Makefile

TOPDIR=..

ifeq ($(FF_PATH),)
	FF_PATH=${TOPDIR}
endif

ifeq ($(FF_DPDK),)
	FF_DPDK=${TOPDIR}/dpdk/x86_64-native-linuxapp-gcc
endif

LIBS+= -L${FF_PATH}/lib -Wl,--whole-archive,-lfstack,--no-whole-archive
LIBS+= -L${FF_DPDK}/lib -Wl,--whole-archive,-ldpdk,--no-whole-archive
LIBS+= -Wl,--no-whole-archive -lrt -lm -ldl -lcrypto -pthread -lnuma

all:
	cc -O -gdwarf-2  -I../lib -o server server.c ${LIBS}

.PHONY: clean
clean:
	rm -f *.o server client

配置文件server.ini

# F-Stack 参数配置 <rongtao@sylincom.com>
[dpdk]
# Hexadecimal bitmask of cores to run on.
# 绑核
lcore_mask=3

# Number of memory channels.
channel=8

# Specify base virtual address to map.
#base_virtaddr=0x7f0000000000

# Promiscuous mode of nic, defualt: enabled.
# 网卡的混杂模式
promiscuous=1
numa_on=1

# TX checksum offload skip, default: disabled.
# We need this switch enabled in the following cases:
# -> The application want to enforce wrong checksum for testing purposes
# -> Some cards advertize the offload capability. However, doesn't calculate checksum.
tx_csum_offoad_skip=0

# TCP segment offload, default: disabled.
tso=0

# HW vlan strip, default: enabled.
vlan_strip=1

# sleep when no pkts incomming
# unit: microseconds
idle_sleep=0

# sent packet delay time(0-100) while send less than 32 pkts.
# default 100 us.
# if set 0, means send pkts immediately.
# if set >100, will dealy 100 us.
# unit: microseconds
pkt_tx_delay=100

# enabled port list
#
# EBNF grammar:
#
#    exp      ::= num_list {"," num_list}
#    num_list ::= <num> | <range>
#    range    ::= <num>"-"<num>
#    num      ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9'
#
# examples
#    0-3       ports 0, 1,2,3 are enabled
#    1-3,4,7   ports 1,2,3,4,7 are enabled
#
# If use bonding, shoule config the bonding port id in port_list
# and not config slave port id in port_list
# such as, port 0 and port 1 trank to a bonding port 2,
# should set `port_list=2` and config `[port2]` section
port_list=0,1

# Number of vdev.
nb_vdev=0

# Number of bond.
nb_bond=0

# Each core write into own pcap file, which is open one time, close one time if enough.
# Support dump the first snaplen bytes of each packet.
# if pcap file is lager than savelen bytes, it will be closed and next file was dumped into.
[pcap]
enable = 0
snaplen= 16777216
savelen= 16777216

# Port config section
# Correspond to dpdk.port_list's index: port0, port1...
[port0]
addr=10.170.7.169
netmask=255.255.255.0
broadcast=10.170.7.255
gateway=10.170.7.254

[port1]
addr=10.170.7.170
netmask=255.255.255.0
broadcast=10.170.7.255
gateway=10.170.7.254

# lcore list used to handle this port
# the format is same as port_list
lcore_list=0,1

# bonding slave port list used to handle this port
# need to config while this port is a bonding port
# the format is same as port_list
#slave_port_list=0,1

# Packet capture path, this will hurt performance
#pcap=./a.pcap

# Vdev config section
# orrespond to dpdk.nb_vdev's index: vdev0, vdev1...
#    iface : Shouldn't set always.
#    path : The vuser device path in container. Required.
#    queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes.
#    queue_size : Queue size.Optional, default 256.
#    mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac.
#    cq : Optional, if queues = 1, default 0; if queues > 1 default 1.
#[vdev0]
##iface=/usr/local/var/run/openvswitch/vhost-user0
#path=/var/run/openvswitch/vhost-user0
#queues=1
#queue_size=256
#mac=00:00:00:00:00:01
#cq=0

# bond config section
# See http://doc.dpdk.org/guides/prog_guide/link_bonding_poll_mode_drv_lib.html
#[bond0]
#mode=4
#slave=0000:0a:00.0,slave=0000:0a:00.1
#primary=0000:0a:00.0
#mac=f0:98:38:xx:xx:xx
## opt argument
#socket_id=0
#xmit_policy=l23
#lsc_poll_period_ms=100
#up_delay=10
#down_delay=50

# Kni config: if enabled and method=reject,
# all packets that do not belong to the following tcp_port and udp_port
# will transmit to kernel; if method=accept, all packets that belong to
# the following tcp_port and udp_port will transmit to kernel.
[kni]
enable=1
method=reject
# The format is same as port_list
tcp_port=1-65535
udp_port=1-65535

# FreeBSD network performance tuning configurations.
# Most native FreeBSD configurations are supported.
[freebsd.boot]
hz=100

# Block out a range of descriptors to avoid overlap
# with the kernel's descriptor space.
# You can increase this value according to your app.
fd_reserve=1024

kern.ipc.maxsockets=262144

net.inet.tcp.syncache.hashsize=4096
net.inet.tcp.syncache.bucketlimit=100

net.inet.tcp.tcbhashsize=65536

kern.ncallout=262144

kern.features.inet6=1
net.inet6.ip6.auto_linklocal=1
net.inet6.ip6.accept_rtadv=2
net.inet6.icmp6.rediraccept=1
net.inet6.ip6.forwarding=0

[freebsd.sysctl]
kern.ipc.somaxconn=32768
kern.ipc.maxsockbuf=16777216

net.link.ether.inet.maxhold=5

net.inet.tcp.fast_finwait2_recycle=1
net.inet.tcp.sendspace=16384
net.inet.tcp.recvspace=8192
#net.inet.tcp.nolocaltimewait=1
net.inet.tcp.cc.algorithm=cubic
net.inet.tcp.sendbuf_max=16777216
net.inet.tcp.recvbuf_max=16777216
net.inet.tcp.sendbuf_auto=1
net.inet.tcp.recvbuf_auto=1
net.inet.tcp.sendbuf_inc=16384
net.inet.tcp.recvbuf_inc=524288
net.inet.tcp.sack.enable=1
net.inet.tcp.blackhole=1
net.inet.tcp.msl=2000
net.inet.tcp.delayed_ack=0

net.inet.udp.blackhole=1
net.inet.ip.redirect=0
net.inet.ip.forwarding=0

客户端代码

client.c

/**
 *  测试F-Stack的UDP发包速率
 *  作者:荣涛 <rongtao@sylincom.com>
 *  时间:2020年7月16日10:08:34
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <netinet/ip.h>

#include "common.h"

int main(int argc, char *argv[])
{
    int sockfd, n;
    struct sockaddr_in servaddr;
    socklen_t servlen = sizeof(servaddr);
    
    struct timeval tvbrfore, tvafter;
    
    long int npkg = 0, nbyte=0;
    
    char sendline[MAXLINE], recvline[MAXLINE + 1];
    
    if(argc != 2) {
        perror("Usage: udpcli <IPAddress>");
        exit(1);
    }

    sockfd = udpsocket_client(argv[1], &servaddr);
    
    printf("Client Input:");
    
    while(fgets(sendline, MAXLINE, stdin) != NULL)
    {
        if(sendto(sockfd, sendline, strlen(sendline), 0, 
                    (struct sockaddr *)&servaddr, servlen) < 0)
        {
            perror("sendto error");
            exit(1);
        } else {
            printf("Client send %d: %s", sockfd, sendline);
        }
        
        npkg = 0, nbyte=0;
        memset(&tvbrfore, 0, sizeof(struct timeval));
        memset(&tvafter, 0, sizeof(struct timeval));
        
        /* 统计时间 */
        my_gettimeval(&tvbrfore);
        
        while(1) {
            if((n = recvfrom(sockfd, recvline, MAXLINE, 0, NULL, &servlen)) < 0)
            {
                perror("recvfrom error");
                break;
            } else {
//                printf("Client recv %d: %s", sockfd, recvline);
                nbyte += n;
                if(npkg++ > 10000) {
                    break;
                }

            }
            recvline[n] = '\0';
        }
        
        /* 统计时间 */
        my_gettimeval(&tvafter);
        
        /* 输出此段时间内的速率 */
        my_statistic_throughput("Recvfrom", &tvbrfore, &tvafter, nbyte, npkg-1);

        /* 准备下次触发f_stack服务端 进行发包测试 */
        printf(">>");
    }

    return 1;
}

common.h

/**
 *  测试F-Stack的UDP发包速率
 *  作者:荣涛 <rongtao@sylincom.com>
 *  时间:2020年7月16日10:08:34
 */
#ifndef _COMMON_H
#define _COMMON_H

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/sysinfo.h>
#include <time.h>
#include <stdint.h>

#define PORT        2152
#define MAXLINE     1500 

int udpsocket_server();
int udpsocket_client(const char *ipv4, struct sockaddr_in *servaddr);

inline long int my_gettimeval(struct timeval *tv);

inline void my_statistic_throughput(char *description, 
            struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg);


#endif /*<_COMMON_H>*/

common.c

/**
 *  测试F-Stack的UDP发包速率
 *  作者:荣涛 <rongtao@sylincom.com>
 *  时间:2020年7月16日10:08:34
 */
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <string.h>
#include <sys/types.h>
#include <netinet/ip.h>

#include "common.h"


int udpsocket_server()
{
    int sockfd;
    
    struct sockaddr_in servaddr;

    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;    
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(PORT);
    
    if((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
    {
        return -1;
    }

    if(bind(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)))
    {
        return -1;
    }
    
    return sockfd;
}

int udpsocket_client(const char *ipv4, struct sockaddr_in *servaddr)
{
    int sockfd, t;
    
    bzero(servaddr, sizeof(struct sockaddr_in));
    servaddr->sin_family = AF_INET;
    servaddr->sin_port = htons(PORT);
    
    if((t = inet_pton(AF_INET, ipv4, &servaddr->sin_addr)) <= 0)
    {
        return -1;
    }
    if((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0)
    {
        return -1;
    }
    
    return sockfd;
}


inline long int my_gettimeval(struct timeval *tv)
{
    gettimeofday(tv, NULL);    
}
inline void my_statistic_throughput(char *description, 
            struct timeval *before, struct timeval *after, unsigned long int bytes, long int npkg)
{
//    printf("\t -- before time: %ld, %ld\n", before->tv_sec, before->tv_usec);
//    printf("\t --  after time: %ld, %ld\n", after->tv_sec, after->tv_usec);
    printf("-- %s: Total %.3lf Mbps, bytes = %ld(bits:%ld), npkg = %ld.\n", 
                            description?description:"Unknown Description", 
                            8*bytes*1.0/((after->tv_sec-before->tv_sec)*1000000
    						            +after->tv_usec-before->tv_usec), bytes, bytes*8, npkg);
}

Makefile

ALL:
	gcc client.c common.c -o client -lm
clean:
	rm *~

运行结果

 

 

 

已标记关键词 清除标记
相关推荐
©️2020 CSDN 皮肤主题: 酷酷鲨 设计师:CSDN官方博客 返回首页