Linux LVS高併發測試程式，核心引數設定，連線數查詢指令
阿新 • • 發佈：2019-01-31
最近在公司參與了一個LVS系統測試的專案，學習到了一些關於高併發測試相關的知識，寫到部落格裡記錄下
Linux核心引數設定
在伺服器端需要調整系統最大檔案控制代碼數
ulimit -n 1000000
在伺服器硬體支援，以及服務較輕量的情況下，最大連線數未必只限於100000,視情況而定吧客戶端需要發起大量連線，理論上是最多可支援65535個連線，但實際上Linux的預設可用埠號並沒有這麼多，需要修改
sudo echo 1024 65534 > /proc/sys/net/ipv4/ip_local_port_range
檢視連線個數
我們一開始用的是
netstat -nap | grep 程式名 | grep ESTABLISHED| wc -l
來查詢，這個指令當連線數上萬後就變得很慢，而且我們發現了一個略坑的問題，通過這組合指令查詢到的結果是波動的，我特意試了10000個連線下，在程式沒有報有連線關閉的情況下，通過netstat查出來的連線個數在9998到10000之間波動。最後我換了個指令
ss state ESTABLISHED dport = :埠號 and dst IP | wc -l 通過ss指令查詢，不僅速度很快，而且不會出現計數不準的情況，需要注意的是，查詢到的連線數要比實際多1
測試程式
/*client1.cpp
*該程式需要libevent庫支援
*測試有無LVS時連線延時區別的程式
*程式發起指定數量的連線後，將每秒給伺服器傳送一個報文，每個連線收到
*伺服器返回的報文後計算耗時
*/ /*server.cpp
*該程式需要libevent庫支援
*簡單的回射伺服器
*/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <assert.h>
#include <event2/event.h>
#include <event2/bufferevent.h>
#define LISTEN_BACKLOG 1024
static int time_out_count = 0;
static int err_conn_count = 0;
static int err_accept_count = 0;
void do_accept(evutil_socket_t listener, short event, void *arg);
void read_cb(struct bufferevent *bev, void *arg);
void error_cb(struct bufferevent *bev, short event, void *arg);
void write_cb(struct bufferevent *bev, void *arg);
int main(int argc, char *argv[])
{
if (argc != 2) {
printf("引數個數不正確 %d\n", argc);
return 0;
}
int port = atoi(argv[1]);
evutil_socket_t listener;
listener = socket(AF_INET, SOCK_STREAM, 0);
assert(listener > 0);
evutil_make_listen_socket_reuseable(listener);
struct sockaddr_in sin;
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = 0;
sin.sin_port = htons(port);
if (bind(listener, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
perror("bind");
return 1;
}
if (listen(listener, LISTEN_BACKLOG) < 0) {
perror("listen");
return 1;
}
printf ("Listening...\n");
evutil_make_socket_nonblocking(listener);
struct event_base *base = event_base_new();
assert(base != NULL);
struct event *listen_event;
listen_event = event_new(base, listener, EV_READ | EV_PERSIST, do_accept, (void*)base);
event_add(listen_event, NULL);
event_base_dispatch(base);
printf("The End.");
return 0;
}
void do_accept(evutil_socket_t listener, short event, void *arg)
{
struct event_base *base = (struct event_base *)arg;
evutil_socket_t fd;
struct sockaddr_in sin;
socklen_t slen;
fd = accept(listener, (struct sockaddr *)&sin, &slen);
evutil_make_socket_nonblocking(fd);
if (fd < 0) {
printf("err in accept, err ccode %d, 發生accept錯誤的個數 %d", errno, ++err_accept_count);
return;
}
printf("ACCEPT: fd = %u\n", fd);
struct bufferevent *bev = bufferevent_socket_new(base, fd, BEV_OPT_CLOSE_ON_FREE);
bufferevent_setcb(bev, read_cb, NULL, error_cb, arg);
bufferevent_enable(bev, EV_READ | EV_WRITE | EV_PERSIST);
}
void read_cb(struct bufferevent *bev, void *arg)
{
#define MAX_LINE 256
char line[MAX_LINE+1];
int n;
evutil_socket_t fd = bufferevent_getfd(bev);
while (n = bufferevent_read(bev, line, MAX_LINE), n > 0) {
line[n] = '\0';
bufferevent_write(bev, line, n);
printf("recv : %s\n", line);
}
}
void write_cb(struct bufferevent *bev, void *arg) {}
void error_cb(struct bufferevent *bev, short event, void *arg)
{
evutil_socket_t fd = bufferevent_getfd(bev);
printf("fd = %u, ", fd);
if (event & BEV_EVENT_TIMEOUT) {
time_out_count++;
printf("Timed out, 發生超時連線個數 %d", time_out_count);
printf(", err code %d\n", errno);
}
else if (event & BEV_EVENT_EOF) {
printf("connection closed\n");
}
else if (event & BEV_EVENT_ERROR) {
err_conn_count++;
printf("some other error, 連線被異常關閉個數 %d", err_conn_count);
printf(", err code %d\n", errno);
}
bufferevent_free(bev);
}/*client2.cpp
*該程式需要libevent庫支援
*測試高併發量的測試客戶端程式，該程式每隔10s就會通過所有連線向服務
*器傳送一個報文，以免長時間沒有互動，連線被LVS斷掉，同時測試伺服器
*的負載
*/
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <unistd.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <iostream>
#include <map>
#include <vector>
#include <string>
#include "event2/bufferevent.h"
#include "event2/buffer.h"
#include "event2/listener.h"
#include "event2/util.h"
#include "event2/event.h"
using namespace std;
typedef struct bufferevent* BufferEvent;
map<struct bufferevent*, long long> bufferevent_map;
map<struct bufferevent*, long long>::iterator iter;
FILE* fp = NULL;
static int index_ = 0;
static int counter = 0;
static int begin_time = 0;
struct cb_arg
{
struct event *ev;
struct timeval tv;
};
void timeout_cb(int fd, short event, void *params)
{
char buf[1024];
struct cb_arg *arg = (struct cb_arg*)params;
struct event *timeout = arg->ev;
struct timeval tv = arg->tv;
for (iter = bufferevent_map.begin(); iter != bufferevent_map.end(); ++iter) {
memset(buf, 0, sizeof(buf));
struct bufferevent *bufevt = iter->first;
sprintf(buf, "%lld", iter->second);
bufferevent_write(bufevt, buf, strlen(buf));
}
evtimer_add(timeout, &tv);
}
void read_cb(struct bufferevent *bufevt, void *arg)
{
char line[1024] = {0};
size_t n;
evutil_socket_t fd = bufferevent_getfd(bufevt);
struct timeval start;
map<struct bufferevent*, long long>::iterator it_temp;
it_temp = bufferevent_map.find(bufevt);
if (it_temp == bufferevent_map.end()) {
return ;
}
while (n = bufferevent_read(bufevt, line, 1024), n > 0)
{
;
}
}
void write_count(int count)
{
char buf[1024] = {0};
int now_time = (int)time(NULL);
sprintf(buf, "pass %d s, now connect count %d\n", now_time - begin_time, count);
fputs(buf, fp);
}
void error_cb(struct bufferevent *bufevt, short events, void *user_data)
{
evutil_socket_t fd = bufferevent_getfd(bufevt);
if (events & BEV_EVENT_EOF)
{
counter--;
printf("Connection closed.\n");
printf("fd %u close ! count %d\n", fd , counter);
printf("now EST connect count %d\n", counter);
bufferevent_free(bufevt);
bufferevent_map.erase(bufevt);
write_count(counter);
}
else if (events & BEV_EVENT_ERROR)
{
counter--;
printf("Got an error on the connection: %d\n", errno);
printf("fd %u close ! count %d\n", fd , counter);
printf("now EST connect count %d\n", counter);
bufferevent_free(bufevt);
bufferevent_map.erase(bufevt);
write_count(counter);
}
}
void create_file()
{
char buf[1024] = {0};
timeval tv;
gettimeofday(&tv, NULL);
int time_ = (int) (1000000 * tv.tv_sec + tv.tv_usec);
sprintf(buf, "result_%d.txt", time_);
fp = fopen(buf, "w+");
if (fp == NULL) {
printf("create result file failed!\n");
exit(-1);
}
}
int main(int argc, char *argv[])
{
if (argc != 4) {
printf("引數個數不正確 %d, 需要3個引數，連線個數，IP，埠號\n", argc - 1);
return 0;
}
vector<string> argvs;
for (int i = 1; i < argc; ++i) {
argvs.push_back(argv[i]);
}
create_file();
timeval tv = {10, 0};
int times = atoi(argvs[0].c_str());
int port = atoi(argvs[2].c_str());
struct sockaddr_in sin;
struct bufferevent* bufevt;
event_base *evbase = event_base_new();
if (evbase == NULL) {
printf("err %d in new event_base\n", errno);
return 0;
}
struct event* timeout;
struct cb_arg arg;
timeout = evtimer_new(evbase, timeout_cb, &arg);
if (timeout == NULL) {
printf("err in new evtimer\n");
return 0;
}
arg.ev = timeout;
arg.tv = tv;
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = inet_addr(argvs[1].c_str());
sin.sin_port = htons(port);
memset(sin.sin_zero, 0x00, 8);
for (int i = 0; i < times; ++i) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
evutil_make_socket_nonblocking(fd);
bufevt = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE);
if (bufevt == NULL) {
printf("err %d in new bufferevent, fd %u connect failed !\n", fd, errno);
continue ;
}
bufferevent_setcb(bufevt, read_cb, NULL, error_cb, NULL);
bufferevent_enable(bufevt, EV_READ | EV_WRITE | EV_PERSIST);
if (bufferevent_socket_connect(bufevt, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
printf("err %d in connect with server, fd %u connect failed !\n", fd, errno);
continue ;
}
char buf[64] = {0};
sprintf(buf, "%d", i);
bufferevent_map.insert(make_pair(bufevt, 0));
bufferevent_write(bufevt, buf, strlen(buf));
usleep(10);
counter++;
}
printf("now EST connect count %d\n", counter);
write_count(counter);
evtimer_add(timeout, &tv);
event_base_dispatch(evbase);
evtimer_del(timeout);
event_base_free(evbase);
return 0;
}