1. 程式人生 > >壓力測試監控資料通過python matplotlib進行視覺化

壓力測試監控資料通過python matplotlib進行視覺化

執行壓力測試過程中,監控伺服器效能變化,通過常用的命令可以觀察當前時刻效能情況,但是無法判斷整個測試過程中效能的變化趨勢。
隨後,將壓力測試過程中伺服器效能指標按照10s一次進行採集並寫入檔案中。完成測試後,通過指令碼解析採集的效能資料,使用python的三方模組matplotlib對指標進行繪圖。

不足之處是,目前這個指令碼只能在壓力測試結束後進行曲線圖繪製,而不能在壓力測試過程中實施繪製

指標檔案資料格式如下:

Network_Flow
In_link Out_link
Linking_Number Net_Traffic(MB)
1.87925 0.119033
4.5772 0.141904
2.62263 0.126162 2.50861 0.12293 4.92332 0.139697 3.24791 0.126943

解析指標檔案的程式碼如下:

# coding:utf-8
__author__ = 'Libiao'
import os,sys
curr_dir = os.getcwd()

def is_multi_lines(linelabel):
    if ' ' in linelabel.strip():
        return True
    else:
        return False

def read_datas(args=()):
    dir_name,files_dict = args
    result = {}
    for
groupname,files_list in files_dict.items(): res = [] res_dir = curr_dir + "\\" + dir_name + "\\" if not os.listdir(res_dir): continue for files in files_list: try: with file(res_dir + files,'r') as f: datas = [data for
data in f] title,line_labels,x_y_labels = datas.pop(0),datas.pop(0),datas.pop(0) x_label,y_label = x_y_labels.strip("\n").split(' ') if is_multi_lines(line_labels): line_label_first,line_label_second = line_labels.strip("\n").split(" ") line_data_first = [d.strip("\n").split(' ')[0] for d in datas] line_data_second = [d.strip("\n").split(' ')[1] for d in datas if d] res.append((title,line_label_first,line_label_second,x_label,y_label,line_data_first,line_data_second)) else: line_data = [d.strip() for d in datas] res.append((title,line_labels,x_label,y_label,line_data)) except IOError as ioerr: print str(ioerr) result[groupname] = res return (dir_name,result)

解析配置檔案的程式碼如下:

# coding:utf-8
__author__ = 'Libiao'

import os
from xml.etree import ElementTree as ET

def read_xml(xmlpath):
    res = []
    root = ET.parse(xmlpath)
    rootnode_list = root.getiterator("dir")
    for node in rootnode_list:
        files_map = {}
        dir_name = node.attrib["name"]
        for second_node in node.getiterator("group"):
            group_name = second_node.attrib['name']
            tmp = []
            for child in second_node.getchildren():
                tmp.append(child.text)
            files_map[group_name] = tmp
        res.append((dir_name,files_map))
    return res

進行曲線圖繪製的程式碼:

#coding:utf-8

import matplotlib.pyplot as plt
from pylab import *
import numpy as np
import random
import read_kpi_data,xml_parser
import threading,subprocess,multiprocessing
import os

curr_dir = os.getcwd()

color = ['r','g','b','y','c','k']

#獲取測試時間值
def get_test_time(dir_name):
    with file(curr_dir + "\\" + dir_name + "\\timeConsum",'r') as f:
        times = int(float(f.readline().strip()))
    return times

#獲取服務連線數
def get_server_num(dir_name):
    ser_num = []
    try:
        with file(curr_dir + "\\" + dir_name + "\\linking_number","r") as f:
            for value in f:
                if value:
                    ser_num.append(int(value))
    except Exception as e:
        print str(e)
    return ser_num

#獲取圖表佈局rows和cols
def get_layout(graph_num):
    rows,cols = None,None
    if graph_num == 0:
        return None
    elif graph_num == 1:
        rows,cols = 1,1
    elif graph_num < 4 and graph_num not in (0,1):
        rows,cols = graph_num,1
    elif graph_num >= 4 and graph_num <= 10:
        if graph_num % 2 == 0:
            rows = graph_num / 2
        else:
            rows = graph_num / 2 + 1
        cols = 2
    else:
        if graph_num % 3 == 0:
            rows = graph_num / 3
        else:
            rows = graph_num / 3 + 1
        cols = 3
    return (rows,cols)

def graph_display(plot_name,all_datas):
    plt.figure(plot_name,figsize=(10,8),dpi=80)
    #設定子圖在畫板中的佈局,主要是子圖高寬(hspace、wspace)、上下邊距(bottom、top),左右邊距(left、right)
    # plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,wspace=0.25,hspace=0.45)
    if len(all_datas) > 10:
        plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,hspace=0.75)
    elif len(all_datas)>6 and len(all_datas) <= 10:
        plt.subplots_adjust(bottom=0.08,top=0.95,hspace=0.35)
    #從time_consum檔案讀取效能指標採集消耗的時間
    #x_axle = get_test_time(plot_name)
    #從linking_number檔案讀取連線數
    x_axle = get_server_num(plot_name)
    #獲取圖表佈局
    rows,cols = get_layout(len(all_datas))
    colorLen = len(color)
    for index, datas in enumerate(all_datas):
        yMax,yMin = 0.0,0.0
        subplt = plt.subplot(rows,cols,index+1)
        title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = None,None,None,None,None,None,None
        try:
            title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = datas
        except:
            title,first_line_label,x_label,y_label,first_line_datas = datas

        tmp = None
        #設定子圖示題
        subplt.set_title(title.strip("\n"))
        x = np.linspace(0,np.array(get_server_num(plot_name)).max(),len(first_line_datas))
        #x = np.array(get_server_num(plot_name))
        if second_line_label == None:
            tmp = [float(d) for d in first_line_datas]
            subplt.plot(x,first_line_datas,color = color[index % colorLen],linewidth=1.5,linestyle='-',label=first_line_label.strip("\n"))
        else:
            tmp1 = [float(d) for d in first_line_datas if d]
            tmp2 = [float(d) for d in second_line_datas if d]
            try:
                tmp = [max(tmp1),max(tmp2),min(tmp1),min(tmp2)]
            except ValueError as e:
                print title
            num = -1 #num=-1,是為了防止color = color[index % colorLen + num]計算中color下標越界
            for line_label,line_data in zip([first_line_label,second_line_label],[first_line_datas,second_line_datas]):
                subplt.plot(x,line_data,color = color[index % colorLen + num],linewidth=1.5,linestyle='-',label=line_label.strip("\n"))
                subplt.legend()
                num += 1
        #讀取連線數,繪製到每個圖表
        with open("SYS_KPI/srs_number",'r') as f:
            datas = f.readlines()
        yMax = max(tmp)
        yMin = min(tmp)
        dx = (x.max() - x.min()) * 0.1
        dy = (yMax - yMin) * 0.1
        subplt.set_ylim(yMin - dy,yMax + dy)
        subplt.set_xlim(x.min() - dx,x.max() + dx)
        #{'position':(1.0,1.0)},其中1.0相當於x軸最右邊,0表示最左邊,0.5便是中間
        subplt.set_xlabel(x_label,{'color':'b','position':(1.0,1.0)})
        subplt.set_ylabel(y_label,{'color':'b'})
        #子圖顯示網格
        subplt.grid(True)
        subplt.legend()
    plt.show()

if __name__ == '__main__':
    xml_path = curr_dir + "\config\kpi.xml"
    files_lists = xml_parser.read_xml(xml_path)

    for files_list in files_lists:
        all_datas = []
        #print read_kpi_data.read_datas(files_list)
        plot_name,plot_datas = read_kpi_data.read_datas(files_list)
        datas = plot_datas.values()
        all_datas.extend(datas)

        for d in all_datas:
            multiprocessing.Process(target=graph_display,args=(plot_name,d)).start()

資料視覺化效果:
壓力測試過程中伺服器cpu、mem、Tps、network以及直播連線數狀態變化
壓力測試過程中,磁碟io狀態