Once the company requirements record, python processes the sa performance data collected by sysstat

In view of the fact that the cpu usage rate, CPU load, memory utilization rate, network traffic and other data of thousands of services of the company need to be extracted every month, which used to be obtained manually through the zabbix platform, it takes 1-2 hours to complete this requirement each time, which is too time-consuming, so it is optimized for systat software to collect data, and then process the data under python script / var/log/sa, and then store it into mysql database. In the future, you only need one line of sql code to get the data, which is convenient and fast. This article takes a screenshot of the script for collecting data, as shown below:

Finally, the complete code is attached:

#! / bin/python2.7

# coding: utf-8

# script usage: enter the time of which day you want to take the data. For example, the input on September 1 should be 20190901.

Import os

Import sys

Import datetime

Import commands

Import time

Def get_cpu (sdate,edate):

Cpunum = int (commands.getoutput ("cat / proc/cpuinfo | grep processor | wc-l"))

Path = os.popen ("find / var/log/sa/-type f-newermt" + sdate+ "!-newermt" + edate+ "| grep-v sar") .readlines ()

Path = [m.rstrip ("\ n") for m in path]

For n in path:

Idle = os.popen ("sar-f" + n + "- u | awk'{print $NF}'| sed '1Magazine 3d' | sed' $d' | sed'/ ^ $/ d'"). Readlines ()

Data = list (idle)

Data = [i.rstrip ("\ n") for i in idle]

Data = [j for j in data if j! ='']

While'% idle' in data: data.remove ("% idle") while 'RESTART' in data: data.remove ("RESTART") data_tmp = list (map (float,data)) data_num = [100anthx for x in data_tmp] max_num = max (data_num) min_num = min (data_num) average_num=sum (data_num) / len (data_num) print ("% s CPU useed max:%.4f" Min:%.4f,average:%.4f "% (nMagneMaxnum minagnum))

Def get_dev (sdate,edate):

Net = commands.getoutput ("route-n | sed '1Magazine 2d' | awk-F'' {print $1score8}'| grep 0.0.0.0 | awk-F''{print $2}'| sed-n '1p'")

RxkB=list ()

TxkB=list ()

Path = os.popen ("find / var/log/sa/-type f-newermt" + sdate+ "!-newermt" + edate+ "| grep-v sar") .readlines ()

Path = [m.rstrip ("\ n") for m in path]

For n in path: rxkB = commands.getoutput ("sar-f" + n + "- n DEV | grep" + net+ "| grep-v Average | awk-F''{print $6}'| sed'$d'"). Split ("\ n") txkB = commands.getoutput ("sar-f" + n + "- n DEV | grep" + net+ "| grep-v Average | awk-F''{print $7}'). Split ("\ n ") data_txkB = list (map (float) TxkB) max_txkB=max (data_txkB) min_txkB=min (data_txkB) average_txkB=sum (data_txkB) / len (data_txkB) data_rxkB = list (map (float,rxkB)) max_rxkB=max (data_rxkB) min_rxkB=min (data_rxkB) average_rxkB=sum (data_rxkB) / len (data_rxkB) print "% s rxval max:%.4f,min:%.4f,average:%.4f,dev:%s"% Min_rxkB,average_rxkB,net) print "% s txval max:%.4f,min:%.4f,average:%.4f,dev:%s"% (n MagneMaxocrtxkB minicomial txkB) averagetemporxkBMagneNet)

Def get_mem (sdate,edate):

Path = os.popen ("find / var/log/sa/-type f-newermt" + sdate+ "!-newermt" + edate+ "| grep-v sar") .readlines ()

Path = [m.rstrip ("\ n") for m in path]

For n in path:

A=int (commands.getoutput ("sar-f" + n + "- r | awk 'END {print NF}') if (a > = 11): memused = os.popen (" sar-f "+ n +"-r | grep-v ^ $| grep-iv Linux | grep-iv memused | grep-iv Average | awk'{print (($(NF-8)-$(NF-5) / ($(NF-9) + $(NF-8))}'). Readlines () else: memused = Os.popen ("sar-f" + n + "- r | grep-v ^ $| grep-iv Linux | grep-iv memused | grep-iv Average | awk'{print (($(NF-5)-$(NF-2)) / ($(NF-6) + $(NF-5))}'"). Readlines () data = list (memused) data = [i.rstrip ("\ n") for i in memused] data_num = list (map (float)) Data) max_num=max (data_num) min_num=min (data_num) average_num=sum (data_num) / len (data_num) print "% s memused max:%.4f,min:%.4f,average:%.4f"%

Def get_ldavg (sdate,edate):

Cpunum = int (commands.getoutput ("cat / proc/cpuinfo | grep processor | wc-l"))

Path = os.popen ("find / var/log/sa/-type f-newermt" + sdate+ "!-newermt" + edate+ "| grep-v sar") .readlines ()

Path = [m.rstrip ("\ n") for m in path]

For n in path:

A=int (commands.getoutput ("sar-f" + n + "- Q | sed-n '10p' | awk' END {print NF}'") if a = 7: ldavg_1 = os.popen ("sar-f" + n + "- Q | awk-F'{print $5}'| sed'1M 3d' | sed'$d' | sed'/ ^ $/ d' | grep-v ldavg-1"). Readlines () ldavg_15 = os.popen ("sar-f" + n + ") -Q | awk-F''{print $7}'| sed '1d' | sed' $d' | sed'/ ^ $/ d' | grep-v ldavg-15 "). Readlines () elif a = 8: ldavg_1 = os.popen (" sar-f "+ n +"-Q | awk-F''{print $5}'| sed '1mt 3d' | sed' $d' | sed'/ ^ $/ d' | grep-v ldavg-1 "). Readlines () ldavg_15 = os.popen ("sar-f" + n + "- Q | awk-F''{print $7}'| sed '1Magazine 3d' | sed' $d' | sed'/ ^ $/ d' | grep-v ldavg-15"). Readlines () elif a = 6: ldavg_1 = os.popen ("sar-f" + n + "- Q | awk-F''{print $4}'| sed '1Magna 3d' | sed' $d' | sed'/ ^ $/ d' | grep-v ldavg-1"). Readlines () Ldavg_15 = os.popen ("sar-f" + n + "- Q | awk-F''{print $6}'| sed '1data1 3d' | sed' $d' | sed'/ ^ $/ d' | grep-v ldavg-15"). Readlines () data1 = [i.rstrip ("\ n") for i in ldavg_1] data15 = [j.rstrip ("\ n") for j in ldavg_15] data1 = list (float (float) Data1) data15= list (map (float,data15)) max_data1=max (data1) min_data1=min (data1) average_data1=sum (data1) / len (data1) max_data15=max (data15) min_data15=min (data15) average_data15=sum (data15) / len (data15) print "% s cpu 1minute load max:%.4f,min:%.4f,average:%.4f"% Average_data1/cpunum) print "% s cpu 15minute load max:%.4f,min:%.4f,average:%.4f"% (nMagneMaxocrydata15Universe cpunumDifferectiondata15According to cpunumMaverageMaindata15Universe)

If name = = "main":

Sdate=sys.argv [1]

Sdate=datetime.datetime.strptime (sdate,'%Y%m%d')

Edate=sdate+datetime.timedelta (days=1)

Sdate=sdate.strftime ('% Y% m% d')

Edate=edate.strftime ('% Y% m% d')

Get_cpu (sdate,edate)

Get_ldavg (sdate,edate)

Get_mem (sdate,edate)

Get_dev (sdate,edate)