You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
713 lines
26 KiB
713 lines
26 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"encoding/json" |
|
"math" |
|
"sort" |
|
"strconv" |
|
"strings" |
|
"time" |
|
|
|
"go-common/app/admin/ep/melloi/conf" |
|
"go-common/app/admin/ep/melloi/model" |
|
"go-common/library/log" |
|
) |
|
|
|
// QueryReportSummarys query report summary |
|
func (s *Service) QueryReportSummarys(c context.Context, sessionID string, qrsr *model.QueryReportSuRequest) (qrs *model.QueryReportSuResponse, err error) { |
|
// 获取服务树节点 |
|
var treeNodes, treeNodesd []string |
|
if treeNodesd, err = s.QueryUserRoleNode(c, sessionID); err != nil { |
|
log.Error("QueryUserRoleNode (%v): ", err) |
|
return |
|
} |
|
treeNodes = append(treeNodesd, "") |
|
if ExistsInSlice(qrsr.Executor, conf.Conf.Melloi.Executor) { |
|
if qrs, err = s.dao.QueryReportSummarysWhiteName(&qrsr.ReportSummary, qrsr.SearchAll, qrsr.PageNum, qrsr.PageSize); err != nil { |
|
return |
|
} |
|
} else { |
|
if qrs, err = s.dao.QueryReportSummarys(&qrsr.ReportSummary, qrsr.SearchAll, qrsr.PageNum, qrsr.PageSize, treeNodes); err != nil { |
|
return |
|
} |
|
} |
|
|
|
// 获取label |
|
for _, report := range qrs.ReportSummarys { |
|
lr := model.LabelRelation{Type: model.ReportType, TargetID: int64(report.ID)} |
|
if report.Labels, err = s.dao.QueryLabelRelation(&lr); err != nil { |
|
return |
|
} |
|
} |
|
return |
|
} |
|
|
|
// QueryReportByID Query Report By ID |
|
func (s *Service) QueryReportByID(id int) (summary *model.ReportSummary, err error) { |
|
return s.dao.QueryReportSuryByID(id) |
|
} |
|
|
|
// CountQueryReportSummarys count query report summarys |
|
func (s *Service) CountQueryReportSummarys(reportSummary *model.ReportSummary) (int, error) { |
|
return s.dao.CountQueryReportSummarys(reportSummary) |
|
} |
|
|
|
// UpdateReportSummary update report summary |
|
func (s *Service) UpdateReportSummary(reportSummary *model.ReportSummary) (status string, err error) { |
|
if err = s.dao.UpdateReportSummary(reportSummary); err != nil { |
|
status = "update fail" |
|
return |
|
} |
|
status = "success" |
|
return |
|
} |
|
|
|
// UpdateReportStatus update report summary status |
|
func (s *Service) UpdateReportStatus(status int) error { |
|
return s.dao.UpdateReportStatus(status) |
|
} |
|
|
|
// AddReSummaryByPtest Add ReSummary By Ptest |
|
func (s *Service) AddReSummaryByPtest(ptestParam model.DoPtestParam, jobName, executeID string, testNameNicks []string, snapID, Debug int) (reportSuID int, err error) { |
|
reportSummary := model.ReportSummary{ |
|
JobName: jobName, |
|
TestStatus: 2, |
|
UserName: ptestParam.UserName, |
|
ResJtl: ptestParam.ResJtl, |
|
JmeterLog: ptestParam.JmeterLog, |
|
Department: ptestParam.Department, |
|
Project: ptestParam.Project, |
|
APP: ptestParam.APP, |
|
ScriptID: ptestParam.ScriptID, |
|
DockerSum: 1, |
|
Active: 1, |
|
Debug: Debug, |
|
ScriptSnapID: snapID, |
|
ExecuteID: executeID, |
|
SceneID: ptestParam.SceneID, |
|
Type: ptestParam.Type, |
|
LoadTime: ptestParam.LoadTime, |
|
} |
|
//Upload==true，这里表示从前端上传的脚本，testNames 和 testNameNicks 长度可能大于1，需要遍历，写入 reportSummary |
|
if ptestParam.Upload { |
|
reportSummary.TestNameNick = SliceToString(testNameNicks, ",") |
|
reportSummary.TestName = ptestParam.ProjectName |
|
reportSummary.Type = model.PROTOCOL_SCENE |
|
} |
|
|
|
// 从页面上直接输入参数生成的单场景脚本，testNames 和 testNameNicks 长度都是 1 |
|
// 生成的场景脚本，ptestParam.TestNameNick 不为空 |
|
if !ptestParam.Upload { |
|
for _, testName := range ptestParam.TestNames { |
|
reportSummary.TestNameNick = QueryTestNameNick(testName, testNameNicks) |
|
reportSummary.TestName = testName |
|
} |
|
//场景脚本的 reportSummary 逻辑 |
|
if ptestParam.Type == model.PROTOCOL_SCENE && ptestParam.TestNameNick != "" { |
|
reportSummary.TestNameNick = ptestParam.TestNameNick |
|
reportSummary.TestName = ptestParam.SceneName |
|
} |
|
} |
|
if reportSuID, err = s.dao.AddReportSummary(&reportSummary); err != nil { |
|
log.Error("s.dao.AddReportSummary err :(%v)", err) |
|
return |
|
} |
|
return |
|
} |
|
|
|
// AddReportSummary add report summary |
|
func (s *Service) AddReportSummary(reportSummary *model.ReportSummary) (status string, reportSuID int, err error) { |
|
var total int |
|
if total, err = s.dao.CountQueryReportSummarys(reportSummary); err != nil { |
|
log.Error("CountQueryReportSummarys error:(%v)", err) |
|
status = "fail" |
|
return |
|
} |
|
if (total == 1) || (reportSummary.TestName == "") { |
|
status = "数据已存在或者无接口名" |
|
} else { |
|
if reportSuID, err = s.dao.AddReportSummary(reportSummary); err != nil { |
|
status = "fail" |
|
return |
|
} |
|
status = "success" |
|
} |
|
return |
|
} |
|
|
|
// QueryReGraph query reGraph |
|
func (s *Service) QueryReGraph(testNameNicks []string) (reportGraphssd [][]model.ReportGraph, err error) { |
|
var ( |
|
TestNames []string |
|
reportGraphss []model.ReportGraph |
|
reportGraphs []model.ReportGraph |
|
) |
|
if reportGraphs, err = s.dao.QueryReportGraph(testNameNicks); err != nil { |
|
log.Error("dao.QueryReportGraph error:(%v)", err) |
|
return |
|
} |
|
if len(reportGraphs) > 0 { |
|
for _, reportGraph := range reportGraphs { |
|
TestNames = append(TestNames, reportGraph.TestName) |
|
} |
|
TestNamesd := RemoveRepByMap(TestNames) |
|
for _, testName := range TestNamesd { |
|
for _, reportGraph := range reportGraphs { |
|
if reportGraph.TestName == testName { |
|
reportGraphss = append(reportGraphss, reportGraph) |
|
} |
|
} |
|
reportGraphssd = append(reportGraphssd, reportGraphss) |
|
reportGraphss = []model.ReportGraph{} |
|
} |
|
} |
|
return |
|
} |
|
|
|
// QueryClientMoni query client moni |
|
func (s *Service) QueryClientMoni(cli *model.ClientMoni) (clientMonisd [][]*model.ClientMoni, err error) { |
|
var ( |
|
jobNames []string |
|
clientMoniss []*model.ClientMoni |
|
clientMonis []*model.ClientMoni |
|
) |
|
if clientMonis, err = s.dao.QueryClientMoni(cli); err != nil { |
|
log.Error("dao.QueryClientMoni error:(%v)", err) |
|
return |
|
} |
|
if len(clientMonis) > 0 { |
|
for _, clientMoni := range clientMonis { |
|
jobNames = append(jobNames, clientMoni.JobName) |
|
} |
|
jobNamesd := RemoveRepByMap(jobNames) |
|
for _, jobNamed := range jobNamesd { |
|
for _, clientMoni := range clientMonis { |
|
if clientMoni.JobName == jobNamed { |
|
clientMoniss = append(clientMoniss, clientMoni) |
|
} |
|
} |
|
clientMonisd = append(clientMonisd, clientMoniss) |
|
clientMoniss = []*model.ClientMoni{} |
|
} |
|
} |
|
return |
|
} |
|
|
|
//QueryReGraphAvg query reGraph |
|
func (s *Service) QueryReGraphAvg(testNameNicks []string) (reportGraphAvgs []model.ReportGraph, err error) { |
|
var reportGraphssd [][]model.ReportGraph |
|
if reportGraphssd, err = s.QueryReGraph(testNameNicks); err != nil { |
|
log.Error("QueryReGraph error :(%s)", err) |
|
return |
|
} |
|
for _, reportGraphsd := range reportGraphssd { |
|
reportGraph := ReportGraphAvg(reportGraphsd) |
|
reportGraphAvgs = append(reportGraphAvgs, reportGraph) |
|
} |
|
return |
|
} |
|
|
|
//ReportGraphAvg report graph avg |
|
func ReportGraphAvg(reportGraphs []model.ReportGraph) (reportGraph model.ReportGraph) { |
|
var sumTime, sumQPS, sumNetIo, min, max, sumCount, sumError, sumCodeEll, sumCodeWll, sumCodeWly, sumCodeWle, sumCodeWls, |
|
sumCodeSll, sumCodeSly, sumCodeSls, sumCodeKong, sumCodeNonHTTP, sumCodeOthers, fiftyTime, ninetyTime, ninetyFiveTime, |
|
ninetyNineTime, sumCode301, sumCode302, timeCount, sumQPSRecent, QPSRecent int |
|
var mins, maxs []int |
|
|
|
if len(reportGraphs) > 0 { |
|
for _, reportGraph := range reportGraphs { |
|
mins = append(mins, reportGraph.Min) |
|
maxs = append(maxs, reportGraph.Max) |
|
sumCount = sumCount + reportGraph.Count |
|
timeCount = reportGraph.Count * reportGraph.AvgTime |
|
sumTime = sumTime + timeCount |
|
sumError = sumError + reportGraph.Error |
|
sumQPS = sumQPS + reportGraph.QPS |
|
sumNetIo = sumNetIo + reportGraph.NetIo |
|
sumCodeEll = sumCodeEll + reportGraph.CodeEll |
|
sumCodeWll = sumCodeWll + reportGraph.CodeWll |
|
sumCodeWly = sumCodeWly + reportGraph.CodeWly |
|
sumCodeWle = sumCodeWle + reportGraph.CodeWle |
|
sumCodeWls = sumCodeWls + reportGraph.CodeWls |
|
sumCodeSll = sumCodeSll + reportGraph.CodeSll |
|
sumCodeSly = sumCodeSly + reportGraph.CodeSly |
|
sumCodeSls = sumCodeSls + reportGraph.CodeSls |
|
sumCodeKong = sumCodeKong + reportGraph.CodeKong |
|
sumCode301 = sumCode301 + reportGraph.Code301 |
|
sumCode302 = sumCode302 + reportGraph.Code302 |
|
sumCodeNonHTTP = sumCodeNonHTTP + reportGraph.CodeNonHTTP |
|
sumCodeOthers = sumCodeOthers + reportGraph.CodeOthers |
|
fiftyTime = fiftyTime + reportGraph.FiftyTime |
|
ninetyTime = ninetyTime + reportGraph.NinetyTime |
|
ninetyFiveTime = ninetyFiveTime + reportGraph.NinetyFiveTime |
|
ninetyNineTime = ninetyNineTime + reportGraph.NinetyNineTime |
|
} |
|
// 取最后 recent次统计的平均 qps，作为最近一段时间的qps |
|
recent := conf.Conf.Melloi.Recent |
|
if len(reportGraphs) > recent { |
|
for i := 1; i <= recent; i++ { |
|
sumQPSRecent += reportGraphs[len(reportGraphs)-i].QPS |
|
QPSRecent = sumQPSRecent / recent |
|
} |
|
} else { |
|
QPSRecent = sumQPS / len(reportGraphs) |
|
} |
|
if len(mins) != 0 { |
|
sort.Ints(mins) |
|
min = mins[0] |
|
sort.Ints(maxs) |
|
max = maxs[len(maxs)-1] |
|
} |
|
failPercentd := float64(sumError) * 100 / float64(sumCount) |
|
n10 := math.Pow10(3) |
|
fail := math.Trunc((failPercentd+0.5/n10)*n10) / n10 |
|
failStr := strconv.FormatFloat(fail, 'f', -1, 64) |
|
failPercent := failStr + "%" |
|
num := len(reportGraphs) |
|
reportGraph = model.ReportGraph{ |
|
TestName: reportGraphs[0].TestName, TestNameNick: reportGraphs[0].TestNameNick, Max: max, Min: min, Count: sumCount, Error: sumError, |
|
NetIo: sumNetIo / num, QPS: sumQPS / num, AvgTime: sumTime / sumCount, Ctime: reportGraphs[len(reportGraphs)-1].Ctime, |
|
CodeEll: sumCodeEll, CodeSll: sumCodeSll, CodeSly: sumCodeSly, CodeSls: sumCodeSls, CodeWll: sumCodeWll, CodeWly: sumCodeWly, |
|
CodeWle: sumCodeWle, CodeWls: sumCodeWls, CodeNonHTTP: sumCodeNonHTTP, CodeKong: sumCodeKong, CodeOthers: sumCodeOthers, |
|
FailPercent: failPercent, FiftyTime: fiftyTime / num, NinetyTime: ninetyTime / num, NinetyFiveTime: ninetyFiveTime / num, |
|
NinetyNineTime: ninetyNineTime / num, Code301: sumCode301, Code302: sumCode302, BeginTime: reportGraphs[0].Ctime, QpsRecent: QPSRecent, |
|
} |
|
} |
|
return |
|
} |
|
|
|
//addReGraphTimer add regraph timer |
|
func (s *Service) addReGraphTimer(c context.Context, addReGrapht model.AddReGraphTimer, res chan interface{}, timeout <-chan time.Time) { |
|
var ( |
|
status = 1 |
|
reportSummary model.ReportSummary |
|
testNamesd []string |
|
podNames []string |
|
beginTimed string |
|
afterTimed string |
|
beginTime = addReGrapht.BeginTime |
|
testNames = addReGrapht.TestNames |
|
jobName = addReGrapht.JobName |
|
token = addReGrapht.Token |
|
testNameNicks = addReGrapht.TestNameNicks |
|
RetMap *model.PaasJobQueryStatus |
|
scriptID = addReGrapht.ScriptID |
|
reportSuID = addReGrapht.ReportSuID |
|
ptestJobs []*model.PtestJob |
|
err error |
|
btm time.Time |
|
firstRetMap *model.PaasJobQueryStatus |
|
reportGraphs []model.ReportGraph |
|
JSON []byte |
|
fusingList = addReGrapht.FusingList |
|
useBusiStopList = addReGrapht.UseBusiStopList |
|
busiStopPercentList = addReGrapht.BusiStopPercentList |
|
) |
|
timer := time.NewTicker(5 * time.Second) |
|
done := make(chan bool, 1) |
|
num := 1 |
|
go func() { |
|
defer close(res) |
|
a := 1 |
|
for { |
|
if btm, err = time.Parse("2006-01-02 15:04:05", beginTime); err != nil { |
|
timeout = time.After(time.Until(time.Now().Add(1 * time.Second))) |
|
return |
|
} |
|
select { |
|
case <-timer.C: |
|
elapsedTime := (num - 1) * 5 |
|
//实时当前任务的所有容器 |
|
if ptestJobs, err = s.QueryOrStopAllPtestByJobName(context.TODO(), reportSuID, false, 3); err != nil { |
|
log.Error("get all Job err (%v)", err) |
|
} |
|
//遍历容器，并查询每个容器的cpu |
|
if len(ptestJobs) > 0 { |
|
for _, ptestJob := range ptestJobs { |
|
if RetMap, err = s.dao.Job(c, token, ptestJob.JobName); err != nil { |
|
log.Error("get job info err (%v)", err) |
|
} |
|
if RetMap != nil && len(RetMap.Data.Pods) > 0 { |
|
for _, pod := range RetMap.Data.Pods { |
|
podNames = append(podNames, pod.Name) |
|
log.Info("containerID :(%s)", pod.ContainerID) |
|
go s.addJobCPU(context.TODO(), pod, scriptID, reportSuID, elapsedTime, ptestJob.JobName) |
|
} |
|
} |
|
} |
|
} |
|
log.Info("podName :(%s)", podNames) |
|
if firstRetMap, err = s.dao.Job(c, token, jobName); err != nil { |
|
log.Error("查询容器状态 error-------(%v)", err) |
|
//接口报错，则删除所有压测容器 |
|
timeout = time.After(time.Until(time.Now().Add(1 * time.Second))) |
|
return |
|
} |
|
if JSON, err = json.Marshal(firstRetMap); err != nil { |
|
timeout = time.After(time.Until(time.Now().Add(1 * time.Second))) |
|
} |
|
result := string(JSON) |
|
log.Info("查询容器状态:(%s)", result) |
|
if firstRetMap.Status == 400 { |
|
status = 3 |
|
timeout = time.After(time.Until(time.Now().Add(1 * time.Second))) |
|
} |
|
if firstRetMap.Status == 200 && firstRetMap.Data.ActiveNum == 1 { |
|
log.Info("第(%d)次同步数据", a) |
|
if num == 1 { |
|
log.Info("job 执行参数 testName:(%s), beginTime :(%s),elapsedTime :(%d), num: (%d)", testNames, beginTime, elapsedTime, num) |
|
for index, testName := range testNames { |
|
testNameNick := QueryTestNameNick(testName, testNameNicks) |
|
reportGraphAdd := model.ReportGraphAdd{ |
|
JobName: jobName, |
|
TestName: testName, |
|
BeginTime: beginTime, |
|
AfterTime: beginTime, |
|
TestNameNick: testNameNick, |
|
PodNames: podNames, |
|
ElapsedTime: elapsedTime, |
|
ReportSuID: reportSuID, |
|
UseBusinessStop: addReGrapht.UseBusinessStop, |
|
BusinessStopPercent: addReGrapht.BusinessStopPercent, |
|
} |
|
if addReGrapht.TestType == model.PROTOCOL_SCENE { |
|
reportGraphAdd.Fusing = fusingList[index] |
|
reportGraphAdd.UseBusinessStop = useBusiStopList[index] |
|
reportGraphAdd.BusinessStopPercent = busiStopPercentList[index] |
|
} else { |
|
reportGraphAdd.Fusing = addReGrapht.Fusing |
|
} |
|
go s.addReportGraph(context.TODO(), reportGraphAdd) |
|
} |
|
} else { |
|
for i := 1; i <= num-1; i++ { |
|
btm = btm.Add(time.Second * 5) |
|
aft := btm.Add(time.Second * 2) |
|
beginTimed = btm.Format("2006-01-02 15:04:05") |
|
afterTimed = aft.Format("2006-01-02 15:04:05") |
|
} |
|
log.Info("job 执行参数 testName:(%s),beginTime :(%s),elapsedTime :(%d)", testNames, beginTimed, elapsedTime) |
|
for index, testName := range testNames { |
|
testNameNick := QueryTestNameNick(testName, testNameNicks) |
|
reportGraphAdd := model.ReportGraphAdd{ |
|
JobName: jobName, |
|
TestName: testName, |
|
BeginTime: beginTimed, |
|
AfterTime: afterTimed, |
|
TestNameNick: testNameNick, |
|
PodNames: podNames, |
|
ElapsedTime: elapsedTime, |
|
ReportSuID: reportSuID, |
|
UseBusinessStop: addReGrapht.UseBusinessStop, |
|
BusinessStopPercent: addReGrapht.BusinessStopPercent, |
|
} |
|
if addReGrapht.TestType == model.PROTOCOL_SCENE { |
|
reportGraphAdd.Fusing = fusingList[index] |
|
reportGraphAdd.UseBusinessStop = useBusiStopList[index] |
|
reportGraphAdd.BusinessStopPercent = busiStopPercentList[index] |
|
} else { |
|
reportGraphAdd.Fusing = addReGrapht.Fusing |
|
} |
|
go s.addReportGraph(context.TODO(), reportGraphAdd) |
|
} |
|
} |
|
|
|
} |
|
podNames = []string{} |
|
if firstRetMap.Status == 200 && firstRetMap.Data.ActiveNum == 0 { |
|
timeout = time.After(time.Until(time.Now().Add(1 * time.Second))) |
|
} |
|
num++ |
|
res <- a |
|
a++ |
|
case <-timeout: |
|
close(done) |
|
log.Info("real stop time (%s)", time.Now().Format("2006-01-02 15:04:05")) |
|
// 关闭所有压测该项目或者接口的容器 |
|
go s.QueryOrStopAllPtestByJobName(context.TODO(), reportSuID, true, status) |
|
if reportGraphs, err = s.dao.QueryReportGraph(testNameNicks); err != nil { |
|
log.Error("query reportGraphs error :(%v) ", err) |
|
return |
|
} |
|
for _, reportGraph := range reportGraphs { |
|
testNamesd = append(testNames, reportGraph.TestName) |
|
} |
|
// 如果reportGraphs 的成员都是同一个testName ，则执行如下代码，表示进行结果合并，生成 reportSummary |
|
if IsUniqObject(testNamesd) { |
|
if len(reportGraphs) > 0 { |
|
reportGraph := ReportGraphAvg(reportGraphs) |
|
reportSummary = model.ReportSummary{ |
|
ID: reportSuID, |
|
TestName: reportGraph.TestName, |
|
TestNameNick: reportGraph.TestNameNick, |
|
Count: reportGraph.Count, |
|
Error: reportGraph.Error, |
|
QPS: reportGraph.QPS, |
|
AvgTime: reportGraph.AvgTime, |
|
NetIo: reportGraph.NetIo, |
|
Min: reportGraph.Min, |
|
Max: reportGraph.Max, |
|
} |
|
reportSummary.TestStatus = status |
|
} else { |
|
log.Info("sorry,your test duration is too short ..... ") |
|
reportSummary = model.ReportSummary{ID: reportSuID, TestStatus: 3} |
|
} |
|
s.dao.UpdateReportSummary(&reportSummary) |
|
} |
|
//场景压测、多个接口走该逻辑 |
|
if !IsUniqObject(testNamesd) && len(testNameNicks) > 0 { |
|
if len(reportGraphs) > 0 { |
|
reportSummary = model.ReportSummary{ID: reportSuID, TestStatus: status} |
|
} else { |
|
reportSummary = model.ReportSummary{ID: reportSuID, TestStatus: 3} |
|
} |
|
} |
|
s.dao.UpdateReportSummary(&reportSummary) |
|
return |
|
} |
|
} |
|
}() |
|
<-done |
|
} |
|
|
|
//addPtestJobHostIp add regraph timer |
|
func (s *Service) addPtestJobHostIp(c context.Context, addReGrapht model.AddReGraphTimer, res chan interface{}, timeout <-chan time.Time) { |
|
var ( |
|
RetMap *model.PaasJobQueryStatus |
|
token = addReGrapht.Token |
|
err error |
|
ptestJobs []*model.PtestJob |
|
ptestJobsd []*model.PtestJob |
|
firstRetMap *model.PaasJobQueryStatus |
|
) |
|
timer := time.NewTicker(10 * time.Second) |
|
done := make(chan bool, 1) |
|
go func() { |
|
defer close(res) |
|
a := 1 |
|
for { |
|
select { |
|
case <-timer.C: |
|
if firstRetMap, err = s.dao.Job(c, token, addReGrapht.JobName); err != nil { |
|
log.Error("查询容器状态 error-------(%v)", err) |
|
//接口报错，则删除所有压测容器 |
|
return |
|
} |
|
if firstRetMap == nil || firstRetMap.Status == 400 || (firstRetMap.Status == 200 && firstRetMap.Data.ActiveNum == 0) { |
|
return |
|
} |
|
ptestJob := model.PtestJob{ReportSuID: addReGrapht.ReportSuID, Active: 1} |
|
if ptestJobs, err = s.dao.QueryPtestJob(&ptestJob); err != nil { |
|
return |
|
} |
|
for _, job := range ptestJobs { |
|
if job.HostIP == "" || job.JobIP == "" || job.JobID == "" { |
|
ptestJobsd = append(ptestJobsd, job) |
|
} |
|
} |
|
if len(ptestJobsd) > 0 { |
|
for _, ptestJob := range ptestJobsd { |
|
//查询容器状态 |
|
if RetMap, err = s.dao.Job(c, token, ptestJob.JobName); err != nil { |
|
log.Error("get job info err (%v)", err) |
|
} |
|
if RetMap != nil && len(RetMap.Data.Pods) > 0 { |
|
for _, pod := range RetMap.Data.Pods { |
|
ptestjo := model.PtestJob{ID: ptestJob.ID, HostIP: pod.HostIP, JobID: pod.ContainerID, JobIP: pod.IP} |
|
//更新容器hostIp |
|
s.dao.UpdatePtestJob(ptestjo) |
|
} |
|
} |
|
} |
|
} |
|
res <- a |
|
a++ |
|
case <-timeout: |
|
close(done) |
|
return |
|
} |
|
} |
|
}() |
|
<-done |
|
} |
|
|
|
//addJobCPU job cpu |
|
func (s *Service) addJobCPU(c context.Context, pod model.PodInfo, scriptID, reportSuID, elapsedTime int, jobName string) (err error) { |
|
|
|
var ( |
|
clientMoni = model.ClientMoni{ |
|
ScriptID: scriptID, |
|
ReportSuID: reportSuID, |
|
JobName: jobName, |
|
JobNameAll: pod.Name, |
|
ElapsdTime: elapsedTime, |
|
} |
|
dockerStats *model.DockerStats |
|
CPUResult *model.PaasQueryJobCPUResult |
|
) |
|
//首先查我们自己的容器cpu 获取服务 |
|
if pod.ContainerID != "" { |
|
if dockerStats, err = s.QueryJobCPUByEP(context.TODO(), pod.ContainerID, pod.HostIP); err != nil { |
|
log.Error("query CPU err ...(%s)", err) |
|
} |
|
if dockerStats != nil && dockerStats.CPU != "" { |
|
cpu := strings.Replace(dockerStats.CPU, "%", "", 1) |
|
cpufloat, _ := strconv.ParseFloat(cpu, 32) |
|
cpuFinal := cpufloat |
|
cpuString := strconv.FormatFloat(cpuFinal, 'f', -1, 32) |
|
clientMoni.CPUUsed = cpuString |
|
} |
|
if dockerStats == nil || dockerStats.CPU == "" { |
|
//如果我们的服务挂了，就查 moni |
|
if CPUResult, err = s.QueryJobCPU(context.TODO(), pod.Name); err != nil { |
|
log.Error("query client cpu err(%v)", err) |
|
return |
|
} |
|
if len(CPUResult.Data) > 0 { |
|
for _, data := range CPUResult.Data { |
|
if len(data.Value) > 1 { |
|
clientMoni.CPUUsed = data.Value[1].(string) |
|
} |
|
} |
|
} |
|
} |
|
} |
|
|
|
//cpu 信息记录到数据库 |
|
if _, err = s.AddClientMoni(&clientMoni); err != nil { |
|
log.Error("add ClientMoni err(%v)", err) |
|
return |
|
} |
|
return |
|
} |
|
|
|
//addReportGraph add reportGraph |
|
func (s *Service) addReportGraph(c context.Context, reportGraphAdd model.ReportGraphAdd) { |
|
var ( |
|
sumTime, sumQPS, sumNetIo, sumCodeEll, sumCodeWll, sumCodeWly, sumCodeWle, sumCodeWls, sumCodeSll, sumCodeSly, sumCodeSls, |
|
sumCodeKong, sumCodeNonHTTP, sumCodeOthers, sumCount, sumError, sumThreads, fiftyTime, ninetyTime, ninetyFiveTime, |
|
timeCount, ninetyNineTime, sumCode301, sumCode302 int |
|
mins, maxs []int |
|
reportTimelys []*model.ReportTimely |
|
err error |
|
) |
|
busiStopPerc := reportGraphAdd.BusinessStopPercent |
|
if reportTimelys, err = s.dao.QueryReTimely(reportGraphAdd.TestName, reportGraphAdd.BeginTime, reportGraphAdd.AfterTime, reportGraphAdd.PodNames); err != nil { |
|
return |
|
} |
|
log.Info("------reportTimelys:(%v)", reportTimelys) |
|
if len(reportTimelys) > 0 { |
|
for _, reportTimely := range reportTimelys { |
|
mins = append(mins, reportTimely.Min) |
|
maxs = append(maxs, reportTimely.Max) |
|
sumCount = sumCount + reportTimely.Count |
|
//计算一个容器的接口响应总时间 |
|
timeCount = reportTimely.Count * reportTimely.AvgTime |
|
//计算所有容器的接口响应时间 |
|
sumTime = sumTime + timeCount |
|
sumQPS = sumQPS + reportTimely.QPS |
|
sumError = sumError + reportTimely.Error |
|
sumNetIo = sumNetIo + reportTimely.NetIo |
|
sumCodeEll = sumCodeEll + reportTimely.CodeEll |
|
sumCodeWll = sumCodeWll + reportTimely.CodeWll |
|
sumCodeWly = sumCodeWly + reportTimely.CodeWly |
|
sumCodeWle = sumCodeWle + reportTimely.CodeWle |
|
sumCodeWls = sumCodeWls + reportTimely.CodeWls |
|
sumCodeSll = sumCodeSll + reportTimely.CodeSll |
|
sumCodeSly = sumCodeSly + reportTimely.CodeSly |
|
sumCodeSls = sumCodeSls + reportTimely.CodeSls |
|
sumCodeKong = sumCodeKong + reportTimely.CodeKong |
|
sumCodeNonHTTP = sumCodeNonHTTP + reportTimely.CodeNonHTTP |
|
sumCodeOthers = sumCodeOthers + reportTimely.CodeOthers |
|
sumCode301 = sumCode301 + reportTimely.Code301 |
|
sumCode302 = sumCode302 + reportTimely.Code302 |
|
sumThreads = sumThreads + reportTimely.ThreadsSum |
|
fiftyTime = fiftyTime + reportTimely.FiftyTime |
|
ninetyTime = ninetyTime + reportTimely.NinetyTime |
|
ninetyFiveTime = ninetyFiveTime + reportTimely.NinetyFiveTime |
|
ninetyNineTime = ninetyNineTime + reportTimely.NinetyNineTime |
|
} |
|
sort.Ints(mins) |
|
min := mins[0] |
|
sort.Ints(maxs) |
|
max := maxs[len(maxs)-1] |
|
log.Info("test_nick_name ----- :(%s)", reportGraphAdd.TestNameNick) |
|
reportGraph := model.ReportGraph{ |
|
TestName: reportGraphAdd.TestName, TestNameNick: reportGraphAdd.TestNameNick, |
|
Count: sumCount, |
|
QPS: sumQPS, |
|
Error: sumError, |
|
AvgTime: sumTime / sumCount, |
|
NetIo: sumNetIo, |
|
Min: min, |
|
Max: max, |
|
ElapsdTime: reportGraphAdd.ElapsedTime, |
|
CodeEll: sumCodeEll, |
|
CodeWll: sumCodeWll, |
|
CodeWly: sumCodeWly, |
|
CodeWle: sumCodeWle, |
|
CodeWls: sumCodeWls, |
|
CodeSll: sumCodeSll, |
|
CodeSly: sumCodeSly, |
|
CodeSls: sumCodeSls, |
|
CodeKong: sumCodeKong, |
|
CodeNonHTTP: sumCodeNonHTTP, |
|
CodeOthers: sumCodeOthers, |
|
ThreadsSum: sumThreads, |
|
FiftyTime: fiftyTime / len(reportTimelys), |
|
NinetyTime: ninetyTime / len(reportTimelys), |
|
NinetyFiveTime: ninetyFiveTime / len(reportTimelys), |
|
NinetyNineTime: ninetyNineTime / len(reportTimelys), |
|
Code301: sumCode301, |
|
Code302: sumCode302, |
|
} |
|
suCodeRate, suBusinessRate := CalcuSuccess(reportGraph) |
|
if reportGraphAdd.Fusing == 0 { |
|
reportGraphAdd.Fusing = conf.Conf.Melloi.DefaultFusing |
|
} |
|
//压测熔断 |
|
//使用业务熔断，只要http code 和 业务断言错误满足一条阈值，则熔断 |
|
if reportGraphAdd.UseBusinessStop { |
|
if busiStopPerc == 0 { |
|
busiStopPerc = conf.Conf.Melloi.DefaultBusinessRate |
|
} |
|
if suCodeRate < reportGraphAdd.Fusing || suBusinessRate < busiStopPerc { |
|
log.Info("接口失败率超过设置阈值，执行自动熔断,jobName ：(%s)", reportGraphAdd.JobName) |
|
if _, err = s.DeleteJob(c, reportGraphAdd.JobName); err != nil { |
|
return |
|
} |
|
reportsu := model.ReportSummary{ID: reportGraphAdd.ReportSuID, IsFusing: true, BusinessValue: busiStopPerc, UseBusinessStop: reportGraphAdd.UseBusinessStop, |
|
FusingTestName: reportGraphAdd.TestName, SuccessCodeRate: suCodeRate, SuccessBusinessRate: suBusinessRate, FusingValue: reportGraphAdd.Fusing} |
|
s.dao.UpdateReportSummary(&reportsu) |
|
} |
|
} else { //不使用业务熔断 |
|
if suCodeRate < reportGraphAdd.Fusing { |
|
log.Info("接口失败率超过设置阈值，执行自动熔断,jobName ：(%s)", reportGraphAdd.JobName) |
|
if _, err = s.DeleteJob(c, reportGraphAdd.JobName); err != nil { |
|
return |
|
} |
|
reportsu := model.ReportSummary{ID: reportGraphAdd.ReportSuID, IsFusing: true, BusinessValue: busiStopPerc, |
|
FusingTestName: reportGraphAdd.TestName, SuccessCodeRate: suCodeRate, SuccessBusinessRate: suBusinessRate, FusingValue: reportGraphAdd.Fusing} |
|
s.dao.UpdateReportSummary(&reportsu) |
|
} |
|
} |
|
|
|
s.dao.AddReportGraph(&reportGraph) |
|
return |
|
} |
|
} |
|
|
|
//CalcuSuccess calcu success |
|
func CalcuSuccess(reportGraph model.ReportGraph) (successCodeRate, successBusinessRate int) { |
|
success := reportGraph.CodeEll + reportGraph.CodeKong + reportGraph.Code301 + reportGraph.Code302 |
|
successCodeRate = success * 100 / reportGraph.Count |
|
successBusinessRate = (reportGraph.Count - reportGraph.Error) * 100 / reportGraph.Count |
|
return |
|
} |
|
|
|
// DelReportSummary delete report summary |
|
func (s *Service) DelReportSummary(id int) error { |
|
return s.dao.DelReportSummary(id) |
|
}
|
|
|