// Copyright 2019 getensh.com. All rights reserved.
// Use of this source code is governed by getensh.com.
package warning
import (
"context"
"gd_crontab/apis"
"gd_crontab/rpc_apis"
"gd_crontab/rpc_apis/gd_management"
"fmt"
"strings"
"sync"
"time"
"gd_crontab/common.in/config"
)
// accessDB is database name for warning in mongodb
var accessDB = "db_gd_access_log"
// accessTabName is collection name for api log
var accessTabName = "t_gd_access_log"
// thirdTabName is collection name for provider log
var thirdTabName = "t_gd_thirdpart_access_log"
// mutex is used for exclusive writing of log
var mutex sync.RWMutex
// AccessLogCheck contains targets and api information for warning
type AccessLogCheck struct {
ApiId int64 `json:"api_id" bson:"api_id"`
ApiName string `json:"api_name" bson:"api_name"`
ApiRouter string `json:"api_router" bson:"api_router"`
Count int `json:"count" bson:"count"`
Success int `json:"success" bson:"success"`
Valid int `json:"valid" bson:"valid"`
NoRecord int `json:"no_record" bson:"no_record"`
AvgElapsed float64 `json:"avg_elapsed" bson:"avg_elapsed"`
Elapsed float64 `json:"avg_elapsed" bson:"elapsed"`
FailRate float64 `json:"fail_rate" bson:"fail_rate"`
NorecordRate float64 `json:"norecord_rate" bson:"norecord_rate"`
}
func GetWarningMongoDb() string {
return accessDB
}
// ThirdLogCheck contains targets and provider api information for warning
type ThirdLogCheck struct {
ProviderName string `json:"provider_name" bson:"provider_name"`
ProviderApiName string `json:"provider_api_name" bson:"api_name"`
ApiRouter string `json:"api_router" bson:"api_router"`
Count int `json:"count" bson:"count"`
Success int `json:"success" bson:"success"`
Valid int `json:"valid" bson:"valid"`
NoRecord int `json:"no_record" bson:"no_record"`
AvgElapsed float64 `json:"avg_elapsed" bson:"avg_elapsed"`
ProviderApiId int64 `json:"provider_api_id" bson:"provider_api_id"`
Elapsed float64 `json:"avg_elapsed" bson:"elapsed"`
FailRate float64 `json:"fail_rate" bson:"fail_rate"`
NorecordRate float64 `json:"norecord_rate" bson:"norecord_rate"`
}
func Percent(data float64) string {
i := int(100.0 * data)
if i == 0 {
return "0"
}
return fmt.Sprintf("%d%%", i)
}
// getThreshold get thresholds from mysql,
// if one value is zero, do not warning
func getThreshold(atype int, id int64) (float64, float64, float64, bool) {
req := gd_management.GetApiThresholdReq{
Type: atype,
ApiId: id,
}
reply, _ := rpc_apis.Management.GetApiThreshold(context.Background(), &req)
if reply.ThresholdTimeout == 0 {
reply.ThresholdTimeout = 9999
}
if reply.ThresholdFailRate == 0 {
reply.ThresholdFailRate = 9999
}
if reply.ThresholdNorecordRate == 0 {
reply.ThresholdNorecordRate = 9999
}
return float64(reply.ThresholdTimeout), reply.ThresholdFailRate, reply.ThresholdNorecordRate, reply.WarningEnable
}
// setCol 设置邮件html表格单元格
func setCol(value string, color string) string {
if color == "" {
return fmt.Sprintf("
%s | ", value)
}
return fmt.Sprintf("%s | ", color, value)
}
// setRow 组装html表格行数据
func setRow(cols []string) string {
value := ""
for _, v := range cols {
value = fmt.Sprintf("%s%s", value, v)
}
return fmt.Sprintf("%s
", value)
}
// setTable 组装html表格
func setTable(rows []string) string {
value := ""
for _, v := range rows {
value = fmt.Sprintf("%s%s", value, v)
}
return fmt.Sprintf("", value)
}
// convertThirdLogToTable 将数据源日志预警信息转为html表格字符串
func convertThirdLogToList(datas []ThirdLogCheck) []string {
lines := make([]string,0)
// 设置行数据
for _, v := range datas {
line := ""
line = fmt.Sprintf("数据源api名称:%s\n",v.ProviderApiName)
line = line + fmt.Sprintf("供应商:%s\n",v.ProviderName)
line = line + fmt.Sprintf("统计量:%d\n",v.Count)
line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed)
line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid)))
line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid)))
lines = append(lines,line)
}
return lines
}
// convertThirdLogToTable 将数据源日志预警信息转为html表格字符串
func convertThirdLogToTable(datas []ThirdLogCheck) string {
rows := make([]string, len(datas))
cols := make([]string, 6)
// 设置表头
cols[0] = setCol("数据源api名称", "")
cols[1] = setCol("供应商", "")
cols[2] = setCol("统计量", "")
cols[3] = setCol("平均响应时长", "")
cols[4] = setCol("失败率", "")
cols[5] = setCol("查无率", "")
row := setRow(cols)
rows[0] = row
// 设置行数据
for index, v := range datas {
elapsed := v.Elapsed
failRate := v.FailRate
norecordRate := v.NorecordRate
col := setCol(v.ProviderApiName, "")
cols[0] = col
col = setCol(v.ProviderName, "")
cols[1] = col
col = setCol(fmt.Sprintf("%d", v.Count), "")
cols[2] = col
if v.AvgElapsed < elapsed {
col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "")
} else {
col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow")
}
cols[3] = col
if float64(v.Success)/float64(v.Valid) > (1 - failRate) {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "")
} else {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow")
}
cols[4] = col
if float64(v.NoRecord)/float64(v.Valid) < norecordRate {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "")
} else {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow")
}
cols[5] = col
row = setRow(cols)
rows[index+1] = row
}
// 组装表格
table := setTable(rows)
return table
}
// accessWarningToMail api日志预警信息发送邮件
func accessWarningToMail(datas []AccessLogCheck, start, end int64) {
if len(datas) == 0 {
return
}
// 将预警信息转化为表格
content := convertAccessLogToList(datas)
req := apis.Warning{}
req.WarningText = content
req.To = strings.Split(config.Conf.Warning.DefaultMails, ";")
req.Subject = "平台api日志告警"
if start > 0 && end > 0 {
req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)",
req.Subject,
time.Unix(start, 0).Format("2006-01-02 15:04:05"),
time.Unix(end, 0).Format("2006-01-02 15:04:05"),
start, end)
}
// 发送预警邮件
Warning(&req)
}
func convertAccessLogToList(datas []AccessLogCheck) []string {
lines := make([]string, 0)
for _, v := range datas {
line := ""
line = fmt.Sprintf("平台api名称:%s\n",v.ApiName)
line = line + fmt.Sprintf("统计量:%d\n",v.Count)
line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed)
line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid)))
line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid)))
lines = append(lines,line)
}
return lines
}
// convertAccessLogToTable 将api日志预警信息转换为表格
func convertAccessLogToTable(datas []AccessLogCheck) string {
rows := make([]string, len(datas)+1)
cols := make([]string, 5)
cols[0] = setCol("平台api名称", "")
cols[1] = setCol("统计量", "")
cols[2] = setCol("平均响应时长", "")
cols[3] = setCol("失败率", "")
cols[4] = setCol("查无率", "")
row := setRow(cols)
rows[0] = row
for index, v := range datas {
elapsed := v.Elapsed
failRate := v.FailRate
norecordRate := v.NorecordRate
col := setCol(v.ApiName, "")
cols[0] = col
col = setCol(fmt.Sprintf("%d", v.Count), "")
cols[1] = col
if v.AvgElapsed < elapsed {
col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "")
} else {
col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow")
}
cols[2] = col
if float64(v.Success)/float64(v.Valid) > (1 - failRate) {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "")
} else {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow")
}
cols[3] = col
if float64(v.NoRecord)/float64(v.Valid) < norecordRate {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "")
} else {
col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow")
}
cols[4] = col
row = setRow(cols)
rows[index+1] = row
}
table := setTable(rows)
return table
}
// thirdWarningToMail 数据源日志预警发送邮件
func thirdWarningToMail(datas []ThirdLogCheck, start, end int64) {
if len(datas) == 0 {
return
}
content := convertThirdLogToList(datas)
req := apis.Warning{}
req.WarningText = content
req.To = strings.Split(config.Conf.Warning.DefaultMails, ";")
req.Subject = "数据源api日志告警"
if start > 0 && end > 0 {
req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)",
req.Subject,
time.Unix(start, 0).Format("2006-01-02:15:04:05"),
time.Unix(end, 0).Format("2006-01-02:15:04:05"),
start, end)
}
Warning(&req)
}
type MgoAccessLog struct {
ApiName string `json:"api_name" bson:"api_name"`
Elapsed float64 `json:"elapsed" bson:"elapsed"`
State int `json:"state" bson:"state"`
Code int `json:"code" bson:"code"`
ApiId int64 `json:"api_id" bson:"api_id"`
Timestamp int64 `json:"timestamp" bson:"timestamp"`
}
type MgoThirdLog struct {
ProviderApiName string `json:"provider_api_name" bson:"provider_api_name"`
ProviderName string `json:"provider_name" bson:"provider_name"`
Elapsed float64 `json:"elapsed" bson:"elapsed"`
State int `json:"state" bson:"state"`
Code int `json:"code" bson:"code"`
ProviderApiId int64 `json:"provider_api_id" bson:"provider_api_id"`
Timestamp int64 `json:"timestamp" bson:"timestamp"`
}