// Copyright 2019 getensh.com. All rights reserved. // Use of this source code is governed by getensh.com. package warning import ( "context" "gd_crontab/apis" "gd_crontab/rpc_apis" "gd_crontab/rpc_apis/gd_management" "fmt" "strings" "sync" "time" "gd_crontab/common.in/config" ) // accessDB is database name for warning in mongodb var accessDB = "db_gd_access_log" // accessTabName is collection name for api log var accessTabName = "t_gd_access_log" // thirdTabName is collection name for provider log var thirdTabName = "t_gd_thirdpart_access_log" // mutex is used for exclusive writing of log var mutex sync.RWMutex // AccessLogCheck contains targets and api information for warning type AccessLogCheck struct { ApiId int64 `json:"api_id" bson:"api_id"` ApiName string `json:"api_name" bson:"api_name"` ApiRouter string `json:"api_router" bson:"api_router"` Count int `json:"count" bson:"count"` Success int `json:"success" bson:"success"` Valid int `json:"valid" bson:"valid"` NoRecord int `json:"no_record" bson:"no_record"` AvgElapsed float64 `json:"avg_elapsed" bson:"avg_elapsed"` Elapsed float64 `json:"avg_elapsed" bson:"elapsed"` FailRate float64 `json:"fail_rate" bson:"fail_rate"` NorecordRate float64 `json:"norecord_rate" bson:"norecord_rate"` } func GetWarningMongoDb() string { return accessDB } // ThirdLogCheck contains targets and provider api information for warning type ThirdLogCheck struct { ProviderName string `json:"provider_name" bson:"provider_name"` ProviderApiName string `json:"provider_api_name" bson:"api_name"` ApiRouter string `json:"api_router" bson:"api_router"` Count int `json:"count" bson:"count"` Success int `json:"success" bson:"success"` Valid int `json:"valid" bson:"valid"` NoRecord int `json:"no_record" bson:"no_record"` AvgElapsed float64 `json:"avg_elapsed" bson:"avg_elapsed"` ProviderApiId int64 `json:"provider_api_id" bson:"provider_api_id"` Elapsed float64 `json:"avg_elapsed" bson:"elapsed"` FailRate float64 `json:"fail_rate" bson:"fail_rate"` NorecordRate float64 `json:"norecord_rate" bson:"norecord_rate"` } func Percent(data float64) string { i := int(100.0 * data) if i == 0 { return "0" } return fmt.Sprintf("%d%%", i) } // getThreshold get thresholds from mysql, // if one value is zero, do not warning func getThreshold(atype int, id int64) (float64, float64, float64, bool) { req := gd_management.GetApiThresholdReq{ Type: atype, ApiId: id, } reply, _ := rpc_apis.Management.GetApiThreshold(context.Background(), &req) if reply.ThresholdTimeout == 0 { reply.ThresholdTimeout = 9999 } if reply.ThresholdFailRate == 0 { reply.ThresholdFailRate = 9999 } if reply.ThresholdNorecordRate == 0 { reply.ThresholdNorecordRate = 9999 } return float64(reply.ThresholdTimeout), reply.ThresholdFailRate, reply.ThresholdNorecordRate, reply.WarningEnable } // setCol 设置邮件html表格单元格 func setCol(value string, color string) string { if color == "" { return fmt.Sprintf("%s", value) } return fmt.Sprintf("%s", color, value) } // setRow 组装html表格行数据 func setRow(cols []string) string { value := "" for _, v := range cols { value = fmt.Sprintf("%s%s", value, v) } return fmt.Sprintf("%s", value) } // setTable 组装html表格 func setTable(rows []string) string { value := "" for _, v := range rows { value = fmt.Sprintf("%s%s", value, v) } return fmt.Sprintf("%s
", value) } // convertThirdLogToTable 将数据源日志预警信息转为html表格字符串 func convertThirdLogToList(datas []ThirdLogCheck) []string { lines := make([]string,0) // 设置行数据 for _, v := range datas { line := "" line = fmt.Sprintf("数据源api名称:%s\n",v.ProviderApiName) line = line + fmt.Sprintf("供应商:%s\n",v.ProviderName) line = line + fmt.Sprintf("统计量:%d\n",v.Count) line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed) line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid))) line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid))) lines = append(lines,line) } return lines } // convertThirdLogToTable 将数据源日志预警信息转为html表格字符串 func convertThirdLogToTable(datas []ThirdLogCheck) string { rows := make([]string, len(datas)) cols := make([]string, 6) // 设置表头 cols[0] = setCol("数据源api名称", "") cols[1] = setCol("供应商", "") cols[2] = setCol("统计量", "") cols[3] = setCol("平均响应时长", "") cols[4] = setCol("失败率", "") cols[5] = setCol("查无率", "") row := setRow(cols) rows[0] = row // 设置行数据 for index, v := range datas { elapsed := v.Elapsed failRate := v.FailRate norecordRate := v.NorecordRate col := setCol(v.ProviderApiName, "") cols[0] = col col = setCol(v.ProviderName, "") cols[1] = col col = setCol(fmt.Sprintf("%d", v.Count), "") cols[2] = col if v.AvgElapsed < elapsed { col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "") } else { col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow") } cols[3] = col if float64(v.Success)/float64(v.Valid) > (1 - failRate) { col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "") } else { col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow") } cols[4] = col if float64(v.NoRecord)/float64(v.Valid) < norecordRate { col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "") } else { col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow") } cols[5] = col row = setRow(cols) rows[index+1] = row } // 组装表格 table := setTable(rows) return table } // accessWarningToMail api日志预警信息发送邮件 func accessWarningToMail(datas []AccessLogCheck, start, end int64) { if len(datas) == 0 { return } // 将预警信息转化为表格 content := convertAccessLogToList(datas) req := apis.Warning{} req.WarningText = content req.To = strings.Split(config.Conf.Warning.DefaultMails, ";") req.Subject = "平台api日志告警" if start > 0 && end > 0 { req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)", req.Subject, time.Unix(start, 0).Format("2006-01-02 15:04:05"), time.Unix(end, 0).Format("2006-01-02 15:04:05"), start, end) } // 发送预警邮件 Warning(&req) } func convertAccessLogToList(datas []AccessLogCheck) []string { lines := make([]string, 0) for _, v := range datas { line := "" line = fmt.Sprintf("平台api名称:%s\n",v.ApiName) line = line + fmt.Sprintf("统计量:%d\n",v.Count) line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed) line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid))) line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid))) lines = append(lines,line) } return lines } // convertAccessLogToTable 将api日志预警信息转换为表格 func convertAccessLogToTable(datas []AccessLogCheck) string { rows := make([]string, len(datas)+1) cols := make([]string, 5) cols[0] = setCol("平台api名称", "") cols[1] = setCol("统计量", "") cols[2] = setCol("平均响应时长", "") cols[3] = setCol("失败率", "") cols[4] = setCol("查无率", "") row := setRow(cols) rows[0] = row for index, v := range datas { elapsed := v.Elapsed failRate := v.FailRate norecordRate := v.NorecordRate col := setCol(v.ApiName, "") cols[0] = col col = setCol(fmt.Sprintf("%d", v.Count), "") cols[1] = col if v.AvgElapsed < elapsed { col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "") } else { col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow") } cols[2] = col if float64(v.Success)/float64(v.Valid) > (1 - failRate) { col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "") } else { col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow") } cols[3] = col if float64(v.NoRecord)/float64(v.Valid) < norecordRate { col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "") } else { col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow") } cols[4] = col row = setRow(cols) rows[index+1] = row } table := setTable(rows) return table } // thirdWarningToMail 数据源日志预警发送邮件 func thirdWarningToMail(datas []ThirdLogCheck, start, end int64) { if len(datas) == 0 { return } content := convertThirdLogToList(datas) req := apis.Warning{} req.WarningText = content req.To = strings.Split(config.Conf.Warning.DefaultMails, ";") req.Subject = "数据源api日志告警" if start > 0 && end > 0 { req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)", req.Subject, time.Unix(start, 0).Format("2006-01-02:15:04:05"), time.Unix(end, 0).Format("2006-01-02:15:04:05"), start, end) } Warning(&req) } type MgoAccessLog struct { ApiName string `json:"api_name" bson:"api_name"` Elapsed float64 `json:"elapsed" bson:"elapsed"` State int `json:"state" bson:"state"` Code int `json:"code" bson:"code"` ApiId int64 `json:"api_id" bson:"api_id"` Timestamp int64 `json:"timestamp" bson:"timestamp"` } type MgoThirdLog struct { ProviderApiName string `json:"provider_api_name" bson:"provider_api_name"` ProviderName string `json:"provider_name" bson:"provider_name"` Elapsed float64 `json:"elapsed" bson:"elapsed"` State int `json:"state" bson:"state"` Code int `json:"code" bson:"code"` ProviderApiId int64 `json:"provider_api_id" bson:"provider_api_id"` Timestamp int64 `json:"timestamp" bson:"timestamp"` }