gd_crontab/impl/warning/warning_check_log.go

// Copyright 2019 getensh.com. All rights reserved.
// Use of this source code is governed by getensh.com.

package warning

import (
	"context"
	"gd_crontab/apis"
	"gd_crontab/rpc_apis"
	"gd_crontab/rpc_apis/gd_management"
	"fmt"
	"strings"
	"sync"
	"time"

	"gd_crontab/common.in/config"
)

// accessDB is database name for warning in mongodb
var accessDB = "db_gd_access_log"

// accessTabName is collection name for api log
var accessTabName = "t_gd_access_log"

// thirdTabName is collection name for provider log
var thirdTabName = "t_gd_thirdpart_access_log"

// mutex is used for exclusive writing of log
var mutex sync.RWMutex

// AccessLogCheck contains targets and api information for warning
type AccessLogCheck struct {
	ApiId        int64   `json:"api_id" bson:"api_id"`
	ApiName      string  `json:"api_name" bson:"api_name"`
	ApiRouter    string  `json:"api_router" bson:"api_router"`
	Count        int     `json:"count" bson:"count"`
	Success      int     `json:"success" bson:"success"`
	Valid        int     `json:"valid" bson:"valid"`
	NoRecord     int     `json:"no_record" bson:"no_record"`
	AvgElapsed   float64 `json:"avg_elapsed" bson:"avg_elapsed"`
	Elapsed      float64 `json:"avg_elapsed" bson:"elapsed"`
	FailRate     float64 `json:"fail_rate" bson:"fail_rate"`
	NorecordRate float64 `json:"norecord_rate" bson:"norecord_rate"`
}

func GetWarningMongoDb() string {
	return accessDB
}

// ThirdLogCheck contains targets and provider api information for warning
type ThirdLogCheck struct {
	ProviderName    string  `json:"provider_name" bson:"provider_name"`
	ProviderApiName string  `json:"provider_api_name" bson:"api_name"`
	ApiRouter       string  `json:"api_router" bson:"api_router"`
	Count           int     `json:"count" bson:"count"`
	Success         int     `json:"success" bson:"success"`
	Valid           int     `json:"valid" bson:"valid"`
	NoRecord        int     `json:"no_record" bson:"no_record"`
	AvgElapsed      float64 `json:"avg_elapsed" bson:"avg_elapsed"`
	ProviderApiId   int64   `json:"provider_api_id" bson:"provider_api_id"`
	Elapsed         float64 `json:"avg_elapsed" bson:"elapsed"`
	FailRate        float64 `json:"fail_rate" bson:"fail_rate"`
	NorecordRate    float64 `json:"norecord_rate" bson:"norecord_rate"`
}

func Percent(data float64) string {
	i := int(100.0 * data)
	if i == 0 {
		return "0"
	}
	return fmt.Sprintf("%d%%", i)
}

// getThreshold get thresholds from mysql,
// if one value is zero, do not warning
func getThreshold(atype int, id int64) (float64, float64, float64, bool) {
	req := gd_management.GetApiThresholdReq{
		Type:  atype,
		ApiId: id,
	}
	reply, _ := rpc_apis.Management.GetApiThreshold(context.Background(), &req)
	if reply.ThresholdTimeout == 0 {
		reply.ThresholdTimeout = 9999
	}
	if reply.ThresholdFailRate == 0 {
		reply.ThresholdFailRate = 9999
	}
	if reply.ThresholdNorecordRate == 0 {
		reply.ThresholdNorecordRate = 9999
	}
	return float64(reply.ThresholdTimeout), reply.ThresholdFailRate, reply.ThresholdNorecordRate, reply.WarningEnable
}

// setCol 设置邮件html表格单元格
func setCol(value string, color string) string {
	if color == "" {
		return fmt.Sprintf("<td>%s</td>", value)
	}
	return fmt.Sprintf("<td bgcolor=\"%s\">%s</td>", color, value)
}

// setRow 组装html表格行数据
func setRow(cols []string) string {
	value := ""
	for _, v := range cols {
		value = fmt.Sprintf("%s%s", value, v)
	}
	return fmt.Sprintf("<tr>%s</tr>", value)
}

// setTable 组装html表格
func setTable(rows []string) string {
	value := ""
	for _, v := range rows {
		value = fmt.Sprintf("%s%s", value, v)
	}
	return fmt.Sprintf("<table border=\"1\">%s</table>", value)
}

// convertThirdLogToTable 将数据源日志预警信息转为html表格字符串
func convertThirdLogToList(datas []ThirdLogCheck) []string {
	lines := make([]string,0)
	// 设置行数据
	for _, v := range datas {
		line := ""

		line = fmt.Sprintf("数据源api名称:%s\n",v.ProviderApiName)
		line = line + fmt.Sprintf("供应商:%s\n",v.ProviderName)
		line = line + fmt.Sprintf("统计量:%d\n",v.Count)
		line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed)
		line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid)))
		line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid)))
		lines = append(lines,line)
	}

	return lines
}


// convertThirdLogToTable 将数据源日志预警信息转为html表格字符串
func convertThirdLogToTable(datas []ThirdLogCheck) string {
	rows := make([]string, len(datas))
	cols := make([]string, 6)

	// 设置表头
	cols[0] = setCol("数据源api名称", "")
	cols[1] = setCol("供应商", "")
	cols[2] = setCol("统计量", "")
	cols[3] = setCol("平均响应时长", "")
	cols[4] = setCol("失败率", "")
	cols[5] = setCol("查无率", "")
	row := setRow(cols)
	rows[0] = row

	// 设置行数据
	for index, v := range datas {
		elapsed := v.Elapsed
		failRate := v.FailRate
		norecordRate := v.NorecordRate
		col := setCol(v.ProviderApiName, "")
		cols[0] = col

		col = setCol(v.ProviderName, "")
		cols[1] = col

		col = setCol(fmt.Sprintf("%d", v.Count), "")
		cols[2] = col

		if v.AvgElapsed < elapsed {
			col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "")
		} else {
			col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow")
		}
		cols[3] = col

		if float64(v.Success)/float64(v.Valid) > (1 - failRate) {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "")
		} else {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow")

		}
		cols[4] = col

		if float64(v.NoRecord)/float64(v.Valid) < norecordRate {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "")
		} else {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow")
		}
		cols[5] = col

		row = setRow(cols)
		rows[index+1] = row
	}

	// 组装表格
	table := setTable(rows)
	return table
}

// accessWarningToMail api日志预警信息发送邮件
func accessWarningToMail(datas []AccessLogCheck, start, end int64) {
	if len(datas) == 0 {
		return
	}

	// 将预警信息转化为表格
	content := convertAccessLogToList(datas)
	req := apis.Warning{}
	req.WarningText = content
	req.To = strings.Split(config.Conf.Warning.DefaultMails, ";")
	req.Subject = "平台api日志告警"
	if start > 0 && end > 0 {
		req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)",
			req.Subject,
			time.Unix(start, 0).Format("2006-01-02 15:04:05"),
			time.Unix(end, 0).Format("2006-01-02 15:04:05"),
			start, end)
	}

	// 发送预警邮件
	Warning(&req)
}

func convertAccessLogToList(datas []AccessLogCheck) []string {
	lines := make([]string, 0)
	for _, v := range datas {
		line := ""
		line = fmt.Sprintf("平台api名称:%s\n",v.ApiName)
		line = line + fmt.Sprintf("统计量:%d\n",v.Count)
		line = line + fmt.Sprintf("平均响应时长:%.2f\n",v.AvgElapsed)
		line = line + fmt.Sprintf("失败率:%s\n",Percent(float64(v.Valid-v.Success)/float64(v.Valid)))
		line = line + fmt.Sprintf("查无率:%s\n",Percent(float64(v.NoRecord)/float64(v.Valid)))
		lines = append(lines,line)
	}
	return lines
}

// convertAccessLogToTable 将api日志预警信息转换为表格
func convertAccessLogToTable(datas []AccessLogCheck) string {
	rows := make([]string, len(datas)+1)
	cols := make([]string, 5)
	cols[0] = setCol("平台api名称", "")
	cols[1] = setCol("统计量", "")
	cols[2] = setCol("平均响应时长", "")
	cols[3] = setCol("失败率", "")
	cols[4] = setCol("查无率", "")
	row := setRow(cols)
	rows[0] = row

	for index, v := range datas {
		elapsed := v.Elapsed
		failRate := v.FailRate
		norecordRate := v.NorecordRate
		col := setCol(v.ApiName, "")
		cols[0] = col
		col = setCol(fmt.Sprintf("%d", v.Count), "")
		cols[1] = col
		if v.AvgElapsed < elapsed {
			col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "")
		} else {
			col = setCol(fmt.Sprintf("%.2f", v.AvgElapsed), "yellow")
		}
		cols[2] = col

		if float64(v.Success)/float64(v.Valid) > (1 - failRate) {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "")
		} else {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.Valid-v.Success)/float64(v.Valid))), "yellow")

		}
		cols[3] = col

		if float64(v.NoRecord)/float64(v.Valid) < norecordRate {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "")
		} else {
			col = setCol(fmt.Sprintf("%s", Percent(float64(v.NoRecord)/float64(v.Valid))), "yellow")
		}
		cols[4] = col
		row = setRow(cols)
		rows[index+1] = row
	}
	table := setTable(rows)
	return table
}

// thirdWarningToMail 数据源日志预警发送邮件
func thirdWarningToMail(datas []ThirdLogCheck, start, end int64) {
	if len(datas) == 0 {
		return
	}
	content := convertThirdLogToList(datas)
	req := apis.Warning{}
	req.WarningText = content
	req.To = strings.Split(config.Conf.Warning.DefaultMails, ";")
	req.Subject = "数据源api日志告警"
	if start > 0 && end > 0 {
		req.Subject = fmt.Sprintf("%s(%s-%s %d-%d)",
			req.Subject,
			time.Unix(start, 0).Format("2006-01-02:15:04:05"),
			time.Unix(end, 0).Format("2006-01-02:15:04:05"),
			start, end)
	}
	Warning(&req)
}


type MgoAccessLog struct {
	ApiName   string  `json:"api_name" bson:"api_name"`
	Elapsed   float64 `json:"elapsed" bson:"elapsed"`
	State     int     `json:"state" bson:"state"`
	Code      int     `json:"code" bson:"code"`
	ApiId     int64   `json:"api_id" bson:"api_id"`
	Timestamp int64   `json:"timestamp" bson:"timestamp"`
}

type MgoThirdLog struct {
	ProviderApiName string  `json:"provider_api_name" bson:"provider_api_name"`
	ProviderName    string  `json:"provider_name" bson:"provider_name"`
	Elapsed         float64 `json:"elapsed" bson:"elapsed"`
	State           int     `json:"state" bson:"state"`
	Code            int     `json:"code" bson:"code"`
	ProviderApiId   int64   `json:"provider_api_id" bson:"provider_api_id"`
	Timestamp       int64   `json:"timestamp" bson:"timestamp"`
}