gd_crontab/impl/warning/warning_log.go

// Copyright 2019 getensh.com. All rights reserved.
// Use of this source code is governed by getensh.com.

package warning

import (
	"context"
	"gd_crontab/apis"
	"gd_crontab/common.in/config"
	"fmt"
	"sync"
	"time"
)


type WarningCount struct{
	ApiName string
	ProviderName string
	Total map[int64]int64
	Valid map[int64]int64
	Success map[int64]int64
	NoRecord map[int64]int64
	Elapsed map[int64]float64
}

type Waring struct{
	RwMutex sync.RWMutex
	Count map[int64]WarningCount
}

type Rate struct {
	Elapsed float64
	FailRate float64
	NorecordRate float64
	Enable bool
}

var ApiMutex sync.Mutex

var ApiRate map[int64]Rate
var ProviderApiRate map[int64]Rate


var AccessLogWaring Waring
var ThirdPartLogWaring Waring

var AccessLogLastTime map[int64]int64
var ThirparthLastTime map[int64]int64
func getAccessLogCode(code int,codeType int ) int64 {
	switch codeType {
	case 1:// 有效
		if code == 0 || code == 20001 || code == 20002 || code == 10001  || code == 10002{
			return 1
		}
	case 2: // 成功
		if code == 0 || code == 20001{
			return 1
		}
	case 3: // 查无
		if code == 20001{
			return 1
		}
	}
	return 0
}

func CountAccessLog(accessLog *apis.ReciveAccessLogReq){
	timeNow := accessLog.Timestamp
	AccessLogWaring.RwMutex.Lock()
	defer AccessLogWaring.RwMutex.Unlock()
	if AccessLogWaring.Count == nil {
		AccessLogWaring.Count = make(map[int64]WarningCount)
	}
	if v,ok := AccessLogWaring.Count[accessLog.ApiId];ok{
		v.ApiName = accessLog.ApiName
		if no ,ok := v.Total[timeNow];ok{
			v.Total[timeNow] = no + 1
		}else {
			v.Total[timeNow] = 1
		}

		if no ,ok := v.Valid[timeNow];ok{
			v.Valid[timeNow] = no + getAccessLogCode(accessLog.Code,1)
		}else {
			v.Valid[timeNow] = getAccessLogCode(accessLog.Code,1)
		}

		if no ,ok := v.Success[timeNow];ok{
			v.Success[timeNow] = no + getAccessLogCode(accessLog.Code,2)
		}else {
			v.Success[timeNow] = getAccessLogCode(accessLog.Code,2)
		}


		if no ,ok := v.NoRecord[timeNow];ok{
			v.NoRecord[timeNow] = no + getAccessLogCode(accessLog.Code,3)
		}else {
			v.NoRecord[timeNow] = getAccessLogCode(accessLog.Code,3)
		}
		if no ,ok := v.Elapsed[timeNow];ok{
			v.Elapsed[timeNow] = no + accessLog.Elapsed
		}else {
			v.Elapsed[timeNow] = accessLog.Elapsed
		}

	}else{
		tmp := WarningCount{}
		tmp.ApiName = accessLog.ApiName
		tmp.Total = make(map[int64]int64)
		tmp.Total[timeNow] = 1

		tmp.Valid = make(map[int64]int64)
		count := getAccessLogCode(accessLog.Code,1)
		if count == 1 {
			tmp.Valid[timeNow] = count
		}

		tmp.Success = make(map[int64]int64)
		count = getAccessLogCode(accessLog.Code,2)
		if count == 1 {
			tmp.Success[timeNow] =count
		}

		tmp.NoRecord = make(map[int64]int64)
		count = getAccessLogCode(accessLog.Code,3)
		if count == 1{
			tmp.NoRecord[timeNow] = count
		}

		tmp.Elapsed = make(map[int64]float64)
		tmp.Elapsed[timeNow] = accessLog.Elapsed
		AccessLogWaring.Count[accessLog.ApiId] = tmp
	}
	//fmt.Println(AccessLogWaring)
}


func CountThirdPartLog(accessLog  *apis.ReciveThirdPartLogReq){
	timeNow := accessLog.Timestamp
	ThirdPartLogWaring.RwMutex.Lock()
	defer ThirdPartLogWaring.RwMutex.Unlock()
	if ThirdPartLogWaring.Count == nil {
		ThirdPartLogWaring.Count = make(map[int64]WarningCount)
	}

	if v,ok := ThirdPartLogWaring.Count[accessLog.ProviderApiId];ok{
		v.ApiName = accessLog.ProviderApiName
		v.ProviderName = accessLog.ProviderName
		if no ,ok := v.Valid[timeNow];ok{
			v.Valid[timeNow] = no + 1
		}else {
			v.Valid[timeNow] = 1
		}

		if no ,ok := v.Success[timeNow];ok{
			if accessLog.State == 1 {
				v.Success[timeNow] = no + 1
			}
		}else {
			if accessLog.State == 1 {
				v.Success[timeNow] =  1
			}
		}


		if no ,ok := v.NoRecord[timeNow];ok{
			v.NoRecord[timeNow] = no + getAccessLogCode(accessLog.Code,3)
		}else {
			v.NoRecord[timeNow] = getAccessLogCode(accessLog.Code,3)
		}
		if no ,ok := v.Elapsed[timeNow];ok{
			v.Elapsed[timeNow] = no + accessLog.Elapsed
		}else {
			v.Elapsed[timeNow] = accessLog.Elapsed
		}

	}else{
		tmp := WarningCount{}
		tmp.ApiName = accessLog.ProviderApiName
		tmp.ProviderName = accessLog.ProviderName
		tmp.Valid = make(map[int64]int64)
		tmp.Valid[timeNow] = 1
		tmp.Success = make(map[int64]int64)
		if accessLog.State == 1{
			tmp.Success[timeNow] = 1
		}

		tmp.NoRecord = make(map[int64]int64)
		count := getAccessLogCode(accessLog.Code,3)
		if count == 1{
			tmp.NoRecord[timeNow] = count
		}

		tmp.Elapsed = make(map[int64]float64)
		tmp.Elapsed[timeNow] = accessLog.Elapsed
		ThirdPartLogWaring.Count[accessLog.ProviderApiId] = tmp
	}
	//fmt.Println(ThirdPartLogWaring)
}


func getTotal(m map[int64]int64,now,period int64) int64 {
	delta := now-period
	count := int64(0)
	for timestamp,v := range m{
		if timestamp >= delta{
			count = count + v
		}else if timestamp < delta {
			delete(m, timestamp)
		}
	}
	return count
}

func getElaspTotal(m map[int64]float64,now,period int64) float64{
	delta := now-period
	count := float64(0)
	for timestamp,v := range m{
		if timestamp >= delta{
			count = count + v
		}else if timestamp < delta {
			delete(m, timestamp)
		}
	}
	return count
}

func CheckThirdPartLastSendTime(old []ThirdLogCheck,now int64,warnSendPeriod int64) []ThirdLogCheck {
	if ThirparthLastTime == nil {
		ThirparthLastTime = make(map[int64]int64)
	}

	new := []ThirdLogCheck{}

	for _,v := range old {
		if timeOld,ok := ThirparthLastTime[v.ProviderApiId];ok{
			if timeOld+warnSendPeriod*60 <= now{
				new = append(new,v)
				ThirparthLastTime[v.ProviderApiId] = now
			}
		}else{
			new = append(new,v)
			ThirparthLastTime[v.ProviderApiId] = now
		}
	}
	return new
}

func CheckProviderLog(now int64,period int64,warnMinCount,warnSendPeriod int64){
	// 发送预警邮件
	warningLogs := CheckProviderLogImpl(now,period,warnMinCount)
	//fmt.Println("3333333333333333:",warningLogs)
	if warningLogs != nil{
		warningLogs = CheckThirdPartLastSendTime(warningLogs,now,warnSendPeriod)
		thirdWarningToMail(warningLogs, now-period, now)
	}
}

func GetPorviderApiIds() []int64 {
	providerApiIds := []int64{}
	ThirdPartLogWaring.RwMutex.Lock()
	defer ThirdPartLogWaring.RwMutex.Unlock()
	for k,_ := range ThirdPartLogWaring.Count{
		providerApiIds = append(providerApiIds,k)
	}
	return providerApiIds
}

func GetApiIds() []int64 {
	apiIds := []int64{}
	AccessLogWaring.RwMutex.Lock()
	defer AccessLogWaring.RwMutex.Unlock()
	for k,_ := range AccessLogWaring.Count{
		apiIds = append(apiIds,k)
	}
	return apiIds
}

func GetApiRate(){
	apiIds := GetApiIds()
	providerApiIds := GetPorviderApiIds()
	//fmt.Println("ids ----------------------",apiIds,providerApiIds)
	ApiRate = make(map[int64]Rate)
	ProviderApiRate = make(map[int64]Rate)
	for _,v := range apiIds{
		elapsed, failRate, norecordRate, enable := getThreshold(0, v)
		tmp := Rate{}
		tmp.Elapsed = elapsed
		tmp.FailRate = failRate
		tmp.NorecordRate = norecordRate
		tmp.Enable = enable
		ApiRate[v] = tmp
		//fmt.Println("API RATE 111111111111111111",ApiRate)
	}

	for _,v := range providerApiIds{
		elapsed, failRate, norecordRate, enable := getThreshold(1, v)
		tmp := Rate{}
		tmp.Elapsed = elapsed
		tmp.FailRate = failRate
		tmp.NorecordRate = norecordRate
		tmp.Enable = enable
		ProviderApiRate[v] = tmp
		//fmt.Println("PROVIDER API RATE 111111111111111111",ProviderApiRate)
	}
}

func CheckProviderLogImpl(now int64,period int64,warnMinCount int64) []ThirdLogCheck {

	if ThirdPartLogWaring.Count == nil {
		return nil
	}

	ThirdPartLogWaring.RwMutex.Lock()
	defer ThirdPartLogWaring.RwMutex.Unlock()

	warningLogs := []ThirdLogCheck{}
	for k,v := range ThirdPartLogWaring.Count{
		valid := getTotal(v.Valid,now,period)
		if valid == 0 {
			continue
		}

		if warnMinCount > 0 && valid <warnMinCount{
			continue
		}

		success := getTotal(v.Success,now,period)
		noRecord := getTotal(v.NoRecord,now,period)
		elapsedTotal := getElaspTotal(v.Elapsed,now,period)
		avgElapsed := elapsedTotal/float64(valid)
		//fmt.Println("----------------------------------:",valid,success,noRecord,elapsedTotal,avgElapsed)
		elapsed, failRate, norecordRate, enable := getThresholdFromMap(1, k)
		//fmt.Println("1111----------------------------------:",elapsed, failRate, norecordRate, enable)
		if enable == false {
			continue
		}

		if valid == 0{
			if avgElapsed < elapsed{
				continue
			}
		}else{
			if avgElapsed < elapsed &&
				float64(noRecord)/float64(valid) < norecordRate &&
				float64(success)/float64(valid) > (1-failRate) {
				continue
			}
		}

		acs := ThirdLogCheck{}
		acs.ProviderApiName = v.ApiName
		acs.ProviderName = v.ProviderName
		acs.Count = int(valid)
		acs.AvgElapsed = avgElapsed
		acs.Valid = int(valid)
		acs.Success = int(success)
		acs.NoRecord = int(noRecord)
		warningLogs = append(warningLogs, acs)
	}
	return warningLogs

}


func CheckAccessLogLastSendTime(old []AccessLogCheck,now int64,warnSendPeriod int64) []AccessLogCheck {
	if AccessLogLastTime == nil {
		AccessLogLastTime = make(map[int64]int64)
	}

	new := []AccessLogCheck{}

	for _,v := range old {
		if timeOld,ok := AccessLogLastTime[v.ApiId];ok{
			if timeOld+warnSendPeriod*60 <= now{
				new = append(new,v)
				AccessLogLastTime[v.ApiId] = now
			}
		}else{
			new = append(new,v)
			AccessLogLastTime[v.ApiId] = now
		}

	}
	return new
}

func CheckAccessLog(now int64,period int64,warnMinCount,warnSendPeriod int64){
	// 发送预警邮件
	warningLogs := CheckAccessLogImpl(now,period,warnMinCount)
	//fmt.Println("22222222222222222222:",warningLogs)
	if warningLogs != nil{
		warningLogs = CheckAccessLogLastSendTime(warningLogs,now,warnSendPeriod)
		accessWarningToMail(warningLogs, now-period, now)
	}
}

func getThresholdFromMap(atype int,id int64)(float64, float64, float64, bool){
	//fmt.Println("000000000000000000000000000",atype,id)
	if atype == 0 {
		if v,ok:= ApiRate[id];ok{
			//fmt.Println("get api rate 1111111111111",v)
			return v.Elapsed,v.FailRate,v.NorecordRate,v.Enable
		}
	}else if atype ==  1 {
		if v,ok:= ProviderApiRate[id];ok{
			//fmt.Println("get provider api rate 1111111111111",v)
			return v.Elapsed,v.FailRate,v.NorecordRate,v.Enable
		}
	}

	return 9999,9999,9999,false
}

func CheckAccessLogImpl(now int64,period int64,warnMinCount int64) []AccessLogCheck {
	if AccessLogWaring.Count == nil {
		return nil
	}

	AccessLogWaring.RwMutex.Lock()
	defer AccessLogWaring.RwMutex.Unlock()
	warningLogs := []AccessLogCheck{}
	for k,v := range AccessLogWaring.Count{
		//fmt.Println("??????????????????????????????",v,now,period)
		total := getTotal(v.Total,now,period)
		if total == 0 {
			continue
		}

		if warnMinCount > 0 && total <warnMinCount{
			continue
		}
		valid := getTotal(v.Valid,now,period)
		success := getTotal(v.Success,now,period)
		noRecord := getTotal(v.NoRecord,now,period)
		elapsedTotal := getElaspTotal(v.Elapsed,now,period)
		avgElapsed := elapsedTotal/float64(total)
		//fmt.Println("222222----------------------------------:",valid,success,noRecord,elapsedTotal,avgElapsed)
		elapsed, failRate, norecordRate, enable := getThresholdFromMap(0, k)
		//fmt.Println("333333----------------------------------:",elapsed, failRate, norecordRate, enable)
		if enable == false {
			continue
		}

		if valid == 0{
			if avgElapsed < elapsed{
				continue
			}
		}else{
			if avgElapsed < elapsed &&
				float64(noRecord)/float64(valid) < norecordRate &&
				float64(success)/float64(valid) > (1-failRate) {
				continue
			}
		}


		acs := AccessLogCheck{}
		acs.ApiName = v.ApiName
		acs.Count = int(total)
		acs.AvgElapsed = avgElapsed
		acs.Valid = int(valid)
		acs.Success = int(success)
		acs.NoRecord = int(noRecord)
		warningLogs = append(warningLogs, acs)
	}
	return warningLogs
}

func CheckLog(){
	defer func() {
		if r := recover(); r != nil {
			err := fmt.Errorf("%+v", r)
			fmt.Println("err:",err)
		}
	}()
	now := time.Now().Unix()
	//fmt.Println("time 11111111111111111111111111111111111111:",now)
	ApiMutex.Lock()
	defer ApiMutex.Unlock()
	GetApiRate()
	warnPeriod,_ := config.Conf.WarnPeriod.Int64()
	warnPeriod =warnPeriod*60

	warnMinCount,_:= config.Conf.WarnMinCount.Int64()
	warnSendPeriod,_:= config.Conf.WarnSendPeriod.Int64()

	CheckAccessLog(now,warnPeriod,warnMinCount,warnSendPeriod)
	CheckProviderLog(now,warnPeriod,warnMinCount,warnSendPeriod)
}

func ReciveAccessLog(ctx context.Context, req *apis.ReciveAccessLogReq, reply *apis.ReciveAccessLogReply) error{
	//fmt.Println("access log req:",req)
	defer func() {
		if r := recover(); r != nil {
			err := fmt.Errorf("%+v", r)
			fmt.Println("err:",err)
		}
	}()
	CountAccessLog(req)
	return nil
}

func ReciveThirdPartLog(ctx context.Context, req *apis.ReciveThirdPartLogReq, reply *apis.ReciveThirdPartLogReply) error{
	//fmt.Println("thirdpart log req:",req)
	defer func() {
		if r := recover(); r != nil {
			err := fmt.Errorf("%+v", r)
			fmt.Println("err:",err)
		}
	}()
	CountThirdPartLog(req)
	return nil
}