// Copyright 2019 getensh.com. All rights reserved. // Use of this source code is governed by getensh.com. package warning import ( "context" "gd_crontab/apis" "gd_crontab/common.in/config" "fmt" "sync" "time" ) type WarningCount struct{ ApiName string ProviderName string Total map[int64]int64 Valid map[int64]int64 Success map[int64]int64 NoRecord map[int64]int64 Elapsed map[int64]float64 } type Waring struct{ RwMutex sync.RWMutex Count map[int64]WarningCount } type Rate struct { Elapsed float64 FailRate float64 NorecordRate float64 Enable bool } var ApiMutex sync.Mutex var ApiRate map[int64]Rate var ProviderApiRate map[int64]Rate var AccessLogWaring Waring var ThirdPartLogWaring Waring var AccessLogLastTime map[int64]int64 var ThirparthLastTime map[int64]int64 func getAccessLogCode(code int,codeType int ) int64 { switch codeType { case 1:// 有效 if code == 0 || code == 20001 || code == 20002 || code == 10001 || code == 10002{ return 1 } case 2: // 成功 if code == 0 || code == 20001{ return 1 } case 3: // 查无 if code == 20001{ return 1 } } return 0 } func CountAccessLog(accessLog *apis.ReciveAccessLogReq){ timeNow := accessLog.Timestamp AccessLogWaring.RwMutex.Lock() defer AccessLogWaring.RwMutex.Unlock() if AccessLogWaring.Count == nil { AccessLogWaring.Count = make(map[int64]WarningCount) } if v,ok := AccessLogWaring.Count[accessLog.ApiId];ok{ v.ApiName = accessLog.ApiName if no ,ok := v.Total[timeNow];ok{ v.Total[timeNow] = no + 1 }else { v.Total[timeNow] = 1 } if no ,ok := v.Valid[timeNow];ok{ v.Valid[timeNow] = no + getAccessLogCode(accessLog.Code,1) }else { v.Valid[timeNow] = getAccessLogCode(accessLog.Code,1) } if no ,ok := v.Success[timeNow];ok{ v.Success[timeNow] = no + getAccessLogCode(accessLog.Code,2) }else { v.Success[timeNow] = getAccessLogCode(accessLog.Code,2) } if no ,ok := v.NoRecord[timeNow];ok{ v.NoRecord[timeNow] = no + getAccessLogCode(accessLog.Code,3) }else { v.NoRecord[timeNow] = getAccessLogCode(accessLog.Code,3) } if no ,ok := v.Elapsed[timeNow];ok{ v.Elapsed[timeNow] = no + accessLog.Elapsed }else { v.Elapsed[timeNow] = accessLog.Elapsed } }else{ tmp := WarningCount{} tmp.ApiName = accessLog.ApiName tmp.Total = make(map[int64]int64) tmp.Total[timeNow] = 1 tmp.Valid = make(map[int64]int64) count := getAccessLogCode(accessLog.Code,1) if count == 1 { tmp.Valid[timeNow] = count } tmp.Success = make(map[int64]int64) count = getAccessLogCode(accessLog.Code,2) if count == 1 { tmp.Success[timeNow] =count } tmp.NoRecord = make(map[int64]int64) count = getAccessLogCode(accessLog.Code,3) if count == 1{ tmp.NoRecord[timeNow] = count } tmp.Elapsed = make(map[int64]float64) tmp.Elapsed[timeNow] = accessLog.Elapsed AccessLogWaring.Count[accessLog.ApiId] = tmp } //fmt.Println(AccessLogWaring) } func CountThirdPartLog(accessLog *apis.ReciveThirdPartLogReq){ timeNow := accessLog.Timestamp ThirdPartLogWaring.RwMutex.Lock() defer ThirdPartLogWaring.RwMutex.Unlock() if ThirdPartLogWaring.Count == nil { ThirdPartLogWaring.Count = make(map[int64]WarningCount) } if v,ok := ThirdPartLogWaring.Count[accessLog.ProviderApiId];ok{ v.ApiName = accessLog.ProviderApiName v.ProviderName = accessLog.ProviderName if no ,ok := v.Valid[timeNow];ok{ v.Valid[timeNow] = no + 1 }else { v.Valid[timeNow] = 1 } if no ,ok := v.Success[timeNow];ok{ if accessLog.State == 1 { v.Success[timeNow] = no + 1 } }else { if accessLog.State == 1 { v.Success[timeNow] = 1 } } if no ,ok := v.NoRecord[timeNow];ok{ v.NoRecord[timeNow] = no + getAccessLogCode(accessLog.Code,3) }else { v.NoRecord[timeNow] = getAccessLogCode(accessLog.Code,3) } if no ,ok := v.Elapsed[timeNow];ok{ v.Elapsed[timeNow] = no + accessLog.Elapsed }else { v.Elapsed[timeNow] = accessLog.Elapsed } }else{ tmp := WarningCount{} tmp.ApiName = accessLog.ProviderApiName tmp.ProviderName = accessLog.ProviderName tmp.Valid = make(map[int64]int64) tmp.Valid[timeNow] = 1 tmp.Success = make(map[int64]int64) if accessLog.State == 1{ tmp.Success[timeNow] = 1 } tmp.NoRecord = make(map[int64]int64) count := getAccessLogCode(accessLog.Code,3) if count == 1{ tmp.NoRecord[timeNow] = count } tmp.Elapsed = make(map[int64]float64) tmp.Elapsed[timeNow] = accessLog.Elapsed ThirdPartLogWaring.Count[accessLog.ProviderApiId] = tmp } //fmt.Println(ThirdPartLogWaring) } func getTotal(m map[int64]int64,now,period int64) int64 { delta := now-period count := int64(0) for timestamp,v := range m{ if timestamp >= delta{ count = count + v }else if timestamp < delta { delete(m, timestamp) } } return count } func getElaspTotal(m map[int64]float64,now,period int64) float64{ delta := now-period count := float64(0) for timestamp,v := range m{ if timestamp >= delta{ count = count + v }else if timestamp < delta { delete(m, timestamp) } } return count } func CheckThirdPartLastSendTime(old []ThirdLogCheck,now int64,warnSendPeriod int64) []ThirdLogCheck { if ThirparthLastTime == nil { ThirparthLastTime = make(map[int64]int64) } new := []ThirdLogCheck{} for _,v := range old { if timeOld,ok := ThirparthLastTime[v.ProviderApiId];ok{ if timeOld+warnSendPeriod*60 <= now{ new = append(new,v) ThirparthLastTime[v.ProviderApiId] = now } }else{ new = append(new,v) ThirparthLastTime[v.ProviderApiId] = now } } return new } func CheckProviderLog(now int64,period int64,warnMinCount,warnSendPeriod int64){ // 发送预警邮件 warningLogs := CheckProviderLogImpl(now,period,warnMinCount) //fmt.Println("3333333333333333:",warningLogs) if warningLogs != nil{ warningLogs = CheckThirdPartLastSendTime(warningLogs,now,warnSendPeriod) thirdWarningToMail(warningLogs, now-period, now) } } func GetPorviderApiIds() []int64 { providerApiIds := []int64{} ThirdPartLogWaring.RwMutex.Lock() defer ThirdPartLogWaring.RwMutex.Unlock() for k,_ := range ThirdPartLogWaring.Count{ providerApiIds = append(providerApiIds,k) } return providerApiIds } func GetApiIds() []int64 { apiIds := []int64{} AccessLogWaring.RwMutex.Lock() defer AccessLogWaring.RwMutex.Unlock() for k,_ := range AccessLogWaring.Count{ apiIds = append(apiIds,k) } return apiIds } func GetApiRate(){ apiIds := GetApiIds() providerApiIds := GetPorviderApiIds() //fmt.Println("ids ----------------------",apiIds,providerApiIds) ApiRate = make(map[int64]Rate) ProviderApiRate = make(map[int64]Rate) for _,v := range apiIds{ elapsed, failRate, norecordRate, enable := getThreshold(0, v) tmp := Rate{} tmp.Elapsed = elapsed tmp.FailRate = failRate tmp.NorecordRate = norecordRate tmp.Enable = enable ApiRate[v] = tmp //fmt.Println("API RATE 111111111111111111",ApiRate) } for _,v := range providerApiIds{ elapsed, failRate, norecordRate, enable := getThreshold(1, v) tmp := Rate{} tmp.Elapsed = elapsed tmp.FailRate = failRate tmp.NorecordRate = norecordRate tmp.Enable = enable ProviderApiRate[v] = tmp //fmt.Println("PROVIDER API RATE 111111111111111111",ProviderApiRate) } } func CheckProviderLogImpl(now int64,period int64,warnMinCount int64) []ThirdLogCheck { if ThirdPartLogWaring.Count == nil { return nil } ThirdPartLogWaring.RwMutex.Lock() defer ThirdPartLogWaring.RwMutex.Unlock() warningLogs := []ThirdLogCheck{} for k,v := range ThirdPartLogWaring.Count{ valid := getTotal(v.Valid,now,period) if valid == 0 { continue } if warnMinCount > 0 && valid (1-failRate) { continue } } acs := ThirdLogCheck{} acs.ProviderApiName = v.ApiName acs.ProviderName = v.ProviderName acs.Count = int(valid) acs.AvgElapsed = avgElapsed acs.Valid = int(valid) acs.Success = int(success) acs.NoRecord = int(noRecord) warningLogs = append(warningLogs, acs) } return warningLogs } func CheckAccessLogLastSendTime(old []AccessLogCheck,now int64,warnSendPeriod int64) []AccessLogCheck { if AccessLogLastTime == nil { AccessLogLastTime = make(map[int64]int64) } new := []AccessLogCheck{} for _,v := range old { if timeOld,ok := AccessLogLastTime[v.ApiId];ok{ if timeOld+warnSendPeriod*60 <= now{ new = append(new,v) AccessLogLastTime[v.ApiId] = now } }else{ new = append(new,v) AccessLogLastTime[v.ApiId] = now } } return new } func CheckAccessLog(now int64,period int64,warnMinCount,warnSendPeriod int64){ // 发送预警邮件 warningLogs := CheckAccessLogImpl(now,period,warnMinCount) //fmt.Println("22222222222222222222:",warningLogs) if warningLogs != nil{ warningLogs = CheckAccessLogLastSendTime(warningLogs,now,warnSendPeriod) accessWarningToMail(warningLogs, now-period, now) } } func getThresholdFromMap(atype int,id int64)(float64, float64, float64, bool){ //fmt.Println("000000000000000000000000000",atype,id) if atype == 0 { if v,ok:= ApiRate[id];ok{ //fmt.Println("get api rate 1111111111111",v) return v.Elapsed,v.FailRate,v.NorecordRate,v.Enable } }else if atype == 1 { if v,ok:= ProviderApiRate[id];ok{ //fmt.Println("get provider api rate 1111111111111",v) return v.Elapsed,v.FailRate,v.NorecordRate,v.Enable } } return 9999,9999,9999,false } func CheckAccessLogImpl(now int64,period int64,warnMinCount int64) []AccessLogCheck { if AccessLogWaring.Count == nil { return nil } AccessLogWaring.RwMutex.Lock() defer AccessLogWaring.RwMutex.Unlock() warningLogs := []AccessLogCheck{} for k,v := range AccessLogWaring.Count{ //fmt.Println("??????????????????????????????",v,now,period) total := getTotal(v.Total,now,period) if total == 0 { continue } if warnMinCount > 0 && total (1-failRate) { continue } } acs := AccessLogCheck{} acs.ApiName = v.ApiName acs.Count = int(total) acs.AvgElapsed = avgElapsed acs.Valid = int(valid) acs.Success = int(success) acs.NoRecord = int(noRecord) warningLogs = append(warningLogs, acs) } return warningLogs } func CheckLog(){ defer func() { if r := recover(); r != nil { err := fmt.Errorf("%+v", r) fmt.Println("err:",err) } }() now := time.Now().Unix() //fmt.Println("time 11111111111111111111111111111111111111:",now) ApiMutex.Lock() defer ApiMutex.Unlock() GetApiRate() warnPeriod,_ := config.Conf.WarnPeriod.Int64() warnPeriod =warnPeriod*60 warnMinCount,_:= config.Conf.WarnMinCount.Int64() warnSendPeriod,_:= config.Conf.WarnSendPeriod.Int64() CheckAccessLog(now,warnPeriod,warnMinCount,warnSendPeriod) CheckProviderLog(now,warnPeriod,warnMinCount,warnSendPeriod) } func ReciveAccessLog(ctx context.Context, req *apis.ReciveAccessLogReq, reply *apis.ReciveAccessLogReply) error{ //fmt.Println("access log req:",req) defer func() { if r := recover(); r != nil { err := fmt.Errorf("%+v", r) fmt.Println("err:",err) } }() CountAccessLog(req) return nil } func ReciveThirdPartLog(ctx context.Context, req *apis.ReciveThirdPartLogReq, reply *apis.ReciveThirdPartLogReply) error{ //fmt.Println("thirdpart log req:",req) defer func() { if r := recover(); r != nil { err := fmt.Errorf("%+v", r) fmt.Println("err:",err) } }() CountThirdPartLog(req) return nil }