You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
431 lines
10 KiB
431 lines
10 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"regexp" |
|
"strconv" |
|
"strings" |
|
|
|
"go-common/app/interface/openplatform/monitor-end/model" |
|
"go-common/app/interface/openplatform/monitor-end/model/monitor" |
|
"go-common/app/interface/openplatform/monitor-end/model/prom" |
|
"go-common/library/log" |
|
|
|
"github.com/json-iterator/go" |
|
) |
|
|
|
const _regex = `.*\d{2,}` |
|
|
|
var json = jsoniter.ConfigCompatibleWithStandardLibrary |
|
|
|
const ( |
|
_typeAPP = "app" |
|
_typeWeb = "web/h5" |
|
_typeAPPH = "app_h" |
|
_eventPage = "page" |
|
_eventAPI = "api" |
|
_eventResource = "resource" |
|
_levelInfo = "info" |
|
_levelWarning = "warning" |
|
_levelError = "error" |
|
_logTypeOne = "1" |
|
_logTypeTwo = "2" |
|
) |
|
|
|
// Report . |
|
func (s *Service) Report(c context.Context, params *model.LogParams, mid int64, ip string, buvid string, userAgent string) (err error) { |
|
var l *monitor.Log |
|
log.Info("get log report(%+v)", params) |
|
if params.IsAPP == 1 { |
|
_, err = s.nativeLog(c, params.Log) |
|
} else { |
|
l, err = s.frontendLog(c, params.Source, params.Log, mid, ip, buvid, userAgent) |
|
go s.promsFE(context.TODO(), l) |
|
} |
|
return |
|
} |
|
func induceSubEvent(s string) string { |
|
if strings.Contains(s, "?") { |
|
s = s[:strings.Index(s, "?")] |
|
} |
|
if strings.Contains(s, "://") { |
|
s = s[(strings.Index(s, "://") + 3):] |
|
} |
|
if len(s) > 512 { |
|
s = s[:512] |
|
} |
|
var ( |
|
ok bool |
|
err error |
|
) |
|
for { |
|
if ok, err = regexp.Match(_regex, []byte(s)); err != nil { |
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, s) |
|
return s |
|
} |
|
if !ok { |
|
break |
|
} |
|
if !strings.Contains(s, "/") { |
|
return s |
|
} |
|
s = s[:strings.LastIndex(s, "/")] |
|
} |
|
return s |
|
} |
|
|
|
func (s *Service) promsFE(c context.Context, l *monitor.Log) { |
|
if !s.c.Prom.Promed || l == nil { |
|
return |
|
} |
|
var ( |
|
ok bool |
|
err error |
|
) |
|
l.SubEvent = induceSubEvent(l.SubEvent) |
|
if l.Event == _eventResource { |
|
regex := `.*i\d{1}.hdslb.com` |
|
if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil { |
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent) |
|
return |
|
} |
|
if ok { |
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9] |
|
} |
|
prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode) |
|
return |
|
} |
|
prom.AddHTTPCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode) |
|
prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode) |
|
if l.Details == nil { |
|
var cost int64 |
|
if l.Duration == "" { |
|
return |
|
} |
|
if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil { |
|
log.Warn("s.Info.ParseInt can not convert duration(%s) to int64", l.Duration) |
|
return |
|
} |
|
prom.AddCommonLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost) |
|
} else { |
|
prom.AddDetailedLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.Details) |
|
} |
|
} |
|
|
|
func (s *Service) nativeLog(c context.Context, data string) (l *monitor.Log, err error) { |
|
l = &monitor.Log{} |
|
if err = json.Unmarshal([]byte(data), l); err != nil { |
|
log.Error("s.nativeLog.unmarshal error(%+v), data(%s)", err, data) |
|
return |
|
} |
|
l.Type = _typeAPPH |
|
s.handleLog(c, l) |
|
return |
|
} |
|
|
|
// HandleMsg . |
|
func (s *Service) HandleMsg(msg []byte) { |
|
l := monitor.LogFromBytes(msg) |
|
l.Type = _typeAPP |
|
s.handleLog(context.TODO(), l) |
|
} |
|
|
|
func (s *Service) handleLog(c context.Context, l *monitor.Log) { |
|
var ( |
|
appID string |
|
kv []log.D |
|
err error |
|
logtype int |
|
isInt bool |
|
cost int64 |
|
ok bool |
|
app = "android" |
|
) |
|
l.TraceidSvr = l.Traceid |
|
l.Traceid = "" |
|
l.CalCode() |
|
if appID, _, kv, err = l.LogData(); err != nil { |
|
log.Error("s.nativeLog.LogData error(%+v), data(%+v)", err, l) |
|
return |
|
} |
|
if l.Result == "1" || l.Result == "" { |
|
s.mh.Info(c, appID, kv...) |
|
} else { |
|
s.mh.Error(c, appID, kv...) |
|
} |
|
if l.SubProduct == "" { |
|
l.SubProduct = l.Product |
|
} |
|
if strings.Contains(l.RequestURI, "ios") { |
|
app = "ios" |
|
isInt = true |
|
} |
|
if logtype, err = bStrToInt(l.LogType, isInt); err != nil { |
|
return |
|
} |
|
if s.c.Prom.IgnoreNA && !s.checkProduct(l) { |
|
return |
|
} |
|
// 丢弃老版本的network日志 |
|
if l.Event == "network" && l.Codes == "" { |
|
return |
|
} |
|
l.SubEvent = induceSubEvent(l.SubEvent) |
|
regex := `.*i\d{1}.hdslb.com` |
|
if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil { |
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent) |
|
return |
|
} |
|
if ok { |
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9] |
|
} |
|
if (logtype & 1) == 1 { |
|
// 性能日志 |
|
if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil { |
|
log.Warn("s.handleLog.ParseInt can not convert duration(%s) to int64", l.Duration) |
|
return |
|
} |
|
prom.AddCommonLog(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost) |
|
} |
|
if (logtype >> 2 & 1) == 1 { |
|
// 成功/失败日志 |
|
if l.Event == "network" { |
|
// 网络类型 上报业务码和http状态 |
|
prom.AddHTTPCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode) |
|
prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode) |
|
} else { |
|
// 其他类型 只上报失败成功 result=1 表示成功 |
|
res := "999" |
|
if l.Result == "1" { |
|
res = "0" |
|
} |
|
prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, res) |
|
} |
|
} |
|
} |
|
|
|
func (s *Service) frontendLog(c context.Context, source string, data string, mid int64, ip string, buvid string, userAgent string) (l *monitor.Log, err error) { |
|
var ( |
|
ms []interface{} |
|
kv []log.D |
|
logtype, loglevel string |
|
url, query string |
|
) |
|
if err = json.Unmarshal([]byte(data), &ms); err != nil { |
|
log.Error("s.frontendLog.unmarshal error(%+v), data(%s)", err, data) |
|
return |
|
} |
|
for _, m := range ms { |
|
switch m := m.(type) { |
|
case map[string]interface{}: |
|
logtype = stringValueByKey(m, "1", "logtype") |
|
loglevel = stringValueByKey(m, "info", "level") |
|
tmp := stringValueByKey(m, "", "url") |
|
if strings.Contains(tmp, "?") { |
|
url = strings.Split(tmp, "?")[0] |
|
query = strings.Split(tmp, "?")[1] |
|
} else { |
|
url = tmp |
|
} |
|
switch logtype { |
|
case _logTypeOne: |
|
l = &monitor.Log{ |
|
LogType: _logTypeOne, |
|
Type: _typeWeb, |
|
Product: source, |
|
IP: ip, |
|
Buvid: buvid, |
|
UserAgent: userAgent, |
|
Event: _eventAPI, |
|
Mid: strconv.FormatInt(mid, 10), |
|
SubEvent: url, |
|
Query: query, |
|
Duration: stringValueByKey(m, "", "cost"), |
|
TraceidSvr: stringValueByKey(m, "", "traceid_svr"), |
|
TraceidEnd: stringValueByKey(m, "", "traceid_end"), |
|
HTTPCode: stringValueByKey(m, "200", "status"), |
|
BusinessCode: stringValueByKey(m, "0", "errno", "code"), |
|
SubProduct: stringValueByKey(m, source, "sub_product"), |
|
Result: "0", |
|
} |
|
srcURL := stringValueByKey(m, "", "srcUrl") |
|
if srcURL != "" { |
|
l.Event = _eventResource |
|
l.SubEvent = srcURL |
|
l.BusinessCode = "-999" |
|
} |
|
if loglevel == _levelInfo { |
|
l.Result = "1" |
|
} |
|
if _, _, kv, err = l.LogData(); err != nil { |
|
return |
|
} |
|
kv = append(kv, extKV(m)...) |
|
case _logTypeTwo: |
|
var msg []byte |
|
l = &monitor.Log{ |
|
LogType: _logTypeTwo, |
|
Type: _typeWeb, |
|
Product: source, |
|
Event: _eventPage, |
|
IP: ip, |
|
Buvid: buvid, |
|
UserAgent: userAgent, |
|
Mid: strconv.FormatInt(mid, 10), |
|
SubEvent: url, |
|
Query: query, |
|
HTTPCode: "200", |
|
Details: details(m), |
|
Result: "1", |
|
BusinessCode: "0", |
|
SubProduct: stringValueByKey(m, source, "sub_product"), |
|
} |
|
if msg, err = json.Marshal(m); err != nil { |
|
log.Error("s.frontendLog.Marshal error(%v), data(%v)", err, m) |
|
continue |
|
} |
|
l.Message = string(msg) |
|
if _, _, kv, err = l.LogData(); err != nil { |
|
return |
|
} |
|
default: |
|
log.Error("s.frontendLog error logype(%v)", logtype) |
|
return |
|
} |
|
switch loglevel { |
|
case _levelInfo: |
|
s.mh.Info(c, source, kv...) |
|
case _levelWarning: |
|
s.mh.Warn(c, source, kv...) |
|
case _levelError: |
|
s.collectFE(c, l) |
|
s.mh.Error(c, source, kv...) |
|
default: |
|
s.mh.Info(c, source, kv...) |
|
} |
|
default: |
|
log.Error("s.frontendLog log data is not json type: " + data) |
|
} |
|
} |
|
return |
|
} |
|
|
|
func (s *Service) collectFE(c context.Context, l *monitor.Log) { |
|
if s.c.CollectFE { |
|
regex := `.*i\d{1}.hdslb.com` |
|
if ok, err := regexp.Match(regex, []byte(l.SubEvent)); err != nil { |
|
log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent) |
|
return |
|
} else if ok { |
|
l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9] |
|
} |
|
go s.Collect(context.TODO(), l) |
|
} |
|
} |
|
|
|
func stringValueByKey(m map[string]interface{}, defValue string, keys ...string) (r string) { |
|
r = defValue |
|
if m == nil { |
|
return |
|
} |
|
var ( |
|
res interface{} |
|
ok bool |
|
) |
|
for _, key := range keys { |
|
if res, ok = m[key]; ok { |
|
break |
|
} |
|
return |
|
} |
|
if res == nil { |
|
return |
|
} |
|
switch d := res.(type) { |
|
case int: |
|
r = strconv.Itoa(d) |
|
case float64: |
|
r = strconv.FormatFloat(d, 'f', -1, 64) |
|
case int64: |
|
r = strconv.FormatInt(d, 10) |
|
case string: |
|
r = d |
|
default: |
|
log.Warn("s.stringValueByKey unexcept type of value(%v)", d) |
|
} |
|
return |
|
} |
|
|
|
func extKV(m map[string]interface{}) (kv []log.D) { |
|
if m == nil { |
|
return |
|
} |
|
var keys = []string{"level", "logtype", "url", "cost", "traceid_svr", "traceid_end", "status", "code", "errno", "sub_product"} |
|
for _, key := range keys { |
|
delete(m, key) |
|
} |
|
for k, v := range m { |
|
kv = append(kv, log.KV(k, v)) |
|
} |
|
return |
|
} |
|
|
|
func details(m map[string]interface{}) (res map[string]int64) { |
|
res = make(map[string]int64) |
|
if m == nil { |
|
return |
|
} |
|
delete(m, "level") |
|
delete(m, "logtype") |
|
delete(m, "url") |
|
for k, v := range m { |
|
var d int64 |
|
if v == nil { |
|
res[k] = 0 |
|
continue |
|
} |
|
switch v := v.(type) { |
|
case int64: |
|
d = v |
|
case float64: |
|
d = int64(v) |
|
case int: |
|
d = int64(v) |
|
case int32: |
|
d = int64(v) |
|
case string: |
|
d, _ = strconv.ParseInt(v, 10, 64) |
|
default: |
|
log.Warn("s.details unexcept type of value(%v)", v) |
|
} |
|
res[k] = d |
|
} |
|
return |
|
} |
|
|
|
func bStrToInt(str string, isInt bool) (r int, err error) { |
|
if isInt { |
|
if r, err = strconv.Atoi(str); err != nil { |
|
log.Warn("s.bStrToInt error(%+v), string(%s)", err, str) |
|
} |
|
return |
|
} |
|
for _, i := range str { |
|
if i == '1' { |
|
r = r*2 + 1 |
|
} else { |
|
r *= 2 |
|
} |
|
} |
|
return |
|
} |
|
|
|
func (s *Service) checkProduct(l *monitor.Log) bool { |
|
if s.naProducts == nil { |
|
return true |
|
} |
|
if s.naProducts[l.Product] { |
|
return true |
|
} |
|
return false |
|
}
|
|
|