You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
341 lines
8.1 KiB
341 lines
8.1 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"sync" |
|
"time" |
|
|
|
"go-common/app/job/main/videoup-report/conf" |
|
arcdao "go-common/app/job/main/videoup-report/dao/archive" |
|
"go-common/app/job/main/videoup-report/dao/data" |
|
"go-common/app/job/main/videoup-report/dao/email" |
|
hbasedao "go-common/app/job/main/videoup-report/dao/hbase" |
|
"go-common/app/job/main/videoup-report/dao/manager" |
|
"go-common/app/job/main/videoup-report/dao/mission" |
|
redisdao "go-common/app/job/main/videoup-report/dao/redis" |
|
"go-common/app/job/main/videoup-report/dao/tag" |
|
arcmdl "go-common/app/job/main/videoup-report/model/archive" |
|
taskmdl "go-common/app/job/main/videoup-report/model/task" |
|
account "go-common/app/service/main/account/api" |
|
arcrpc "go-common/app/service/main/archive/api/gorpc" |
|
upsrpc "go-common/app/service/main/up/api/v1" |
|
"go-common/library/log" |
|
"go-common/library/queue/databus" |
|
"strings" |
|
) |
|
|
|
const ( |
|
_archiveTable = "archive" |
|
_videoTable = "archive_video" |
|
_upsTable = "ups" |
|
_jumpChanSize = int(4000) |
|
_logChanSize = int(200000) |
|
) |
|
|
|
// Service is service. |
|
type Service struct { |
|
c *conf.Config |
|
arc *arcdao.Dao |
|
redis *redisdao.Dao |
|
hbase *hbasedao.Dao |
|
dataDao *data.Dao |
|
email *email.Dao |
|
mng *manager.Dao |
|
arcUpChs []chan *arcmdl.UpInfo |
|
videoUpInfoChs []chan *arcmdl.VideoUpInfo |
|
// waiter |
|
waiter sync.WaitGroup |
|
// databus |
|
archiveSub *databus.Databus |
|
arcResultSub *databus.Databus |
|
videoupSub *databus.Databus |
|
ManagerDBSub *databus.Databus |
|
// cache |
|
sfTpsCache map[int16]*arcmdl.Type |
|
adtTpsCache map[int16]struct{} |
|
taskCache *arcmdl.TaskCache |
|
videoAuditCache *arcmdl.VideoAuditCache |
|
arcMoveTypeCache *arcmdl.ArcMoveTypeCache |
|
arcRoundFlowCache *arcmdl.ArcRoundFlowCache |
|
xcodeTimeCache *arcmdl.XcodeTimeCache |
|
assignCache map[int64]*taskmdl.AssignConfig |
|
upperCache map[int8]map[int64]struct{} |
|
weightCache map[int8]map[int64]*taskmdl.ConfigItem |
|
missTagsCache map[string]int |
|
|
|
lastjumpMap map[int64]struct{} //上轮插队的这一轮也更新，否则会出现权重只增不减 |
|
jumplist *taskmdl.JumpList //插队序列 |
|
jumpchan chan *taskmdl.WeightLog |
|
tasklogchan chan *taskmdl.WeightLog |
|
//rpc |
|
arcRPCGroup2 *arcrpc.Service2 |
|
|
|
//grpc |
|
accRPC account.AccountClient |
|
upsRPC upsrpc.UpClient |
|
|
|
// closed |
|
closed bool |
|
tagDao *tag.Dao |
|
missionDao *mission.Dao |
|
} |
|
|
|
// New is videoup-report-job service implementation. |
|
func New(c *conf.Config) (s *Service) { |
|
s = &Service{ |
|
c: c, |
|
//dao |
|
arc: arcdao.New(c), |
|
redis: redisdao.New(c), |
|
hbase: hbasedao.New(c), |
|
//databus |
|
archiveSub: databus.New(c.ArchiveSub), |
|
arcResultSub: databus.New(c.ArchiveResultSub), |
|
videoupSub: databus.New(c.VideoupSub), |
|
ManagerDBSub: databus.New(c.ManagerDBSub), |
|
dataDao: data.New(c), |
|
email: email.New(c), |
|
mng: manager.New(c), |
|
// cache |
|
taskCache: &arcmdl.TaskCache{ |
|
Task: make(map[int64]*arcmdl.Task), |
|
}, |
|
videoAuditCache: &arcmdl.VideoAuditCache{ |
|
Data: make(map[int16]map[string]int), |
|
}, |
|
arcMoveTypeCache: &arcmdl.ArcMoveTypeCache{ |
|
Data: make(map[int8]map[int16]map[string]int), |
|
}, |
|
arcRoundFlowCache: &arcmdl.ArcRoundFlowCache{ |
|
Data: make(map[int8]map[int64]map[string]int), |
|
}, |
|
xcodeTimeCache: &arcmdl.XcodeTimeCache{ |
|
Data: make(map[int8][]int), |
|
}, |
|
arcRPCGroup2: arcrpc.New2(c.ArchiveRPCGroup2), |
|
lastjumpMap: make(map[int64]struct{}), |
|
jumplist: taskmdl.NewJumpList(), |
|
jumpchan: make(chan *taskmdl.WeightLog, _jumpChanSize), |
|
tasklogchan: make(chan *taskmdl.WeightLog, _logChanSize), |
|
tagDao: tag.New(c), |
|
missionDao: mission.New(c), |
|
} |
|
var err error |
|
if s.accRPC, err = account.NewClient(conf.Conf.GRPC.AccRPC); err != nil { |
|
panic(err) |
|
} |
|
|
|
if s.upsRPC, err = upsrpc.NewClient(conf.Conf.GRPC.UpsRPC); err != nil { |
|
panic(err) |
|
} |
|
|
|
for i := 0; i < s.c.ChanSize; i++ { |
|
log.Info("videoup-report-job chanSize starting(%d)", i) |
|
s.arcUpChs = append(s.arcUpChs, make(chan *arcmdl.UpInfo, 10240)) |
|
s.waiter.Add(1) |
|
go s.arcUpdateproc(i) |
|
s.videoUpInfoChs = append(s.videoUpInfoChs, make(chan *arcmdl.VideoUpInfo, 10240)) |
|
s.waiter.Add(1) |
|
go s.upVideoproc(i) |
|
} |
|
s.loadConf() |
|
s.loadType() |
|
// load cache. |
|
s.loadTask() |
|
s.loadTaskTookSort() |
|
s.hdlTraffic() |
|
s.loadMission() |
|
go s.cacheproc() |
|
go s.monitorNotifyProc() |
|
s.waiter.Add(1) |
|
go s.hotarchiveproc() |
|
s.waiter.Add(1) |
|
go s.arcCanalConsume() |
|
s.waiter.Add(1) |
|
go s.arcResultConsume() |
|
s.waiter.Add(1) |
|
go s.taskWeightConsumer() |
|
s.waiter.Add(1) |
|
go s.taskweightproc() |
|
s.waiter.Add(1) |
|
go s.movetaskproc() |
|
go s.deltaskproc() |
|
s.waiter.Add(1) |
|
go s.videoupConsumer() |
|
s.waiter.Add(1) |
|
go s.emailProc() |
|
s.waiter.Add(1) |
|
go s.emailFastProc() |
|
s.waiter.Add(1) |
|
go s.retryProc() |
|
s.waiter.Add(1) |
|
go s.managerDBConsume() |
|
|
|
return s |
|
} |
|
|
|
func (s *Service) loadType() { |
|
tpm, err := s.arc.TypeMapping(context.TODO()) |
|
if err != nil { |
|
log.Error("s.dede.TypeMapping error(%v)", err) |
|
return |
|
} |
|
s.sfTpsCache = tpm |
|
// audit types |
|
adt, err := s.arc.AuditTypesConf(context.TODO()) |
|
if err != nil { |
|
log.Error("s.dede.AuditTypesConf error(%v)", err) |
|
return |
|
} |
|
s.adtTpsCache = adt |
|
|
|
wvc, err := s.arc.WeightValueConf(context.TODO()) |
|
if err != nil { |
|
log.Error("s.arc.WeightValueConf error(%v)", err) |
|
return |
|
} |
|
taskmdl.WLVConf = wvc |
|
} |
|
|
|
func (s *Service) isAuditType(tpID int16) bool { |
|
_, isAt := s.adtTpsCache[tpID] |
|
return isAt |
|
} |
|
|
|
func (s *Service) topType(tpID int16) (id int16) { |
|
if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil { |
|
id = tp.PID |
|
} |
|
return |
|
} |
|
|
|
func (s *Service) typeName(tpID int16) (name string) { |
|
if tp, ok := s.sfTpsCache[tpID]; ok && tp != nil { |
|
name = tp.Name |
|
} |
|
return |
|
} |
|
|
|
func (s *Service) topTypeName(tpID int16) (name string) { |
|
pid := s.topType(tpID) |
|
name = s.typeName(pid) |
|
return |
|
} |
|
|
|
func (s *Service) cacheproc() { |
|
for { |
|
time.Sleep(1 * time.Minute) |
|
// config |
|
s.loadConf() |
|
// task |
|
s.loadTask() |
|
s.loadTaskTookSort() |
|
// handle task took |
|
s.hdlTaskTook() |
|
s.hdlTaskTookByHourHalf() |
|
// handle video audit |
|
s.hdlVideoAuditCount() |
|
s.hdlMoveTypeCount() |
|
s.hdlRoundFlowCount() |
|
//handle calculate video xcode time stats, and save to DB |
|
s.hdlXcodeStats() |
|
s.hdlTraffic() |
|
s.loadMission() |
|
} |
|
} |
|
|
|
func (s *Service) monitorNotifyProc() { |
|
for { |
|
s.monitorNotify() |
|
time.Sleep(30 * time.Minute) |
|
} |
|
} |
|
|
|
// s.missTagsCache: missionName or first tag |
|
func (s *Service) loadMission() { |
|
mm, err := s.missionDao.Missions(context.TODO()) |
|
if err != nil { |
|
log.Error("s.missionDao.Mission error(%v)", err) |
|
return |
|
} |
|
s.missTagsCache = make(map[string]int) |
|
for _, m := range mm { |
|
if len(m.Tags) > 0 { |
|
splitedTags := strings.Split(m.Tags, ",") |
|
s.missTagsCache[splitedTags[0]] = m.ID |
|
} else { |
|
s.missTagsCache[m.Name] = m.ID |
|
} |
|
} |
|
} |
|
|
|
// hotarchiveproc get hot archive which need to recheck |
|
func (s *Service) hotarchiveproc() { |
|
defer s.waiter.Done() |
|
for { |
|
if s.closed { |
|
return |
|
} |
|
s.addHotRecheck() |
|
time.Sleep(10 * time.Minute) |
|
} |
|
} |
|
|
|
// Close consumer close. |
|
func (s *Service) Close() { |
|
s.closed = true |
|
s.archiveSub.Close() |
|
s.arcResultSub.Close() |
|
s.videoupSub.Close() |
|
time.Sleep(2 * time.Second) |
|
for i := 0; i < s.c.ChanSize; i++ { |
|
log.Info("videoup-report-job chanSize closing(%d)", i) |
|
close(s.arcUpChs[i]) |
|
close(s.videoUpInfoChs[i]) |
|
} |
|
s.arc.Close() |
|
s.mng.Close() |
|
s.redis.Close() |
|
s.email.Close() |
|
s.hbase.Close() |
|
s.waiter.Wait() |
|
} |
|
|
|
// Ping check server ok. |
|
func (s *Service) Ping(c context.Context) (err error) { |
|
if err = s.arc.Ping(c); err != nil { |
|
return |
|
} |
|
if err = s.mng.Ping(c); err != nil { |
|
return |
|
} |
|
return |
|
} |
|
|
|
func (s *Service) loadConf() { |
|
var ( |
|
err error |
|
assignConf map[int64]*taskmdl.AssignConfig |
|
weightConf map[int8]map[int64]*taskmdl.ConfigItem |
|
upperCache map[int8]map[int64]struct{} |
|
) |
|
|
|
if assignConf, err = s.assignConf(context.TODO()); err != nil { |
|
log.Error("s.assignConf error(%v)", err) |
|
return |
|
} |
|
s.assignCache = assignConf |
|
|
|
upperCache, err = s.upSpecial(context.TODO()) |
|
if err != nil { |
|
log.Error("s.upSpecial error(%v)", err) |
|
} else { |
|
s.upperCache = upperCache |
|
} |
|
|
|
if weightConf, err = s.weightConf(context.TODO()); err != nil { |
|
log.Error(" s.weightConf error(%v)", err) |
|
return |
|
} |
|
s.weightCache = weightConf |
|
}
|
|
|