You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
135 lines
3.1 KiB
135 lines
3.1 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"encoding/hex" |
|
"os" |
|
"os/exec" |
|
|
|
"go-common/app/job/bbq/recall/proto" |
|
"go-common/library/log" |
|
|
|
"github.com/golang/snappy" |
|
) |
|
|
|
// GenForwardIndex 生产正排索引 |
|
func (s *Service) GenForwardIndex() { |
|
log.Info("run [%s]", "GenForwardIndex") |
|
c := context.Background() |
|
vInfo, err := s.videoBasicInfo(c) |
|
if err != nil { |
|
log.Error("video info: %v", err) |
|
return |
|
} |
|
|
|
outputFile, err := os.OpenFile(s.c.Job.ForwardIndex.Output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) |
|
if err != nil { |
|
log.Error("open file: %v", err) |
|
return |
|
} |
|
shadowFile, _ := os.OpenFile(s.c.Job.ForwardIndex.Output+".bak", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) |
|
defer outputFile.Close() |
|
defer shadowFile.Close() |
|
|
|
for _, v := range vInfo { |
|
qu, _ := s.dao.FetchVideoQuality(c, v.SVID) |
|
tmp := &proto.ForwardIndex{ |
|
SVID: v.SVID, |
|
BasicInfo: v, |
|
VideoQuality: qu, |
|
} |
|
raw, err := tmp.Marshal() |
|
if err != nil { |
|
log.Error("json marshal: %v", err) |
|
continue |
|
} |
|
_, err = outputFile.WriteString(hex.EncodeToString(snappy.Encode(nil, raw))) |
|
if err != nil { |
|
log.Error("output: %v", err) |
|
} |
|
outputFile.Write([]byte("\n")) |
|
if err != nil { |
|
log.Error("output endline: %v", err) |
|
} |
|
shadowFile.WriteString(tmp.String()) |
|
if err != nil { |
|
log.Error("shadow: %v", err) |
|
} |
|
shadowFile.Write([]byte("\n")) |
|
if err != nil { |
|
log.Error("shadow endline: %v", err) |
|
} |
|
} |
|
exec.Command(s.c.Job.ForwardIndex.Output + ".sh").Run() |
|
|
|
log.Info("finish [GenForwardIndex]") |
|
|
|
s.GenRealTimeInvertedIndex() |
|
} |
|
|
|
func (s *Service) videoBasicInfo(c context.Context) (result []*proto.VideoInfo, err error) { |
|
// fetch tag info from db |
|
tags, err := s.dao.FetchVideoTagAll(c) |
|
if err != nil { |
|
return |
|
} |
|
tagIDMap := make(map[int32]*proto.Tag) |
|
tagNameMap := make(map[string]*proto.Tag) |
|
for _, v := range tags { |
|
tagIDMap[v.TagID] = v |
|
tagNameMap[v.TagName] = v |
|
} |
|
|
|
// fetch video info from db |
|
offset := 0 |
|
size := 1000 |
|
basic, err := s.dao.FetchVideoInfo(c, offset, size) |
|
if err != nil { |
|
log.Error("FetchVideoInfo: %v", err) |
|
return |
|
} |
|
for len(basic) > 0 && err == nil { |
|
log.Info("FetchVideoInfo: %v", len(result)) |
|
for _, v := range basic { |
|
vInfo := &proto.VideoInfo{ |
|
SVID: uint64(v.SVID), |
|
Title: v.Title, |
|
Content: v.Content, |
|
MID: uint64(v.MID), |
|
AVID: uint64(v.AVID), |
|
CID: uint64(v.CID), |
|
PubTime: v.PubTime.Time().Unix(), |
|
CTime: v.CTime.Time().Unix(), |
|
MTime: v.MTime.Time().Unix(), |
|
Duration: uint32(v.Duration), |
|
State: int32(v.State), |
|
} |
|
vTags := make([]*proto.Tag, 0) |
|
// 一级标签 |
|
if tag, ok := tagIDMap[v.TID]; ok { |
|
vTags = append(vTags, tag) |
|
} |
|
// 二级标签 |
|
if subTag, ok := tagIDMap[v.SubTID]; ok { |
|
vTags = append(vTags, subTag) |
|
} |
|
// 三级标签 |
|
if textTags, e := s.dao.FetchVideoTextTag(c, v.SVID); e == nil { |
|
for _, v := range textTags { |
|
if tmp, ok := tagNameMap[v]; ok { |
|
vTags = append(vTags, tmp) |
|
} |
|
} |
|
} |
|
|
|
vInfo.Tags = vTags |
|
result = append(result, vInfo) |
|
} |
|
offset += size |
|
basic, err = s.dao.FetchVideoInfo(c, offset, size) |
|
if err != nil { |
|
log.Error("FetchVideoInfo: %v", err) |
|
} |
|
} |
|
return |
|
}
|
|
|