You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
590 lines
16 KiB
590 lines
16 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"encoding/json" |
|
"fmt" |
|
"regexp" |
|
"strings" |
|
"time" |
|
|
|
"go-common/app/admin/ep/saga/model" |
|
"go-common/app/admin/ep/saga/service/utils" |
|
"go-common/library/log" |
|
|
|
"github.com/xanzy/go-gitlab" |
|
) |
|
|
|
const _specialMrID = 10199 |
|
|
|
// QueryProjectMr query project commit info according to project id. |
|
func (s *Service) QueryProjectMr(c context.Context, req *model.ProjectDataReq) (resp *model.ProjectDataResp, err error) { |
|
|
|
if resp, err = s.QueryProject(c, model.ObjectMR, req); err != nil { |
|
return |
|
} |
|
return |
|
} |
|
|
|
// QueryTeamMr query team commit info according to department and business |
|
func (s *Service) QueryTeamMr(c context.Context, req *model.TeamDataRequest) (resp *model.TeamDataResp, err error) { |
|
|
|
if resp, err = s.QueryTeam(c, model.ObjectMR, req); err != nil { |
|
return |
|
} |
|
return |
|
} |
|
|
|
// QueryProjectMrReport query mr review |
|
func (s *Service) QueryProjectMrReport(c context.Context, req *model.ProjectMrReportReq) (resp *model.ProjectMrReportResp, err error) { |
|
var ( |
|
info []*model.MrInfo |
|
changeAdd int |
|
changeDel int |
|
mrCount int |
|
stateCount int |
|
discussionCount int |
|
discussionResolved int |
|
mrTime int |
|
spentTime time.Duration |
|
avarageTime time.Duration |
|
reviewers []string |
|
reviews []string |
|
reviewChangeAdd int |
|
reviewChangeDel int |
|
reviewTime int |
|
reviewTotalTime time.Duration |
|
) |
|
if info, err = s.QueryAllMergeRequestInfo(c, req.ProjectID); err != nil { |
|
return |
|
} |
|
|
|
for _, i := range info { |
|
for _, r := range i.Reviewers { |
|
if r.Name == req.Member { |
|
reviews = append(reviews, i.Author) |
|
reviewChangeAdd += i.ChangeAdd |
|
reviewChangeDel += i.ChangeDel |
|
reviewTime += i.SpentTime |
|
} |
|
} |
|
if i.Author == req.Member { |
|
mrCount++ |
|
if i.State == model.StatusMerged { |
|
stateCount++ |
|
mrTime += i.SpentTime |
|
changeAdd += i.ChangeAdd |
|
changeDel += i.ChangeDel |
|
discussionCount += i.TotalDiscussion |
|
discussionResolved += i.SolvedDiscussion |
|
for _, r := range i.Reviewers { |
|
reviewers = append(reviewers, r.Name) |
|
} |
|
} |
|
} |
|
} |
|
// 判断mr为零的情况 |
|
if mrCount == 0 { |
|
resp = &model.ProjectMrReportResp{} |
|
return |
|
} |
|
|
|
if spentTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime)); err != nil { |
|
return |
|
} |
|
if avarageTime, err = time.ParseDuration(fmt.Sprintf("%ds", mrTime/mrCount)); err != nil { |
|
return |
|
} |
|
if reviewTotalTime, err = time.ParseDuration(fmt.Sprintf("%ds", reviewTime)); err != nil { |
|
return |
|
} |
|
|
|
resp = &model.ProjectMrReportResp{ |
|
ChangeAdd: changeAdd, |
|
ChangeDel: changeDel, |
|
MrCount: mrCount, |
|
SpentTime: spentTime.String(), |
|
StateCount: stateCount, |
|
AverageMerge: avarageTime.String(), |
|
Reviewers: reviewers, |
|
Discussion: discussionCount, |
|
Resolve: discussionResolved, |
|
ReviewerOther: reviews, |
|
ReviewChangeAdd: reviewChangeAdd, |
|
ReviewChangeDel: reviewChangeDel, |
|
ReviewTotalTime: reviewTotalTime.String(), |
|
} |
|
|
|
return |
|
} |
|
|
|
// QueryAllMergeRequestInfo ... |
|
func (s *Service) QueryAllMergeRequestInfo(c context.Context, projID int) (info []*model.MrInfo, err error) { |
|
var ( |
|
until = time.Now() |
|
since = until.AddDate(0, -1, 0) |
|
mrs []*gitlab.MergeRequest |
|
resp *gitlab.Response |
|
) |
|
|
|
for page := 1; ; page++ { |
|
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projID, &since, &until, page); err != nil { |
|
return |
|
} |
|
for _, m := range mrs { |
|
|
|
mr := &model.MrInfo{ProjectID: projID, MrID: m.IID, Author: m.Author.Name, State: m.State} |
|
|
|
if m.State == model.StatusMerged { |
|
spent := m.UpdatedAt.Sub(*m.CreatedAt) |
|
mr.SpentTime = int(spent.Seconds()) |
|
} |
|
|
|
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projID, m.IID); err != nil { |
|
return |
|
} |
|
|
|
if mr.Reviewers, err = s.QueryMergeRequestReview(c, projID, m.IID); err != nil { |
|
return |
|
} |
|
|
|
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projID, m.IID); err != nil { |
|
return |
|
} |
|
|
|
info = append(info, mr) |
|
} |
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
|
|
return |
|
} |
|
|
|
// QueryMergeRequestReview 查询mr reviewer信息 |
|
func (s *Service) QueryMergeRequestReview(c context.Context, projectID, mrIID int) (reviewers []*model.MrReviewer, err error) { |
|
var ( |
|
notes []*model.StatisticsNotes |
|
emojis []*model.StatisticsMRAwardEmojis |
|
owners []string |
|
r *regexp.Regexp |
|
) |
|
|
|
//query note |
|
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil { |
|
return |
|
} |
|
|
|
// query emoji |
|
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil { |
|
return |
|
} |
|
|
|
//评论中解析获取owner |
|
if len(notes) == 0 { |
|
return |
|
} |
|
if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil { |
|
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body) |
|
if len(matchResult) > 0 { |
|
owners = strings.Split(matchResult[1], " 或 @") |
|
} |
|
} |
|
|
|
mrCreatedAt := notes[len(notes)-1].CreatedAt |
|
|
|
// 从评论和表情中解析 reviewers |
|
for _, note := range notes { |
|
if note.Body == "+1" { |
|
reviewers = append(reviewers, &model.MrReviewer{Name: note.AuthorName, FinishedAt: note.CreatedAt, SpentTime: int(note.CreatedAt.Sub(*mrCreatedAt))}) |
|
} |
|
} |
|
for _, emoji := range emojis { |
|
if emoji.Name == "thumbsup" { |
|
reviewers = append(reviewers, &model.MrReviewer{Name: emoji.UserName, FinishedAt: emoji.CreatedAt, SpentTime: int(emoji.CreatedAt.Sub(*mrCreatedAt))}) |
|
} |
|
} |
|
|
|
// 判断reviewer类型 |
|
for _, r := range reviewers { |
|
for _, owner := range owners { |
|
if owner == r.Name { |
|
r.UserType = "owner" |
|
break |
|
} |
|
} |
|
if r.UserType == "" { |
|
r.UserType = "other" |
|
} |
|
} |
|
|
|
return |
|
} |
|
|
|
// QueryMergeRequestDiscussion 查询获取mr的discussion |
|
func (s *Service) QueryMergeRequestDiscussion(c context.Context, projectID, mrIID int) (total, solved int, err error) { |
|
var discussions []*model.StatisticsDiscussions |
|
if discussions, err = s.dao.DiscussionsByMRIID(c, projectID, mrIID); err != nil { |
|
return |
|
} |
|
for _, d := range discussions { |
|
var notesArray []int |
|
if err = json.Unmarshal([]byte(d.Notes), ¬esArray); err != nil { |
|
return |
|
} |
|
for _, n := range notesArray { |
|
var note *model.StatisticsNotes |
|
if note, err = s.dao.NoteByID(c, projectID, mrIID, n); err != nil { |
|
return |
|
} |
|
if note.Resolvable { |
|
total++ |
|
if note.Resolved { |
|
solved++ |
|
} |
|
} |
|
} |
|
} |
|
return |
|
} |
|
|
|
// QueryMergeRequestDiff 查询获取到mr修改文件的总行数, 参数 p project_ID m MR_ID |
|
func (s *Service) QueryMergeRequestDiff(c context.Context, p, m int) (ChangeAdd, ChangeDel int, err error) { |
|
var mr *gitlab.MergeRequest |
|
|
|
// 此MR数据量太大，曾导致过gitlab服务器崩溃，访问会返回502服务器错误，因此特殊处理。 |
|
if m == _specialMrID { |
|
return 0, 0, nil |
|
} |
|
|
|
if mr, _, err = s.gitlab.GetMergeRequestDiff(p, m); err != nil { |
|
return |
|
} |
|
for _, change := range mr.Changes { |
|
rows := strings.Split(change.Diff, "\n") |
|
if len(rows) < 3 { |
|
// 处理diff为空 |
|
continue |
|
} |
|
|
|
for _, row := range rows[3:] { |
|
if strings.HasPrefix(row, "+") { |
|
ChangeAdd++ |
|
} else if strings.HasPrefix(row, "-") { |
|
ChangeDel++ |
|
} |
|
} |
|
} |
|
|
|
return |
|
} |
|
|
|
/*-------------------------------------- sync MR ----------------------------------------*/ |
|
|
|
// SyncProjectMR ... |
|
func (s *Service) SyncProjectMR(c context.Context, projectID int) (result *model.SyncResult, err error) { |
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
mrs []*gitlab.MergeRequest |
|
resp *gitlab.Response |
|
since *time.Time |
|
until *time.Time |
|
projectInfo *model.ProjectInfo |
|
) |
|
result = &model.SyncResult{} |
|
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil { |
|
return |
|
} |
|
|
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
log.Info("sync project(%d) MR time since: %v, until: %v", projectID, since, until) |
|
for page := 1; ; page++ { |
|
result.TotalPage++ |
|
if mrs, resp, err = s.gitlab.ListProjectMergeRequests(projectID, since, until, page); err != nil { |
|
return |
|
} |
|
|
|
for _, mr := range mrs { |
|
if err = s.structureDatabaseMR(c, projectID, projectInfo.Name, mr); err != nil { |
|
log.Error("mr Save Database err: projectID(%d), MRIID(%d)", projectID, mr.IID) |
|
err = nil |
|
|
|
errData := &model.FailData{ |
|
ChildID: mr.IID, |
|
} |
|
result.FailData = append(result.FailData, errData) |
|
continue |
|
} |
|
result.TotalNum++ |
|
} |
|
|
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
return |
|
} |
|
|
|
// structureDatabaseMR ... |
|
func (s *Service) structureDatabaseMR(c context.Context, projectID int, projectName string, mr *gitlab.MergeRequest) (err error) { |
|
var ( |
|
milestoneID int |
|
mrLables string |
|
mrChanges string |
|
mrLablesByte []byte |
|
mrChangesByte []byte |
|
) |
|
|
|
if mr.Milestone != nil { |
|
milestoneID = mr.Milestone.ID |
|
} |
|
if mrLablesByte, err = json.Marshal(mr.Labels); err != nil { |
|
mrLables = model.JsonMarshalErrorText |
|
} else { |
|
mrLables = string(mrLablesByte) |
|
} |
|
|
|
if mrChangesByte, err = json.Marshal(mr.Changes); err != nil { |
|
mrChanges = model.JsonMarshalErrorText |
|
} else { |
|
mrChanges = string(mrChangesByte) |
|
} |
|
|
|
mrDB := &model.StatisticsMrs{ |
|
MRID: mr.ID, |
|
MRIID: mr.IID, |
|
TargetBranch: mr.TargetBranch, |
|
SourceBranch: mr.SourceBranch, |
|
ProjectID: mr.ProjectID, |
|
ProjectName: projectName, |
|
Title: mr.Title, |
|
State: mr.State, |
|
CreatedAt: mr.CreatedAt, |
|
UpdatedAt: mr.UpdatedAt, |
|
Upvotes: mr.Upvotes, |
|
Downvotes: mr.Downvotes, |
|
AuthorID: mr.Author.ID, |
|
AuthorName: mr.Author.Name, |
|
AssigneeID: mr.Assignee.ID, |
|
AssigneeName: mr.Assignee.Name, |
|
SourceProjectID: mr.SourceProjectID, |
|
TargetProjectID: mr.TargetProjectID, |
|
Labels: mrLables, |
|
Description: mr.Description, |
|
WorkInProgress: mr.WorkInProgress, |
|
MilestoneID: milestoneID, |
|
MergeWhenPipelineSucceeds: mr.MergeWhenPipelineSucceeds, |
|
MergeStatus: mr.MergeStatus, |
|
MergedByID: mr.MergedBy.ID, |
|
MergedByName: mr.MergedBy.Name, |
|
MergedAt: mr.MergedAt, |
|
ClosedByID: mr.ClosedBy.ID, |
|
ClosedAt: mr.ClosedAt, |
|
Subscribed: mr.Subscribed, |
|
SHA: mr.SHA, |
|
MergeCommitSHA: mr.MergeCommitSHA, |
|
UserNotesCount: mr.UserNotesCount, |
|
ChangesCount: mr.ChangesCount, |
|
ShouldRemoveSourceBranch: mr.ShouldRemoveSourceBranch, |
|
ForceRemoveSourceBranch: mr.ForceRemoveSourceBranch, |
|
WebURL: mr.WebURL, |
|
DiscussionLocked: mr.DiscussionLocked, |
|
Changes: mrChanges, |
|
TimeStatsHumanTimeEstimate: mr.TimeStats.HumanTimeEstimate, |
|
TimeStatsHumanTotalTimeSpent: mr.TimeStats.HumanTotalTimeSpent, |
|
TimeStatsTimeEstimate: mr.TimeStats.TimeEstimate, |
|
TimeStatsTotalTimeSpent: mr.TimeStats.TotalTimeSpent, |
|
Squash: mr.Squash, |
|
PipelineID: mr.Pipeline.ID, |
|
} |
|
if len(mrDB.Labels) > model.MessageMaxLen { |
|
mrDB.Labels = mrDB.Labels[0 : model.MessageMaxLen-1] |
|
} |
|
if len(mrDB.Description) > model.MessageMaxLen { |
|
mrDB.Description = mrDB.Description[0 : model.MessageMaxLen-1] |
|
} |
|
return s.SaveDatabaseMR(c, mrDB) |
|
} |
|
|
|
// SaveDatabaseMR ... |
|
func (s *Service) SaveDatabaseMR(c context.Context, mrDB *model.StatisticsMrs) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasMR(c, mrDB.ProjectID, mrDB.MRIID); err != nil { |
|
log.Error("SaveDatabaseMR HasMR(%+v)", err) |
|
return |
|
} |
|
|
|
// found only one, so update |
|
if total == 1 { |
|
return s.dao.UpdateMR(c, mrDB.ProjectID, mrDB.MRIID, mrDB) |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseMR mr has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
return s.dao.CreateMR(c, mrDB) |
|
} |
|
|
|
/*-------------------------------------- agg MR ----------------------------------------*/ |
|
|
|
// AggregateProjectMR ... |
|
func (s *Service) AggregateProjectMR(c context.Context, projectID int) (err error) { |
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
mrs []*model.StatisticsMrs |
|
projectInfo *model.ProjectInfo |
|
since *time.Time |
|
until *time.Time |
|
) |
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil { |
|
return |
|
} |
|
|
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil { |
|
return |
|
} |
|
|
|
for _, mr := range mrs { |
|
if err = s.MRAddedInfoDB(c, projectID, mr); err != nil { |
|
log.Error("MRAddedInfoDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID) |
|
err = nil |
|
} |
|
if err = s.MRReviewerDB(c, projectID, mr.MRIID, projectInfo.Name, mr); err != nil { |
|
log.Error("MRReviewerDB Save Database err: projectID(%d), MRIID(%d)", projectID, mr.MRIID) |
|
err = nil |
|
} |
|
} |
|
return |
|
} |
|
|
|
// MRAddedInfoDB ... |
|
func (s *Service) MRAddedInfoDB(c context.Context, projectID int, mr *model.StatisticsMrs) (err error) { |
|
if mr.ChangeAdd, mr.ChangeDel, err = s.QueryMergeRequestDiff(c, projectID, mr.MRIID); err != nil { |
|
return |
|
} |
|
if mr.TotalDiscussion, mr.SolvedDiscussion, err = s.QueryMergeRequestDiscussion(c, projectID, mr.MRIID); err != nil { |
|
return |
|
} |
|
return s.dao.UpdateMR(c, projectID, mr.MRIID, mr) |
|
} |
|
|
|
// MRReviewerDB 查询mr reviewer信息 |
|
func (s *Service) MRReviewerDB(c context.Context, projectID, mrIID int, projectName string, mr *model.StatisticsMrs) (err error) { |
|
var ( |
|
notes []*model.StatisticsNotes |
|
emojis []*model.StatisticsMRAwardEmojis |
|
owners []string |
|
r *regexp.Regexp |
|
reviewers []*model.AggregateMrReviewer |
|
) |
|
|
|
//query note |
|
if notes, err = s.dao.NoteByMRIID(c, projectID, mrIID); err != nil { |
|
return |
|
} |
|
|
|
// query emoji |
|
if emojis, err = s.dao.AwardEmojiByMRIID(c, projectID, mrIID); err != nil { |
|
return |
|
} |
|
|
|
//评论中解析获取owner |
|
if len(notes) == 0 { |
|
return |
|
} |
|
if r, err = regexp.Compile("OWNER:.*?@(.*)(；|)"); err == nil { |
|
matchResult := r.FindStringSubmatch(notes[len(notes)-1].Body) |
|
if len(matchResult) > 0 { |
|
owners = strings.Split(matchResult[1], " 或 @") |
|
} |
|
} |
|
|
|
mrCreatedAt := notes[len(notes)-1].CreatedAt |
|
|
|
// 从评论和表情中解析 reviewers |
|
for _, note := range notes { |
|
if note.Body == "+1" { |
|
reviewer := &model.AggregateMrReviewer{ |
|
ReviewerID: note.AuthorID, |
|
ReviewerName: note.AuthorName, |
|
ReviewType: "note", |
|
ReviewID: note.NoteID, |
|
ReviewCommand: "+1", |
|
CreatedAt: note.CreatedAt, |
|
ApproveTime: int(note.CreatedAt.Sub(*mrCreatedAt).Seconds()), |
|
MergeTime: int(mr.UpdatedAt.Sub(*note.CreatedAt).Seconds()), |
|
} |
|
reviewers = append(reviewers, reviewer) |
|
} |
|
} |
|
for _, emoji := range emojis { |
|
if emoji.Name == "thumbsup" { |
|
reviewer := &model.AggregateMrReviewer{ |
|
ReviewerID: emoji.UserID, |
|
ReviewerName: emoji.UserName, |
|
ReviewType: "emoji", |
|
ReviewID: emoji.AwardEmojiID, |
|
ReviewCommand: "thumbsup", |
|
CreatedAt: emoji.CreatedAt, |
|
ApproveTime: int(emoji.CreatedAt.Sub(*mrCreatedAt).Seconds()), |
|
MergeTime: int(mr.UpdatedAt.Sub(*emoji.CreatedAt).Seconds()), |
|
} |
|
reviewers = append(reviewers, reviewer) |
|
} |
|
} |
|
|
|
// 判断reviewer类型 |
|
for _, r := range reviewers { |
|
r.ProjectID = projectID |
|
r.ProjectName = projectName |
|
r.MrIID = mrIID |
|
r.Title = mr.Title |
|
r.WebUrl = mr.WebURL |
|
r.AuthorName = mr.AuthorName |
|
if utils.InSlice(r.ReviewerName, owners) { |
|
r.UserType = "owner" |
|
} else { |
|
r.UserType = "other" |
|
} |
|
if err = s.SaveDatabaseAggMR(c, r); err != nil { |
|
log.Error("mrReviewer 存数据库报错(%+V)", err) |
|
continue |
|
} |
|
} |
|
return |
|
} |
|
|
|
// SaveDatabaseAggMR ... |
|
func (s *Service) SaveDatabaseAggMR(c context.Context, mrDB *model.AggregateMrReviewer) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID); err != nil { |
|
log.Error("SaveDatabaseAggMR HasAggMR(%+v)", err) |
|
return |
|
} |
|
|
|
// found only one, so update |
|
if total == 1 { |
|
return s.dao.UpdateAggregateReviewer(c, mrDB.ProjectID, mrDB.MrIID, mrDB.ReviewerID, mrDB.ReviewID, mrDB) |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseAggMR aggMR has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
return s.dao.CreateAggregateReviewer(c, mrDB) |
|
}
|
|
|