You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
485 lines
14 KiB
485 lines
14 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"encoding/json" |
|
"strconv" |
|
"strings" |
|
"time" |
|
|
|
"go-common/app/admin/ep/saga/model" |
|
"go-common/app/admin/ep/saga/service/utils" |
|
"go-common/library/log" |
|
|
|
"github.com/xanzy/go-gitlab" |
|
) |
|
|
|
/*-------------------------------------- sync issue ----------------------------------------*/ |
|
|
|
// SyncAllIssues ... |
|
func (s *Service) SyncAllIssues(projectID int) (totalPage, totalNum int, err error) { |
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
issues []*gitlab.Issue |
|
resp *gitlab.Response |
|
since *time.Time |
|
until *time.Time |
|
) |
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
|
|
for page := 1; ; page++ { |
|
totalPage++ |
|
if issues, resp, err = s.gitlab.ListProjectIssues(projectID, page, since, until); err != nil { |
|
return |
|
} |
|
for _, issue := range issues { |
|
var ( |
|
issueAssignees string |
|
issueLabels string |
|
issueTimeStats string |
|
milestoneID int |
|
) |
|
if issue.Milestone != nil { |
|
milestoneID = issue.Milestone.ID |
|
} |
|
issueAssigneesByte, _ := json.Marshal(issue.Assignees) |
|
issueAssignees = string(issueAssigneesByte) |
|
issueLabelsByte, _ := json.Marshal(issue.Labels) |
|
issueLabels = string(issueLabelsByte) |
|
issueTimeStatsByte, _ := json.Marshal(issue.TimeStats) |
|
issueTimeStats = string(issueTimeStatsByte) |
|
issueDB := &model.StatisticsIssues{ |
|
ProjectID: projectID, |
|
IssueID: issue.ID, |
|
IssueIID: issue.IID, |
|
MilestoneID: milestoneID, |
|
Description: issue.Description, |
|
State: issue.State, |
|
Assignees: issueAssignees, |
|
Upvotes: issue.Upvotes, |
|
Downvotes: issue.Downvotes, |
|
Labels: issueLabels, |
|
Title: issue.Title, |
|
UpdatedAt: issue.UpdatedAt, |
|
CreatedAt: issue.CreatedAt, |
|
ClosedAt: issue.ClosedAt, |
|
Subscribed: issue.Subscribed, |
|
UserNotesCount: issue.UserNotesCount, |
|
DueDate: issue.DueDate, |
|
WebURL: issue.WebURL, |
|
TimeStats: issueTimeStats, |
|
Confidential: issue.Confidential, |
|
Weight: issue.Weight, |
|
DiscussionLocked: issue.DiscussionLocked, |
|
IssueLinkID: issue.IssueLinkID, |
|
} |
|
|
|
if issue.Author != nil { |
|
issueDB.AuthorID = issue.Author.ID |
|
issueDB.AuthorName = issue.Author.Name |
|
} |
|
if issue.Assignee != nil { |
|
issueDB.AssigneeID = issue.Assignee.ID |
|
issueDB.AssigneeName = issue.Assignee.Name |
|
} |
|
|
|
if len(issueDB.Description) > model.MessageMaxLen { |
|
issueDB.Description = issueDB.Description[0 : model.MessageMaxLen-1] |
|
} |
|
|
|
if err = s.SaveDatabaseIssue(issueDB); err != nil { |
|
log.Error("issue Save Database err: projectID(%d), IssueID(%d)", projectID, issue.ID) |
|
err = nil |
|
continue |
|
} |
|
totalNum++ |
|
} |
|
|
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
return |
|
} |
|
|
|
// SaveDatabaseIssue ... |
|
func (s *Service) SaveDatabaseIssue(issueDB *model.StatisticsIssues) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasIssue(issueDB.ProjectID, issueDB.IssueID); err != nil { |
|
log.Error("SaveDatabaseIssue HasIssue(%+v)", err) |
|
return |
|
} |
|
|
|
// update found row |
|
if total == 1 { |
|
if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil { |
|
if strings.Contains(err.Error(), model.DatabaseErrorText) { |
|
issueDB.Title = strconv.QuoteToASCII(issueDB.Title) |
|
issueDB.Description = strconv.QuoteToASCII(issueDB.Description) |
|
issueDB.Title = utils.Unicode2Chinese(issueDB.Title) |
|
issueDB.Description = utils.Unicode2Chinese(issueDB.Description) |
|
} |
|
if err = s.dao.UpdateIssue(issueDB.ProjectID, issueDB.IssueID, issueDB); err != nil { |
|
log.Error("SaveDatabaseIssue UpdateIssue(%+v)", err) |
|
return |
|
} |
|
} |
|
return |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseIssue issue has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
if err = s.dao.CreateIssue(issueDB); err != nil { |
|
if strings.Contains(err.Error(), model.DatabaseErrorText) { |
|
issueDB.Title = strconv.QuoteToASCII(issueDB.Title) |
|
issueDB.Description = strconv.QuoteToASCII(issueDB.Description) |
|
issueDB.Title = utils.Unicode2Chinese(issueDB.Title) |
|
issueDB.Description = utils.Unicode2Chinese(issueDB.Description) |
|
} |
|
if err = s.dao.CreateIssue(issueDB); err != nil { |
|
log.Error("SaveDatabaseIssue CreateIssue(%+v)", err) |
|
return |
|
} |
|
} |
|
|
|
return |
|
} |
|
|
|
/*-------------------------------------- sync note ----------------------------------------*/ |
|
|
|
// SyncProjectNotes ... |
|
func (s *Service) SyncProjectNotes(c context.Context, projectID int) (totalPage, totalNum int, err error) { |
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
resp *gitlab.Response |
|
projectInfo *model.ProjectInfo |
|
mrs []*model.StatisticsMrs |
|
notes []*gitlab.Note |
|
since *time.Time |
|
until *time.Time |
|
) |
|
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil { |
|
return |
|
} |
|
|
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil { |
|
return |
|
} |
|
|
|
for _, mr := range mrs { |
|
for page := 1; ; page++ { |
|
totalPage++ |
|
if notes, resp, err = s.gitlab.ListMRNotes(c, projectID, mr.MRIID, page); err != nil { |
|
return |
|
} |
|
for _, note := range notes { |
|
var ( |
|
notePosition string |
|
notePositionByte []byte |
|
) |
|
if notePositionByte, err = json.Marshal(note.Position); err != nil { |
|
return |
|
} |
|
notePosition = string(notePositionByte) |
|
noteDB := &model.StatisticsNotes{ |
|
ProjectID: projectID, |
|
ProjectName: projectInfo.Name, |
|
MrIID: mr.MRIID, |
|
NoteID: note.ID, |
|
Body: note.Body, |
|
Attachment: note.Attachment, |
|
Title: note.Title, |
|
FileName: note.FileName, |
|
AuthorID: note.Author.ID, |
|
AuthorName: note.Author.Name, |
|
System: note.System, |
|
ExpiresAt: note.ExpiresAt, |
|
UpdatedAt: note.UpdatedAt, |
|
CreatedAt: note.CreatedAt, |
|
NoteableID: note.NoteableID, |
|
NoteableType: note.NoteableType, |
|
Position: notePosition, |
|
Resolvable: note.Resolvable, |
|
Resolved: note.Resolved, |
|
ResolvedByID: note.ResolvedBy.ID, |
|
ResolvedByName: note.ResolvedBy.Username, |
|
NoteableIID: note.NoteableIID, |
|
} |
|
if err = s.SaveDatabaseNote(c, noteDB); err != nil { |
|
log.Error("note Save Database err: projectID(%d), NoteID(%d)", projectID, note.ID) |
|
err = nil |
|
continue |
|
} |
|
totalNum++ |
|
} |
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
} |
|
return |
|
} |
|
|
|
// HandleNoteDBError ... |
|
func (s *Service) HandleNoteDBError(c context.Context, noteDB *model.StatisticsNotes, errText string) (note *model.StatisticsNotes) { |
|
if strings.Contains(errText, model.DatabaseErrorText) { |
|
noteDB.Title = strconv.QuoteToASCII(noteDB.Title) |
|
noteDB.Body = strconv.QuoteToASCII(noteDB.Body) |
|
noteDB.Title = utils.Unicode2Chinese(noteDB.Title) |
|
noteDB.Body = utils.Unicode2Chinese(noteDB.Body) |
|
if len(noteDB.Body) > model.MessageMaxLen { |
|
noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1] |
|
} |
|
} else if strings.Contains(errText, model.DatabaseMaxLenthErrorText) { |
|
noteDB.Body = noteDB.Body[0 : model.MessageMaxLen-1] |
|
} |
|
return noteDB |
|
} |
|
|
|
// SaveDatabaseNote ... |
|
func (s *Service) SaveDatabaseNote(c context.Context, noteDB *model.StatisticsNotes) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasNote(c, noteDB.ProjectID, noteDB.NoteID); err != nil { |
|
log.Error("SaveDatabaseNote HasNote(%+v)", err) |
|
return |
|
} |
|
|
|
// update found row |
|
if total == 1 { |
|
if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil { |
|
noteDB = s.HandleNoteDBError(c, noteDB, err.Error()) |
|
if err = s.dao.UpdateNote(c, noteDB.ProjectID, noteDB.NoteID, noteDB); err != nil { |
|
log.Error("SaveDatabaseNote UpdateNote(%+v)", err) |
|
return |
|
} |
|
} |
|
return |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseNote Note has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
if err = s.dao.CreateNote(c, noteDB); err != nil { |
|
noteDB = s.HandleNoteDBError(c, noteDB, err.Error()) |
|
if err = s.dao.CreateNote(c, noteDB); err != nil { |
|
log.Error("SaveDatabaseNote CreateNote(%+v)", err) |
|
return |
|
} |
|
} |
|
|
|
return |
|
} |
|
|
|
/*-------------------------------------- sync emoji ----------------------------------------*/ |
|
|
|
// SyncProjectAwardEmoji ... |
|
func (s *Service) SyncProjectAwardEmoji(c context.Context, projectID int) (totalPage, totalNum int, err error) { |
|
|
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
resp *gitlab.Response |
|
projectInfo *model.ProjectInfo |
|
mrs []*model.StatisticsMrs |
|
awardEmojis []*gitlab.AwardEmoji |
|
since *time.Time |
|
until *time.Time |
|
) |
|
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil { |
|
return |
|
} |
|
|
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil { |
|
return |
|
} |
|
|
|
for _, mr := range mrs { |
|
for page := 1; ; page++ { |
|
totalPage++ |
|
if awardEmojis, resp, err = s.gitlab.ListMRAwardEmoji(projectID, mr.MRIID, page); err != nil { |
|
return |
|
} |
|
for _, awardEmoji := range awardEmojis { |
|
awardEmojiDB := &model.StatisticsMRAwardEmojis{ |
|
ProjectID: projectID, |
|
ProjectName: projectInfo.Name, |
|
MrIID: mr.MRIID, |
|
AwardEmojiID: awardEmoji.ID, |
|
Name: awardEmoji.Name, |
|
UserID: awardEmoji.User.ID, |
|
UserName: awardEmoji.User.Name, |
|
CreatedAt: awardEmoji.CreatedAt, |
|
UpdatedAt: awardEmoji.UpdatedAt, |
|
AwardableID: awardEmoji.AwardableID, |
|
AwardableType: awardEmoji.AwardableType, |
|
} |
|
if err = s.SaveDatabaseAwardEmoji(c, awardEmojiDB); err != nil { |
|
log.Error("awardEmoji Save Database err: projectID(%d), AwardEmojiID(%d)", projectID, awardEmoji.ID) |
|
err = nil |
|
continue |
|
} |
|
totalNum++ |
|
} |
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
} |
|
return |
|
} |
|
|
|
// SaveDatabaseAwardEmoji ... |
|
func (s *Service) SaveDatabaseAwardEmoji(c context.Context, awardEmojiDB *model.StatisticsMRAwardEmojis) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID); err != nil { |
|
log.Error("SaveDatabaseAwardEmoji HasAwardEmoji(%+v)", err) |
|
return |
|
} |
|
|
|
// update found row |
|
if total == 1 { |
|
if err = s.dao.UpdateMRAwardEmoji(c, awardEmojiDB.ProjectID, awardEmojiDB.MrIID, awardEmojiDB.AwardEmojiID, awardEmojiDB); err != nil { |
|
log.Error("SaveDatabaseAwardEmoji UpdateAwardEmoji(%+v)", err) |
|
} |
|
return |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseAwardEmoji Note has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
if err = s.dao.CreateMRAwardEmoji(c, awardEmojiDB); err != nil { |
|
log.Error("SaveDatabaseAwardEmoji CreateAwardEmoji(%+v)", err) |
|
return |
|
|
|
} |
|
|
|
return |
|
} |
|
|
|
/*-------------------------------------- sync Discussion ----------------------------------------*/ |
|
|
|
// SyncProjectDiscussion ... |
|
func (s *Service) SyncProjectDiscussion(c context.Context, projectID int) (totalPage, totalNum int, err error) { |
|
|
|
var ( |
|
//syncAllTime = conf.Conf.Property.SyncData.SyncAllTime |
|
syncAllTime = false |
|
resp *gitlab.Response |
|
projectInfo *model.ProjectInfo |
|
mrs []*model.StatisticsMrs |
|
discussions []*gitlab.Discussion |
|
since *time.Time |
|
until *time.Time |
|
) |
|
|
|
if projectInfo, err = s.dao.ProjectInfoByID(projectID); err != nil { |
|
return |
|
} |
|
|
|
if !syncAllTime { |
|
since, until = utils.CalSyncTime() |
|
} |
|
|
|
if mrs, err = s.dao.MRByProjectID(c, projectID, since, until); err != nil { |
|
return |
|
} |
|
|
|
for _, mr := range mrs { |
|
for page := 1; ; page++ { |
|
totalPage++ |
|
if discussions, resp, err = s.gitlab.ListMRDiscussions(projectID, mr.MRIID, page); err != nil { |
|
return |
|
} |
|
for _, discussion := range discussions { |
|
var ( |
|
notesArray []int |
|
notesByte []byte |
|
notes string |
|
) |
|
if discussion.Notes != nil { |
|
for _, note := range discussion.Notes { |
|
notesArray = append(notesArray, note.ID) |
|
} |
|
} |
|
if notesByte, err = json.Marshal(notesArray); err != nil { |
|
notes = model.JsonMarshalErrorText |
|
} else { |
|
notes = string(notesByte) |
|
} |
|
discussionDB := &model.StatisticsDiscussions{ |
|
ProjectID: projectID, |
|
ProjectName: projectInfo.Name, |
|
MrIID: mr.MRIID, |
|
DiscussionID: discussion.ID, |
|
IndividualNote: discussion.IndividualNote, |
|
Notes: notes, |
|
} |
|
if err = s.SaveDatabaseDiscussion(c, discussionDB); err != nil { |
|
log.Error("discussion Save Database err: projectID(%d), DiscussionID(%s)", projectID, discussion.ID) |
|
err = nil |
|
continue |
|
} |
|
totalNum++ |
|
} |
|
if resp.NextPage == 0 { |
|
break |
|
} |
|
} |
|
} |
|
return |
|
} |
|
|
|
// SaveDatabaseDiscussion ... |
|
func (s *Service) SaveDatabaseDiscussion(c context.Context, discussionDB *model.StatisticsDiscussions) (err error) { |
|
var total int |
|
|
|
if total, err = s.dao.HasDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID); err != nil { |
|
log.Error("SaveDatabaseDiscussion HasDiscussion(%+v)", err) |
|
return |
|
} |
|
|
|
// update found row |
|
if total == 1 { |
|
if err = s.dao.UpdateDiscussion(c, discussionDB.ProjectID, discussionDB.MrIID, discussionDB.DiscussionID, discussionDB); err != nil { |
|
log.Error("SaveDatabaseDiscussion UpdateDiscussion(%+v)", err) |
|
} |
|
return |
|
} else if total > 1 { |
|
// found repeated row, this situation will not exist under normal |
|
log.Warn("SaveDatabaseDiscussion Note has more rows(%d)", total) |
|
return |
|
} |
|
|
|
// insert row now |
|
if err = s.dao.CreateDiscussion(c, discussionDB); err != nil { |
|
log.Error("SaveDatabaseDiscussion CreateDiscussion(%+v)", err) |
|
return |
|
|
|
} |
|
|
|
return |
|
}
|
|
|