1945df5fa5
* TP-28766 | Houston summary * TP-28766 | cron changes * TP-28766 | update * TP-28766 | changes in format * TP-28766 | Team metric * TP-28766 | change config * TP-28766 | change config * TP-28766 | team metric interval change
281 lines
11 KiB
Go
281 lines
11 KiB
Go
package cron
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"time"
|
|
|
|
"houston/common/util"
|
|
"houston/internal/processor/action"
|
|
"houston/model/incident"
|
|
"houston/model/severity"
|
|
"houston/model/shedlock"
|
|
"houston/model/team"
|
|
"houston/pkg/slackbot"
|
|
|
|
"github.com/slack-go/slack"
|
|
"github.com/slack-go/slack/socketmode"
|
|
"github.com/spf13/viper"
|
|
"go.uber.org/zap"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
func RunJob(socketModeClient *socketmode.Client, db *gorm.DB, logger *zap.Logger, incidentService *incident.Repository, severityService *severity.Repository, teamService *team.Repository, shedlockService *shedlock.Repository) {
|
|
|
|
//HOUSTON ESCALATE
|
|
shedlockConfig := NewLockerDbWithLockTime(viper.GetInt("cron.job.lock.default.time.in.sec"))
|
|
|
|
err := shedlockConfig.AddFun(viper.GetString("cron.job.name"), viper.GetString("cron.job.update.incident.interval"), shedlockService, func() {
|
|
UpdateIncidentByCronJob(socketModeClient, db, logger, incidentService, teamService, severityService, viper.GetString("cron.job.name"))
|
|
})
|
|
if err != nil {
|
|
logger.Error("HOUSTON_ESCALATE error: " + err.Error())
|
|
}
|
|
|
|
//HOUSTON DAILY TEAM UPDATE
|
|
err = shedlockConfig.AddFun(viper.GetString("cron.job.team.metric"), viper.GetString("cron.job.team.metric.interval"), shedlockService, func() {
|
|
postTeamMetrics(socketModeClient, db, logger, incidentService, teamService, severityService, viper.GetString("cron.job.team.metric"))
|
|
})
|
|
if err != nil {
|
|
logger.Error("HOUSTON_TEAM_METRICS error :" + err.Error())
|
|
}
|
|
|
|
shedlockConfig.Start()
|
|
|
|
}
|
|
|
|
func UpdateIncidentByCronJob(socketModeClient *socketmode.Client, db *gorm.DB, logger *zap.Logger, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, name string) {
|
|
|
|
fmt.Println("Running job at", time.Now().Format(time.RFC3339))
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error)))
|
|
}
|
|
}()
|
|
//Do not need to fetch Severity every time, hence keeping a map outside.
|
|
incidentSeverityList, err := severityService.GetAllActiveSeverity()
|
|
if err != nil || incidentSeverityList == nil {
|
|
logger.Error("GetAllActiveSeverity error in cron Job",
|
|
zap.String("cron_name", name), zap.Error(err))
|
|
return
|
|
}
|
|
|
|
incidentStatusEntity, _ := incidentService.FindIncidentStatusByName(incident.Resolved)
|
|
//FETCH INCIDENTS WHICH ARE NOT RESOLVED AND CURRENT TIMESTAMP>=SEVERITY TAT FIELD
|
|
if incidentStatusEntity == nil || incidentStatusEntity.ID == 0 {
|
|
logger.Error("Error : RESOLVED Incident Status not found in cron job")
|
|
return
|
|
}
|
|
|
|
incidents, err := incidentService.FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0(db, int(incidentStatusEntity.ID))
|
|
if err != nil {
|
|
logger.Error("FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0 error",
|
|
zap.Error(err))
|
|
return
|
|
}
|
|
|
|
if len(incidents) == 0 {
|
|
logger.Info("No incidents found to be updated by cron job")
|
|
}
|
|
|
|
//Looping through All incidents which are not resolved and sev tat is breaching
|
|
for i := 0; i < len(incidents); i++ {
|
|
logger.Info("TAT breached Not Resolved Incident found. Name: " + incidents[i].IncidentName + ". Severity: " + strconv.Itoa(int(incidents[i].SeverityId)))
|
|
updatingSevForEachInc(logger, incidents, i, incidentSeverityList, err, incidentService, socketModeClient, teamService, severityService)
|
|
}
|
|
}
|
|
|
|
func updatingSevForEachInc(logger *zap.Logger, incidents []incident.IncidentEntity, i int, incidentSeverityList *[]severity.SeverityEntity, err error, incidentService *incident.Repository, socketModeClient *socketmode.Client, teamService *team.Repository, severityService *severity.Repository) {
|
|
|
|
var currentSeverityId = incidents[i].SeverityId
|
|
var severityString string = ""
|
|
var usersToBeAdded = make([]string, 0)
|
|
|
|
incidents[i].SeverityId = currentSeverityId - 1
|
|
|
|
//Find the severity object from list and update the tat of incident
|
|
for _, s := range *incidentSeverityList {
|
|
if s.ID == incidents[i].SeverityId {
|
|
incidents[i].SeverityTat = time.Now().AddDate(0, 0, s.Sla)
|
|
usersToBeAdded = s.SlackUserIds
|
|
severityString = fmt.Sprintln(s.Name + " (" + s.Description + ")")
|
|
}
|
|
}
|
|
|
|
//Updating Incident
|
|
incidents[i].UpdatedAt = time.Now()
|
|
err = incidentService.UpdateIncident(&incidents[i])
|
|
if err != nil {
|
|
logger.Error("failed to update incident in cron job",
|
|
zap.String("channel", incidents[i].SlackChannel), zap.Error(err))
|
|
}
|
|
|
|
//Default User Addition According To Sev Tat
|
|
for _, o := range usersToBeAdded {
|
|
//throws error if the customer is already present in channel
|
|
_, err := socketModeClient.Client.InviteUsersToConversation(incidents[i].SlackChannel, o)
|
|
if err != nil {
|
|
logger.Error("Slack Client InviteUsersToConversation error in cron job")
|
|
}
|
|
}
|
|
|
|
//UPDATING MESSAGE
|
|
s := action.NewIncidentChannelMessageUpdateAction(socketModeClient, logger, incidentService, teamService, severityService)
|
|
slackbotClient := slackbot.NewSlackClient(logger, socketModeClient)
|
|
team, err := teamService.FindTeamById(incidents[i].TeamId)
|
|
if err != nil {
|
|
logger.Error("error in fetching team by id", zap.Uint("teamId", incidents[i].TeamId), zap.Error(err), zap.String("s", severityString))
|
|
}
|
|
incidentSeverityEntity, err := severityService.FindSeverityById(incidents[i].SeverityId)
|
|
if err != nil {
|
|
logger.Error("error in finding severity entity", zap.Error(err), zap.Uint("severityId", incidents[i].SeverityId))
|
|
}
|
|
s.ProcessAction(incidents[i].SlackChannel)
|
|
msgOption := slack.MsgOptionText(fmt.Sprintf("houston escalated incident to %s", severityString), false)
|
|
_, _, errMessage := socketModeClient.PostMessage(incidents[i].SlackChannel, msgOption)
|
|
topic := fmt.Sprintf("%s-%s(%s) Incident-%d | %s", team.Name, incidentSeverityEntity.Name, incidentSeverityEntity.Description, incidents[i].ID, incidents[i].Title)
|
|
slackbotClient.SetChannelTopic(incidents[i].SlackChannel, topic)
|
|
if errMessage != nil {
|
|
logger.Error("PostMessage failed for cronJob ", zap.Error(errMessage), zap.Int("incidentId", int(incidents[i].ID)))
|
|
}
|
|
}
|
|
|
|
func postTeamMetrics(socketModeClient *socketmode.Client, db *gorm.DB, logger *zap.Logger, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, name string) {
|
|
fmt.Println("Running Team Metrics cron at", time.Now().Format(time.RFC3339))
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error)))
|
|
}
|
|
}()
|
|
|
|
teams, err := teamService.GetAllActiveTeams()
|
|
if err != nil {
|
|
logger.Error("GetAllActiveTeams error", zap.Error(err))
|
|
return
|
|
}
|
|
|
|
teamsList := *teams
|
|
//Do not need to fetch Severity every time, hence keeping a map outside.
|
|
incidentSeverityList, err := severityService.GetAllActiveSeverity()
|
|
if err != nil || incidentSeverityList == nil {
|
|
logger.Error("GetAllActiveSeverity error in cron Job",
|
|
zap.String("cron_name", name), zap.Error(err))
|
|
return
|
|
}
|
|
severityMap := convertSeverityListToMap(*incidentSeverityList)
|
|
|
|
for i := 0; i < len(teamsList); i++ {
|
|
incidents, err := incidentService.GetIncidentsByTeamIdAndNotResolved(teamsList[i].ID)
|
|
if err != nil {
|
|
logger.Error("GetIncidentsByTeamIdAndNotResolved error in cron Job",
|
|
zap.String("cron_name", name), zap.Error(err))
|
|
continue
|
|
}
|
|
incidentsList := *incidents
|
|
|
|
list := make([]incident.SlackChannelWithResponderId, 0, len(incidentsList))
|
|
for j := 0; j < len(incidentsList); j++ {
|
|
//RESPONDER
|
|
incidentRole, err := incidentService.GetIncidentRoleByIncidentIdAndRole(incidentsList[j].ID, incident.Responder)
|
|
if err != nil {
|
|
logger.Error("GetIncidentRoleByIncidentIdAndRole error in cron Job",
|
|
zap.String("cron_name", name), zap.Error(err))
|
|
continue
|
|
}
|
|
|
|
// map to pojo and to list
|
|
obj := incident.SlackChannelWithResponderId{
|
|
SlackChannel: incidentsList[j].SlackChannel,
|
|
ResponderId: incidentRole.AssignedTo,
|
|
CreatedAt: incidentsList[j].CreatedAt,
|
|
Severity: severityMap[incidentsList[j].SeverityId],
|
|
ManagerId: teamsList[i].ManagerHandle,
|
|
}
|
|
// Appending the obj to the list
|
|
list = append(list, obj)
|
|
}
|
|
//Post message
|
|
teamName := builderTeamNameHeader(teamsList[i].Name)
|
|
openIncidentCount := buildOpenIncidentText(len(incidentsList))
|
|
onCallAndManagerBlock := buildOnCallAndManagerBlock(teamsList[i].OncallHandle, teamsList[i].ManagerHandle)
|
|
blocks := IncidentMetricBlock(teamName, openIncidentCount, onCallAndManagerBlock)
|
|
//On the basis of no of incidents, change colour
|
|
color := util.GetColourByOpenIncidents(len(incidentsList))
|
|
att := slack.Attachment{Blocks: blocks, Color: color}
|
|
_, _, err = socketModeClient.PostMessage(viper.GetString("team.metric.update.channel"), slack.MsgOptionAttachments(att))
|
|
|
|
text := ""
|
|
for index := 0; index < len(list); index++ {
|
|
// Calculate the time difference
|
|
currentTime := time.Now()
|
|
duration := 5*time.Hour + 30*time.Minute
|
|
newTime := currentTime.Add(duration)
|
|
timeDiff := newTime.Sub(list[index].CreatedAt)
|
|
|
|
// Convert the duration into days and hours
|
|
days := int(timeDiff.Hours() / 24)
|
|
hours := int(timeDiff.Hours()) % 24
|
|
if list[index].ResponderId == "" && list[index].ManagerId != "" {
|
|
text += fmt.Sprintf("<#%s>'s *Severity* is `%s` assigned to *Manager* <@%s>. *Open Since*- `%dd%dhr` \n", list[index].SlackChannel, list[index].Severity, list[index].ManagerId, days, hours)
|
|
} else if list[index].ResponderId != "" {
|
|
text += fmt.Sprintf("<#%s>'s *Severity* is `%s` assigned to *Responder* <@%s>. *Open Since*- `%dd%dhr` \n", list[index].SlackChannel, list[index].Severity, list[index].ResponderId, days, hours)
|
|
} else if list[index].ResponderId == "" && list[index].ManagerId == "" {
|
|
text += fmt.Sprintf("<#%s>'s *Severity* is `%s` assigned to *No-One*. *Open Since*- `%dd%dhr` \n", list[index].SlackChannel, list[index].Severity, days, hours)
|
|
}
|
|
}
|
|
|
|
if text != "" {
|
|
msgOption := slack.MsgOptionText(text, false)
|
|
_, _, errMessage := socketModeClient.PostMessage(viper.GetString("team.metric.update.channel"), msgOption)
|
|
if errMessage != nil {
|
|
logger.Error("PostMessage failed for cronJob ", zap.Error(errMessage))
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func convertSeverityListToMap(severities []severity.SeverityEntity) map[uint]string {
|
|
severityMap := make(map[uint]string)
|
|
|
|
for _, severity := range severities {
|
|
severityMap[severity.ID] = severity.Name
|
|
}
|
|
|
|
return severityMap
|
|
}
|
|
|
|
func IncidentMetricBlock(teamName *slack.HeaderBlock, incidentOpenCount *slack.SectionBlock, onCallAndManagerBlock *slack.SectionBlock) slack.Blocks {
|
|
return slack.Blocks{
|
|
BlockSet: []slack.Block{
|
|
teamName,
|
|
incidentOpenCount,
|
|
onCallAndManagerBlock,
|
|
},
|
|
}
|
|
}
|
|
|
|
func builderTeamNameHeader(teamName string) *slack.HeaderBlock {
|
|
headerText := slack.NewTextBlockObject(slack.PlainTextType, teamName, true, false)
|
|
headerSection := slack.NewHeaderBlock(headerText)
|
|
|
|
return headerSection
|
|
}
|
|
|
|
func buildOpenIncidentText(count int) *slack.SectionBlock {
|
|
sectionBlock := slack.NewTextBlockObject(slack.PlainTextType, fmt.Sprintf("Open Incidents: %d", count), true, false)
|
|
headerSection := slack.NewSectionBlock(sectionBlock, nil, nil)
|
|
|
|
return headerSection
|
|
}
|
|
|
|
func buildOnCallAndManagerBlock(onCallHandle string, managerHandle string) *slack.SectionBlock {
|
|
fields := []*slack.TextBlockObject{
|
|
slack.NewTextBlockObject("mrkdwn", fmt.Sprintf("*OnCall*\n<@%s>", onCallHandle), false, false),
|
|
slack.NewTextBlockObject("mrkdwn", fmt.Sprintf("*Manager*\n<@%s>", managerHandle), false, false),
|
|
}
|
|
block := slack.NewSectionBlock(nil, fields, nil)
|
|
|
|
return block
|
|
}
|