varnitgoyal95/houston-be
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

INFRA-3098 : Auto escalate refactor (#409)

Browse Source 
* INFRA-3098 : Auto escalate refactor

* INFRA-3098 : Delete old cron code

* INFRA-2887 : File and api name changes

* INFRA-2887 : Code review comments
This commit is contained in:
Vijay Joshi
2024-03-27 19:06:22 +05:30
committed by GitHub
parent 164a29cb86
commit a8a0d44da9
12 changed files with 280 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/app/handler/incident_handler.go
View File

@@ -135,6 +135,18 @@ func (handler *IncidentHandler) HandleResolveIncident(c *gin.Context) {
c.JSON(http.StatusOK, common.SuccessResponse("Incident resolved successfully", http.StatusOK))
}
func (handler *IncidentHandler) HandleEscalateIncident(c *gin.Context) {
err := handler.service.EscalateIncidents()
if err != nil {
logger.Error(fmt.Sprintf("%s failed to escalate incidents", logTag), zap.Error(err))
metrics.PublishHoustonFlowFailureMetrics(incidentService.ESCALATE_INCIDENT, err.Error())
c.JSON(http.StatusInternalServerError, common.ErrorResponse(err, http.StatusInternalServerError, nil))
return
}
c.JSON(http.StatusOK, common.SuccessResponse("Incidents escalated successfully", http.StatusOK))
}
func (handler *IncidentHandler) HandleJiraLinking(c *gin.Context) {
var linkJiraRequest request.LinkJiraRequest
if err := c.ShouldBindJSON(&linkJiraRequest); err != nil {

1
cmd/app/server.go
View File

@@ -218,6 +218,7 @@ func (s *Server) incidentClientHandlerV2(houstonGroup *gin.RouterGroup) {
s.gin.POST("/incidents", authService.IfValidHoustonUser(incidentHandler.HandleUpdateIncident))
houstonGroup.POST("/incidents", incidentHandler.HandleUpdateIncident)
houstonGroup.POST("/incidents/resolve", authService.IfValidHoustonUser(incidentHandler.HandleResolveIncident))
houstonGroup.POST("/incidents/escalate", incidentHandler.HandleEscalateIncident)
houstonGroup.POST("/link-jira-to-incident", incidentHandler.HandleJiraLinking)
houstonGroup.POST("/unlink-jira-from-incident", incidentHandler.HandleJiraUnLinking)
houstonGroup.GET("/get-jira-statuses", incidentHandler.HandleGetJiraStatuses)

149
internal/cron/cron.go
View File

@@ -1,32 +1,27 @@
package cron
import (
"errors"
"fmt"
"houston/common/metrics"
"houston/logger"
incidentService "houston/service/incident/impl"
slackService "houston/service/slack"
"sort"
"strconv"
"strings"
"sync"
"time"
"houston/common/util"
"houston/internal/processor/action"
"houston/model/incident"
"houston/model/severity"
"houston/model/shedlock"
"houston/model/team"
"houston/model/user"
"houston/pkg/slackbot"
"github.com/slack-go/slack"
"github.com/slack-go/slack/socketmode"
"github.com/spf13/viper"
"go.uber.org/zap"
"gorm.io/gorm"
"houston/common/util"
"houston/model/incident"
"houston/model/severity"
"houston/model/shedlock"
"houston/model/team"
"houston/model/user"
)
func RunJob(
@@ -43,17 +38,8 @@ func RunJob(
//HOUSTON ESCALATE
shedlockConfig := NewLockerDbWithLockTime(viper.GetInt("cron.job.lock.default.time.in.sec"))
err := shedlockConfig.AddFun(viper.GetString("cron.job.name"), viper.GetString("cron.job.update.incident.interval"), shedlockRepository, func() {
RunJobWithExecutionMetrics(AUTO_ESCALATE, func() {
UpdateIncidentByCronJob(socketModeClient, db, incidentRepository, teamRepository, severityRepository, viper.GetString("cron.job.name"))
})
})
if err != nil {
logger.Error("HOUSTON_ESCALATE error: " + err.Error())
}
//HOUSTON ADDING USER
err = shedlockConfig.AddFun(viper.GetString("cron.job.upsert_user"), viper.GetString("cron.job.upsert_user_interval"), shedlockRepository, func() {
err := shedlockConfig.AddFun(viper.GetString("cron.job.upsert_user"), viper.GetString("cron.job.upsert_user_interval"), shedlockRepository, func() {
RunJobWithExecutionMetrics(UPSERT_USERS, func() {
UpsertUsers(socketModeClient, userRepository)
})
@@ -101,127 +87,6 @@ func RunJobWithExecutionMetrics(jobName string, jobFunc func()) {
metrics.PublishCronJobExecutionCounterMetrics(jobName, duration)
}
func UpdateIncidentByCronJob(socketModeClient *socketmode.Client, db *gorm.DB, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, name string) {
fmt.Println("Running job at", time.Now().Format(time.RFC3339))
defer func() {
if r := recover(); r != nil {
logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error)))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, r.(error).Error())
}
}()
//Do not need to fetch Severity every time, hence keeping a map outside.
incidentSeverityList, err := severityService.GetAllActiveSeverity()
if err != nil {
logger.Error("GetAllActiveSeverity error in cron Job",
zap.String("cron_name", name), zap.Error(err))
panic(err)
return
}
if incidentSeverityList == nil {
errMessage := "Error : No active severity found in cron job"
logger.Error(errMessage)
panic(errors.New(errMessage))
return
}
incidentStatusEntity, _ := incidentService.FindIncidentStatusByName(incident.Resolved)
//FETCH INCIDENTS WHICH ARE NOT RESOLVED AND CURRENT TIMESTAMP>=SEVERITY TAT FIELD
if incidentStatusEntity == nil || incidentStatusEntity.ID == 0 {
errMessage := "Error : RESOLVED Incident Status not found in cron job"
logger.Error(errMessage)
panic(errors.New(errMessage))
return
}
incidents, err := incidentService.FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0(db, int(incidentStatusEntity.ID))
if err != nil {
logger.Error("FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0 error",
zap.Error(err))
panic(err)
return
}
if len(incidents) == 0 {
logger.Info("No incidents found to be updated by cron job")
}
//Looping through All incidents which are not resolved and sev tat is breaching
for i := 0; i < len(incidents); i++ {
logger.Info("TAT breached Not Resolved Incident found. Name: " + incidents[i].IncidentName + ". Severity: " + strconv.Itoa(int(incidents[i].SeverityId)))
updatingSevForEachInc(incidents, i, incidentSeverityList, err, incidentService, socketModeClient, teamService, severityService)
}
}
func updatingSevForEachInc(incidents []incident.IncidentEntity, i int, incidentSeverityList *[]severity.SeverityEntity, err error, incidentService *incident.Repository, socketModeClient *socketmode.Client, teamService *team.Repository, severityService *severity.Repository) {
var currentSeverityId = incidents[i].SeverityId
var severityString string = ""
var usersToBeAdded = make([]string, 0)
incidents[i].SeverityId = currentSeverityId - 1
//Find the severity object from list and update the tat of incident
for _, s := range *incidentSeverityList {
if s.ID == incidents[i].SeverityId {
incidents[i].SeverityTat = time.Now().AddDate(0, 0, s.Sla)
usersToBeAdded = s.SlackUserIds
severityString = fmt.Sprintln(s.Name + " (" + s.Description + ")")
}
}
//Updating Incident
incidents[i].UpdatedAt = time.Now()
incidents[i].UpdatedBy = viper.GetString("HOUSTON_BOT_SLACK_ID")
err = incidentService.UpdateIncident(&incidents[i])
if err != nil {
logger.Error("failed to update incident in cron job",
zap.String("channel", incidents[i].SlackChannel), zap.Error(err))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
//Default User Addition According To Sev Tat
for _, o := range usersToBeAdded {
//throws error if the customer is already present in channel
_, err := socketModeClient.Client.InviteUsersToConversation(incidents[i].SlackChannel, o)
if err != nil {
logger.Error("Slack Client InviteUsersToConversation error in cron job")
if err.Error() != util.AlreadyInChannelError {
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
}
}
//UPDATING MESSAGE
s := action.NewIncidentChannelMessageUpdateAction(socketModeClient, incidentService, teamService, severityService)
slackbotClient := slackbot.NewSlackClient(socketModeClient)
team, err := teamService.FindTeamById(incidents[i].TeamId)
if err != nil {
logger.Error("error in fetching team by id", zap.Uint("teamId", incidents[i].TeamId), zap.Error(err), zap.String("s", severityString))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
incidentSeverityEntity, err := severityService.FindSeverityById(incidents[i].SeverityId)
if err != nil {
logger.Error("error in finding severity entity", zap.Error(err), zap.Uint("severityId", incidents[i].SeverityId))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
s.ProcessAction(incidents[i].SlackChannel)
msgOption := slack.MsgOptionText(fmt.Sprintf("houston escalated incident to %s", severityString), false)
_, _, err = socketModeClient.PostMessage(incidents[i].SlackChannel, msgOption)
if err != nil {
logger.Error("PostMessage failed for cronJob ", zap.Error(err), zap.Int("incidentId", int(incidents[i].ID)))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
topic := fmt.Sprintf("%s-%s(%s) Incident-%d | %s", team.Name, incidentSeverityEntity.Name, incidentSeverityEntity.Description, incidents[i].ID, incidents[i].Title)
_, err = slackbotClient.SetChannelTopic(incidents[i].SlackChannel, topic)
if err != nil {
logger.Error("Set Channel Topic failed for cronJob", zap.Error(err), zap.Int("incidentId", int(incidents[i].ID)))
metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error())
}
}
func IncidentMetricBlock(teamName *slack.HeaderBlock, incidentOpenCount, onCallAndManagerBlock *slack.SectionBlock) slack.Blocks {
return slack.Blocks{
BlockSet: []slack.Block{

16
model/incident/incident.go
View File

@@ -661,6 +661,20 @@ func (r *Repository) FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCu
return incidentEntity, nil
}
func (r *Repository) GetIncidentsForEscalation() ([]IncidentEntity, error) {
var incidentEntity []IncidentEntity
currentTime := time.Now()
result := r.gormClient.Find(&incidentEntity, "status IN (?) AND severity_id <> ? AND severity_tat <= ?", statusesForEscalation, 1, currentTime)
if result.Error != nil {
return nil, result.Error
}
return incidentEntity, nil
}
func (r *Repository) GetIncidentsByTeamIdAndNotResolvedAndOfSev0OrSev1OrSev2(team_id uint) (*[]IncidentEntity, error) {
var incidentEntity []IncidentEntity
@@ -818,6 +832,8 @@ func buildIntervalExpr(severityIdToArchivalHoursMap map[int]int) string {
)
}
var statusesForEscalation = []uint{1, 2}
const (
SLA_BREACH_ALLOWED_STATUSES = "1,2"
)

1
model/incident/incident_repository_interface.go
View File

@@ -36,6 +36,7 @@ type IIncidentRepository interface {
GetIncidentTagsByIncidentId(incidentId uint) (*[]IncidentTagEntity, error)
GetIncidentTagByTagId(incidentId uint, tagId uint) (*IncidentTagEntity, error)
SaveIncidentTag(entity IncidentTagEntity) (*IncidentTagEntity, error)
GetIncidentsForEscalation() ([]IncidentEntity, error)
GetIncidentsByTeamIdAndNotResolvedAndOfSev0OrSev1OrSev2(team_id uint) (*[]IncidentEntity, error)
GetCountOfIncidentsByTeamIdAndNotResolved(team_id uint) (int, error)
GetIncidentRoleByIncidentIdAndRole(incident_id uint, role string) (*IncidentRoleEntity, error)

63
service/incident/impl/escalate_incident.go Normal file
View File

@@ -0,0 +1,63 @@
package impl
import (
"errors"
"fmt"
"github.com/spf13/viper"
"houston/logger"
incidentModel "houston/model/incident"
service "houston/service/request"
"strconv"
)
const escalateLogTag = "[escalate]"
func (i *IncidentServiceV2) EscalateIncidents() error {
logger.Info(fmt.Sprintf("%s received request for escalating incidents", escalateLogTag))
incidents, err := i.incidentRepository.GetIncidentsForEscalation()
if err != nil {
logger.Error(fmt.Sprintf("%s failed to get incidents for escalation %v", escalateLogTag, err))
return err
}
escalationMap, err := i.severityService.GetSeverityEscalationMap()
if err != nil {
logger.Error(fmt.Sprintf("%s failed to get severity escalation map %v", escalateLogTag, err))
return err
}
return i.runEscalationOnIncidents(incidents, escalationMap)
}
func (i *IncidentServiceV2) runEscalationOnIncidents(incidents []incidentModel.IncidentEntity, escalationMap map[uint]*uint) error {
logger.Info(fmt.Sprintf("%s found %d incidents for escalation", escalateLogTag, len(incidents)))
var incidentsWithFailures []string
for _, incident := range incidents {
logger.Info(fmt.Sprintf("%s escalating incident with id: %d", escalateLogTag, incident.ID))
nextSeverityID := escalationMap[incident.SeverityId]
if nextSeverityID == nil {
logger.Error(fmt.Sprintf("%s no escalation found for incident with id: %d", escalateLogTag, incident.ID))
incidentsWithFailures = append(incidentsWithFailures, incident.IncidentName)
continue
}
_, err := i.UpdateIncident(service.UpdateIncidentRequest{
Id: incident.ID,
SeverityId: strconv.Itoa(int(*nextSeverityID)),
}, viper.GetString("HOUSTON_BOT_SLACK_ID"))
if err != nil {
logger.Error(fmt.Sprintf("%s failed to escalate incident with id: %d %v", escalateLogTag, incident.ID, err))
incidentsWithFailures = append(incidentsWithFailures, incident.IncidentName)
}
}
if len(incidentsWithFailures) > 0 {
return errors.New(fmt.Sprintf("failed to escalate incidents: %v", incidentsWithFailures))
}
return nil
}

122
service/incident/impl/escalate_incident_test.go Normal file
View File

@@ -0,0 +1,122 @@
package impl
import (
"errors"
"github.com/slack-go/slack"
"houston/model/incident"
teamUserModel "houston/model/teamUser"
"houston/model/user"
)
func (suite *IncidentServiceSuite) Test_Escalate_GetIncidentsError() {
suite.incidentRepository.GetIncidentsForEscalationMock.Return(nil, errors.New("error"))
err := suite.incidentService.EscalateIncidents()
suite.Error(err, "service should return error")
}
func (suite *IncidentServiceSuite) Test_Escalate_GetSeverityEscalationMapError() {
incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
suite.severityService.GetSeverityEscalationMapMock.Return(nil, errors.New("error"))
err := suite.incidentService.EscalateIncidents()
suite.Error(err, "service should return error")
}
func (suite *IncidentServiceSuite) Test_Escalate_NoIncidents() {
incidents := []incident.IncidentEntity{}
suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
suite.severityService.GetSeverityEscalationMapMock.Return(nil, nil)
err := suite.incidentService.EscalateIncidents()
suite.NoError(err, "service should not return error")
}
func (suite *IncidentServiceSuite) Test_Escalate_SeverityMapNil() {
incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
suite.severityService.GetSeverityEscalationMapMock.Return(nil, nil)
err := suite.incidentService.EscalateIncidents()
suite.Error(err, "service should return error")
}
func (suite *IncidentServiceSuite) Test_Escalate_IncidentUpdateError() {
incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil)
suite.incidentRepository.FindIncidentByIdMock.Return(nil, errors.New("error"))
err := suite.incidentService.EscalateIncidents()
suite.Error(err, "service should return error")
}
func (suite *IncidentServiceSuite) Test_Escalate_Success() {
mockIncident := GetMockIncident()
mockUser := GetMockUser()
mockSeverity := GetMockSeverityWithId(suite.previousSeverityId)
mockUpdatedSeverity := GetMockSeverityWithId(suite.updatedSeverityId)
mockChannels := GetMockChannels()
krakatoaIncidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.incidentRepository.FindIncidentByChannelIdMock.When(suite.mockIncidentSlackChannelID).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailOrIDMock.Return(mockUser, nil)
suite.severityRepository.FindSeverityByIdMock.When(suite.previousSeverityId).
Then(mockSeverity, nil)
suite.severityRepository.FindSeverityByIdMock.When(suite.updatedSeverityId).
Then(mockUpdatedSeverity, nil)
suite.incidentChannelService.GetIncidentChannelsMock.When(mockIncident.ID).
Then(mockChannels, nil)
suite.incidentRepository.UpdateIncidentMock.When(mockIncident).
Then(nil)
suite.incidentRepository.UpdateIncidentWithJustificationMock.When(mockIncident, "").Then(nil)
suite.slackService.PostMessageByChannelIDMock.Return("", nil)
suite.teamRepository.FindTeamByIdMock.Return(GetMockTeamWithId(1), nil)
suite.incidentRepository.FindIncidentStatusByIdMock.Return(GetMockIncidentStatus(), nil)
suite.slackService.UpdateMessageWithAttachmentsMock.Return(nil)
suite.slackService.InviteUsersToConversationMock.Return(nil)
suite.slackService.GetConversationRepliesMock.Return([]slack.Message{}, false, "", nil)
suite.incidentRepository.UpsertIncidentRoleMock.Return(nil)
suite.slackService.SetChannelTopicByChannelIdMock.Return(nil)
suite.slackService.PostMessageWithAttachmentsMock.Return("", nil)
suite.slackService.PostMessageOptionMock.Return("", nil)
suite.incidentRepository.GetOpenIncidentsByCreatorIdForGivenTeamMock.Return(&krakatoaIncidents, nil)
suite.krakatoaService.ExecuteKrakatoaWorkflowMock.Return(nil)
tu := teamUserModel.TeamUserDTO{User: user.UserDTO{ID: 1, Name: "test", Email: "", SlackUserId: "1", Active: true}}
suite.teamUserService.GetTeamUsersForGivenSeverityMock.Return([]teamUserModel.TeamUserDTO{tu}, nil)
suite.teamUserService.GetTeamUsersWithMinimumSeverityIdLessThanOrEqualToGivenSeverityMock.Return([]teamUserModel.TeamUserDTO{tu}, nil)
incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()}
suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil)
suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil)
err := suite.incidentService.EscalateIncidents()
suite.NoError(err, "service should not return error")
}
func getMockSeverityEscalationMap() map[uint]*uint {
return map[uint]*uint{
2: uintPtr(1),
3: uintPtr(2),
4: uintPtr(3),
}
}
func uintPtr(i uint) *uint {
return &i
}

1
service/incident/impl/incident_constants.go
View File

@@ -8,4 +8,5 @@ const (
UPDATE_INCIDENT_METADATA = "update_incident_metadata"
DUPLICATE_INCIDENT = "duplicate_incident"
RESOLVE_INCIDENT = "resolve_incident"
ESCALATE_INCIDENT = "escalate_incident"
)

17
service/incident/impl/incident_service_test.go
View File

@@ -37,6 +37,7 @@ type IncidentServiceSuite struct {
tagService mocks.ITagServiceMock
calendarService mocks.ICalendarServiceMock
rcaService mocks.IRcaServiceMock
severityService mocks.ISeverityServiceMock
mockDirectory string
mockPngName string
mockCsvName string
@@ -71,7 +72,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_Success() {
suite.incidentRepository.FindIncidentByChannelIdMock.When(suite.mockIncidentSlackChannelID).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.When(suite.previousTeamId).
@@ -153,7 +154,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidTeam() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(nil, errors.New("record not found"))
@@ -175,7 +176,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidSeverity() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil)
@@ -200,7 +201,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidStatus() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil)
@@ -228,7 +229,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidChannel() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil)
@@ -259,7 +260,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_DBError() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil)
@@ -292,7 +293,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_SlackError() {
suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId).
Then(mockIncident, nil)
suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail).
suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail).
Then(mockUser, nil)
suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil)
@@ -496,6 +497,7 @@ func (suite *IncidentServiceSuite) SetupTest() {
suite.previousTeamId = 1
suite.updatedTeamId = 2
suite.teamUserService = *mocks.NewITeamUserServiceMock(suite.T())
suite.severityService = *mocks.NewISeverityServiceMock(suite.T())
suite.incidentService = &IncidentServiceV2{
db: nil,
slackService: &suite.slackService,
@@ -510,6 +512,7 @@ func (suite *IncidentServiceSuite) SetupTest() {
calendarService: &suite.calendarService,
rcaService: &suite.rcaService,
teamUserService: &suite.teamUserService,
severityService: &suite.severityService,
}
suite.mockDirectory = "test_file"

15
service/incident/impl/incident_service_v2.go
View File

@@ -49,6 +49,7 @@ import (
incidentRequest "houston/service/request/incident"
service "houston/service/response"
common "houston/service/response/common"
severityService "houston/service/severity"
severityServiceImpl "houston/service/severity/impl"
"houston/service/slack"
"houston/service/tag"
@@ -84,6 +85,7 @@ type IncidentServiceV2 struct {
teamServiceV2 teamService.ITeamServiceV2
alertService alertService.IAlertService
teamUserService teamUser2.ITeamUserService
severityService severityService.ISeverityService
}
/*
@@ -98,6 +100,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 {
incidentRepository := incident.NewIncidentRepository(
db, severityRepository, logRepository, teamRepository, slackService.SocketModeClientWrapper.GetClient(),
)
severityService := severityServiceImpl.NewSeverityService(severity.NewSeverityRepository(db))
incidentChannelService := incidentChannel.NewIncidentChannelService(db)
krakatoaService := krakatoa.NewKrakatoaService()
calendarActions := conference.GetCalendarActions()
@@ -109,7 +112,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 {
externalTeamRepo.NewExternalTeamRepository(db),
teamSeverityServiceImpl.NewTeamSeverityService(
teamSeverityRepo.NewTeamSeverityRepository(db),
severityServiceImpl.NewSeverityService(severity.NewSeverityRepository(db)),
severityService,
),
userService.NewUserService(user.NewUserRepository(db)),
teamUserServiceImpl.NewTeamUserService(teamUser.NewTeamUserRepository(db)),
@@ -134,6 +137,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 {
teamServiceV2: teamServiceV2,
alertService: alertService.NewAlertService(),
teamUserService: teamUserService,
severityService: severityService,
}
driveActions, _ := googleDrive.NewGoogleDriveActions()
incidentService.rcaService = rcaServiceImpl.NewRcaService(
@@ -1331,7 +1335,7 @@ func (i *IncidentServiceV2) FetchAllUpdateIncidentData(incidentId uint, userEmai
)
}
userInfo, err := i.slackService.GetUserByEmail(userEmail)
userInfo, err := i.slackService.GetUserByEmailOrID(userEmail)
if err != nil {
logger.Error(
fmt.Sprintf(
@@ -1576,8 +1580,13 @@ func (i *IncidentServiceV2) UpdateSeverityWorkflow(
waitGroup.Add(updateSeveritySlackActionCount)
go util.ExecuteConcurrentAction(&waitGroup, func() {
severityChangeMessage := fmt.Sprintf("<@%s> > set severity to %s", userId, severityEntity.Name)
if userId == viper.GetString("HOUSTON_BOT_SLACK_ID") {
severityChangeMessage = fmt.Sprintf("houston escalated incident to %s (%s)", severityEntity.Name, severityEntity.Description)
}
_, err := i.slackService.PostMessageByChannelID(
fmt.Sprintf("<@%s> > set severity to %s", userId, severityEntity.Name),
severityChangeMessage,
false,
incidentEntity.SlackChannel,
)

34
service/severity/impl/severity_service.go
View File

@@ -5,6 +5,7 @@ import (
"houston/logger"
"houston/model/severity"
"houston/service/dtoConverter"
"sort"
)
type SeverityService struct {
@@ -27,6 +28,15 @@ func (service *SeverityService) GetAllActiveSeverities() ([]severity.SeverityDTO
return dtoConverter.SeverityEntitiesToDTOs(*severityEntities), err
}
func (service *SeverityService) GetSeverityEscalationMap() (map[uint]*uint, error) {
severities, err := service.GetAllActiveSeverities()
if err != nil {
return nil, err
}
return service.createSeverityEscalationMap(severities), nil
}
func (service *SeverityService) FindSeverityById(severityId uint) (*severity.SeverityDTO, error) {
severityEntity, err := service.severityRepository.FindSeverityById(severityId)
if err != nil {
@@ -38,3 +48,27 @@ func (service *SeverityService) FindSeverityById(severityId uint) (*severity.Sev
return &severityDTO, err
}
func (service *SeverityService) createSeverityEscalationMap(severities []severity.SeverityDTO) map[uint]*uint {
sort.Slice(severities, func(i, j int) bool {
return severities[i].Priority > severities[j].Priority
})
escalationMap := make(map[uint]*uint)
for index, severity := range severities {
var nextSeverityID *uint
for nextIndex := index + 1; nextIndex < len(severities); nextIndex++ {
if severities[nextIndex].Priority < severity.Priority {
nextSeverityID = &severities[nextIndex].ID
break
}
}
if nextSeverityID != nil {
escalationMap[severity.ID] = nextSeverityID
}
}
return escalationMap
}

1
service/severity/severity_service_interface.go
View File

@@ -4,5 +4,6 @@ import "houston/model/severity"
type ISeverityService interface {
GetAllActiveSeverities() ([]severity.SeverityDTO, error)
GetSeverityEscalationMap() (map[uint]*uint, error)
FindSeverityById(severityId uint) (*severity.SeverityDTO, error)
}