diff --git a/cmd/app/handler/incident_handler.go b/cmd/app/handler/incident_handler.go index 7a25b9e..2f15c41 100644 --- a/cmd/app/handler/incident_handler.go +++ b/cmd/app/handler/incident_handler.go @@ -135,6 +135,18 @@ func (handler *IncidentHandler) HandleResolveIncident(c *gin.Context) { c.JSON(http.StatusOK, common.SuccessResponse("Incident resolved successfully", http.StatusOK)) } +func (handler *IncidentHandler) HandleEscalateIncident(c *gin.Context) { + err := handler.service.EscalateIncidents() + if err != nil { + logger.Error(fmt.Sprintf("%s failed to escalate incidents", logTag), zap.Error(err)) + metrics.PublishHoustonFlowFailureMetrics(incidentService.ESCALATE_INCIDENT, err.Error()) + c.JSON(http.StatusInternalServerError, common.ErrorResponse(err, http.StatusInternalServerError, nil)) + return + } + + c.JSON(http.StatusOK, common.SuccessResponse("Incidents escalated successfully", http.StatusOK)) +} + func (handler *IncidentHandler) HandleJiraLinking(c *gin.Context) { var linkJiraRequest request.LinkJiraRequest if err := c.ShouldBindJSON(&linkJiraRequest); err != nil { diff --git a/cmd/app/server.go b/cmd/app/server.go index c80365f..754f3d0 100644 --- a/cmd/app/server.go +++ b/cmd/app/server.go @@ -218,6 +218,7 @@ func (s *Server) incidentClientHandlerV2(houstonGroup *gin.RouterGroup) { s.gin.POST("/incidents", authService.IfValidHoustonUser(incidentHandler.HandleUpdateIncident)) houstonGroup.POST("/incidents", incidentHandler.HandleUpdateIncident) houstonGroup.POST("/incidents/resolve", authService.IfValidHoustonUser(incidentHandler.HandleResolveIncident)) + houstonGroup.POST("/incidents/escalate", incidentHandler.HandleEscalateIncident) houstonGroup.POST("/link-jira-to-incident", incidentHandler.HandleJiraLinking) houstonGroup.POST("/unlink-jira-from-incident", incidentHandler.HandleJiraUnLinking) houstonGroup.GET("/get-jira-statuses", incidentHandler.HandleGetJiraStatuses) diff --git a/internal/cron/cron.go b/internal/cron/cron.go index 0ddd60c..66f0803 100644 --- a/internal/cron/cron.go +++ b/internal/cron/cron.go @@ -1,32 +1,27 @@ package cron import ( - "errors" "fmt" "houston/common/metrics" "houston/logger" incidentService "houston/service/incident/impl" slackService "houston/service/slack" "sort" - "strconv" "strings" "sync" "time" - "houston/common/util" - "houston/internal/processor/action" - "houston/model/incident" - "houston/model/severity" - "houston/model/shedlock" - "houston/model/team" - "houston/model/user" - "houston/pkg/slackbot" - "github.com/slack-go/slack" "github.com/slack-go/slack/socketmode" "github.com/spf13/viper" "go.uber.org/zap" "gorm.io/gorm" + "houston/common/util" + "houston/model/incident" + "houston/model/severity" + "houston/model/shedlock" + "houston/model/team" + "houston/model/user" ) func RunJob( @@ -43,17 +38,8 @@ func RunJob( //HOUSTON ESCALATE shedlockConfig := NewLockerDbWithLockTime(viper.GetInt("cron.job.lock.default.time.in.sec")) - err := shedlockConfig.AddFun(viper.GetString("cron.job.name"), viper.GetString("cron.job.update.incident.interval"), shedlockRepository, func() { - RunJobWithExecutionMetrics(AUTO_ESCALATE, func() { - UpdateIncidentByCronJob(socketModeClient, db, incidentRepository, teamRepository, severityRepository, viper.GetString("cron.job.name")) - }) - }) - if err != nil { - logger.Error("HOUSTON_ESCALATE error: " + err.Error()) - } - //HOUSTON ADDING USER - err = shedlockConfig.AddFun(viper.GetString("cron.job.upsert_user"), viper.GetString("cron.job.upsert_user_interval"), shedlockRepository, func() { + err := shedlockConfig.AddFun(viper.GetString("cron.job.upsert_user"), viper.GetString("cron.job.upsert_user_interval"), shedlockRepository, func() { RunJobWithExecutionMetrics(UPSERT_USERS, func() { UpsertUsers(socketModeClient, userRepository) }) @@ -101,127 +87,6 @@ func RunJobWithExecutionMetrics(jobName string, jobFunc func()) { metrics.PublishCronJobExecutionCounterMetrics(jobName, duration) } -func UpdateIncidentByCronJob(socketModeClient *socketmode.Client, db *gorm.DB, incidentService *incident.Repository, teamService *team.Repository, severityService *severity.Repository, name string) { - - fmt.Println("Running job at", time.Now().Format(time.RFC3339)) - defer func() { - if r := recover(); r != nil { - logger.Error(fmt.Sprintf("Exception occurred in cron: %v", r.(error))) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, r.(error).Error()) - } - }() - //Do not need to fetch Severity every time, hence keeping a map outside. - incidentSeverityList, err := severityService.GetAllActiveSeverity() - if err != nil { - logger.Error("GetAllActiveSeverity error in cron Job", - zap.String("cron_name", name), zap.Error(err)) - panic(err) - return - } - - if incidentSeverityList == nil { - errMessage := "Error : No active severity found in cron job" - logger.Error(errMessage) - panic(errors.New(errMessage)) - return - } - - incidentStatusEntity, _ := incidentService.FindIncidentStatusByName(incident.Resolved) - //FETCH INCIDENTS WHICH ARE NOT RESOLVED AND CURRENT TIMESTAMP>=SEVERITY TAT FIELD - if incidentStatusEntity == nil || incidentStatusEntity.ID == 0 { - errMessage := "Error : RESOLVED Incident Status not found in cron job" - logger.Error(errMessage) - panic(errors.New(errMessage)) - return - } - - incidents, err := incidentService.FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0(db, int(incidentStatusEntity.ID)) - if err != nil { - logger.Error("FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCurrentDateAndNotSev0 error", - zap.Error(err)) - panic(err) - return - } - - if len(incidents) == 0 { - logger.Info("No incidents found to be updated by cron job") - } - - //Looping through All incidents which are not resolved and sev tat is breaching - for i := 0; i < len(incidents); i++ { - logger.Info("TAT breached Not Resolved Incident found. Name: " + incidents[i].IncidentName + ". Severity: " + strconv.Itoa(int(incidents[i].SeverityId))) - updatingSevForEachInc(incidents, i, incidentSeverityList, err, incidentService, socketModeClient, teamService, severityService) - } -} - -func updatingSevForEachInc(incidents []incident.IncidentEntity, i int, incidentSeverityList *[]severity.SeverityEntity, err error, incidentService *incident.Repository, socketModeClient *socketmode.Client, teamService *team.Repository, severityService *severity.Repository) { - - var currentSeverityId = incidents[i].SeverityId - var severityString string = "" - var usersToBeAdded = make([]string, 0) - - incidents[i].SeverityId = currentSeverityId - 1 - - //Find the severity object from list and update the tat of incident - for _, s := range *incidentSeverityList { - if s.ID == incidents[i].SeverityId { - incidents[i].SeverityTat = time.Now().AddDate(0, 0, s.Sla) - usersToBeAdded = s.SlackUserIds - severityString = fmt.Sprintln(s.Name + " (" + s.Description + ")") - } - } - - //Updating Incident - incidents[i].UpdatedAt = time.Now() - incidents[i].UpdatedBy = viper.GetString("HOUSTON_BOT_SLACK_ID") - err = incidentService.UpdateIncident(&incidents[i]) - if err != nil { - logger.Error("failed to update incident in cron job", - zap.String("channel", incidents[i].SlackChannel), zap.Error(err)) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - } - - //Default User Addition According To Sev Tat - for _, o := range usersToBeAdded { - //throws error if the customer is already present in channel - _, err := socketModeClient.Client.InviteUsersToConversation(incidents[i].SlackChannel, o) - if err != nil { - logger.Error("Slack Client InviteUsersToConversation error in cron job") - if err.Error() != util.AlreadyInChannelError { - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - } - } - } - - //UPDATING MESSAGE - s := action.NewIncidentChannelMessageUpdateAction(socketModeClient, incidentService, teamService, severityService) - slackbotClient := slackbot.NewSlackClient(socketModeClient) - team, err := teamService.FindTeamById(incidents[i].TeamId) - if err != nil { - logger.Error("error in fetching team by id", zap.Uint("teamId", incidents[i].TeamId), zap.Error(err), zap.String("s", severityString)) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - } - incidentSeverityEntity, err := severityService.FindSeverityById(incidents[i].SeverityId) - if err != nil { - logger.Error("error in finding severity entity", zap.Error(err), zap.Uint("severityId", incidents[i].SeverityId)) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - } - s.ProcessAction(incidents[i].SlackChannel) - msgOption := slack.MsgOptionText(fmt.Sprintf("houston escalated incident to %s", severityString), false) - _, _, err = socketModeClient.PostMessage(incidents[i].SlackChannel, msgOption) - if err != nil { - logger.Error("PostMessage failed for cronJob ", zap.Error(err), zap.Int("incidentId", int(incidents[i].ID))) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - } - topic := fmt.Sprintf("%s-%s(%s) Incident-%d | %s", team.Name, incidentSeverityEntity.Name, incidentSeverityEntity.Description, incidents[i].ID, incidents[i].Title) - _, err = slackbotClient.SetChannelTopic(incidents[i].SlackChannel, topic) - if err != nil { - logger.Error("Set Channel Topic failed for cronJob", zap.Error(err), zap.Int("incidentId", int(incidents[i].ID))) - metrics.PublishCronJobFailureMetrics(AUTO_ESCALATE, err.Error()) - - } -} - func IncidentMetricBlock(teamName *slack.HeaderBlock, incidentOpenCount, onCallAndManagerBlock *slack.SectionBlock) slack.Blocks { return slack.Blocks{ BlockSet: []slack.Block{ diff --git a/model/incident/incident.go b/model/incident/incident.go index 00d0ed0..eb359ff 100644 --- a/model/incident/incident.go +++ b/model/incident/incident.go @@ -661,6 +661,20 @@ func (r *Repository) FindIncidentsByNotResolvedStatusAndGreaterSeverityTatThanCu return incidentEntity, nil } +func (r *Repository) GetIncidentsForEscalation() ([]IncidentEntity, error) { + var incidentEntity []IncidentEntity + + currentTime := time.Now() + + result := r.gormClient.Find(&incidentEntity, "status IN (?) AND severity_id <> ? AND severity_tat <= ?", statusesForEscalation, 1, currentTime) + + if result.Error != nil { + return nil, result.Error + } + + return incidentEntity, nil +} + func (r *Repository) GetIncidentsByTeamIdAndNotResolvedAndOfSev0OrSev1OrSev2(team_id uint) (*[]IncidentEntity, error) { var incidentEntity []IncidentEntity @@ -818,6 +832,8 @@ func buildIntervalExpr(severityIdToArchivalHoursMap map[int]int) string { ) } +var statusesForEscalation = []uint{1, 2} + const ( SLA_BREACH_ALLOWED_STATUSES = "1,2" ) diff --git a/model/incident/incident_repository_interface.go b/model/incident/incident_repository_interface.go index 3f7e26a..5f9b590 100644 --- a/model/incident/incident_repository_interface.go +++ b/model/incident/incident_repository_interface.go @@ -36,6 +36,7 @@ type IIncidentRepository interface { GetIncidentTagsByIncidentId(incidentId uint) (*[]IncidentTagEntity, error) GetIncidentTagByTagId(incidentId uint, tagId uint) (*IncidentTagEntity, error) SaveIncidentTag(entity IncidentTagEntity) (*IncidentTagEntity, error) + GetIncidentsForEscalation() ([]IncidentEntity, error) GetIncidentsByTeamIdAndNotResolvedAndOfSev0OrSev1OrSev2(team_id uint) (*[]IncidentEntity, error) GetCountOfIncidentsByTeamIdAndNotResolved(team_id uint) (int, error) GetIncidentRoleByIncidentIdAndRole(incident_id uint, role string) (*IncidentRoleEntity, error) diff --git a/service/incident/impl/escalate_incident.go b/service/incident/impl/escalate_incident.go new file mode 100644 index 0000000..80bf014 --- /dev/null +++ b/service/incident/impl/escalate_incident.go @@ -0,0 +1,63 @@ +package impl + +import ( + "errors" + "fmt" + "github.com/spf13/viper" + "houston/logger" + incidentModel "houston/model/incident" + service "houston/service/request" + "strconv" +) + +const escalateLogTag = "[escalate]" + +func (i *IncidentServiceV2) EscalateIncidents() error { + logger.Info(fmt.Sprintf("%s received request for escalating incidents", escalateLogTag)) + + incidents, err := i.incidentRepository.GetIncidentsForEscalation() + if err != nil { + logger.Error(fmt.Sprintf("%s failed to get incidents for escalation %v", escalateLogTag, err)) + return err + } + + escalationMap, err := i.severityService.GetSeverityEscalationMap() + if err != nil { + logger.Error(fmt.Sprintf("%s failed to get severity escalation map %v", escalateLogTag, err)) + return err + } + + return i.runEscalationOnIncidents(incidents, escalationMap) +} + +func (i *IncidentServiceV2) runEscalationOnIncidents(incidents []incidentModel.IncidentEntity, escalationMap map[uint]*uint) error { + logger.Info(fmt.Sprintf("%s found %d incidents for escalation", escalateLogTag, len(incidents))) + + var incidentsWithFailures []string + + for _, incident := range incidents { + logger.Info(fmt.Sprintf("%s escalating incident with id: %d", escalateLogTag, incident.ID)) + nextSeverityID := escalationMap[incident.SeverityId] + if nextSeverityID == nil { + logger.Error(fmt.Sprintf("%s no escalation found for incident with id: %d", escalateLogTag, incident.ID)) + incidentsWithFailures = append(incidentsWithFailures, incident.IncidentName) + continue + } + + _, err := i.UpdateIncident(service.UpdateIncidentRequest{ + Id: incident.ID, + SeverityId: strconv.Itoa(int(*nextSeverityID)), + }, viper.GetString("HOUSTON_BOT_SLACK_ID")) + + if err != nil { + logger.Error(fmt.Sprintf("%s failed to escalate incident with id: %d %v", escalateLogTag, incident.ID, err)) + incidentsWithFailures = append(incidentsWithFailures, incident.IncidentName) + } + } + + if len(incidentsWithFailures) > 0 { + return errors.New(fmt.Sprintf("failed to escalate incidents: %v", incidentsWithFailures)) + } + + return nil +} diff --git a/service/incident/impl/escalate_incident_test.go b/service/incident/impl/escalate_incident_test.go new file mode 100644 index 0000000..c7b2e4f --- /dev/null +++ b/service/incident/impl/escalate_incident_test.go @@ -0,0 +1,122 @@ +package impl + +import ( + "errors" + "github.com/slack-go/slack" + "houston/model/incident" + teamUserModel "houston/model/teamUser" + "houston/model/user" +) + +func (suite *IncidentServiceSuite) Test_Escalate_GetIncidentsError() { + suite.incidentRepository.GetIncidentsForEscalationMock.Return(nil, errors.New("error")) + err := suite.incidentService.EscalateIncidents() + suite.Error(err, "service should return error") +} + +func (suite *IncidentServiceSuite) Test_Escalate_GetSeverityEscalationMapError() { + incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()} + suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil) + suite.severityService.GetSeverityEscalationMapMock.Return(nil, errors.New("error")) + err := suite.incidentService.EscalateIncidents() + suite.Error(err, "service should return error") +} + +func (suite *IncidentServiceSuite) Test_Escalate_NoIncidents() { + incidents := []incident.IncidentEntity{} + suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil) + suite.severityService.GetSeverityEscalationMapMock.Return(nil, nil) + err := suite.incidentService.EscalateIncidents() + suite.NoError(err, "service should not return error") +} + +func (suite *IncidentServiceSuite) Test_Escalate_SeverityMapNil() { + incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()} + suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil) + suite.severityService.GetSeverityEscalationMapMock.Return(nil, nil) + err := suite.incidentService.EscalateIncidents() + suite.Error(err, "service should return error") +} + +func (suite *IncidentServiceSuite) Test_Escalate_IncidentUpdateError() { + incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()} + suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil) + suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil) + suite.incidentRepository.FindIncidentByIdMock.Return(nil, errors.New("error")) + err := suite.incidentService.EscalateIncidents() + suite.Error(err, "service should return error") +} + +func (suite *IncidentServiceSuite) Test_Escalate_Success() { + mockIncident := GetMockIncident() + mockUser := GetMockUser() + mockSeverity := GetMockSeverityWithId(suite.previousSeverityId) + mockUpdatedSeverity := GetMockSeverityWithId(suite.updatedSeverityId) + mockChannels := GetMockChannels() + krakatoaIncidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()} + + suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). + Then(mockIncident, nil) + + suite.incidentRepository.FindIncidentByChannelIdMock.When(suite.mockIncidentSlackChannelID). + Then(mockIncident, nil) + + suite.slackService.GetUserByEmailOrIDMock.Return(mockUser, nil) + + suite.severityRepository.FindSeverityByIdMock.When(suite.previousSeverityId). + Then(mockSeverity, nil) + + suite.severityRepository.FindSeverityByIdMock.When(suite.updatedSeverityId). + Then(mockUpdatedSeverity, nil) + + suite.incidentChannelService.GetIncidentChannelsMock.When(mockIncident.ID). + Then(mockChannels, nil) + + suite.incidentRepository.UpdateIncidentMock.When(mockIncident). + Then(nil) + suite.incidentRepository.UpdateIncidentWithJustificationMock.When(mockIncident, "").Then(nil) + suite.slackService.PostMessageByChannelIDMock.Return("", nil) + suite.teamRepository.FindTeamByIdMock.Return(GetMockTeamWithId(1), nil) + suite.incidentRepository.FindIncidentStatusByIdMock.Return(GetMockIncidentStatus(), nil) + + suite.slackService.UpdateMessageWithAttachmentsMock.Return(nil) + + suite.slackService.InviteUsersToConversationMock.Return(nil) + + suite.slackService.GetConversationRepliesMock.Return([]slack.Message{}, false, "", nil) + + suite.incidentRepository.UpsertIncidentRoleMock.Return(nil) + + suite.slackService.SetChannelTopicByChannelIdMock.Return(nil) + + suite.slackService.PostMessageWithAttachmentsMock.Return("", nil) + + suite.slackService.PostMessageOptionMock.Return("", nil) + + suite.incidentRepository.GetOpenIncidentsByCreatorIdForGivenTeamMock.Return(&krakatoaIncidents, nil) + + suite.krakatoaService.ExecuteKrakatoaWorkflowMock.Return(nil) + tu := teamUserModel.TeamUserDTO{User: user.UserDTO{ID: 1, Name: "test", Email: "", SlackUserId: "1", Active: true}} + + suite.teamUserService.GetTeamUsersForGivenSeverityMock.Return([]teamUserModel.TeamUserDTO{tu}, nil) + suite.teamUserService.GetTeamUsersWithMinimumSeverityIdLessThanOrEqualToGivenSeverityMock.Return([]teamUserModel.TeamUserDTO{tu}, nil) + + incidents := []incident.IncidentEntity{*GetMockIncident(), *GetMockIncident()} + suite.incidentRepository.GetIncidentsForEscalationMock.Return(incidents, nil) + suite.severityService.GetSeverityEscalationMapMock.Return(getMockSeverityEscalationMap(), nil) + + err := suite.incidentService.EscalateIncidents() + suite.NoError(err, "service should not return error") +} + +func getMockSeverityEscalationMap() map[uint]*uint { + return map[uint]*uint{ + 2: uintPtr(1), + 3: uintPtr(2), + 4: uintPtr(3), + } +} + +func uintPtr(i uint) *uint { + return &i +} diff --git a/service/incident/impl/incident_constants.go b/service/incident/impl/incident_constants.go index ebe204d..b2d50ec 100644 --- a/service/incident/impl/incident_constants.go +++ b/service/incident/impl/incident_constants.go @@ -8,4 +8,5 @@ const ( UPDATE_INCIDENT_METADATA = "update_incident_metadata" DUPLICATE_INCIDENT = "duplicate_incident" RESOLVE_INCIDENT = "resolve_incident" + ESCALATE_INCIDENT = "escalate_incident" ) diff --git a/service/incident/impl/incident_service_test.go b/service/incident/impl/incident_service_test.go index cc1bb19..6120a45 100644 --- a/service/incident/impl/incident_service_test.go +++ b/service/incident/impl/incident_service_test.go @@ -37,6 +37,7 @@ type IncidentServiceSuite struct { tagService mocks.ITagServiceMock calendarService mocks.ICalendarServiceMock rcaService mocks.IRcaServiceMock + severityService mocks.ISeverityServiceMock mockDirectory string mockPngName string mockCsvName string @@ -71,7 +72,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_Success() { suite.incidentRepository.FindIncidentByChannelIdMock.When(suite.mockIncidentSlackChannelID). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.When(suite.previousTeamId). @@ -153,7 +154,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidTeam() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(nil, errors.New("record not found")) @@ -175,7 +176,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidSeverity() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil) @@ -200,7 +201,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidStatus() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil) @@ -228,7 +229,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_InvalidChannel() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil) @@ -259,7 +260,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_DBError() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil) @@ -292,7 +293,7 @@ func (suite *IncidentServiceSuite) Test_UpdateIncident_SlackError() { suite.incidentRepository.FindIncidentByIdMock.When(suite.mockIncidentId). Then(mockIncident, nil) - suite.slackService.GetUserByEmailMock.When(suite.mockUserEmail). + suite.slackService.GetUserByEmailOrIDMock.When(suite.mockUserEmail). Then(mockUser, nil) suite.teamRepository.FindTeamByIdMock.Return(mockTeam, nil) @@ -496,6 +497,7 @@ func (suite *IncidentServiceSuite) SetupTest() { suite.previousTeamId = 1 suite.updatedTeamId = 2 suite.teamUserService = *mocks.NewITeamUserServiceMock(suite.T()) + suite.severityService = *mocks.NewISeverityServiceMock(suite.T()) suite.incidentService = &IncidentServiceV2{ db: nil, slackService: &suite.slackService, @@ -510,6 +512,7 @@ func (suite *IncidentServiceSuite) SetupTest() { calendarService: &suite.calendarService, rcaService: &suite.rcaService, teamUserService: &suite.teamUserService, + severityService: &suite.severityService, } suite.mockDirectory = "test_file" diff --git a/service/incident/impl/incident_service_v2.go b/service/incident/impl/incident_service_v2.go index 6de7b1d..639d709 100644 --- a/service/incident/impl/incident_service_v2.go +++ b/service/incident/impl/incident_service_v2.go @@ -49,6 +49,7 @@ import ( incidentRequest "houston/service/request/incident" service "houston/service/response" common "houston/service/response/common" + severityService "houston/service/severity" severityServiceImpl "houston/service/severity/impl" "houston/service/slack" "houston/service/tag" @@ -84,6 +85,7 @@ type IncidentServiceV2 struct { teamServiceV2 teamService.ITeamServiceV2 alertService alertService.IAlertService teamUserService teamUser2.ITeamUserService + severityService severityService.ISeverityService } /* @@ -98,6 +100,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 { incidentRepository := incident.NewIncidentRepository( db, severityRepository, logRepository, teamRepository, slackService.SocketModeClientWrapper.GetClient(), ) + severityService := severityServiceImpl.NewSeverityService(severity.NewSeverityRepository(db)) incidentChannelService := incidentChannel.NewIncidentChannelService(db) krakatoaService := krakatoa.NewKrakatoaService() calendarActions := conference.GetCalendarActions() @@ -109,7 +112,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 { externalTeamRepo.NewExternalTeamRepository(db), teamSeverityServiceImpl.NewTeamSeverityService( teamSeverityRepo.NewTeamSeverityRepository(db), - severityServiceImpl.NewSeverityService(severity.NewSeverityRepository(db)), + severityService, ), userService.NewUserService(user.NewUserRepository(db)), teamUserServiceImpl.NewTeamUserService(teamUser.NewTeamUserRepository(db)), @@ -134,6 +137,7 @@ func NewIncidentServiceV2(db *gorm.DB) *IncidentServiceV2 { teamServiceV2: teamServiceV2, alertService: alertService.NewAlertService(), teamUserService: teamUserService, + severityService: severityService, } driveActions, _ := googleDrive.NewGoogleDriveActions() incidentService.rcaService = rcaServiceImpl.NewRcaService( @@ -1331,7 +1335,7 @@ func (i *IncidentServiceV2) FetchAllUpdateIncidentData(incidentId uint, userEmai ) } - userInfo, err := i.slackService.GetUserByEmail(userEmail) + userInfo, err := i.slackService.GetUserByEmailOrID(userEmail) if err != nil { logger.Error( fmt.Sprintf( @@ -1576,8 +1580,13 @@ func (i *IncidentServiceV2) UpdateSeverityWorkflow( waitGroup.Add(updateSeveritySlackActionCount) go util.ExecuteConcurrentAction(&waitGroup, func() { + severityChangeMessage := fmt.Sprintf("<@%s> > set severity to %s", userId, severityEntity.Name) + if userId == viper.GetString("HOUSTON_BOT_SLACK_ID") { + severityChangeMessage = fmt.Sprintf("houston escalated incident to %s (%s)", severityEntity.Name, severityEntity.Description) + } + _, err := i.slackService.PostMessageByChannelID( - fmt.Sprintf("<@%s> > set severity to %s", userId, severityEntity.Name), + severityChangeMessage, false, incidentEntity.SlackChannel, ) diff --git a/service/severity/impl/severity_service.go b/service/severity/impl/severity_service.go index 906dc17..ad8a944 100644 --- a/service/severity/impl/severity_service.go +++ b/service/severity/impl/severity_service.go @@ -5,6 +5,7 @@ import ( "houston/logger" "houston/model/severity" "houston/service/dtoConverter" + "sort" ) type SeverityService struct { @@ -27,6 +28,15 @@ func (service *SeverityService) GetAllActiveSeverities() ([]severity.SeverityDTO return dtoConverter.SeverityEntitiesToDTOs(*severityEntities), err } +func (service *SeverityService) GetSeverityEscalationMap() (map[uint]*uint, error) { + severities, err := service.GetAllActiveSeverities() + if err != nil { + return nil, err + } + + return service.createSeverityEscalationMap(severities), nil +} + func (service *SeverityService) FindSeverityById(severityId uint) (*severity.SeverityDTO, error) { severityEntity, err := service.severityRepository.FindSeverityById(severityId) if err != nil { @@ -38,3 +48,27 @@ func (service *SeverityService) FindSeverityById(severityId uint) (*severity.Sev return &severityDTO, err } + +func (service *SeverityService) createSeverityEscalationMap(severities []severity.SeverityDTO) map[uint]*uint { + sort.Slice(severities, func(i, j int) bool { + return severities[i].Priority > severities[j].Priority + }) + + escalationMap := make(map[uint]*uint) + + for index, severity := range severities { + var nextSeverityID *uint + for nextIndex := index + 1; nextIndex < len(severities); nextIndex++ { + if severities[nextIndex].Priority < severity.Priority { + nextSeverityID = &severities[nextIndex].ID + break + } + } + + if nextSeverityID != nil { + escalationMap[severity.ID] = nextSeverityID + } + } + + return escalationMap +} diff --git a/service/severity/severity_service_interface.go b/service/severity/severity_service_interface.go index bc58b5d..cb1b6be 100644 --- a/service/severity/severity_service_interface.go +++ b/service/severity/severity_service_interface.go @@ -4,5 +4,6 @@ import "houston/model/severity" type ISeverityService interface { GetAllActiveSeverities() ([]severity.SeverityDTO, error) + GetSeverityEscalationMap() (map[uint]*uint, error) FindSeverityById(severityId uint) (*severity.SeverityDTO, error) }